_ Git - rust-lightning/blob - lightning/src/ln/reload_tests.rs

   1 // This file is Copyright its original authors, visible in version control
   2 // history.
   3 //
   4 // This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
   5 // or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
   6 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
   7 // You may not use this file except in accordance with one or both of these
   8 // licenses.
   9
  10 //! Functional tests which test for correct behavior across node restarts.
  11
  12 use crate::chain::{ChannelMonitorUpdateStatus, Watch};
  13 use crate::chain::chaininterface::LowerBoundedFeeEstimator;
  14 use crate::chain::channelmonitor::{CLOSED_CHANNEL_UPDATE_ID, ChannelMonitor};
  15 use crate::sign::EntropySource;
  16 use crate::chain::transaction::OutPoint;
  17 use crate::events::{ClosureReason, Event, HTLCDestination, MessageSendEvent, MessageSendEventsProvider};
  18 use crate::ln::channelmanager::{ChannelManager, ChannelManagerReadArgs, PaymentId, RecipientOnionFields};
  19 use crate::ln::msgs;
  20 use crate::ln::msgs::{ChannelMessageHandler, RoutingMessageHandler, ErrorAction};
  21 use crate::util::test_channel_signer::TestChannelSigner;
  22 use crate::util::test_utils;
  23 use crate::util::errors::APIError;
  24 use crate::util::ser::{Writeable, ReadableArgs};
  25 use crate::util::config::UserConfig;
  26 use crate::util::string::UntrustedString;
  27
  28 use bitcoin::hash_types::BlockHash;
  29
  30 use crate::prelude::*;
  31 use core::default::Default;
  32 use crate::sync::Mutex;
  33
  34 use crate::ln::functional_test_utils::*;
  35
  36 #[test]
  37 fn test_funding_peer_disconnect() {
  38         // Test that we can lock in our funding tx while disconnected
  39         let chanmon_cfgs = create_chanmon_cfgs(2);
  40         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
  41         let persister;
  42         let new_chain_monitor;
  43
  44         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
  45         let nodes_0_deserialized;
  46         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
  47         let tx = create_chan_between_nodes_with_value_init(&nodes[0], &nodes[1], 100000, 10001);
  48
  49         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id());
  50         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id());
  51
  52         confirm_transaction(&nodes[0], &tx);
  53         let events_1 = nodes[0].node.get_and_clear_pending_msg_events();
  54         assert!(events_1.is_empty());
  55
  56         let mut reconnect_args = ReconnectArgs::new(&nodes[0], &nodes[1]);
  57         reconnect_args.send_channel_ready.1 = true;
  58         reconnect_nodes(reconnect_args);
  59
  60         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id());
  61         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id());
  62
  63         confirm_transaction(&nodes[1], &tx);
  64         let events_2 = nodes[1].node.get_and_clear_pending_msg_events();
  65         assert!(events_2.is_empty());
  66
  67         nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init {
  68                 features: nodes[1].node.init_features(), networks: None, remote_network_address: None
  69         }, true).unwrap();
  70         let as_reestablish = get_chan_reestablish_msgs!(nodes[0], nodes[1]).pop().unwrap();
  71         nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init {
  72                 features: nodes[0].node.init_features(), networks: None, remote_network_address: None
  73         }, false).unwrap();
  74         let bs_reestablish = get_chan_reestablish_msgs!(nodes[1], nodes[0]).pop().unwrap();
  75
  76         // nodes[0] hasn't yet received a channel_ready, so it only sends that on reconnect.
  77         nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &bs_reestablish);
  78         let events_3 = nodes[0].node.get_and_clear_pending_msg_events();
  79         assert_eq!(events_3.len(), 1);
  80         let as_channel_ready = match events_3[0] {
  81                 MessageSendEvent::SendChannelReady { ref node_id, ref msg } => {
  82                         assert_eq!(*node_id, nodes[1].node.get_our_node_id());
  83                         msg.clone()
  84                 },
  85                 _ => panic!("Unexpected event {:?}", events_3[0]),
  86         };
  87
  88         // nodes[1] received nodes[0]'s channel_ready on the first reconnect above, so it should send
  89         // announcement_signatures as well as channel_update.
  90         nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &as_reestablish);
  91         let events_4 = nodes[1].node.get_and_clear_pending_msg_events();
  92         assert_eq!(events_4.len(), 3);
  93         let chan_id;
  94         let bs_channel_ready = match events_4[0] {
  95                 MessageSendEvent::SendChannelReady { ref node_id, ref msg } => {
  96                         assert_eq!(*node_id, nodes[0].node.get_our_node_id());
  97                         chan_id = msg.channel_id;
  98                         msg.clone()
  99                 },
 100                 _ => panic!("Unexpected event {:?}", events_4[0]),
 101         };
 102         let bs_announcement_sigs = match events_4[1] {
 103                 MessageSendEvent::SendAnnouncementSignatures { ref node_id, ref msg } => {
 104                         assert_eq!(*node_id, nodes[0].node.get_our_node_id());
 105                         msg.clone()
 106                 },
 107                 _ => panic!("Unexpected event {:?}", events_4[1]),
 108         };
 109         match events_4[2] {
 110                 MessageSendEvent::SendChannelUpdate { ref node_id, msg: _ } => {
 111                         assert_eq!(*node_id, nodes[0].node.get_our_node_id());
 112                 },
 113                 _ => panic!("Unexpected event {:?}", events_4[2]),
 114         }
 115
 116         // Re-deliver nodes[0]'s channel_ready, which nodes[1] can safely ignore. It currently
 117         // generates a duplicative private channel_update
 118         nodes[1].node.handle_channel_ready(&nodes[0].node.get_our_node_id(), &as_channel_ready);
 119         let events_5 = nodes[1].node.get_and_clear_pending_msg_events();
 120         assert_eq!(events_5.len(), 1);
 121         match events_5[0] {
 122                 MessageSendEvent::SendChannelUpdate { ref node_id, msg: _ } => {
 123                         assert_eq!(*node_id, nodes[0].node.get_our_node_id());
 124                 },
 125                 _ => panic!("Unexpected event {:?}", events_5[0]),
 126         };
 127
 128         // When we deliver nodes[1]'s channel_ready, however, nodes[0] will generate its
 129         // announcement_signatures.
 130         nodes[0].node.handle_channel_ready(&nodes[1].node.get_our_node_id(), &bs_channel_ready);
 131         let events_6 = nodes[0].node.get_and_clear_pending_msg_events();
 132         assert_eq!(events_6.len(), 1);
 133         let as_announcement_sigs = match events_6[0] {
 134                 MessageSendEvent::SendAnnouncementSignatures { ref node_id, ref msg } => {
 135                         assert_eq!(*node_id, nodes[1].node.get_our_node_id());
 136                         msg.clone()
 137                 },
 138                 _ => panic!("Unexpected event {:?}", events_6[0]),
 139         };
 140         expect_channel_ready_event(&nodes[0], &nodes[1].node.get_our_node_id());
 141         expect_channel_ready_event(&nodes[1], &nodes[0].node.get_our_node_id());
 142
 143         // When we deliver nodes[1]'s announcement_signatures to nodes[0], nodes[0] should immediately
 144         // broadcast the channel announcement globally, as well as re-send its (now-public)
 145         // channel_update.
 146         nodes[0].node.handle_announcement_signatures(&nodes[1].node.get_our_node_id(), &bs_announcement_sigs);
 147         let events_7 = nodes[0].node.get_and_clear_pending_msg_events();
 148         assert_eq!(events_7.len(), 1);
 149         let (chan_announcement, as_update) = match events_7[0] {
 150                 MessageSendEvent::BroadcastChannelAnnouncement { ref msg, ref update_msg } => {
 151                         (msg.clone(), update_msg.clone().unwrap())
 152                 },
 153                 _ => panic!("Unexpected event {:?}", events_7[0]),
 154         };
 155
 156         // Finally, deliver nodes[0]'s announcement_signatures to nodes[1] and make sure it creates the
 157         // same channel_announcement.
 158         nodes[1].node.handle_announcement_signatures(&nodes[0].node.get_our_node_id(), &as_announcement_sigs);
 159         let events_8 = nodes[1].node.get_and_clear_pending_msg_events();
 160         assert_eq!(events_8.len(), 1);
 161         let bs_update = match events_8[0] {
 162                 MessageSendEvent::BroadcastChannelAnnouncement { ref msg, ref update_msg } => {
 163                         assert_eq!(*msg, chan_announcement);
 164                         update_msg.clone().unwrap()
 165                 },
 166                 _ => panic!("Unexpected event {:?}", events_8[0]),
 167         };
 168
 169         // Provide the channel announcement and public updates to the network graph
 170         nodes[0].gossip_sync.handle_channel_announcement(&chan_announcement).unwrap();
 171         nodes[0].gossip_sync.handle_channel_update(&bs_update).unwrap();
 172         nodes[0].gossip_sync.handle_channel_update(&as_update).unwrap();
 173
 174         let (route, _, _, _) = get_route_and_payment_hash!(nodes[0], nodes[1], 1000000);
 175         let payment_preimage = send_along_route(&nodes[0], route, &[&nodes[1]], 1000000).0;
 176         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage);
 177
 178         // Check that after deserialization and reconnection we can still generate an identical
 179         // channel_announcement from the cached signatures.
 180         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id());
 181
 182         let chan_0_monitor_serialized = get_monitor!(nodes[0], chan_id).encode();
 183
 184         reload_node!(nodes[0], &nodes[0].node.encode(), &[&chan_0_monitor_serialized], persister, new_chain_monitor, nodes_0_deserialized);
 185
 186         reconnect_nodes(ReconnectArgs::new(&nodes[0], &nodes[1]));
 187 }
 188
 189 #[test]
 190 fn test_no_txn_manager_serialize_deserialize() {
 191         let chanmon_cfgs = create_chanmon_cfgs(2);
 192         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
 193         let persister;
 194         let new_chain_monitor;
 195
 196         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
 197         let nodes_0_deserialized;
 198         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
 199
 200         let tx = create_chan_between_nodes_with_value_init(&nodes[0], &nodes[1], 100000, 10001);
 201
 202         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id());
 203
 204         let chan_0_monitor_serialized =
 205                 get_monitor!(nodes[0], OutPoint { txid: tx.txid(), index: 0 }.to_channel_id()).encode();
 206         reload_node!(nodes[0], nodes[0].node.encode(), &[&chan_0_monitor_serialized], persister, new_chain_monitor, nodes_0_deserialized);
 207
 208         nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init {
 209                 features: nodes[1].node.init_features(), networks: None, remote_network_address: None
 210         }, true).unwrap();
 211         let reestablish_1 = get_chan_reestablish_msgs!(nodes[0], nodes[1]);
 212         nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init {
 213                 features: nodes[0].node.init_features(), networks: None, remote_network_address: None
 214         }, false).unwrap();
 215         let reestablish_2 = get_chan_reestablish_msgs!(nodes[1], nodes[0]);
 216
 217         nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &reestablish_1[0]);
 218         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 219         nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &reestablish_2[0]);
 220         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 221
 222         let (channel_ready, _) = create_chan_between_nodes_with_value_confirm(&nodes[0], &nodes[1], &tx);
 223         let (announcement, as_update, bs_update) = create_chan_between_nodes_with_value_b(&nodes[0], &nodes[1], &channel_ready);
 224         for node in nodes.iter() {
 225                 assert!(node.gossip_sync.handle_channel_announcement(&announcement).unwrap());
 226                 node.gossip_sync.handle_channel_update(&as_update).unwrap();
 227                 node.gossip_sync.handle_channel_update(&bs_update).unwrap();
 228         }
 229
 230         send_payment(&nodes[0], &[&nodes[1]], 1000000);
 231 }
 232
 233 #[test]
 234 fn test_manager_serialize_deserialize_events() {
 235         // This test makes sure the events field in ChannelManager survives de/serialization
 236         let chanmon_cfgs = create_chanmon_cfgs(2);
 237         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
 238         let persister;
 239         let new_chain_monitor;
 240
 241         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
 242         let nodes_0_deserialized;
 243         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
 244
 245         // Start creating a channel, but stop right before broadcasting the funding transaction
 246         let channel_value = 100000;
 247         let push_msat = 10001;
 248         let node_a = nodes.remove(0);
 249         let node_b = nodes.remove(0);
 250         node_a.node.create_channel(node_b.node.get_our_node_id(), channel_value, push_msat, 42, None, None).unwrap();
 251         node_b.node.handle_open_channel(&node_a.node.get_our_node_id(), &get_event_msg!(node_a, MessageSendEvent::SendOpenChannel, node_b.node.get_our_node_id()));
 252         node_a.node.handle_accept_channel(&node_b.node.get_our_node_id(), &get_event_msg!(node_b, MessageSendEvent::SendAcceptChannel, node_a.node.get_our_node_id()));
 253
 254         let (temporary_channel_id, tx, funding_output) = create_funding_transaction(&node_a, &node_b.node.get_our_node_id(), channel_value, 42);
 255
 256         node_a.node.funding_transaction_generated(&temporary_channel_id, &node_b.node.get_our_node_id(), tx.clone()).unwrap();
 257         check_added_monitors!(node_a, 0);
 258
 259         node_b.node.handle_funding_created(&node_a.node.get_our_node_id(), &get_event_msg!(node_a, MessageSendEvent::SendFundingCreated, node_b.node.get_our_node_id()));
 260         {
 261                 let mut added_monitors = node_b.chain_monitor.added_monitors.lock().unwrap();
 262                 assert_eq!(added_monitors.len(), 1);
 263                 assert_eq!(added_monitors[0].0, funding_output);
 264                 added_monitors.clear();
 265         }
 266
 267         let bs_funding_signed = get_event_msg!(node_b, MessageSendEvent::SendFundingSigned, node_a.node.get_our_node_id());
 268         node_a.node.handle_funding_signed(&node_b.node.get_our_node_id(), &bs_funding_signed);
 269         {
 270                 let mut added_monitors = node_a.chain_monitor.added_monitors.lock().unwrap();
 271                 assert_eq!(added_monitors.len(), 1);
 272                 assert_eq!(added_monitors[0].0, funding_output);
 273                 added_monitors.clear();
 274         }
 275         // Normally, this is where node_a would broadcast the funding transaction, but the test de/serializes first instead
 276
 277         expect_channel_pending_event(&node_a, &node_b.node.get_our_node_id());
 278         expect_channel_pending_event(&node_b, &node_a.node.get_our_node_id());
 279
 280         nodes.push(node_a);
 281         nodes.push(node_b);
 282
 283         // Start the de/seriailization process mid-channel creation to check that the channel manager will hold onto events that are serialized
 284         let chan_0_monitor_serialized = get_monitor!(nodes[0], bs_funding_signed.channel_id).encode();
 285         reload_node!(nodes[0], nodes[0].node.encode(), &[&chan_0_monitor_serialized], persister, new_chain_monitor, nodes_0_deserialized);
 286
 287         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id());
 288
 289         // After deserializing, make sure the funding_transaction is still held by the channel manager
 290         let events_4 = nodes[0].node.get_and_clear_pending_events();
 291         assert_eq!(events_4.len(), 0);
 292         assert_eq!(nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().len(), 1);
 293         assert_eq!(nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap()[0].txid(), funding_output.txid);
 294
 295         // Make sure the channel is functioning as though the de/serialization never happened
 296         assert_eq!(nodes[0].node.list_channels().len(), 1);
 297
 298         nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init {
 299                 features: nodes[1].node.init_features(), networks: None, remote_network_address: None
 300         }, true).unwrap();
 301         let reestablish_1 = get_chan_reestablish_msgs!(nodes[0], nodes[1]);
 302         nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init {
 303                 features: nodes[0].node.init_features(), networks: None, remote_network_address: None
 304         }, false).unwrap();
 305         let reestablish_2 = get_chan_reestablish_msgs!(nodes[1], nodes[0]);
 306
 307         nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &reestablish_1[0]);
 308         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 309         nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &reestablish_2[0]);
 310         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 311
 312         let (channel_ready, _) = create_chan_between_nodes_with_value_confirm(&nodes[0], &nodes[1], &tx);
 313         let (announcement, as_update, bs_update) = create_chan_between_nodes_with_value_b(&nodes[0], &nodes[1], &channel_ready);
 314         for node in nodes.iter() {
 315                 assert!(node.gossip_sync.handle_channel_announcement(&announcement).unwrap());
 316                 node.gossip_sync.handle_channel_update(&as_update).unwrap();
 317                 node.gossip_sync.handle_channel_update(&bs_update).unwrap();
 318         }
 319
 320         send_payment(&nodes[0], &[&nodes[1]], 1000000);
 321 }
 322
 323 #[test]
 324 fn test_simple_manager_serialize_deserialize() {
 325         let chanmon_cfgs = create_chanmon_cfgs(2);
 326         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
 327         let persister;
 328         let new_chain_monitor;
 329
 330         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
 331         let nodes_0_deserialized;
 332         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
 333         let chan_id = create_announced_chan_between_nodes(&nodes, 0, 1).2;
 334
 335         let (our_payment_preimage, ..) = route_payment(&nodes[0], &[&nodes[1]], 1000000);
 336         let (_, our_payment_hash, ..) = route_payment(&nodes[0], &[&nodes[1]], 1000000);
 337
 338         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id());
 339
 340         let chan_0_monitor_serialized = get_monitor!(nodes[0], chan_id).encode();
 341         reload_node!(nodes[0], nodes[0].node.encode(), &[&chan_0_monitor_serialized], persister, new_chain_monitor, nodes_0_deserialized);
 342
 343         reconnect_nodes(ReconnectArgs::new(&nodes[0], &nodes[1]));
 344
 345         fail_payment(&nodes[0], &[&nodes[1]], our_payment_hash);
 346         claim_payment(&nodes[0], &[&nodes[1]], our_payment_preimage);
 347 }
 348
 349 #[test]
 350 fn test_manager_serialize_deserialize_inconsistent_monitor() {
 351         // Test deserializing a ChannelManager with an out-of-date ChannelMonitor
 352         let chanmon_cfgs = create_chanmon_cfgs(4);
 353         let node_cfgs = create_node_cfgs(4, &chanmon_cfgs);
 354         let logger;
 355         let fee_estimator;
 356         let persister;
 357         let new_chain_monitor;
 358
 359         let node_chanmgrs = create_node_chanmgrs(4, &node_cfgs, &[None, None, None, None]);
 360         let nodes_0_deserialized;
 361         let mut nodes = create_network(4, &node_cfgs, &node_chanmgrs);
 362
 363         let chan_id_1 = create_announced_chan_between_nodes(&nodes, 0, 1).2;
 364         let chan_id_2 = create_announced_chan_between_nodes(&nodes, 2, 0).2;
 365         let (_, _, channel_id, funding_tx) = create_announced_chan_between_nodes(&nodes, 0, 3);
 366
 367         let mut node_0_stale_monitors_serialized = Vec::new();
 368         for chan_id_iter in &[chan_id_1, chan_id_2, channel_id] {
 369                 let mut writer = test_utils::TestVecWriter(Vec::new());
 370                 get_monitor!(nodes[0], chan_id_iter).write(&mut writer).unwrap();
 371                 node_0_stale_monitors_serialized.push(writer.0);
 372         }
 373
 374         let (our_payment_preimage, ..) = route_payment(&nodes[2], &[&nodes[0], &nodes[1]], 1000000);
 375
 376         // Serialize the ChannelManager here, but the monitor we keep up-to-date
 377         let nodes_0_serialized = nodes[0].node.encode();
 378
 379         route_payment(&nodes[0], &[&nodes[3]], 1000000);
 380         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id());
 381         nodes[2].node.peer_disconnected(&nodes[0].node.get_our_node_id());
 382         nodes[3].node.peer_disconnected(&nodes[0].node.get_our_node_id());
 383
 384         // Now the ChannelMonitor (which is now out-of-sync with ChannelManager for channel w/
 385         // nodes[3])
 386         let mut node_0_monitors_serialized = Vec::new();
 387         for chan_id_iter in &[chan_id_1, chan_id_2, channel_id] {
 388                 node_0_monitors_serialized.push(get_monitor!(nodes[0], chan_id_iter).encode());
 389         }
 390
 391         logger = test_utils::TestLogger::new();
 392         fee_estimator = test_utils::TestFeeEstimator { sat_per_kw: Mutex::new(253) };
 393         persister = test_utils::TestPersister::new();
 394         let keys_manager = &chanmon_cfgs[0].keys_manager;
 395         new_chain_monitor = test_utils::TestChainMonitor::new(Some(nodes[0].chain_source), nodes[0].tx_broadcaster, &logger, &fee_estimator, &persister, keys_manager);
 396         nodes[0].chain_monitor = &new_chain_monitor;
 397
 398
 399         let mut node_0_stale_monitors = Vec::new();
 400         for serialized in node_0_stale_monitors_serialized.iter() {
 401                 let mut read = &serialized[..];
 402                 let (_, monitor) = <(BlockHash, ChannelMonitor<TestChannelSigner>)>::read(&mut read, (keys_manager, keys_manager)).unwrap();
 403                 assert!(read.is_empty());
 404                 node_0_stale_monitors.push(monitor);
 405         }
 406
 407         let mut node_0_monitors = Vec::new();
 408         for serialized in node_0_monitors_serialized.iter() {
 409                 let mut read = &serialized[..];
 410                 let (_, monitor) = <(BlockHash, ChannelMonitor<TestChannelSigner>)>::read(&mut read, (keys_manager, keys_manager)).unwrap();
 411                 assert!(read.is_empty());
 412                 node_0_monitors.push(monitor);
 413         }
 414
 415         let mut nodes_0_read = &nodes_0_serialized[..];
 416         if let Err(msgs::DecodeError::InvalidValue) =
 417                 <(BlockHash, ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestRouter, &test_utils::TestLogger>)>::read(&mut nodes_0_read, ChannelManagerReadArgs {
 418                 default_config: UserConfig::default(),
 419                 entropy_source: keys_manager,
 420                 node_signer: keys_manager,
 421                 signer_provider: keys_manager,
 422                 fee_estimator: &fee_estimator,
 423                 router: &nodes[0].router,
 424                 chain_monitor: nodes[0].chain_monitor,
 425                 tx_broadcaster: nodes[0].tx_broadcaster.clone(),
 426                 logger: &logger,
 427                 channel_monitors: node_0_stale_monitors.iter_mut().map(|monitor| { (monitor.get_funding_txo().0, monitor) }).collect(),
 428         }) { } else {
 429                 panic!("If the monitor(s) are stale, this indicates a bug and we should get an Err return");
 430         };
 431
 432         let mut nodes_0_read = &nodes_0_serialized[..];
 433         let (_, nodes_0_deserialized_tmp) =
 434                 <(BlockHash, ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestRouter, &test_utils::TestLogger>)>::read(&mut nodes_0_read, ChannelManagerReadArgs {
 435                 default_config: UserConfig::default(),
 436                 entropy_source: keys_manager,
 437                 node_signer: keys_manager,
 438                 signer_provider: keys_manager,
 439                 fee_estimator: &fee_estimator,
 440                 router: nodes[0].router,
 441                 chain_monitor: nodes[0].chain_monitor,
 442                 tx_broadcaster: nodes[0].tx_broadcaster.clone(),
 443                 logger: &logger,
 444                 channel_monitors: node_0_monitors.iter_mut().map(|monitor| { (monitor.get_funding_txo().0, monitor) }).collect(),
 445         }).unwrap();
 446         nodes_0_deserialized = nodes_0_deserialized_tmp;
 447         assert!(nodes_0_read.is_empty());
 448
 449         for monitor in node_0_monitors.drain(..) {
 450                 assert_eq!(nodes[0].chain_monitor.watch_channel(monitor.get_funding_txo().0, monitor),
 451                         Ok(ChannelMonitorUpdateStatus::Completed));
 452                 check_added_monitors!(nodes[0], 1);
 453         }
 454         nodes[0].node = &nodes_0_deserialized;
 455
 456         check_closed_event!(nodes[0], 1, ClosureReason::OutdatedChannelManager, [nodes[3].node.get_our_node_id()], 100000);
 457         { // Channel close should result in a commitment tx
 458                 nodes[0].node.timer_tick_occurred();
 459                 let txn = nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap();
 460                 assert_eq!(txn.len(), 1);
 461                 check_spends!(txn[0], funding_tx);
 462                 assert_eq!(txn[0].input[0].previous_output.txid, funding_tx.txid());
 463         }
 464         check_added_monitors!(nodes[0], 1);
 465
 466         // nodes[1] and nodes[2] have no lost state with nodes[0]...
 467         reconnect_nodes(ReconnectArgs::new(&nodes[0], &nodes[1]));
 468         reconnect_nodes(ReconnectArgs::new(&nodes[0], &nodes[2]));
 469         //... and we can even still claim the payment!
 470         claim_payment(&nodes[2], &[&nodes[0], &nodes[1]], our_payment_preimage);
 471
 472         nodes[3].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init {
 473                 features: nodes[0].node.init_features(), networks: None, remote_network_address: None
 474         }, true).unwrap();
 475         let reestablish = get_chan_reestablish_msgs!(nodes[3], nodes[0]).pop().unwrap();
 476         nodes[0].node.peer_connected(&nodes[3].node.get_our_node_id(), &msgs::Init {
 477                 features: nodes[3].node.init_features(), networks: None, remote_network_address: None
 478         }, false).unwrap();
 479         nodes[0].node.handle_channel_reestablish(&nodes[3].node.get_our_node_id(), &reestablish);
 480         let mut found_err = false;
 481         for msg_event in nodes[0].node.get_and_clear_pending_msg_events() {
 482                 if let MessageSendEvent::HandleError { ref action, .. } = msg_event {
 483                         match action {
 484                                 &ErrorAction::SendErrorMessage { ref msg } => {
 485                                         assert_eq!(msg.channel_id, channel_id);
 486                                         assert!(!found_err);
 487                                         found_err = true;
 488                                 },
 489                                 _ => panic!("Unexpected event!"),
 490                         }
 491                 }
 492         }
 493         assert!(found_err);
 494 }
 495
 496 #[cfg(feature = "std")]
 497 fn do_test_data_loss_protect(reconnect_panicing: bool) {
 498         // When we get a data_loss_protect proving we're behind, we immediately panic as the
 499         // chain::Watch API requirements have been violated (e.g. the user restored from a backup). The
 500         // panic message informs the user they should force-close without broadcasting, which is tested
 501         // if `reconnect_panicing` is not set.
 502         let mut chanmon_cfgs = create_chanmon_cfgs(2);
 503         // We broadcast during Drop because chanmon is out of sync with chanmgr, which would cause a panic
 504         // during signing due to revoked tx
 505         chanmon_cfgs[0].keys_manager.disable_revocation_policy_check = true;
 506         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
 507         let persister;
 508         let new_chain_monitor;
 509
 510         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
 511         let nodes_0_deserialized;
 512
 513         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
 514
 515         let chan = create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 1000000, 1000000);
 516
 517         // Cache node A state before any channel update
 518         let previous_node_state = nodes[0].node.encode();
 519         let previous_chain_monitor_state = get_monitor!(nodes[0], chan.2).encode();
 520
 521         send_payment(&nodes[0], &vec!(&nodes[1])[..], 8000000);
 522         send_payment(&nodes[0], &vec!(&nodes[1])[..], 8000000);
 523
 524         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id());
 525         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id());
 526
 527         reload_node!(nodes[0], previous_node_state, &[&previous_chain_monitor_state], persister, new_chain_monitor, nodes_0_deserialized);
 528
 529         if reconnect_panicing {
 530                 nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init {
 531                         features: nodes[1].node.init_features(), networks: None, remote_network_address: None
 532                 }, true).unwrap();
 533                 nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init {
 534                         features: nodes[0].node.init_features(), networks: None, remote_network_address: None
 535                 }, false).unwrap();
 536
 537                 let reestablish_1 = get_chan_reestablish_msgs!(nodes[0], nodes[1]);
 538
 539                 // Check we close channel detecting A is fallen-behind
 540                 // Check that we sent the warning message when we detected that A has fallen behind,
 541                 // and give the possibility for A to recover from the warning.
 542                 nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &reestablish_1[0]);
 543                 let warn_msg = "Peer attempted to reestablish channel with a very old local commitment transaction: 0 (received) vs 4 (expected)".to_owned();
 544
 545                 let warn_reestablish = nodes[1].node.get_and_clear_pending_msg_events();
 546                 assert_eq!(warn_reestablish.len(), 2);
 547                 match warn_reestablish[1] {
 548                         MessageSendEvent::HandleError { action: ErrorAction::SendWarningMessage { ref msg, .. }, .. } => {
 549                                 assert_eq!(msg.data, warn_msg);
 550                         },
 551                         _ => panic!("Unexpected event"),
 552                 }
 553                 let reestablish_msg = match &warn_reestablish[0] {
 554                         MessageSendEvent::SendChannelReestablish { msg, .. } => msg.clone(),
 555                         _ => panic!("Unexpected event"),
 556                 };
 557
 558                 {
 559                         let mut node_txn = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap().clone();
 560                         // The node B should not broadcast the transaction to force close the channel!
 561                         assert!(node_txn.is_empty());
 562                 }
 563
 564                 // Check A panics upon seeing proof it has fallen behind.
 565                 assert!(std::panic::catch_unwind(|| {
 566                         nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &reestablish_msg);
 567                 }).is_err());
 568                 std::mem::forget(nodes); // Skip the `Drop` handler for `Node`
 569                 return; // By this point we should have panic'ed!
 570         }
 571
 572         nodes[0].node.force_close_without_broadcasting_txn(&chan.2, &nodes[1].node.get_our_node_id()).unwrap();
 573         check_added_monitors!(nodes[0], 1);
 574         check_closed_event!(nodes[0], 1, ClosureReason::HolderForceClosed, [nodes[1].node.get_our_node_id()], 1000000);
 575         {
 576                 let node_txn = nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap();
 577                 assert_eq!(node_txn.len(), 0);
 578         }
 579
 580         for msg in nodes[0].node.get_and_clear_pending_msg_events() {
 581                 if let MessageSendEvent::BroadcastChannelUpdate { .. } = msg {
 582                 } else if let MessageSendEvent::HandleError { ref action, .. } = msg {
 583                         match action {
 584                                 &ErrorAction::DisconnectPeer { ref msg } => {
 585                                         assert_eq!(msg.as_ref().unwrap().data, "Channel force-closed");
 586                                 },
 587                                 _ => panic!("Unexpected event!"),
 588                         }
 589                 } else {
 590                         panic!("Unexpected event {:?}", msg)
 591                 }
 592         }
 593
 594         // after the warning message sent by B, we should not able to
 595         // use the channel, or reconnect with success to the channel.
 596         assert!(nodes[0].node.list_usable_channels().is_empty());
 597         nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init {
 598                 features: nodes[1].node.init_features(), networks: None, remote_network_address: None
 599         }, true).unwrap();
 600         nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init {
 601                 features: nodes[0].node.init_features(), networks: None, remote_network_address: None
 602         }, false).unwrap();
 603         let retry_reestablish = get_chan_reestablish_msgs!(nodes[1], nodes[0]);
 604
 605         nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &retry_reestablish[0]);
 606         let mut err_msgs_0 = Vec::with_capacity(1);
 607         if let MessageSendEvent::HandleError { ref action, .. } = nodes[0].node.get_and_clear_pending_msg_events()[1] {
 608                 match action {
 609                         &ErrorAction::SendErrorMessage { ref msg } => {
 610                                 assert_eq!(msg.data, format!("Got a message for a channel from the wrong node! No such channel for the passed counterparty_node_id {}", &nodes[1].node.get_our_node_id()));
 611                                 err_msgs_0.push(msg.clone());
 612                         },
 613                         _ => panic!("Unexpected event!"),
 614                 }
 615         } else {
 616                 panic!("Unexpected event!");
 617         }
 618         assert_eq!(err_msgs_0.len(), 1);
 619         nodes[1].node.handle_error(&nodes[0].node.get_our_node_id(), &err_msgs_0[0]);
 620         assert!(nodes[1].node.list_usable_channels().is_empty());
 621         check_added_monitors!(nodes[1], 1);
 622         check_closed_event!(nodes[1], 1, ClosureReason::CounterpartyForceClosed { peer_msg: UntrustedString(format!("Got a message for a channel from the wrong node! No such channel for the passed counterparty_node_id {}", &nodes[1].node.get_our_node_id())) }
 623                 , [nodes[0].node.get_our_node_id()], 1000000);
 624         check_closed_broadcast!(nodes[1], false);
 625 }
 626
 627 #[test]
 628 #[cfg(feature = "std")]
 629 fn test_data_loss_protect() {
 630         do_test_data_loss_protect(true);
 631         do_test_data_loss_protect(false);
 632 }
 633
 634 #[test]
 635 fn test_forwardable_regen() {
 636         // Tests that if we reload a ChannelManager while forwards are pending we will regenerate the
 637         // PendingHTLCsForwardable event automatically, ensuring we don't forget to forward/receive
 638         // HTLCs.
 639         // We test it for both payment receipt and payment forwarding.
 640
 641         let chanmon_cfgs = create_chanmon_cfgs(3);
 642         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
 643         let persister;
 644         let new_chain_monitor;
 645         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
 646         let nodes_1_deserialized;
 647         let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
 648         let chan_id_1 = create_announced_chan_between_nodes(&nodes, 0, 1).2;
 649         let chan_id_2 = create_announced_chan_between_nodes(&nodes, 1, 2).2;
 650
 651         // First send a payment to nodes[1]
 652         let (route, payment_hash, payment_preimage, payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[1], 100_000);
 653         nodes[0].node.send_payment_with_route(&route, payment_hash,
 654                 RecipientOnionFields::secret_only(payment_secret), PaymentId(payment_hash.0)).unwrap();
 655         check_added_monitors!(nodes[0], 1);
 656
 657         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
 658         assert_eq!(events.len(), 1);
 659         let payment_event = SendEvent::from_event(events.pop().unwrap());
 660         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
 661         commitment_signed_dance!(nodes[1], nodes[0], payment_event.commitment_msg, false);
 662
 663         expect_pending_htlcs_forwardable_ignore!(nodes[1]);
 664
 665         // Next send a payment which is forwarded by nodes[1]
 666         let (route_2, payment_hash_2, payment_preimage_2, payment_secret_2) = get_route_and_payment_hash!(nodes[0], nodes[2], 200_000);
 667         nodes[0].node.send_payment_with_route(&route_2, payment_hash_2,
 668                 RecipientOnionFields::secret_only(payment_secret_2), PaymentId(payment_hash_2.0)).unwrap();
 669         check_added_monitors!(nodes[0], 1);
 670
 671         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
 672         assert_eq!(events.len(), 1);
 673         let payment_event = SendEvent::from_event(events.pop().unwrap());
 674         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
 675         commitment_signed_dance!(nodes[1], nodes[0], payment_event.commitment_msg, false);
 676
 677         // There is already a PendingHTLCsForwardable event "pending" so another one will not be
 678         // generated
 679         assert!(nodes[1].node.get_and_clear_pending_events().is_empty());
 680
 681         // Now restart nodes[1] and make sure it regenerates a single PendingHTLCsForwardable
 682         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id());
 683         nodes[2].node.peer_disconnected(&nodes[1].node.get_our_node_id());
 684
 685         let chan_0_monitor_serialized = get_monitor!(nodes[1], chan_id_1).encode();
 686         let chan_1_monitor_serialized = get_monitor!(nodes[1], chan_id_2).encode();
 687         reload_node!(nodes[1], nodes[1].node.encode(), &[&chan_0_monitor_serialized, &chan_1_monitor_serialized], persister, new_chain_monitor, nodes_1_deserialized);
 688
 689         reconnect_nodes(ReconnectArgs::new(&nodes[0], &nodes[1]));
 690         // Note that nodes[1] and nodes[2] resend their channel_ready here since they haven't updated
 691         // the commitment state.
 692         let mut reconnect_args = ReconnectArgs::new(&nodes[1], &nodes[2]);
 693         reconnect_args.send_channel_ready = (true, true);
 694         reconnect_nodes(reconnect_args);
 695
 696         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 697
 698         expect_pending_htlcs_forwardable!(nodes[1]);
 699         expect_payment_claimable!(nodes[1], payment_hash, payment_secret, 100_000);
 700         check_added_monitors!(nodes[1], 1);
 701
 702         let mut events = nodes[1].node.get_and_clear_pending_msg_events();
 703         assert_eq!(events.len(), 1);
 704         let payment_event = SendEvent::from_event(events.pop().unwrap());
 705         nodes[2].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &payment_event.msgs[0]);
 706         commitment_signed_dance!(nodes[2], nodes[1], payment_event.commitment_msg, false);
 707         expect_pending_htlcs_forwardable!(nodes[2]);
 708         expect_payment_claimable!(nodes[2], payment_hash_2, payment_secret_2, 200_000);
 709
 710         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage);
 711         claim_payment(&nodes[0], &[&nodes[1], &nodes[2]], payment_preimage_2);
 712 }
 713
 714 fn do_test_partial_claim_before_restart(persist_both_monitors: bool) {
 715         // Test what happens if a node receives an MPP payment, claims it, but crashes before
 716         // persisting the ChannelManager. If `persist_both_monitors` is false, also crash after only
 717         // updating one of the two channels' ChannelMonitors. As a result, on startup, we'll (a) still
 718         // have the PaymentClaimable event, (b) have one (or two) channel(s) that goes on chain with the
 719         // HTLC preimage in them, and (c) optionally have one channel that is live off-chain but does
 720         // not have the preimage tied to the still-pending HTLC.
 721         //
 722         // To get to the correct state, on startup we should propagate the preimage to the
 723         // still-off-chain channel, claiming the HTLC as soon as the peer connects, with the monitor
 724         // receiving the preimage without a state update.
 725         //
 726         // Further, we should generate a `PaymentClaimed` event to inform the user that the payment was
 727         // definitely claimed.
 728         let chanmon_cfgs = create_chanmon_cfgs(4);
 729         let node_cfgs = create_node_cfgs(4, &chanmon_cfgs);
 730         let persister;
 731         let new_chain_monitor;
 732
 733         let node_chanmgrs = create_node_chanmgrs(4, &node_cfgs, &[None, None, None, None]);
 734         let nodes_3_deserialized;
 735
 736         let mut nodes = create_network(4, &node_cfgs, &node_chanmgrs);
 737
 738         create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 100_000, 0);
 739         create_announced_chan_between_nodes_with_value(&nodes, 0, 2, 100_000, 0);
 740         let chan_id_persisted = create_announced_chan_between_nodes_with_value(&nodes, 1, 3, 100_000, 0).2;
 741         let chan_id_not_persisted = create_announced_chan_between_nodes_with_value(&nodes, 2, 3, 100_000, 0).2;
 742
 743         // Create an MPP route for 15k sats, more than the default htlc-max of 10%
 744         let (mut route, payment_hash, payment_preimage, payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[3], 15_000_000);
 745         assert_eq!(route.paths.len(), 2);
 746         route.paths.sort_by(|path_a, _| {
 747                 // Sort the path so that the path through nodes[1] comes first
 748                 if path_a.hops[0].pubkey == nodes[1].node.get_our_node_id() {
 749                         core::cmp::Ordering::Less } else { core::cmp::Ordering::Greater }
 750         });
 751
 752         nodes[0].node.send_payment_with_route(&route, payment_hash,
 753                 RecipientOnionFields::secret_only(payment_secret), PaymentId(payment_hash.0)).unwrap();
 754         check_added_monitors!(nodes[0], 2);
 755
 756         // Send the payment through to nodes[3] *without* clearing the PaymentClaimable event
 757         let mut send_events = nodes[0].node.get_and_clear_pending_msg_events();
 758         assert_eq!(send_events.len(), 2);
 759         let node_1_msgs = remove_first_msg_event_to_node(&nodes[1].node.get_our_node_id(), &mut send_events);
 760         let node_2_msgs = remove_first_msg_event_to_node(&nodes[2].node.get_our_node_id(), &mut send_events);
 761         do_pass_along_path(&nodes[0], &[&nodes[1], &nodes[3]], 15_000_000, payment_hash, Some(payment_secret), node_1_msgs, true, false, None, false);
 762         do_pass_along_path(&nodes[0], &[&nodes[2], &nodes[3]], 15_000_000, payment_hash, Some(payment_secret), node_2_msgs, true, false, None, false);
 763
 764         // Now that we have an MPP payment pending, get the latest encoded copies of nodes[3]'s
 765         // monitors and ChannelManager, for use later, if we don't want to persist both monitors.
 766         let mut original_monitor = test_utils::TestVecWriter(Vec::new());
 767         if !persist_both_monitors {
 768                 for outpoint in nodes[3].chain_monitor.chain_monitor.list_monitors() {
 769                         if outpoint.to_channel_id() == chan_id_not_persisted {
 770                                 assert!(original_monitor.0.is_empty());
 771                                 nodes[3].chain_monitor.chain_monitor.get_monitor(outpoint).unwrap().write(&mut original_monitor).unwrap();
 772                         }
 773                 }
 774         }
 775
 776         let original_manager = nodes[3].node.encode();
 777
 778         expect_payment_claimable!(nodes[3], payment_hash, payment_secret, 15_000_000);
 779
 780         nodes[3].node.claim_funds(payment_preimage);
 781         check_added_monitors!(nodes[3], 2);
 782         expect_payment_claimed!(nodes[3], payment_hash, 15_000_000);
 783
 784         // Now fetch one of the two updated ChannelMonitors from nodes[3], and restart pretending we
 785         // crashed in between the two persistence calls - using one old ChannelMonitor and one new one,
 786         // with the old ChannelManager.
 787         let mut updated_monitor = test_utils::TestVecWriter(Vec::new());
 788         for outpoint in nodes[3].chain_monitor.chain_monitor.list_monitors() {
 789                 if outpoint.to_channel_id() == chan_id_persisted {
 790                         assert!(updated_monitor.0.is_empty());
 791                         nodes[3].chain_monitor.chain_monitor.get_monitor(outpoint).unwrap().write(&mut updated_monitor).unwrap();
 792                 }
 793         }
 794         // If `persist_both_monitors` is set, get the second monitor here as well
 795         if persist_both_monitors {
 796                 for outpoint in nodes[3].chain_monitor.chain_monitor.list_monitors() {
 797                         if outpoint.to_channel_id() == chan_id_not_persisted {
 798                                 assert!(original_monitor.0.is_empty());
 799                                 nodes[3].chain_monitor.chain_monitor.get_monitor(outpoint).unwrap().write(&mut original_monitor).unwrap();
 800                         }
 801                 }
 802         }
 803
 804         // Now restart nodes[3].
 805         reload_node!(nodes[3], original_manager, &[&updated_monitor.0, &original_monitor.0], persister, new_chain_monitor, nodes_3_deserialized);
 806
 807         // On startup the preimage should have been copied into the non-persisted monitor:
 808         assert!(get_monitor!(nodes[3], chan_id_persisted).get_stored_preimages().contains_key(&payment_hash));
 809         assert!(get_monitor!(nodes[3], chan_id_not_persisted).get_stored_preimages().contains_key(&payment_hash));
 810
 811         nodes[1].node.peer_disconnected(&nodes[3].node.get_our_node_id());
 812         nodes[2].node.peer_disconnected(&nodes[3].node.get_our_node_id());
 813
 814         // During deserialization, we should have closed one channel and broadcast its latest
 815         // commitment transaction. We should also still have the original PaymentClaimable event we
 816         // never finished processing.
 817         let events = nodes[3].node.get_and_clear_pending_events();
 818         assert_eq!(events.len(), if persist_both_monitors { 4 } else { 3 });
 819         if let Event::PaymentClaimable { amount_msat: 15_000_000, .. } = events[0] { } else { panic!(); }
 820         if let Event::ChannelClosed { reason: ClosureReason::OutdatedChannelManager, .. } = events[1] { } else { panic!(); }
 821         if persist_both_monitors {
 822                 if let Event::ChannelClosed { reason: ClosureReason::OutdatedChannelManager, .. } = events[2] { } else { panic!(); }
 823                 check_added_monitors(&nodes[3], 2);
 824         } else {
 825                 check_added_monitors(&nodes[3], 1);
 826         }
 827
 828         // On restart, we should also get a duplicate PaymentClaimed event as we persisted the
 829         // ChannelManager prior to handling the original one.
 830         if let Event::PaymentClaimed { payment_hash: our_payment_hash, amount_msat: 15_000_000, .. } =
 831                 events[if persist_both_monitors { 3 } else { 2 }]
 832         {
 833                 assert_eq!(payment_hash, our_payment_hash);
 834         } else { panic!(); }
 835
 836         assert_eq!(nodes[3].node.list_channels().len(), if persist_both_monitors { 0 } else { 1 });
 837         if !persist_both_monitors {
 838                 // If one of the two channels is still live, reveal the payment preimage over it.
 839
 840                 nodes[3].node.peer_connected(&nodes[2].node.get_our_node_id(), &msgs::Init {
 841                         features: nodes[2].node.init_features(), networks: None, remote_network_address: None
 842                 }, true).unwrap();
 843                 let reestablish_1 = get_chan_reestablish_msgs!(nodes[3], nodes[2]);
 844                 nodes[2].node.peer_connected(&nodes[3].node.get_our_node_id(), &msgs::Init {
 845                         features: nodes[3].node.init_features(), networks: None, remote_network_address: None
 846                 }, false).unwrap();
 847                 let reestablish_2 = get_chan_reestablish_msgs!(nodes[2], nodes[3]);
 848
 849                 nodes[2].node.handle_channel_reestablish(&nodes[3].node.get_our_node_id(), &reestablish_1[0]);
 850                 get_event_msg!(nodes[2], MessageSendEvent::SendChannelUpdate, nodes[3].node.get_our_node_id());
 851                 assert!(nodes[2].node.get_and_clear_pending_msg_events().is_empty());
 852
 853                 nodes[3].node.handle_channel_reestablish(&nodes[2].node.get_our_node_id(), &reestablish_2[0]);
 854
 855                 // Once we call `get_and_clear_pending_msg_events` the holding cell is cleared and the HTLC
 856                 // claim should fly.
 857                 let ds_msgs = nodes[3].node.get_and_clear_pending_msg_events();
 858                 check_added_monitors!(nodes[3], 1);
 859                 assert_eq!(ds_msgs.len(), 2);
 860                 if let MessageSendEvent::SendChannelUpdate { .. } = ds_msgs[0] {} else { panic!(); }
 861
 862                 let cs_updates = match ds_msgs[1] {
 863                         MessageSendEvent::UpdateHTLCs { ref updates, .. } => {
 864                                 nodes[2].node.handle_update_fulfill_htlc(&nodes[3].node.get_our_node_id(), &updates.update_fulfill_htlcs[0]);
 865                                 check_added_monitors!(nodes[2], 1);
 866                                 let cs_updates = get_htlc_update_msgs!(nodes[2], nodes[0].node.get_our_node_id());
 867                                 expect_payment_forwarded!(nodes[2], nodes[0], nodes[3], Some(1000), false, false);
 868                                 commitment_signed_dance!(nodes[2], nodes[3], updates.commitment_signed, false, true);
 869                                 cs_updates
 870                         }
 871                         _ => panic!(),
 872                 };
 873
 874                 nodes[0].node.handle_update_fulfill_htlc(&nodes[2].node.get_our_node_id(), &cs_updates.update_fulfill_htlcs[0]);
 875                 commitment_signed_dance!(nodes[0], nodes[2], cs_updates.commitment_signed, false, true);
 876                 expect_payment_sent!(nodes[0], payment_preimage);
 877         }
 878 }
 879
 880 #[test]
 881 fn test_partial_claim_before_restart() {
 882         do_test_partial_claim_before_restart(false);
 883         do_test_partial_claim_before_restart(true);
 884 }
 885
 886 fn do_forwarded_payment_no_manager_persistence(use_cs_commitment: bool, claim_htlc: bool, use_intercept: bool) {
 887         if !use_cs_commitment { assert!(!claim_htlc); }
 888         // If we go to forward a payment, and the ChannelMonitor persistence completes, but the
 889         // ChannelManager does not, we shouldn't try to forward the payment again, nor should we fail
 890         // it back until the ChannelMonitor decides the fate of the HTLC.
 891         // This was never an issue, but it may be easy to regress here going forward.
 892         let chanmon_cfgs = create_chanmon_cfgs(3);
 893         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
 894         let persister;
 895         let new_chain_monitor;
 896
 897         let mut intercept_forwards_config = test_default_channel_config();
 898         intercept_forwards_config.accept_intercept_htlcs = true;
 899         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, Some(intercept_forwards_config), None]);
 900         let nodes_1_deserialized;
 901
 902         let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
 903
 904         let chan_id_1 = create_announced_chan_between_nodes(&nodes, 0, 1).2;
 905         let chan_id_2 = create_announced_chan_between_nodes(&nodes, 1, 2).2;
 906
 907         let intercept_scid = nodes[1].node.get_intercept_scid();
 908
 909         let (mut route, payment_hash, payment_preimage, payment_secret) =
 910                 get_route_and_payment_hash!(nodes[0], nodes[2], 1_000_000);
 911         if use_intercept {
 912                 route.paths[0].hops[1].short_channel_id = intercept_scid;
 913         }
 914         let payment_id = PaymentId(nodes[0].keys_manager.backing.get_secure_random_bytes());
 915         let htlc_expiry = nodes[0].best_block_info().1 + TEST_FINAL_CLTV;
 916         nodes[0].node.send_payment_with_route(&route, payment_hash,
 917                 RecipientOnionFields::secret_only(payment_secret), payment_id).unwrap();
 918         check_added_monitors!(nodes[0], 1);
 919
 920         let payment_event = SendEvent::from_node(&nodes[0]);
 921         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
 922         commitment_signed_dance!(nodes[1], nodes[0], payment_event.commitment_msg, false);
 923
 924         // Store the `ChannelManager` before handling the `PendingHTLCsForwardable`/`HTLCIntercepted`
 925         // events, expecting either event (and the HTLC itself) to be missing on reload even though its
 926         // present when we serialized.
 927         let node_encoded = nodes[1].node.encode();
 928
 929         let mut intercept_id = None;
 930         let mut expected_outbound_amount_msat = None;
 931         if use_intercept {
 932                 let events = nodes[1].node.get_and_clear_pending_events();
 933                 assert_eq!(events.len(), 1);
 934                 match events[0] {
 935                         Event::HTLCIntercepted { intercept_id: ev_id, expected_outbound_amount_msat: ev_amt, .. } => {
 936                                 intercept_id = Some(ev_id);
 937                                 expected_outbound_amount_msat = Some(ev_amt);
 938                         },
 939                         _ => panic!()
 940                 }
 941                 nodes[1].node.forward_intercepted_htlc(intercept_id.unwrap(), &chan_id_2,
 942                         nodes[2].node.get_our_node_id(), expected_outbound_amount_msat.unwrap()).unwrap();
 943         }
 944
 945         expect_pending_htlcs_forwardable!(nodes[1]);
 946
 947         let payment_event = SendEvent::from_node(&nodes[1]);
 948         nodes[2].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &payment_event.msgs[0]);
 949         nodes[2].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &payment_event.commitment_msg);
 950         check_added_monitors!(nodes[2], 1);
 951
 952         if claim_htlc {
 953                 get_monitor!(nodes[2], chan_id_2).provide_payment_preimage(&payment_hash, &payment_preimage,
 954                         &nodes[2].tx_broadcaster, &LowerBoundedFeeEstimator(nodes[2].fee_estimator), &nodes[2].logger);
 955         }
 956         assert!(nodes[2].tx_broadcaster.txn_broadcasted.lock().unwrap().is_empty());
 957
 958         let _ = nodes[2].node.get_and_clear_pending_msg_events();
 959
 960         nodes[2].node.force_close_broadcasting_latest_txn(&chan_id_2, &nodes[1].node.get_our_node_id()).unwrap();
 961         let cs_commitment_tx = nodes[2].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
 962         assert_eq!(cs_commitment_tx.len(), if claim_htlc { 2 } else { 1 });
 963
 964         check_added_monitors!(nodes[2], 1);
 965         check_closed_event!(nodes[2], 1, ClosureReason::HolderForceClosed, [nodes[1].node.get_our_node_id()], 100000);
 966         check_closed_broadcast!(nodes[2], true);
 967
 968         let chan_0_monitor_serialized = get_monitor!(nodes[1], chan_id_1).encode();
 969         let chan_1_monitor_serialized = get_monitor!(nodes[1], chan_id_2).encode();
 970         reload_node!(nodes[1], node_encoded, &[&chan_0_monitor_serialized, &chan_1_monitor_serialized], persister, new_chain_monitor, nodes_1_deserialized);
 971
 972         // Note that this checks that this is the only event on nodes[1], implying the
 973         // `HTLCIntercepted` event has been removed in the `use_intercept` case.
 974         check_closed_event!(nodes[1], 1, ClosureReason::OutdatedChannelManager, [nodes[2].node.get_our_node_id()], 100000);
 975
 976         if use_intercept {
 977                 // Attempt to forward the HTLC back out over nodes[1]' still-open channel, ensuring we get
 978                 // a intercept-doesn't-exist error.
 979                 let forward_err = nodes[1].node.forward_intercepted_htlc(intercept_id.unwrap(), &chan_id_1,
 980                         nodes[0].node.get_our_node_id(), expected_outbound_amount_msat.unwrap()).unwrap_err();
 981                 assert_eq!(forward_err, APIError::APIMisuseError {
 982                         err: format!("Payment with intercept id {} not found", log_bytes!(intercept_id.unwrap().0))
 983                 });
 984         }
 985
 986         nodes[1].node.timer_tick_occurred();
 987         let bs_commitment_tx = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
 988         assert_eq!(bs_commitment_tx.len(), 1);
 989         check_added_monitors!(nodes[1], 1);
 990
 991         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id());
 992         reconnect_nodes(ReconnectArgs::new(&nodes[0], &nodes[1]));
 993
 994         if use_cs_commitment {
 995                 // If we confirm a commitment transaction that has the HTLC on-chain, nodes[1] should wait
 996                 // for an HTLC-spending transaction before it does anything with the HTLC upstream.
 997                 confirm_transaction(&nodes[1], &cs_commitment_tx[0]);
 998                 assert!(nodes[1].node.get_and_clear_pending_events().is_empty());
 999                 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1000
1001                 if claim_htlc {
1002                         confirm_transaction(&nodes[1], &cs_commitment_tx[1]);
1003                 } else {
1004                         connect_blocks(&nodes[1], htlc_expiry - nodes[1].best_block_info().1 + 1);
1005                         let bs_htlc_timeout_tx = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
1006                         assert_eq!(bs_htlc_timeout_tx.len(), 1);
1007                         confirm_transaction(&nodes[1], &bs_htlc_timeout_tx[0]);
1008                 }
1009         } else {
1010                 confirm_transaction(&nodes[1], &bs_commitment_tx[0]);
1011         }
1012
1013         if !claim_htlc {
1014                 expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[1], [HTLCDestination::NextHopChannel { node_id: Some(nodes[2].node.get_our_node_id()), channel_id: chan_id_2 }]);
1015         } else {
1016                 expect_payment_forwarded!(nodes[1], nodes[0], nodes[2], Some(1000), false, true);
1017         }
1018         check_added_monitors!(nodes[1], 1);
1019
1020         let events = nodes[1].node.get_and_clear_pending_msg_events();
1021         assert_eq!(events.len(), 1);
1022         match &events[0] {
1023                 MessageSendEvent::UpdateHTLCs { updates: msgs::CommitmentUpdate { update_fulfill_htlcs, update_fail_htlcs, commitment_signed, .. }, .. } => {
1024                         if claim_htlc {
1025                                 nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &update_fulfill_htlcs[0]);
1026                         } else {
1027                                 nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &update_fail_htlcs[0]);
1028                         }
1029                         commitment_signed_dance!(nodes[0], nodes[1], commitment_signed, false);
1030                 },
1031                 _ => panic!("Unexpected event"),
1032         }
1033
1034         if claim_htlc {
1035                 expect_payment_sent!(nodes[0], payment_preimage);
1036         } else {
1037                 expect_payment_failed!(nodes[0], payment_hash, false);
1038         }
1039 }
1040
1041 #[test]
1042 fn forwarded_payment_no_manager_persistence() {
1043         do_forwarded_payment_no_manager_persistence(true, true, false);
1044         do_forwarded_payment_no_manager_persistence(true, false, false);
1045         do_forwarded_payment_no_manager_persistence(false, false, false);
1046 }
1047
1048 #[test]
1049 fn intercepted_payment_no_manager_persistence() {
1050         do_forwarded_payment_no_manager_persistence(true, true, true);
1051         do_forwarded_payment_no_manager_persistence(true, false, true);
1052         do_forwarded_payment_no_manager_persistence(false, false, true);
1053 }
1054
1055 #[test]
1056 fn removed_payment_no_manager_persistence() {
1057         // If an HTLC is failed to us on a channel, and the ChannelMonitor persistence completes, but
1058         // the corresponding ChannelManager persistence does not, we need to ensure that the HTLC is
1059         // still failed back to the previous hop even though the ChannelMonitor now no longer is aware
1060         // of the HTLC. This was previously broken as no attempt was made to figure out which HTLCs
1061         // were left dangling when a channel was force-closed due to a stale ChannelManager.
1062         let chanmon_cfgs = create_chanmon_cfgs(3);
1063         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
1064         let persister;
1065         let new_chain_monitor;
1066
1067         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
1068         let nodes_1_deserialized;
1069
1070         let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
1071
1072         let chan_id_1 = create_announced_chan_between_nodes(&nodes, 0, 1).2;
1073         let chan_id_2 = create_announced_chan_between_nodes(&nodes, 1, 2).2;
1074
1075         let (_, payment_hash, ..) = route_payment(&nodes[0], &[&nodes[1], &nodes[2]], 1_000_000);
1076
1077         let node_encoded = nodes[1].node.encode();
1078
1079         nodes[2].node.fail_htlc_backwards(&payment_hash);
1080         expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[2], [HTLCDestination::FailedPayment { payment_hash }]);
1081         check_added_monitors!(nodes[2], 1);
1082         let events = nodes[2].node.get_and_clear_pending_msg_events();
1083         assert_eq!(events.len(), 1);
1084         match &events[0] {
1085                 MessageSendEvent::UpdateHTLCs { updates: msgs::CommitmentUpdate { update_fail_htlcs, commitment_signed, .. }, .. } => {
1086                         nodes[1].node.handle_update_fail_htlc(&nodes[2].node.get_our_node_id(), &update_fail_htlcs[0]);
1087                         commitment_signed_dance!(nodes[1], nodes[2], commitment_signed, false);
1088                 },
1089                 _ => panic!("Unexpected event"),
1090         }
1091
1092         let chan_0_monitor_serialized = get_monitor!(nodes[1], chan_id_1).encode();
1093         let chan_1_monitor_serialized = get_monitor!(nodes[1], chan_id_2).encode();
1094         reload_node!(nodes[1], node_encoded, &[&chan_0_monitor_serialized, &chan_1_monitor_serialized], persister, new_chain_monitor, nodes_1_deserialized);
1095
1096         match nodes[1].node.pop_pending_event().unwrap() {
1097                 Event::ChannelClosed { ref reason, .. } => {
1098                         assert_eq!(*reason, ClosureReason::OutdatedChannelManager);
1099                 },
1100                 _ => panic!("Unexpected event"),
1101         }
1102
1103         nodes[1].node.test_process_background_events();
1104         check_added_monitors(&nodes[1], 1);
1105
1106         // Now that the ChannelManager has force-closed the channel which had the HTLC removed, it is
1107         // now forgotten everywhere. The ChannelManager should have, as a side-effect of reload,
1108         // learned that the HTLC is gone from the ChannelMonitor and added it to the to-fail-back set.
1109         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id());
1110         reconnect_nodes(ReconnectArgs::new(&nodes[0], &nodes[1]));
1111
1112         expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[1], [HTLCDestination::NextHopChannel { node_id: Some(nodes[2].node.get_our_node_id()), channel_id: chan_id_2 }]);
1113         check_added_monitors!(nodes[1], 1);
1114         let events = nodes[1].node.get_and_clear_pending_msg_events();
1115         assert_eq!(events.len(), 1);
1116         match &events[0] {
1117                 MessageSendEvent::UpdateHTLCs { updates: msgs::CommitmentUpdate { update_fail_htlcs, commitment_signed, .. }, .. } => {
1118                         nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &update_fail_htlcs[0]);
1119                         commitment_signed_dance!(nodes[0], nodes[1], commitment_signed, false);
1120                 },
1121                 _ => panic!("Unexpected event"),
1122         }
1123
1124         expect_payment_failed!(nodes[0], payment_hash, false);
1125 }
1126
1127 #[test]
1128 fn test_reload_partial_funding_batch() {
1129         let chanmon_cfgs = create_chanmon_cfgs(3);
1130         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
1131         let new_persister;
1132         let new_chain_monitor;
1133
1134         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
1135         let new_channel_manager;
1136         let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
1137
1138         // Initiate channel opening and create the batch channel funding transaction.
1139         let (tx, funding_created_msgs) = create_batch_channel_funding(&nodes[0], &[
1140                 (&nodes[1], 100_000, 0, 42, None),
1141                 (&nodes[2], 200_000, 0, 43, None),
1142         ]);
1143
1144         // Go through the funding_created and funding_signed flow with node 1.
1145         nodes[1].node.handle_funding_created(&nodes[0].node.get_our_node_id(), &funding_created_msgs[0]);
1146         check_added_monitors(&nodes[1], 1);
1147         expect_channel_pending_event(&nodes[1], &nodes[0].node.get_our_node_id());
1148
1149         // The monitor is persisted when receiving funding_signed.
1150         let funding_signed_msg = get_event_msg!(nodes[1], MessageSendEvent::SendFundingSigned, nodes[0].node.get_our_node_id());
1151         nodes[0].node.handle_funding_signed(&nodes[1].node.get_our_node_id(), &funding_signed_msg);
1152         check_added_monitors(&nodes[0], 1);
1153
1154         // The transaction should not have been broadcast before all channels are ready.
1155         assert_eq!(nodes[0].tx_broadcaster.txn_broadcast().len(), 0);
1156
1157         // Reload the node while a subset of the channels in the funding batch have persisted monitors.
1158         let channel_id_1 = OutPoint { txid: tx.txid(), index: 0 }.to_channel_id();
1159         let node_encoded = nodes[0].node.encode();
1160         let channel_monitor_1_serialized = get_monitor!(nodes[0], channel_id_1).encode();
1161         reload_node!(nodes[0], node_encoded, &[&channel_monitor_1_serialized], new_persister, new_chain_monitor, new_channel_manager);
1162
1163         // Process monitor events.
1164         assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
1165
1166         // The monitor should become closed.
1167         check_added_monitors(&nodes[0], 1);
1168         {
1169                 let mut monitor_updates = nodes[0].chain_monitor.monitor_updates.lock().unwrap();
1170                 let monitor_updates_1 = monitor_updates.get(&channel_id_1).unwrap();
1171                 assert_eq!(monitor_updates_1.len(), 1);
1172                 assert_eq!(monitor_updates_1[0].update_id, CLOSED_CHANNEL_UPDATE_ID);
1173         }
1174
1175         // The funding transaction should not have been broadcast, but we broadcast the force-close
1176         // transaction as part of closing the monitor.
1177         {
1178                 let broadcasted_txs = nodes[0].tx_broadcaster.txn_broadcast();
1179                 assert_eq!(broadcasted_txs.len(), 1);
1180                 assert!(broadcasted_txs[0].txid() != tx.txid());
1181                 assert_eq!(broadcasted_txs[0].input.len(), 1);
1182                 assert_eq!(broadcasted_txs[0].input[0].previous_output.txid, tx.txid());
1183         }
1184
1185         // Ensure the channels don't exist anymore.
1186         assert!(nodes[0].node.list_channels().is_empty());
1187 }