+
+#[derive(PartialEq)]
+enum HTLCStatusAtDupClaim {
+ Received,
+ HoldingCell,
+ Cleared,
+}
+fn do_test_reconnect_dup_htlc_claims(htlc_status: HTLCStatusAtDupClaim, second_fails: bool) {
+ // When receiving an update_fulfill_htlc message, we immediately forward the claim backwards
+ // along the payment path before waiting for a full commitment_signed dance. This is great, but
+ // can cause duplicative claims if a node sends an update_fulfill_htlc message, disconnects,
+ // reconnects, and then has to re-send its update_fulfill_htlc message again.
+ // In previous code, we didn't handle the double-claim correctly, spuriously closing the
+ // channel on which the inbound HTLC was received.
+ let chanmon_cfgs = create_chanmon_cfgs(3);
+ let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
+ let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
+ let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
+
+ create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::known(), InitFeatures::known());
+ let chan_id_2 = create_announced_chan_between_nodes(&nodes, 1, 2, InitFeatures::known(), InitFeatures::known()).2;
+
+ let (payment_preimage, payment_hash, _) = route_payment(&nodes[0], &[&nodes[1], &nodes[2]], 100_000);
+
+ let mut as_raa = None;
+ if htlc_status == HTLCStatusAtDupClaim::HoldingCell {
+ // In order to get the HTLC claim into the holding cell at nodes[1], we need nodes[1] to be
+ // awaiting a remote revoke_and_ack from nodes[0].
+ let (route, second_payment_hash, _, second_payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[1], 100_000);
+ nodes[0].node.send_payment(&route, second_payment_hash, &Some(second_payment_secret)).unwrap();
+ check_added_monitors!(nodes[0], 1);
+
+ let send_event = SendEvent::from_event(nodes[0].node.get_and_clear_pending_msg_events().remove(0));
+ nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &send_event.msgs[0]);
+ nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &send_event.commitment_msg);
+ check_added_monitors!(nodes[1], 1);
+
+ let (bs_raa, bs_cs) = get_revoke_commit_msgs!(nodes[1], nodes[0].node.get_our_node_id());
+ nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_raa);
+ check_added_monitors!(nodes[0], 1);
+ nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_cs);
+ check_added_monitors!(nodes[0], 1);
+
+ as_raa = Some(get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id()));
+ }
+
+ let fulfill_msg = msgs::UpdateFulfillHTLC {
+ channel_id: chan_id_2,
+ htlc_id: 0,
+ payment_preimage,
+ };
+ if second_fails {
+ nodes[2].node.fail_htlc_backwards(&payment_hash);
+ expect_pending_htlcs_forwardable!(nodes[2]);
+ check_added_monitors!(nodes[2], 1);
+ get_htlc_update_msgs!(nodes[2], nodes[1].node.get_our_node_id());
+ } else {
+ nodes[2].node.claim_funds(payment_preimage);
+ check_added_monitors!(nodes[2], 1);
+ expect_payment_claimed!(nodes[2], payment_hash, 100_000);
+
+ let cs_updates = get_htlc_update_msgs!(nodes[2], nodes[1].node.get_our_node_id());
+ assert_eq!(cs_updates.update_fulfill_htlcs.len(), 1);
+ // Check that the message we're about to deliver matches the one generated:
+ assert_eq!(fulfill_msg, cs_updates.update_fulfill_htlcs[0]);
+ }
+ nodes[1].node.handle_update_fulfill_htlc(&nodes[2].node.get_our_node_id(), &fulfill_msg);
+ expect_payment_forwarded!(nodes[1], nodes[0], nodes[2], Some(1000), false, false);
+ check_added_monitors!(nodes[1], 1);
+
+ let mut bs_updates = None;
+ if htlc_status != HTLCStatusAtDupClaim::HoldingCell {
+ bs_updates = Some(get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id()));
+ assert_eq!(bs_updates.as_ref().unwrap().update_fulfill_htlcs.len(), 1);
+ nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &bs_updates.as_ref().unwrap().update_fulfill_htlcs[0]);
+ expect_payment_sent_without_paths!(nodes[0], payment_preimage);
+ if htlc_status == HTLCStatusAtDupClaim::Cleared {
+ commitment_signed_dance!(nodes[0], nodes[1], &bs_updates.as_ref().unwrap().commitment_signed, false);
+ expect_payment_path_successful!(nodes[0]);
+ }
+ } else {
+ assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
+ }
+
+ nodes[1].node.peer_disconnected(&nodes[2].node.get_our_node_id(), false);
+ nodes[2].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
+
+ if second_fails {
+ reconnect_nodes(&nodes[1], &nodes[2], (false, false), (0, 0), (0, 0), (1, 0), (0, 0), (0, 0), (false, false));
+ expect_pending_htlcs_forwardable!(nodes[1]);
+ } else {
+ reconnect_nodes(&nodes[1], &nodes[2], (false, false), (0, 0), (1, 0), (0, 0), (0, 0), (0, 0), (false, false));
+ }
+
+ if htlc_status == HTLCStatusAtDupClaim::HoldingCell {
+ nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_raa.unwrap());
+ check_added_monitors!(nodes[1], 1);
+ expect_pending_htlcs_forwardable_ignore!(nodes[1]); // We finally receive the second payment, but don't claim it
+
+ bs_updates = Some(get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id()));
+ assert_eq!(bs_updates.as_ref().unwrap().update_fulfill_htlcs.len(), 1);
+ nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &bs_updates.as_ref().unwrap().update_fulfill_htlcs[0]);
+ expect_payment_sent_without_paths!(nodes[0], payment_preimage);
+ }
+ if htlc_status != HTLCStatusAtDupClaim::Cleared {
+ commitment_signed_dance!(nodes[0], nodes[1], &bs_updates.as_ref().unwrap().commitment_signed, false);
+ expect_payment_path_successful!(nodes[0]);
+ }
+}
+
+#[test]
+fn test_reconnect_dup_htlc_claims() {
+ do_test_reconnect_dup_htlc_claims(HTLCStatusAtDupClaim::Received, false);
+ do_test_reconnect_dup_htlc_claims(HTLCStatusAtDupClaim::HoldingCell, false);
+ do_test_reconnect_dup_htlc_claims(HTLCStatusAtDupClaim::Cleared, false);
+ do_test_reconnect_dup_htlc_claims(HTLCStatusAtDupClaim::Received, true);
+ do_test_reconnect_dup_htlc_claims(HTLCStatusAtDupClaim::HoldingCell, true);
+ do_test_reconnect_dup_htlc_claims(HTLCStatusAtDupClaim::Cleared, true);
+}
+
+#[test]
+fn test_temporary_error_during_shutdown() {
+ // Test that temporary failures when updating the monitor's shutdown script delay cooperative
+ // close.
+ let mut config = test_default_channel_config();
+ config.channel_options.commit_upfront_shutdown_pubkey = false;
+
+ let chanmon_cfgs = create_chanmon_cfgs(2);
+ let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
+ let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[Some(config), Some(config)]);
+ let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
+
+ let (_, _, channel_id, funding_tx) = create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::known(), InitFeatures::known());
+
+ chanmon_cfgs[0].persister.set_update_ret(Err(ChannelMonitorUpdateErr::TemporaryFailure));
+ chanmon_cfgs[1].persister.set_update_ret(Err(ChannelMonitorUpdateErr::TemporaryFailure));
+
+ nodes[0].node.close_channel(&channel_id, &nodes[1].node.get_our_node_id()).unwrap();
+ nodes[1].node.handle_shutdown(&nodes[0].node.get_our_node_id(), &InitFeatures::known(), &get_event_msg!(nodes[0], MessageSendEvent::SendShutdown, nodes[1].node.get_our_node_id()));
+ check_added_monitors!(nodes[1], 1);
+
+ nodes[0].node.handle_shutdown(&nodes[1].node.get_our_node_id(), &InitFeatures::known(), &get_event_msg!(nodes[1], MessageSendEvent::SendShutdown, nodes[0].node.get_our_node_id()));
+ check_added_monitors!(nodes[0], 1);
+
+ assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
+
+ chanmon_cfgs[0].persister.set_update_ret(Ok(()));
+ chanmon_cfgs[1].persister.set_update_ret(Ok(()));
+
+ let (outpoint, latest_update, _) = nodes[0].chain_monitor.latest_monitor_update_id.lock().unwrap().get(&channel_id).unwrap().clone();
+ nodes[0].chain_monitor.chain_monitor.force_channel_monitor_updated(outpoint, latest_update);
+ nodes[1].node.handle_closing_signed(&nodes[0].node.get_our_node_id(), &get_event_msg!(nodes[0], MessageSendEvent::SendClosingSigned, nodes[1].node.get_our_node_id()));
+
+ assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
+
+ chanmon_cfgs[1].persister.set_update_ret(Ok(()));
+ let (outpoint, latest_update, _) = nodes[1].chain_monitor.latest_monitor_update_id.lock().unwrap().get(&channel_id).unwrap().clone();
+ nodes[1].chain_monitor.chain_monitor.force_channel_monitor_updated(outpoint, latest_update);
+
+ nodes[0].node.handle_closing_signed(&nodes[1].node.get_our_node_id(), &get_event_msg!(nodes[1], MessageSendEvent::SendClosingSigned, nodes[0].node.get_our_node_id()));
+ let (_, closing_signed_a) = get_closing_signed_broadcast!(nodes[0].node, nodes[1].node.get_our_node_id());
+ let txn_a = nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
+
+ nodes[1].node.handle_closing_signed(&nodes[0].node.get_our_node_id(), &closing_signed_a.unwrap());
+ let (_, none_b) = get_closing_signed_broadcast!(nodes[1].node, nodes[0].node.get_our_node_id());
+ assert!(none_b.is_none());
+ let txn_b = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
+
+ assert_eq!(txn_a, txn_b);
+ assert_eq!(txn_a.len(), 1);
+ check_spends!(txn_a[0], funding_tx);
+ check_closed_event!(nodes[1], 1, ClosureReason::CooperativeClosure);
+ check_closed_event!(nodes[0], 1, ClosureReason::CooperativeClosure);
+}
+
+#[test]
+fn test_permanent_error_during_sending_shutdown() {
+ // Test that permanent failures when updating the monitor's shutdown script result in a force
+ // close when initiating a cooperative close.
+ let mut config = test_default_channel_config();
+ config.channel_options.commit_upfront_shutdown_pubkey = false;
+
+ let chanmon_cfgs = create_chanmon_cfgs(2);
+ let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
+ let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[Some(config), None]);
+ let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
+
+ let channel_id = create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::known(), InitFeatures::known()).2;
+ chanmon_cfgs[0].persister.set_update_ret(Err(ChannelMonitorUpdateErr::PermanentFailure));
+
+ assert!(nodes[0].node.close_channel(&channel_id, &nodes[1].node.get_our_node_id()).is_ok());
+ check_closed_broadcast!(nodes[0], true);
+ check_added_monitors!(nodes[0], 2);
+ check_closed_event!(nodes[0], 1, ClosureReason::ProcessingError { err: "ChannelMonitor storage failure".to_string() });
+}
+
+#[test]
+fn test_permanent_error_during_handling_shutdown() {
+ // Test that permanent failures when updating the monitor's shutdown script result in a force
+ // close when handling a cooperative close.
+ let mut config = test_default_channel_config();
+ config.channel_options.commit_upfront_shutdown_pubkey = false;
+
+ let chanmon_cfgs = create_chanmon_cfgs(2);
+ let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
+ let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, Some(config)]);
+ let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
+
+ let channel_id = create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::known(), InitFeatures::known()).2;
+ chanmon_cfgs[1].persister.set_update_ret(Err(ChannelMonitorUpdateErr::PermanentFailure));
+
+ assert!(nodes[0].node.close_channel(&channel_id, &nodes[1].node.get_our_node_id()).is_ok());
+ let shutdown = get_event_msg!(nodes[0], MessageSendEvent::SendShutdown, nodes[1].node.get_our_node_id());
+ nodes[1].node.handle_shutdown(&nodes[0].node.get_our_node_id(), &InitFeatures::known(), &shutdown);
+ check_closed_broadcast!(nodes[1], true);
+ check_added_monitors!(nodes[1], 2);
+ check_closed_event!(nodes[1], 1, ClosureReason::ProcessingError { err: "ChannelMonitor storage failure".to_string() });
+}
+
+#[test]
+fn double_temp_error() {
+ // Test that it's OK to have multiple `ChainMonitor::update_channel` calls fail in a row.
+ let chanmon_cfgs = create_chanmon_cfgs(2);
+ let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
+ let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
+ let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
+
+ let (_, _, channel_id, _) = create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::known(), InitFeatures::known());
+
+ let (payment_preimage_1, payment_hash_1, _) = route_payment(&nodes[0], &[&nodes[1]], 1_000_000);
+ let (payment_preimage_2, payment_hash_2, _) = route_payment(&nodes[0], &[&nodes[1]], 1_000_000);
+
+ chanmon_cfgs[1].persister.set_update_ret(Err(ChannelMonitorUpdateErr::TemporaryFailure));
+ // `claim_funds` results in a ChannelMonitorUpdate.
+ nodes[1].node.claim_funds(payment_preimage_1);
+ check_added_monitors!(nodes[1], 1);
+ expect_payment_claimed!(nodes[1], payment_hash_1, 1_000_000);
+ let (funding_tx, latest_update_1, _) = nodes[1].chain_monitor.latest_monitor_update_id.lock().unwrap().get(&channel_id).unwrap().clone();
+
+ chanmon_cfgs[1].persister.set_update_ret(Err(ChannelMonitorUpdateErr::TemporaryFailure));
+ // Previously, this would've panicked due to a double-call to `Channel::monitor_update_failed`,
+ // which had some asserts that prevented it from being called twice.
+ nodes[1].node.claim_funds(payment_preimage_2);
+ check_added_monitors!(nodes[1], 1);
+ expect_payment_claimed!(nodes[1], payment_hash_2, 1_000_000);
+ chanmon_cfgs[1].persister.set_update_ret(Ok(()));
+
+ let (_, latest_update_2, _) = nodes[1].chain_monitor.latest_monitor_update_id.lock().unwrap().get(&channel_id).unwrap().clone();
+ nodes[1].chain_monitor.chain_monitor.force_channel_monitor_updated(funding_tx, latest_update_1);
+ assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
+ check_added_monitors!(nodes[1], 0);
+ nodes[1].chain_monitor.chain_monitor.force_channel_monitor_updated(funding_tx, latest_update_2);
+
+ // Complete the first HTLC.
+ let events = nodes[1].node.get_and_clear_pending_msg_events();
+ assert_eq!(events.len(), 1);
+ let (update_fulfill_1, commitment_signed_b1, node_id) = {
+ match &events[0] {
+ &MessageSendEvent::UpdateHTLCs { ref node_id, updates: msgs::CommitmentUpdate { ref update_add_htlcs, ref update_fulfill_htlcs, ref update_fail_htlcs, ref update_fail_malformed_htlcs, ref update_fee, ref commitment_signed } } => {
+ assert!(update_add_htlcs.is_empty());
+ assert_eq!(update_fulfill_htlcs.len(), 1);
+ assert!(update_fail_htlcs.is_empty());
+ assert!(update_fail_malformed_htlcs.is_empty());
+ assert!(update_fee.is_none());
+ (update_fulfill_htlcs[0].clone(), commitment_signed.clone(), node_id.clone())
+ },
+ _ => panic!("Unexpected event"),
+ }
+ };
+ assert_eq!(node_id, nodes[0].node.get_our_node_id());
+ nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &update_fulfill_1);
+ check_added_monitors!(nodes[0], 0);
+ expect_payment_sent_without_paths!(nodes[0], payment_preimage_1);
+ nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &commitment_signed_b1);
+ check_added_monitors!(nodes[0], 1);
+ nodes[0].node.process_pending_htlc_forwards();
+ let (raa_a1, commitment_signed_a1) = get_revoke_commit_msgs!(nodes[0], nodes[1].node.get_our_node_id());
+ check_added_monitors!(nodes[1], 0);
+ assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
+ nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &raa_a1);
+ check_added_monitors!(nodes[1], 1);
+ nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &commitment_signed_a1);
+ check_added_monitors!(nodes[1], 1);
+
+ // Complete the second HTLC.
+ let ((update_fulfill_2, commitment_signed_b2), raa_b2) = {
+ let events = nodes[1].node.get_and_clear_pending_msg_events();
+ assert_eq!(events.len(), 2);
+ (match &events[0] {
+ MessageSendEvent::UpdateHTLCs { node_id, updates } => {
+ assert_eq!(*node_id, nodes[0].node.get_our_node_id());
+ assert!(updates.update_add_htlcs.is_empty());
+ assert!(updates.update_fail_htlcs.is_empty());
+ assert!(updates.update_fail_malformed_htlcs.is_empty());
+ assert!(updates.update_fee.is_none());
+ assert_eq!(updates.update_fulfill_htlcs.len(), 1);
+ (updates.update_fulfill_htlcs[0].clone(), updates.commitment_signed.clone())
+ },
+ _ => panic!("Unexpected event"),
+ },
+ match events[1] {
+ MessageSendEvent::SendRevokeAndACK { ref node_id, ref msg } => {
+ assert_eq!(*node_id, nodes[0].node.get_our_node_id());
+ (*msg).clone()
+ },
+ _ => panic!("Unexpected event"),
+ })
+ };
+ nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &raa_b2);
+ check_added_monitors!(nodes[0], 1);
+ expect_payment_path_successful!(nodes[0]);
+
+ nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &update_fulfill_2);
+ check_added_monitors!(nodes[0], 0);
+ assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
+ commitment_signed_dance!(nodes[0], nodes[1], commitment_signed_b2, false);
+ expect_payment_sent!(nodes[0], payment_preimage_2);
+}