Merge pull request #1436 from TheBlueMatt/2022-04-event-process-try-lock

Reorder the BP loop to make manager persistence more reliable

diff --combined lightning-background-processor/src/lib.rs
index 7a1512b79b8af92140474d1fc4c3f9b2346a15f4,255fb245e0543a16ad0a38fbd0e3e267b25954ec..e23737c0adcf16fec641c0d48aac940645c54716
--- 1/lightning-background-processor/src/lib.rs
--- 2/lightning-background-processor/src/lib.rs
+++ b/lightning-background-processor/src/lib.rs
@@@ -203,10 -203,22 +203,22 @@@ impl BackgroundProcessor 
                        let mut have_pruned = false;
  
                        loop {
-                               peer_manager.process_events(); // Note that this may block on ChannelManager's locking
                                channel_manager.process_pending_events(&event_handler);
                                chain_monitor.process_pending_events(&event_handler);
  
+                               // Note that the PeerManager::process_events may block on ChannelManager's locks,
+                               // hence it comes last here. When the ChannelManager finishes whatever it's doing,
+                               // we want to ensure we get into `persist_manager` as quickly as we can, especially
+                               // without running the normal event processing above and handing events to users.
+                               //
+                               // Specifically, on an *extremely* slow machine, we may see ChannelManager start
+                               // processing a message effectively at any point during this loop. In order to
+                               // minimize the time between such processing completing and persisting the updated
+                               // ChannelManager, we want to minimize methods blocking on a ChannelManager
+                               // generally, and as a fallback place such blocking only immediately before
+                               // persistence.
+                               peer_manager.process_events();
+ 
                                // We wait up to 100ms, but track how long it takes to detect being put to sleep,
                                // see `await_start`'s use below.
                                let await_start = Instant::now();
@@@ -335,9 -347,10 +347,9 @@@ mod tests 
        use bitcoin::blockdata::constants::genesis_block;
        use bitcoin::blockdata::transaction::{Transaction, TxOut};
        use bitcoin::network::constants::Network;
 -      use lightning::chain::chaininterface::{BroadcasterInterface, FeeEstimator};
 -      use lightning::chain::{BestBlock, Confirm, chainmonitor, self};
 +      use lightning::chain::{BestBlock, Confirm, chainmonitor};
        use lightning::chain::channelmonitor::ANTI_REORG_DELAY;
 -      use lightning::chain::keysinterface::{InMemorySigner, Recipient, KeysInterface, KeysManager, Sign};
 +      use lightning::chain::keysinterface::{InMemorySigner, Recipient, KeysInterface, KeysManager};
        use lightning::chain::transaction::OutPoint;
        use lightning::get_event_msg;
        use lightning::ln::channelmanager::{BREAKDOWN_TIMEOUT, ChainParameters, ChannelManager, SimpleArcChannelManager};
@@@ -347,13 -360,15 +359,13 @@@
        use lightning::routing::network_graph::{NetworkGraph, NetGraphMsgHandler};
        use lightning::util::config::UserConfig;
        use lightning::util::events::{Event, MessageSendEventsProvider, MessageSendEvent};
 -      use lightning::util::logger::Logger;
        use lightning::util::ser::Writeable;
        use lightning::util::test_utils;
        use lightning::util::persist::KVStorePersister;
        use lightning_invoice::payment::{InvoicePayer, RetryAttempts};
        use lightning_invoice::utils::DefaultRouter;
        use lightning_persister::FilesystemPersister;
 -      use std::fs::{self, File};
 -      use std::ops::Deref;
 +      use std::fs;
        use std::path::PathBuf;
        use std::sync::{Arc, Mutex};
        use std::time::Duration;
@@@ -396,6 -411,7 +408,6 @@@
        }
  
        struct Persister {
 -              data_dir: String,
                graph_error: Option<(std::io::ErrorKind, &'static str)>,
                manager_error: Option<(std::io::ErrorKind, &'static str)>,
                filesystem_persister: FilesystemPersister,
@@@ -404,7 -420,7 +416,7 @@@
        impl Persister {
                fn new(data_dir: String) -> Self {
                        let filesystem_persister = FilesystemPersister::new(data_dir.clone());
 -                      Self { data_dir, graph_error: None, manager_error: None, filesystem_persister }
 +                      Self { graph_error: None, manager_error: None, filesystem_persister }
                }
  
                fn with_graph_error(self, error: std::io::ErrorKind, message: &'static str) -> Self {