]> git.bitcoin.ninja Git - rust-lightning/commitdiff
Merge pull request #856 from TheBlueMatt/2021-03-check-tx
authorMatt Corallo <649246+TheBlueMatt@users.noreply.github.com>
Sat, 10 Apr 2021 20:27:24 +0000 (20:27 +0000)
committerGitHub <noreply@github.com>
Sat, 10 Apr 2021 20:27:24 +0000 (20:27 +0000)
Take the full funding transaction from the user on generation

36 files changed:
.github/workflows/build.yml
Cargo.toml
background-processor/src/lib.rs
fuzz/src/chanmon_consistency.rs
fuzz/src/full_stack.rs
fuzz/src/router.rs
lightning-invoice/.gitignore [new file with mode: 0644]
lightning-invoice/Cargo.toml [new file with mode: 0644]
lightning-invoice/README.md [new file with mode: 0644]
lightning-invoice/fuzz/.gitignore [new file with mode: 0644]
lightning-invoice/fuzz/Cargo.toml [new file with mode: 0644]
lightning-invoice/fuzz/ci-fuzz.sh [new file with mode: 0755]
lightning-invoice/fuzz/fuzz_targets/serde_data_part.rs [new file with mode: 0644]
lightning-invoice/src/de.rs [new file with mode: 0644]
lightning-invoice/src/lib.rs [new file with mode: 0644]
lightning-invoice/src/ser.rs [new file with mode: 0644]
lightning-invoice/src/tb.rs [new file with mode: 0644]
lightning-invoice/tests/ser_de.rs [new file with mode: 0644]
lightning-persister/Cargo.toml
lightning-persister/src/lib.rs
lightning/src/chain/chainmonitor.rs
lightning/src/chain/channelmonitor.rs
lightning/src/chain/keysinterface.rs
lightning/src/chain/mod.rs
lightning/src/lib.rs
lightning/src/ln/chanmon_update_fail_tests.rs
lightning/src/ln/channel.rs
lightning/src/ln/channelmanager.rs
lightning/src/ln/functional_test_utils.rs
lightning/src/ln/functional_tests.rs
lightning/src/ln/mod.rs
lightning/src/ln/onion_utils.rs
lightning/src/ln/reorg_tests.rs
lightning/src/routing/router.rs
lightning/src/util/enforcing_trait_impls.rs
lightning/src/util/test_utils.rs

index 09cef0ffceb5c7d34c26c79f1f875b56a6a151f4..a379d1cd64ba73c4c07c57336f698ce245214724 100644 (file)
@@ -153,11 +153,14 @@ jobs:
             echo "Bad hash"
             exit 1
           fi
-      - name: Run benchmarks on Rust ${{ matrix.toolchain }}
+      - name: Test with Network Graph on Rust ${{ matrix.toolchain }}
         run: |
           cd lightning
-          cargo bench --features unstable
+          RUSTFLAGS="--cfg=require_route_graph_test" cargo test
           cd ..
+      - name: Run benchmarks on Rust ${{ matrix.toolchain }}
+        run: |
+          cargo bench --features unstable
 
   check_commits:
     runs-on: ubuntu-latest
@@ -204,7 +207,9 @@ jobs:
       - name: Sanity check fuzz targets on Rust ${{ env.TOOLCHAIN }}
         run: cd fuzz && RUSTFLAGS="--cfg=fuzzing" cargo test --verbose --color always
       - name: Run fuzzers
-        run: cd fuzz && ./ci-fuzz.sh
+        run: cd fuzz && ./ci-fuzz.sh && cd ..
+      - name: Run lightning-invoice fuzzers
+        run: cd lightning-invoice/fuzz && RUSTFLAGS="--cfg=fuzzing" cargo test --verbose && ./ci-fuzz.sh
 
   linting:
     runs-on: ubuntu-latest
index 1ffd75a8710494bd0694decc9bcb5e5ae03f6bfa..e1807eab684e29e4a7bc6f0dd7839281379dfb65 100644 (file)
@@ -3,6 +3,7 @@
 members = [
     "lightning",
     "lightning-block-sync",
+    "lightning-invoice",
     "lightning-net-tokio",
     "lightning-persister",
     "background-processor",
index 49be55cf9b60c59e3f646501c4a0741ccb1009df..248870073658ecb2b9119bcda45e2872a427fe4a 100644 (file)
@@ -12,6 +12,8 @@ use lightning::chain;
 use lightning::chain::chaininterface::{BroadcasterInterface, FeeEstimator};
 use lightning::chain::keysinterface::{Sign, KeysInterface};
 use lightning::ln::channelmanager::ChannelManager;
+use lightning::ln::msgs::{ChannelMessageHandler, RoutingMessageHandler};
+use lightning::ln::peer_handler::{PeerManager, SocketDescriptor};
 use lightning::util::logger::Logger;
 use std::sync::Arc;
 use std::sync::atomic::{AtomicBool, Ordering};
@@ -63,40 +65,50 @@ impl BackgroundProcessor {
        /// [`ChannelManager`]: lightning::ln::channelmanager::ChannelManager
        /// [`ChannelManager::write`]: lightning::ln::channelmanager::ChannelManager#impl-Writeable
        /// [`FilesystemPersister::persist_manager`]: lightning_persister::FilesystemPersister::persist_manager
-       pub fn start<PM, Signer, M, T, K, F, L>(persist_manager: PM, manager: Arc<ChannelManager<Signer, Arc<M>, Arc<T>, Arc<K>, Arc<F>, Arc<L>>>, logger: Arc<L>) -> Self
-       where Signer: 'static + Sign,
-             M: 'static + chain::Watch<Signer>,
-             T: 'static + BroadcasterInterface,
-             K: 'static + KeysInterface<Signer=Signer>,
-             F: 'static + FeeEstimator,
-             L: 'static + Logger,
-             PM: 'static + Send + Fn(&ChannelManager<Signer, Arc<M>, Arc<T>, Arc<K>, Arc<F>, Arc<L>>) -> Result<(), std::io::Error>,
+       pub fn start<PM, Signer, M, T, K, F, L, Descriptor: 'static + SocketDescriptor + Send, CM, RM>(
+               persist_channel_manager: PM,
+               channel_manager: Arc<ChannelManager<Signer, Arc<M>, Arc<T>, Arc<K>, Arc<F>, Arc<L>>>,
+               peer_manager: Arc<PeerManager<Descriptor, Arc<CM>, Arc<RM>, Arc<L>>>, logger: Arc<L>,
+       ) -> Self
+       where
+               Signer: 'static + Sign,
+               M: 'static + chain::Watch<Signer>,
+               T: 'static + BroadcasterInterface,
+               K: 'static + KeysInterface<Signer = Signer>,
+               F: 'static + FeeEstimator,
+               L: 'static + Logger,
+               CM: 'static + ChannelMessageHandler,
+               RM: 'static + RoutingMessageHandler,
+               PM: 'static
+                       + Send
+                       + Fn(
+                               &ChannelManager<Signer, Arc<M>, Arc<T>, Arc<K>, Arc<F>, Arc<L>>,
+                       ) -> Result<(), std::io::Error>,
        {
                let stop_thread = Arc::new(AtomicBool::new(false));
                let stop_thread_clone = stop_thread.clone();
                let handle = thread::spawn(move || -> Result<(), std::io::Error> {
                        let mut current_time = Instant::now();
                        loop {
-                               let updates_available = manager.await_persistable_update_timeout(Duration::from_millis(100));
+                               peer_manager.process_events();
+                               let updates_available =
+                                       channel_manager.await_persistable_update_timeout(Duration::from_millis(100));
                                if updates_available {
-                                       persist_manager(&*manager)?;
+                                       persist_channel_manager(&*channel_manager)?;
                                }
                                // Exit the loop if the background processor was requested to stop.
                                if stop_thread.load(Ordering::Acquire) == true {
                                        log_trace!(logger, "Terminating background processor.");
-                                       return Ok(())
+                                       return Ok(());
                                }
                                if current_time.elapsed().as_secs() > CHAN_FRESHNESS_TIMER {
                                        log_trace!(logger, "Calling manager's timer_chan_freshness_every_min");
-                                       manager.timer_chan_freshness_every_min();
+                                       channel_manager.timer_chan_freshness_every_min();
                                        current_time = Instant::now();
                                }
                        }
                });
-               Self {
-                       stop_thread: stop_thread_clone,
-                       thread_handle: handle,
-               }
+               Self { stop_thread: stop_thread_clone, thread_handle: handle }
        }
 
        /// Stop `BackgroundProcessor`'s thread.
@@ -120,6 +132,7 @@ mod tests {
        use lightning::ln::channelmanager::{ChainParameters, ChannelManager, SimpleArcChannelManager};
        use lightning::ln::features::InitFeatures;
        use lightning::ln::msgs::ChannelMessageHandler;
+       use lightning::ln::peer_handler::{PeerManager, MessageHandler, SocketDescriptor};
        use lightning::util::config::UserConfig;
        use lightning::util::events::{Event, EventsProvider, MessageSendEventsProvider, MessageSendEvent};
        use lightning::util::logger::Logger;
@@ -132,10 +145,21 @@ mod tests {
        use std::time::Duration;
        use super::BackgroundProcessor;
 
+       #[derive(Clone, Eq, Hash, PartialEq)]
+       struct TestDescriptor{}
+       impl SocketDescriptor for TestDescriptor {
+               fn send_data(&mut self, _data: &[u8], _resume_read: bool) -> usize {
+                       0
+               }
+
+               fn disconnect_socket(&mut self) {}
+       }
+
        type ChainMonitor = chainmonitor::ChainMonitor<InMemorySigner, Arc<test_utils::TestChainSource>, Arc<test_utils::TestBroadcaster>, Arc<test_utils::TestFeeEstimator>, Arc<test_utils::TestLogger>, Arc<FilesystemPersister>>;
 
        struct Node {
                node: Arc<SimpleArcChannelManager<ChainMonitor, test_utils::TestBroadcaster, test_utils::TestFeeEstimator, test_utils::TestLogger>>,
+               peer_manager: Arc<PeerManager<TestDescriptor, Arc<test_utils::TestChannelMessageHandler>, Arc<test_utils::TestRoutingMessageHandler>, Arc<test_utils::TestLogger>>>,
                persister: Arc<FilesystemPersister>,
                logger: Arc<test_utils::TestLogger>,
        }
@@ -176,7 +200,9 @@ mod tests {
                                latest_height: 0,
                        };
                        let manager = Arc::new(ChannelManager::new(fee_estimator.clone(), chain_monitor.clone(), tx_broadcaster, logger.clone(), keys_manager.clone(), UserConfig::default(), params));
-                       let node = Node { node: manager, persister, logger };
+                       let msg_handler = MessageHandler { chan_handler: Arc::new(test_utils::TestChannelMessageHandler::new()), route_handler: Arc::new(test_utils::TestRoutingMessageHandler::new() )};
+                       let peer_manager = Arc::new(PeerManager::new(msg_handler, keys_manager.get_node_secret(), &seed, logger.clone()));
+                       let node = Node { node: manager, peer_manager, persister, logger };
                        nodes.push(node);
                }
                nodes
@@ -219,7 +245,7 @@ mod tests {
                // Initiate the background processors to watch each node.
                let data_dir = nodes[0].persister.get_data_dir();
                let callback = move |node: &ChannelManager<InMemorySigner, Arc<ChainMonitor>, Arc<test_utils::TestBroadcaster>, Arc<KeysManager>, Arc<test_utils::TestFeeEstimator>, Arc<test_utils::TestLogger>>| FilesystemPersister::persist_manager(data_dir.clone(), node);
-               let bg_processor = BackgroundProcessor::start(callback, nodes[0].node.clone(), nodes[0].logger.clone());
+               let bg_processor = BackgroundProcessor::start(callback, nodes[0].node.clone(), nodes[0].peer_manager.clone(), nodes[0].logger.clone());
 
                // Go through the channel creation process until each node should have something persisted.
                let tx = open_channel!(nodes[0], nodes[1], 100000);
@@ -274,7 +300,7 @@ mod tests {
                let nodes = create_nodes(1, "test_chan_freshness_called".to_string());
                let data_dir = nodes[0].persister.get_data_dir();
                let callback = move |node: &ChannelManager<InMemorySigner, Arc<ChainMonitor>, Arc<test_utils::TestBroadcaster>, Arc<KeysManager>, Arc<test_utils::TestFeeEstimator>, Arc<test_utils::TestLogger>>| FilesystemPersister::persist_manager(data_dir.clone(), node);
-               let bg_processor = BackgroundProcessor::start(callback, nodes[0].node.clone(), nodes[0].logger.clone());
+               let bg_processor = BackgroundProcessor::start(callback, nodes[0].node.clone(), nodes[0].peer_manager.clone(), nodes[0].logger.clone());
                loop {
                        let log_entries = nodes[0].logger.lines.lock().unwrap();
                        let desired_log = "Calling manager's timer_chan_freshness_every_min".to_string();
@@ -301,7 +327,7 @@ mod tests {
                }
 
                let nodes = create_nodes(2, "test_persist_error".to_string());
-               let bg_processor = BackgroundProcessor::start(persist_manager, nodes[0].node.clone(), nodes[0].logger.clone());
+               let bg_processor = BackgroundProcessor::start(persist_manager, nodes[0].node.clone(), nodes[0].peer_manager.clone(), nodes[0].logger.clone());
                open_channel!(nodes[0], nodes[1], 100000);
 
                let _ = bg_processor.thread_handle.join().unwrap().expect_err("Errored persisting manager: test");
index 3feeaf46d5e1636d01736a904c9a52aeb8f2b5fa..a6a6a853ed17d877cf51084c035a5b6aaaa55d4a 100644 (file)
@@ -234,7 +234,7 @@ fn check_api_err(api_err: APIError) {
                                _ if err.starts_with("Cannot send value that would put our balance under counterparty-announced channel reserve value") => {},
                                _ if err.starts_with("Cannot send value that would overdraw remaining funds.") => {},
                                _ if err.starts_with("Cannot send value that would not leave enough to pay for fees.") => {},
-                               _ => panic!(err),
+                               _ => panic!("{}", err),
                        }
                },
                APIError::MonitorUpdateFailed => {
@@ -429,11 +429,11 @@ pub fn do_test<Out: test_logger::Output>(data: &[u8], out: Out) {
                        let chain_hash = genesis_block(Network::Bitcoin).block_hash();
                        let mut header = BlockHeader { version: 0x20000000, prev_blockhash: chain_hash, merkle_root: Default::default(), time: 42, bits: 42, nonce: 42 };
                        let txdata: Vec<_> = channel_txn.iter().enumerate().map(|(i, tx)| (i + 1, tx)).collect();
-                       $node.block_connected(&header, &txdata, 1);
-                       for i in 2..100 {
+                       $node.transactions_confirmed(&header, 1, &txdata);
+                       for _ in 2..100 {
                                header = BlockHeader { version: 0x20000000, prev_blockhash: header.block_hash(), merkle_root: Default::default(), time: 42, bits: 42, nonce: 42 };
-                               $node.block_connected(&header, &[], i);
                        }
+                       $node.update_best_block(&header, 99);
                } }
        }
 
index 3acf7ba53ca7c642c6d3bcf4685480eab1131741..9e4f18b5df8611d442c6ab13c2afc0341aed8e31 100644 (file)
@@ -27,6 +27,7 @@ use bitcoin::hashes::sha256::Hash as Sha256;
 use bitcoin::hash_types::{Txid, BlockHash, WPubkeyHash};
 
 use lightning::chain;
+use lightning::chain::Listen;
 use lightning::chain::chaininterface::{BroadcasterInterface, ConfirmationTarget, FeeEstimator};
 use lightning::chain::chainmonitor;
 use lightning::chain::transaction::OutPoint;
@@ -206,7 +207,8 @@ impl<'a> MoneyLossDetector<'a> {
                self.blocks_connected += 1;
                let header = BlockHeader { version: 0x20000000, prev_blockhash: self.header_hashes[self.height].0, merkle_root: Default::default(), time: self.blocks_connected, bits: 42, nonce: 42 };
                self.height += 1;
-               self.manager.block_connected(&header, &txdata, self.height as u32);
+               self.manager.transactions_confirmed(&header, self.height as u32, &txdata);
+               self.manager.update_best_block(&header, self.height as u32);
                (*self.monitor).block_connected(&header, &txdata, self.height as u32);
                if self.header_hashes.len() > self.height {
                        self.header_hashes[self.height] = (header.block_hash(), self.blocks_connected);
@@ -220,7 +222,7 @@ impl<'a> MoneyLossDetector<'a> {
        fn disconnect_block(&mut self) {
                if self.height > 0 && (self.max_height < 6 || self.height >= self.max_height - 6) {
                        let header = BlockHeader { version: 0x20000000, prev_blockhash: self.header_hashes[self.height - 1].0, merkle_root: Default::default(), time: self.header_hashes[self.height].1, bits: 42, nonce: 42 };
-                       self.manager.block_disconnected(&header);
+                       self.manager.block_disconnected(&header, self.height as u32);
                        self.monitor.block_disconnected(&header, self.height as u32);
                        self.height -= 1;
                        let removal_height = self.height;
index e93618ca7137905ab7677a775e3697e390b6cd02..fb720c9916c2d7b38ee653df9fea675f3f8ea23f 100644 (file)
@@ -130,7 +130,7 @@ pub fn do_test<Out: test_logger::Output>(data: &[u8], out: Out) {
                                        msgs::DecodeError::InvalidValue => return,
                                        msgs::DecodeError::BadLengthDescriptor => return,
                                        msgs::DecodeError::ShortRead => panic!("We picked the length..."),
-                                       msgs::DecodeError::Io(e) => panic!(format!("{:?}", e)),
+                                       msgs::DecodeError::Io(e) => panic!("{:?}", e),
                                }
                        }
                }}
diff --git a/lightning-invoice/.gitignore b/lightning-invoice/.gitignore
new file mode 100644 (file)
index 0000000..f798150
--- /dev/null
@@ -0,0 +1,3 @@
+target
+**/*.rs.bk
+Cargo.lock
\ No newline at end of file
diff --git a/lightning-invoice/Cargo.toml b/lightning-invoice/Cargo.toml
new file mode 100644 (file)
index 0000000..47bf9d7
--- /dev/null
@@ -0,0 +1,19 @@
+[package]
+name = "lightning-invoice"
+description = "Data structures to parse and serialize BOLT11 lightning invoices"
+version = "0.4.0"
+authors = ["Sebastian Geisler <sgeisler@wh2.tu-dresden.de>"]
+documentation = "https://docs.rs/lightning-invoice/"
+license = "MIT OR Apache-2.0"
+keywords = [ "lightning", "bitcoin", "invoice", "BOLT11" ]
+readme = "README.md"
+
+[dependencies]
+bech32 = "0.7"
+secp256k1 = { version = "0.20", features = ["recovery"] }
+num-traits = "0.2.8"
+bitcoin_hashes = "0.9.4"
+
+[lib]
+crate-type = ["cdylib", "rlib"]
+
diff --git a/lightning-invoice/README.md b/lightning-invoice/README.md
new file mode 100644 (file)
index 0000000..e9968d2
--- /dev/null
@@ -0,0 +1,9 @@
+# lightning-invoice
+ [![Docs.rs](https://docs.rs/lightning-invoice/badge.svg)](https://docs.rs/lightning-invoice/)
+
+This repo provides data structures for BOLT 11 lightning invoices and
+functions to parse and serialize these from and to bech32.
+
+**Please be sure to run the test suite since we need to check assumptions
+regarding `SystemTime`'s bounds on your platform. You can also call `check_platform`
+on startup or in your test suite to do so.**
diff --git a/lightning-invoice/fuzz/.gitignore b/lightning-invoice/fuzz/.gitignore
new file mode 100644 (file)
index 0000000..38a9008
--- /dev/null
@@ -0,0 +1,2 @@
+target
+hfuzz_*
diff --git a/lightning-invoice/fuzz/Cargo.toml b/lightning-invoice/fuzz/Cargo.toml
new file mode 100644 (file)
index 0000000..68a0d4e
--- /dev/null
@@ -0,0 +1,26 @@
+[package]
+name = "lightning-invoice-fuzz"
+version = "0.0.1"
+authors = ["Automatically generated"]
+publish = false
+
+[package.metadata]
+cargo-fuzz = true
+
+[features]
+afl_fuzz = ["afl"]
+honggfuzz_fuzz = ["honggfuzz"]
+
+[dependencies]
+honggfuzz = { version = "0.5", optional = true }
+afl = { version = "0.4", optional = true }
+lightning-invoice = { path = ".."}
+bech32 = "0.7"
+
+# Prevent this from interfering with workspaces
+[workspace]
+members = ["."]
+
+[[bin]]
+name = "serde_data_part"
+path = "fuzz_targets/serde_data_part.rs"
diff --git a/lightning-invoice/fuzz/ci-fuzz.sh b/lightning-invoice/fuzz/ci-fuzz.sh
new file mode 100755 (executable)
index 0000000..ae85ea9
--- /dev/null
@@ -0,0 +1,19 @@
+#!/bin/bash
+set -e
+cargo install --force honggfuzz
+for TARGET in fuzz_targets/*; do
+    FILENAME=$(basename $TARGET)
+       FILE="${FILENAME%.*}"
+       if [ -d hfuzz_input/$FILE ]; then
+           HFUZZ_INPUT_ARGS="-f hfuzz_input/$FILE/input"
+       fi
+       HFUZZ_BUILD_ARGS="--features honggfuzz_fuzz" HFUZZ_RUN_ARGS="-N1000000 --exit_upon_crash -v $HFUZZ_INPUT_ARGS" cargo hfuzz run $FILE
+
+       if [ -f hfuzz_workspace/$FILE/HONGGFUZZ.REPORT.TXT ]; then
+               cat hfuzz_workspace/$FILE/HONGGFUZZ.REPORT.TXT
+               for CASE in hfuzz_workspace/$FILE/SIG*; do
+                       cat $CASE | xxd -p
+               done
+               exit 1
+       fi
+done
diff --git a/lightning-invoice/fuzz/fuzz_targets/serde_data_part.rs b/lightning-invoice/fuzz/fuzz_targets/serde_data_part.rs
new file mode 100644 (file)
index 0000000..406f967
--- /dev/null
@@ -0,0 +1,69 @@
+extern crate lightning_invoice;
+extern crate bech32;
+
+use lightning_invoice::RawDataPart;
+use bech32::{FromBase32, ToBase32, u5};
+
+fn do_test(data: &[u8]) {
+    let bech32 = data.iter().map(|x| u5::try_from_u8(x % 32).unwrap()).collect::<Vec<_>>();
+    let invoice = match RawDataPart::from_base32(&bech32) {
+        Ok(invoice) => invoice,
+        Err(_) => return,
+    };
+
+    // Our encoding is not worse than the input
+    assert!(invoice.to_base32().len() <= bech32.len());
+
+    // Our serialization is loss-less
+    assert_eq!(
+        RawDataPart::from_base32(&invoice.to_base32()).expect("faild parsing out own encoding"),
+        invoice
+    );
+}
+
+#[cfg(feature = "afl")]
+#[macro_use] extern crate afl;
+#[cfg(feature = "afl")]
+fn main() {
+    fuzz!(|data| {
+        do_test(&data);
+    });
+}
+
+#[cfg(feature = "honggfuzz")]
+#[macro_use] extern crate honggfuzz;
+#[cfg(feature = "honggfuzz")]
+fn main() {
+    loop {
+        fuzz!(|data| {
+            do_test(data);
+        });
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    fn extend_vec_from_hex(hex: &str, out: &mut Vec<u8>) {
+        let mut b = 0;
+        for (idx, c) in hex.as_bytes().iter().filter(|&&c| c != b'\n').enumerate() {
+            b <<= 4;
+            match *c {
+                b'A'...b'F' => b |= c - b'A' + 10,
+                b'a'...b'f' => b |= c - b'a' + 10,
+                b'0'...b'9' => b |= c - b'0',
+                _ => panic!("Bad hex"),
+            }
+            if (idx & 1) == 1 {
+                out.push(b);
+                b = 0;
+            }
+        }
+    }
+
+    #[test]
+    fn duplicate_crash() {
+        let mut a = Vec::new();
+        extend_vec_from_hex("000000", &mut a);
+        super::do_test(&a);
+    }
+}
diff --git a/lightning-invoice/src/de.rs b/lightning-invoice/src/de.rs
new file mode 100644 (file)
index 0000000..d6cb920
--- /dev/null
@@ -0,0 +1,1076 @@
+use std::error;
+use std::fmt;
+use std::fmt::{Display, Formatter};
+use std::num::ParseIntError;
+use std::str;
+use std::str::FromStr;
+
+use bech32;
+use bech32::{u5, FromBase32};
+
+use bitcoin_hashes::Hash;
+use bitcoin_hashes::sha256;
+
+use num_traits::{CheckedAdd, CheckedMul};
+
+use secp256k1;
+use secp256k1::recovery::{RecoveryId, RecoverableSignature};
+use secp256k1::key::PublicKey;
+
+use super::*;
+
+use self::hrp_sm::parse_hrp;
+
+/// State machine to parse the hrp
+mod hrp_sm {
+       use std::ops::Range;
+
+       #[derive(PartialEq, Eq, Debug)]
+       enum States {
+               Start,
+               ParseL,
+               ParseN,
+               ParseCurrencyPrefix,
+               ParseAmountNumber,
+               ParseAmountSiPrefix,
+       }
+
+       impl States {
+               fn next_state(&self, read_symbol: char) -> Result<States, super::ParseError> {
+                       match *self {
+                               States::Start => {
+                                       if read_symbol == 'l' {
+                                               Ok(States::ParseL)
+                                       } else {
+                                               Err(super::ParseError::MalformedHRP)
+                                       }
+                               }
+                               States::ParseL => {
+                                       if read_symbol == 'n' {
+                                               Ok(States::ParseN)
+                                       } else {
+                                               Err(super::ParseError::MalformedHRP)
+                                       }
+                               },
+                               States::ParseN => {
+                                       if !read_symbol.is_numeric() {
+                                               Ok(States::ParseCurrencyPrefix)
+                                       } else {
+                                               Ok(States::ParseAmountNumber)
+                                       }
+                               },
+                               States::ParseCurrencyPrefix => {
+                                       if !read_symbol.is_numeric() {
+                                               Ok(States::ParseCurrencyPrefix)
+                                       } else {
+                                               Ok(States::ParseAmountNumber)
+                                       }
+                               },
+                               States::ParseAmountNumber => {
+                                       if read_symbol.is_numeric() {
+                                               Ok(States::ParseAmountNumber)
+                                       } else if ['m', 'u', 'n', 'p'].contains(&read_symbol) {
+                                               Ok(States::ParseAmountSiPrefix)
+                                       } else {
+                                               Err(super::ParseError::MalformedHRP)
+                                       }
+                               },
+                               States::ParseAmountSiPrefix => Err(super::ParseError::MalformedHRP),
+                       }
+               }
+
+               fn is_final(&self) -> bool {
+                       !(*self == States::ParseL || *self == States::ParseN)
+               }
+       }
+
+
+       struct StateMachine {
+               state: States,
+               position: usize,
+               currency_prefix: Option<Range<usize>>,
+               amount_number: Option<Range<usize>>,
+               amount_si_prefix: Option<Range<usize>>,
+       }
+
+       impl StateMachine {
+               fn new() -> StateMachine {
+                       StateMachine {
+                               state: States::Start,
+                               position: 0,
+                               currency_prefix: None,
+                               amount_number: None,
+                               amount_si_prefix: None,
+                       }
+               }
+
+               fn update_range(range: &mut Option<Range<usize>>, position: usize) {
+                       let new_range = match *range {
+                               None => Range {start: position, end: position + 1},
+                               Some(ref r) => Range {start: r.start, end: r.end + 1},
+                       };
+                       *range = Some(new_range);
+               }
+
+               fn step(&mut self, c: char) -> Result<(), super::ParseError> {
+                       let next_state = self.state.next_state(c)?;
+                       match next_state {
+                               States::ParseCurrencyPrefix => {
+                                       StateMachine::update_range(&mut self.currency_prefix, self.position)
+                               }
+                               States::ParseAmountNumber => {
+                                       StateMachine::update_range(&mut self.amount_number, self.position)
+                               },
+                               States::ParseAmountSiPrefix => {
+                                       StateMachine::update_range(&mut self.amount_si_prefix, self.position)
+                               },
+                               _ => {}
+                       }
+
+                       self.position += 1;
+                       self.state = next_state;
+                       Ok(())
+               }
+
+               fn is_final(&self) -> bool {
+                       self.state.is_final()
+               }
+
+               fn currency_prefix(&self) -> &Option<Range<usize>> {
+                       &self.currency_prefix
+               }
+
+               fn amount_number(&self) -> &Option<Range<usize>> {
+                       &self.amount_number
+               }
+
+               fn amount_si_prefix(&self) -> &Option<Range<usize>> {
+                       &self.amount_si_prefix
+               }
+       }
+
+       pub fn parse_hrp(input: &str) -> Result<(&str, &str, &str), super::ParseError> {
+               let mut sm = StateMachine::new();
+               for c in input.chars() {
+                       sm.step(c)?;
+               }
+
+               if !sm.is_final() {
+                       return Err(super::ParseError::MalformedHRP);
+               }
+
+               let currency = sm.currency_prefix().clone()
+                       .map(|r| &input[r]).unwrap_or("");
+               let amount = sm.amount_number().clone()
+                       .map(|r| &input[r]).unwrap_or("");
+               let si = sm.amount_si_prefix().clone()
+                       .map(|r| &input[r]).unwrap_or("");
+
+               Ok((currency, amount, si))
+       }
+}
+
+
+impl FromStr for super::Currency {
+       type Err = ParseError;
+
+       fn from_str(currency_prefix: &str) -> Result<Self, ParseError> {
+               match currency_prefix {
+                       "bc" => Ok(Currency::Bitcoin),
+                       "tb" => Ok(Currency::BitcoinTestnet),
+                       "bcrt" => Ok(Currency::Regtest),
+                       "sb" => Ok(Currency::Simnet),
+                       _ => Err(ParseError::UnknownCurrency)
+               }
+       }
+}
+
+impl FromStr for SiPrefix {
+       type Err = ParseError;
+
+       fn from_str(currency_prefix: &str) -> Result<Self, ParseError> {
+               use SiPrefix::*;
+               match currency_prefix {
+                       "m" => Ok(Milli),
+                       "u" => Ok(Micro),
+                       "n" => Ok(Nano),
+                       "p" => Ok(Pico),
+                       _ => Err(ParseError::UnknownSiPrefix)
+               }
+       }
+}
+
+/// ```
+/// use lightning_invoice::Invoice;
+///
+/// let invoice = "lnbc1pvjluezpp5qqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqypqdp\
+///    l2pkx2ctnv5sxxmmwwd5kgetjypeh2ursdae8g6twvus8g6rfwvs8qun0dfjkxaq8rkx3yf5tcsyz3d7\
+///    3gafnh3cax9rn449d9p5uxz9ezhhypd0elx87sjle52x86fux2ypatgddc6k63n7erqz25le42c4u4ec\
+///    ky03ylcqca784w";
+///
+/// assert!(invoice.parse::<Invoice>().is_ok());
+/// ```
+impl FromStr for Invoice {
+       type Err = ParseOrSemanticError;
+
+       fn from_str(s: &str) -> Result<Self, <Self as FromStr>::Err> {
+               let signed = s.parse::<SignedRawInvoice>()?;
+               Ok(Invoice::from_signed(signed)?)
+       }
+}
+
+/// ```
+/// use lightning_invoice::*;
+///
+/// let invoice = "lnbc1pvjluezpp5qqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqypqdp\
+///    l2pkx2ctnv5sxxmmwwd5kgetjypeh2ursdae8g6twvus8g6rfwvs8qun0dfjkxaq8rkx3yf5tcsyz3d7\
+///    3gafnh3cax9rn449d9p5uxz9ezhhypd0elx87sjle52x86fux2ypatgddc6k63n7erqz25le42c4u4ec\
+///    ky03ylcqca784w";
+///
+/// let parsed_1 = invoice.parse::<Invoice>();
+///
+/// let parsed_2 = match invoice.parse::<SignedRawInvoice>() {
+///    Ok(signed) => match Invoice::from_signed(signed) {
+///            Ok(invoice) => Ok(invoice),
+///            Err(e) => Err(ParseOrSemanticError::SemanticError(e)),
+///    },
+///    Err(e) => Err(ParseOrSemanticError::ParseError(e)),
+/// };
+///
+/// assert!(parsed_1.is_ok());
+/// assert_eq!(parsed_1, parsed_2);
+/// ```
+impl FromStr for SignedRawInvoice {
+       type Err = ParseError;
+
+       fn from_str(s: &str) -> Result<Self, Self::Err> {
+               let (hrp, data) = bech32::decode(s)?;
+
+               if data.len() < 104 {
+                       return Err(ParseError::TooShortDataPart);
+               }
+
+               let raw_hrp: RawHrp = hrp.parse()?;
+               let data_part = RawDataPart::from_base32(&data[..data.len()-104])?;
+
+               Ok(SignedRawInvoice {
+                       raw_invoice: RawInvoice {
+                               hrp: raw_hrp,
+                               data: data_part,
+                       },
+                       hash: RawInvoice::hash_from_parts(
+                               hrp.as_bytes(),
+                               &data[..data.len()-104]
+                       ),
+                       signature: Signature::from_base32(&data[data.len()-104..])?,
+               })
+       }
+}
+
+impl FromStr for RawHrp {
+       type Err = ParseError;
+
+       fn from_str(hrp: &str) -> Result<Self, <Self as FromStr>::Err> {
+               let parts = parse_hrp(hrp)?;
+
+               let currency = parts.0.parse::<Currency>()?;
+
+               let amount = if !parts.1.is_empty() {
+                       Some(parts.1.parse::<u64>()?)
+               } else {
+                       None
+               };
+
+               let si_prefix: Option<SiPrefix> = if parts.2.is_empty() {
+                       None
+               } else {
+                       let si: SiPrefix = parts.2.parse()?;
+                       if let Some(amt) = amount {
+                               if amt.checked_mul(si.multiplier()).is_none() {
+                                       return Err(ParseError::IntegerOverflowError);
+                               }
+                       }
+                       Some(si)
+               };
+
+               Ok(RawHrp {
+                       currency: currency,
+                       raw_amount: amount,
+                       si_prefix: si_prefix,
+               })
+       }
+}
+
+impl FromBase32 for RawDataPart {
+       type Err = ParseError;
+
+       fn from_base32(data: &[u5]) -> Result<Self, Self::Err> {
+               if data.len() < 7 { // timestamp length
+                       return Err(ParseError::TooShortDataPart);
+               }
+
+               let timestamp = PositiveTimestamp::from_base32(&data[0..7])?;
+               let tagged = parse_tagged_parts(&data[7..])?;
+
+               Ok(RawDataPart {
+                       timestamp: timestamp,
+                       tagged_fields: tagged,
+               })
+       }
+}
+
+impl FromBase32 for PositiveTimestamp {
+       type Err = ParseError;
+
+       fn from_base32(b32: &[u5]) -> Result<Self, Self::Err> {
+               if b32.len() != 7 {
+                       return Err(ParseError::InvalidSliceLength("PositiveTimestamp::from_base32()".into()));
+               }
+               let timestamp: u64 = parse_int_be(b32, 32)
+                       .expect("7*5bit < 64bit, no overflow possible");
+               match PositiveTimestamp::from_unix_timestamp(timestamp) {
+                       Ok(t) => Ok(t),
+                       Err(CreationError::TimestampOutOfBounds) => Err(ParseError::TimestampOverflow),
+                       Err(_) => unreachable!(),
+               }
+       }
+}
+
+impl FromBase32 for Signature {
+       type Err = ParseError;
+       fn from_base32(signature: &[u5]) -> Result<Self, Self::Err> {
+               if signature.len() != 104 {
+                       return Err(ParseError::InvalidSliceLength("Signature::from_base32()".into()));
+               }
+               let recoverable_signature_bytes = Vec::<u8>::from_base32(signature)?;
+               let signature = &recoverable_signature_bytes[0..64];
+               let recovery_id = RecoveryId::from_i32(recoverable_signature_bytes[64] as i32)?;
+
+               Ok(Signature(RecoverableSignature::from_compact(
+                       signature,
+                       recovery_id
+               )?))
+       }
+}
+
+fn parse_int_be<T, U>(digits: &[U], base: T) -> Option<T>
+       where T: CheckedAdd + CheckedMul + From<u8> + Default,
+             U: Into<u8> + Copy
+{
+       digits.iter().fold(Some(Default::default()), |acc, b|
+               acc
+                       .and_then(|x| x.checked_mul(&base))
+                       .and_then(|x| x.checked_add(&(Into::<u8>::into(*b)).into()))
+       )
+}
+
+fn parse_tagged_parts(data: &[u5]) -> Result<Vec<RawTaggedField>, ParseError> {
+       let mut parts = Vec::<RawTaggedField>::new();
+       let mut data = data;
+
+       while !data.is_empty() {
+               if data.len() < 3 {
+                       return Err(ParseError::UnexpectedEndOfTaggedFields);
+               }
+
+               // Ignore tag at data[0], it will be handled in the TaggedField parsers and
+               // parse the length to find the end of the tagged field's data
+               let len = parse_int_be(&data[1..3], 32).expect("can't overflow");
+               let last_element = 3 + len;
+
+               if data.len() < last_element {
+                       return Err(ParseError::UnexpectedEndOfTaggedFields);
+               }
+
+               // Get the tagged field's data slice
+               let field = &data[0..last_element];
+
+               // Set data slice to remaining data
+               data = &data[last_element..];
+
+               match TaggedField::from_base32(field) {
+                       Ok(field) => {
+                               parts.push(RawTaggedField::KnownSemantics(field))
+                       },
+                       Err(ParseError::Skip) => {
+                               parts.push(RawTaggedField::UnknownSemantics(field.into()))
+                       },
+                       Err(e) => {return Err(e)}
+               }
+       }
+       Ok(parts)
+}
+
+impl FromBase32 for TaggedField {
+       type Err = ParseError;
+
+       fn from_base32(field: &[u5]) -> Result<TaggedField, ParseError> {
+               if field.len() < 3 {
+                       return Err(ParseError::UnexpectedEndOfTaggedFields);
+               }
+
+               let tag = field[0];
+               let field_data =  &field[3..];
+
+               match tag.to_u8() {
+                       constants::TAG_PAYMENT_HASH =>
+                               Ok(TaggedField::PaymentHash(Sha256::from_base32(field_data)?)),
+                       constants::TAG_DESCRIPTION =>
+                               Ok(TaggedField::Description(Description::from_base32(field_data)?)),
+                       constants::TAG_PAYEE_PUB_KEY =>
+                               Ok(TaggedField::PayeePubKey(PayeePubKey::from_base32(field_data)?)),
+                       constants::TAG_DESCRIPTION_HASH =>
+                               Ok(TaggedField::DescriptionHash(Sha256::from_base32(field_data)?)),
+                       constants::TAG_EXPIRY_TIME =>
+                               Ok(TaggedField::ExpiryTime(ExpiryTime::from_base32(field_data)?)),
+                       constants::TAG_MIN_FINAL_CLTV_EXPIRY =>
+                               Ok(TaggedField::MinFinalCltvExpiry(MinFinalCltvExpiry::from_base32(field_data)?)),
+                       constants::TAG_FALLBACK =>
+                               Ok(TaggedField::Fallback(Fallback::from_base32(field_data)?)),
+                       constants::TAG_ROUTE =>
+                               Ok(TaggedField::Route(Route::from_base32(field_data)?)),
+                       constants::TAG_PAYMENT_SECRET =>
+                               Ok(TaggedField::PaymentSecret(PaymentSecret::from_base32(field_data)?)),
+                       _ => {
+                               // "A reader MUST skip over unknown fields"
+                               Err(ParseError::Skip)
+                       }
+               }
+       }
+}
+
+impl FromBase32 for Sha256 {
+       type Err = ParseError;
+
+       fn from_base32(field_data: &[u5]) -> Result<Sha256, ParseError> {
+               if field_data.len() != 52 {
+                       // "A reader MUST skip over […] a p, [or] h […] field that does not have data_length 52 […]."
+                       Err(ParseError::Skip)
+               } else {
+                       Ok(Sha256(sha256::Hash::from_slice(&Vec::<u8>::from_base32(field_data)?)
+                               .expect("length was checked before (52 u5 -> 32 u8)")))
+               }
+       }
+}
+
+impl FromBase32 for Description {
+       type Err = ParseError;
+
+       fn from_base32(field_data: &[u5]) -> Result<Description, ParseError> {
+               let bytes = Vec::<u8>::from_base32(field_data)?;
+               let description = String::from(str::from_utf8(&bytes)?);
+               Ok(Description::new(description).expect(
+                       "Max len is 639=floor(1023*5/8) since the len field is only 10bits long"
+               ))
+       }
+}
+
+impl FromBase32 for PayeePubKey {
+       type Err = ParseError;
+
+       fn from_base32(field_data: &[u5]) -> Result<PayeePubKey, ParseError> {
+               if field_data.len() != 53 {
+                       // "A reader MUST skip over […] a n […] field that does not have data_length 53 […]."
+                       Err(ParseError::Skip)
+               } else {
+                       let data_bytes = Vec::<u8>::from_base32(field_data)?;
+                       let pub_key = PublicKey::from_slice(&data_bytes)?;
+                       Ok(pub_key.into())
+               }
+       }
+}
+
+impl FromBase32 for PaymentSecret {
+       type Err = ParseError;
+
+       fn from_base32(field_data: &[u5]) -> Result<PaymentSecret, ParseError> {
+               if field_data.len() != 52 {
+                       Err(ParseError::Skip)
+               } else {
+                       let data_bytes = Vec::<u8>::from_base32(field_data)?;
+                       let mut payment_secret = [0; 32];
+                       payment_secret.copy_from_slice(&data_bytes);
+                       Ok(PaymentSecret(payment_secret))
+               }
+       }
+}
+
+impl FromBase32 for ExpiryTime {
+       type Err = ParseError;
+
+       fn from_base32(field_data: &[u5]) -> Result<ExpiryTime, ParseError> {
+               match parse_int_be::<u64, u5>(field_data, 32)
+                       .and_then(|t| ExpiryTime::from_seconds(t).ok()) // ok, since the only error is out of bounds
+               {
+                       Some(t) => Ok(t),
+                       None => Err(ParseError::IntegerOverflowError),
+               }
+       }
+}
+
+impl FromBase32 for MinFinalCltvExpiry {
+       type Err = ParseError;
+
+       fn from_base32(field_data: &[u5]) -> Result<MinFinalCltvExpiry, ParseError> {
+               let expiry = parse_int_be::<u64, u5>(field_data, 32);
+               if let Some(expiry) = expiry {
+                       Ok(MinFinalCltvExpiry(expiry))
+               } else {
+                       Err(ParseError::IntegerOverflowError)
+               }
+       }
+}
+
+impl FromBase32 for Fallback {
+       type Err = ParseError;
+
+       fn from_base32(field_data: &[u5]) -> Result<Fallback, ParseError> {
+               if field_data.len() < 1 {
+                       return Err(ParseError::UnexpectedEndOfTaggedFields);
+               }
+
+               let version = field_data[0];
+               let bytes = Vec::<u8>::from_base32(&field_data[1..])?;
+
+               match version.to_u8() {
+                       0..=16 => {
+                               if bytes.len() < 2 || bytes.len() > 40 {
+                                       return Err(ParseError::InvalidSegWitProgramLength);
+                               }
+
+                               Ok(Fallback::SegWitProgram {
+                                       version: version,
+                                       program: bytes
+                               })
+                       },
+                       17 => {
+                               if bytes.len() != 20 {
+                                       return Err(ParseError::InvalidPubKeyHashLength);
+                               }
+                               //TODO: refactor once const generics are available
+                               let mut pkh = [0u8; 20];
+                               pkh.copy_from_slice(&bytes);
+                               Ok(Fallback::PubKeyHash(pkh))
+                       }
+                       18 => {
+                               if bytes.len() != 20 {
+                                       return Err(ParseError::InvalidScriptHashLength);
+                               }
+                               let mut sh = [0u8; 20];
+                               sh.copy_from_slice(&bytes);
+                               Ok(Fallback::ScriptHash(sh))
+                       }
+                       _ => Err(ParseError::Skip)
+               }
+       }
+}
+
+impl FromBase32 for Route {
+       type Err = ParseError;
+
+       fn from_base32(field_data: &[u5]) -> Result<Route, ParseError> {
+               let bytes = Vec::<u8>::from_base32(field_data)?;
+
+               if bytes.len() % 51 != 0 {
+                       return Err(ParseError::UnexpectedEndOfTaggedFields);
+               }
+
+               let mut route_hops = Vec::<RouteHop>::new();
+
+               let mut bytes = bytes.as_slice();
+               while !bytes.is_empty() {
+                       let hop_bytes = &bytes[0..51];
+                       bytes = &bytes[51..];
+
+                       let mut channel_id: [u8; 8] = Default::default();
+                       channel_id.copy_from_slice(&hop_bytes[33..41]);
+
+                       let hop = RouteHop {
+                               pubkey: PublicKey::from_slice(&hop_bytes[0..33])?,
+                               short_channel_id: channel_id,
+                               fee_base_msat: parse_int_be(&hop_bytes[41..45], 256).expect("slice too big?"),
+                               fee_proportional_millionths: parse_int_be(&hop_bytes[45..49], 256).expect("slice too big?"),
+                               cltv_expiry_delta: parse_int_be(&hop_bytes[49..51], 256).expect("slice too big?")
+                       };
+
+                       route_hops.push(hop);
+               }
+
+               Ok(Route(route_hops))
+       }
+}
+
+/// Errors that indicate what is wrong with the invoice. They have some granularity for debug
+/// reasons, but should generally result in an "invalid BOLT11 invoice" message for the user.
+#[allow(missing_docs)]
+#[derive(PartialEq, Debug, Clone)]
+pub enum ParseError {
+       Bech32Error(bech32::Error),
+       ParseAmountError(ParseIntError),
+       MalformedSignature(secp256k1::Error),
+       BadPrefix,
+       UnknownCurrency,
+       UnknownSiPrefix,
+       MalformedHRP,
+       TooShortDataPart,
+       UnexpectedEndOfTaggedFields,
+       DescriptionDecodeError(str::Utf8Error),
+       PaddingError,
+       IntegerOverflowError,
+       InvalidSegWitProgramLength,
+       InvalidPubKeyHashLength,
+       InvalidScriptHashLength,
+       InvalidRecoveryId,
+       InvalidSliceLength(String),
+
+       /// Not an error, but used internally to signal that a part of the invoice should be ignored
+       /// according to BOLT11
+       Skip,
+       TimestampOverflow,
+}
+
+/// Indicates that something went wrong while parsing or validating the invoice. Parsing errors
+/// should be mostly seen as opaque and are only there for debugging reasons. Semantic errors
+/// like wrong signatures, missing fields etc. could mean that someone tampered with the invoice.
+#[derive(PartialEq, Debug, Clone)]
+pub enum ParseOrSemanticError {
+       /// The invoice couldn't be decoded
+       ParseError(ParseError),
+
+       /// The invoice could be decoded but violates the BOLT11 standard
+       SemanticError(::SemanticError),
+}
+
+impl Display for ParseError {
+       fn fmt(&self, f: &mut Formatter) -> fmt::Result {
+               match *self {
+                       // TODO: find a way to combine the first three arms (e as error::Error?)
+                       ParseError::Bech32Error(ref e) => {
+                               write!(f, "Invalid bech32: {}", e)
+                       }
+                       ParseError::ParseAmountError(ref e) => {
+                               write!(f, "Invalid amount in hrp ({})", e)
+                       }
+                       ParseError::MalformedSignature(ref e) => {
+                               write!(f, "Invalid secp256k1 signature: {}", e)
+                       }
+                       ParseError::DescriptionDecodeError(ref e) => {
+                               write!(f, "Description is not a valid utf-8 string: {}", e)
+                       }
+                       ParseError::InvalidSliceLength(ref function) => {
+                               write!(f, "Slice in function {} had the wrong length", function)
+                       }
+                       ParseError::BadPrefix => f.write_str("did not begin with 'ln'"),
+                       ParseError::UnknownCurrency => f.write_str("currency code unknown"),
+                       ParseError::UnknownSiPrefix => f.write_str("unknown SI prefix"),
+                       ParseError::MalformedHRP => f.write_str("malformed human readable part"),
+                       ParseError::TooShortDataPart => {
+                               f.write_str("data part too short (should be at least 111 bech32 chars long)")
+                       },
+                       ParseError::UnexpectedEndOfTaggedFields => {
+                               f.write_str("tagged fields part ended unexpectedly")
+                       },
+                       ParseError::PaddingError => f.write_str("some data field had bad padding"),
+                       ParseError::IntegerOverflowError => {
+                               f.write_str("parsed integer doesn't fit into receiving type")
+                       },
+                       ParseError::InvalidSegWitProgramLength => {
+                               f.write_str("fallback SegWit program is too long or too short")
+                       },
+                       ParseError::InvalidPubKeyHashLength => {
+                               f.write_str("fallback public key hash has a length unequal 20 bytes")
+                       },
+                       ParseError::InvalidScriptHashLength => {
+                               f.write_str("fallback script hash has a length unequal 32 bytes")
+                       },
+                       ParseError::InvalidRecoveryId => {
+                               f.write_str("recovery id is out of range (should be in [0,3])")
+                       },
+                       ParseError::Skip => {
+                               f.write_str("the tagged field has to be skipped because of an unexpected, but allowed property")
+                       },
+                       ParseError::TimestampOverflow => {
+                f.write_str("the invoice's timestamp could not be represented as SystemTime")
+            },
+               }
+       }
+}
+
+impl Display for ParseOrSemanticError {
+       fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
+               match self {
+                       ParseOrSemanticError::ParseError(err) => err.fmt(f),
+                       ParseOrSemanticError::SemanticError(err) => err.fmt(f),
+               }
+       }
+}
+
+impl error::Error for ParseError {}
+
+impl error::Error for ParseOrSemanticError {}
+
+macro_rules! from_error {
+    ($my_error:expr, $extern_error:ty) => {
+        impl From<$extern_error> for ParseError {
+            fn from(e: $extern_error) -> Self {
+                $my_error(e)
+            }
+        }
+    }
+}
+
+from_error!(ParseError::MalformedSignature, secp256k1::Error);
+from_error!(ParseError::ParseAmountError, ParseIntError);
+from_error!(ParseError::DescriptionDecodeError, str::Utf8Error);
+
+impl From<bech32::Error> for ParseError {
+       fn from(e: bech32::Error) -> Self {
+               match e {
+                       bech32::Error::InvalidPadding => ParseError::PaddingError,
+                       _ => ParseError::Bech32Error(e)
+               }
+       }
+}
+
+impl From<ParseError> for ParseOrSemanticError {
+       fn from(e: ParseError) -> Self {
+               ParseOrSemanticError::ParseError(e)
+       }
+}
+
+impl From<::SemanticError> for ParseOrSemanticError {
+       fn from(e: SemanticError) -> Self {
+               ParseOrSemanticError::SemanticError(e)
+       }
+}
+
+#[cfg(test)]
+mod test {
+       use de::ParseError;
+       use secp256k1::PublicKey;
+       use bech32::u5;
+       use bitcoin_hashes::hex::FromHex;
+       use bitcoin_hashes::sha256;
+
+       const CHARSET_REV: [i8; 128] = [
+               -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
+               -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
+               -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
+               15, -1, 10, 17, 21, 20, 26, 30, 7, 5, -1, -1, -1, -1, -1, -1,
+               -1, 29, -1, 24, 13, 25, 9, 8, 23, -1, 18, 22, 31, 27, 19, -1,
+               1, 0, 3, 16, 11, 28, 12, 14, 6, 4, 2, -1, -1, -1, -1, -1,
+               -1, 29, -1, 24, 13, 25, 9, 8, 23, -1, 18, 22, 31, 27, 19, -1,
+               1, 0, 3, 16, 11, 28, 12, 14, 6, 4, 2, -1, -1, -1, -1, -1
+       ];
+
+       fn from_bech32(bytes_5b: &[u8]) -> Vec<u5> {
+               bytes_5b
+                       .iter()
+                       .map(|c| u5::try_from_u8(CHARSET_REV[*c as usize] as u8).unwrap())
+                       .collect()
+       }
+
+       #[test]
+       fn test_parse_currency_prefix() {
+               use Currency;
+
+               assert_eq!("bc".parse::<Currency>(), Ok(Currency::Bitcoin));
+               assert_eq!("tb".parse::<Currency>(), Ok(Currency::BitcoinTestnet));
+               assert_eq!("bcrt".parse::<Currency>(), Ok(Currency::Regtest));
+               assert_eq!("sb".parse::<Currency>(), Ok(Currency::Simnet));
+               assert_eq!("something_else".parse::<Currency>(), Err(ParseError::UnknownCurrency))
+       }
+
+       #[test]
+       fn test_parse_int_from_bytes_be() {
+               use de::parse_int_be;
+
+               assert_eq!(parse_int_be::<u32, u8>(&[1, 2, 3, 4], 256), Some(16909060));
+               assert_eq!(parse_int_be::<u32, u8>(&[1, 3], 32), Some(35));
+               assert_eq!(parse_int_be::<u32, u8>(&[255, 255, 255, 255], 256), Some(4294967295));
+               assert_eq!(parse_int_be::<u32, u8>(&[1, 0, 0, 0, 0], 256), None);
+       }
+
+       #[test]
+       fn test_parse_sha256_hash() {
+               use Sha256;
+               use bech32::FromBase32;
+
+               let input = from_bech32(
+                       "qqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqypq".as_bytes()
+               );
+
+               let hash = sha256::Hash::from_hex(
+                       "0001020304050607080900010203040506070809000102030405060708090102"
+               ).unwrap();
+               let expected = Ok(Sha256(hash));
+
+               assert_eq!(Sha256::from_base32(&input), expected);
+
+               // make sure hashes of unknown length get skipped
+               let input_unexpected_length = from_bech32(
+                       "qqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqypyq".as_bytes()
+               );
+               assert_eq!(Sha256::from_base32(&input_unexpected_length), Err(ParseError::Skip));
+       }
+
+       #[test]
+       fn test_parse_description() {
+               use ::Description;
+               use bech32::FromBase32;
+
+               let input = from_bech32("xysxxatsyp3k7enxv4js".as_bytes());
+               let expected = Ok(Description::new("1 cup coffee".to_owned()).unwrap());
+               assert_eq!(Description::from_base32(&input), expected);
+       }
+
+       #[test]
+       fn test_parse_payee_pub_key() {
+               use ::PayeePubKey;
+               use bech32::FromBase32;
+
+               let input = from_bech32("q0n326hr8v9zprg8gsvezcch06gfaqqhde2aj730yg0durunfhv66".as_bytes());
+               let pk_bytes = [
+                       0x03, 0xe7, 0x15, 0x6a, 0xe3, 0x3b, 0x0a, 0x20, 0x8d, 0x07, 0x44, 0x19, 0x91, 0x63,
+                       0x17, 0x7e, 0x90, 0x9e, 0x80, 0x17, 0x6e, 0x55, 0xd9, 0x7a, 0x2f, 0x22, 0x1e, 0xde,
+                       0x0f, 0x93, 0x4d, 0xd9, 0xad
+               ];
+               let expected = Ok(PayeePubKey(
+                       PublicKey::from_slice(&pk_bytes[..]).unwrap()
+               ));
+
+               assert_eq!(PayeePubKey::from_base32(&input), expected);
+
+               // expects 33 bytes
+               let input_unexpected_length = from_bech32(
+                       "q0n326hr8v9zprg8gsvezcch06gfaqqhde2aj730yg0durunfhvq".as_bytes()
+               );
+               assert_eq!(PayeePubKey::from_base32(&input_unexpected_length), Err(ParseError::Skip));
+       }
+
+       #[test]
+       fn test_parse_expiry_time() {
+               use ::ExpiryTime;
+               use bech32::FromBase32;
+
+               let input = from_bech32("pu".as_bytes());
+               let expected = Ok(ExpiryTime::from_seconds(60).unwrap());
+               assert_eq!(ExpiryTime::from_base32(&input), expected);
+
+               let input_too_large = from_bech32("sqqqqqqqqqqqq".as_bytes());
+               assert_eq!(ExpiryTime::from_base32(&input_too_large), Err(ParseError::IntegerOverflowError));
+       }
+
+       #[test]
+       fn test_parse_min_final_cltv_expiry() {
+               use ::MinFinalCltvExpiry;
+               use bech32::FromBase32;
+
+               let input = from_bech32("pr".as_bytes());
+               let expected = Ok(MinFinalCltvExpiry(35));
+
+               assert_eq!(MinFinalCltvExpiry::from_base32(&input), expected);
+       }
+
+       #[test]
+       fn test_parse_fallback() {
+               use Fallback;
+               use bech32::FromBase32;
+
+               let cases = vec![
+                       (
+                               from_bech32("3x9et2e20v6pu37c5d9vax37wxq72un98".as_bytes()),
+                               Ok(Fallback::PubKeyHash([
+                                       0x31, 0x72, 0xb5, 0x65, 0x4f, 0x66, 0x83, 0xc8, 0xfb, 0x14, 0x69, 0x59, 0xd3,
+                                       0x47, 0xce, 0x30, 0x3c, 0xae, 0x4c, 0xa7
+                               ]))
+                       ),
+                       (
+                               from_bech32("j3a24vwu6r8ejrss3axul8rxldph2q7z9".as_bytes()),
+                               Ok(Fallback::ScriptHash([
+                                       0x8f, 0x55, 0x56, 0x3b, 0x9a, 0x19, 0xf3, 0x21, 0xc2, 0x11, 0xe9, 0xb9, 0xf3,
+                                       0x8c, 0xdf, 0x68, 0x6e, 0xa0, 0x78, 0x45
+                               ]))
+                       ),
+                       (
+                               from_bech32("qw508d6qejxtdg4y5r3zarvary0c5xw7k".as_bytes()),
+                               Ok(Fallback::SegWitProgram {
+                                       version: u5::try_from_u8(0).unwrap(),
+                                       program: Vec::from(&[
+                                               0x75u8, 0x1e, 0x76, 0xe8, 0x19, 0x91, 0x96, 0xd4, 0x54, 0x94, 0x1c, 0x45,
+                                               0xd1, 0xb3, 0xa3, 0x23, 0xf1, 0x43, 0x3b, 0xd6
+                                       ][..])
+                               })
+                       ),
+                       (
+                               vec![u5::try_from_u8(21).unwrap(); 41],
+                               Err(ParseError::Skip)
+                       ),
+                       (
+                               vec![],
+                               Err(ParseError::UnexpectedEndOfTaggedFields)
+                       ),
+                       (
+                               vec![u5::try_from_u8(1).unwrap(); 81],
+                               Err(ParseError::InvalidSegWitProgramLength)
+                       ),
+                       (
+                               vec![u5::try_from_u8(17).unwrap(); 1],
+                               Err(ParseError::InvalidPubKeyHashLength)
+                       ),
+                       (
+                               vec![u5::try_from_u8(18).unwrap(); 1],
+                               Err(ParseError::InvalidScriptHashLength)
+                       )
+               ];
+
+               for (input, expected) in cases.into_iter() {
+                       assert_eq!(Fallback::from_base32(&input), expected);
+               }
+       }
+
+       #[test]
+       fn test_parse_route() {
+               use RouteHop;
+               use ::Route;
+               use bech32::FromBase32;
+
+               let input = from_bech32(
+                       "q20q82gphp2nflc7jtzrcazrra7wwgzxqc8u7754cdlpfrmccae92qgzqvzq2ps8pqqqqqqpqqqqq9qqqvpeuqa\
+                       fqxu92d8lr6fvg0r5gv0heeeqgcrqlnm6jhphu9y00rrhy4grqszsvpcgpy9qqqqqqgqqqqq7qqzq".as_bytes()
+               );
+
+               let mut expected = Vec::<RouteHop>::new();
+               expected.push(RouteHop {
+                       pubkey: PublicKey::from_slice(
+                               &[
+                                       0x02u8, 0x9e, 0x03, 0xa9, 0x01, 0xb8, 0x55, 0x34, 0xff, 0x1e, 0x92, 0xc4, 0x3c,
+                                       0x74, 0x43, 0x1f, 0x7c, 0xe7, 0x20, 0x46, 0x06, 0x0f, 0xcf, 0x7a, 0x95, 0xc3,
+                                       0x7e, 0x14, 0x8f, 0x78, 0xc7, 0x72, 0x55
+                               ][..]
+                       ).unwrap(),
+                       short_channel_id: [0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08],
+                       fee_base_msat: 1,
+                       fee_proportional_millionths: 20,
+                       cltv_expiry_delta: 3
+               });
+               expected.push(RouteHop {
+                       pubkey: PublicKey::from_slice(
+                               &[
+                                       0x03u8, 0x9e, 0x03, 0xa9, 0x01, 0xb8, 0x55, 0x34, 0xff, 0x1e, 0x92, 0xc4, 0x3c,
+                                       0x74, 0x43, 0x1f, 0x7c, 0xe7, 0x20, 0x46, 0x06, 0x0f, 0xcf, 0x7a, 0x95, 0xc3,
+                                       0x7e, 0x14, 0x8f, 0x78, 0xc7, 0x72, 0x55
+                               ][..]
+                       ).unwrap(),
+                       short_channel_id: [0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a],
+                       fee_base_msat: 2,
+                       fee_proportional_millionths: 30,
+                       cltv_expiry_delta: 4
+               });
+
+               assert_eq!(Route::from_base32(&input), Ok(Route(expected)));
+
+               assert_eq!(
+                       Route::from_base32(&[u5::try_from_u8(0).unwrap(); 40][..]),
+                       Err(ParseError::UnexpectedEndOfTaggedFields)
+               );
+       }
+
+       #[test]
+       fn test_payment_secret_deserialization() {
+               use bech32::CheckBase32;
+               use secp256k1::recovery::{RecoveryId, RecoverableSignature};
+               use TaggedField::*;
+               use {SiPrefix, SignedRawInvoice, Signature, RawInvoice, RawTaggedField, RawHrp, RawDataPart,
+                                Currency, Sha256, PositiveTimestamp};
+
+               assert_eq!( // BOLT 11 payment secret invoice. The unknown fields are invoice features.
+                       "lnbc25m1pvjluezpp5qqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqypqdq5vdhkven9v5sxyetpdeessp5zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zygs9q5sqqqqqqqqqqqqqqqpqsq67gye39hfg3zd8rgc80k32tvy9xk2xunwm5lzexnvpx6fd77en8qaq424dxgt56cag2dpt359k3ssyhetktkpqh24jqnjyw6uqd08sgptq44qu".parse(),
+                       Ok(SignedRawInvoice {
+                                       raw_invoice: RawInvoice {
+                                               hrp: RawHrp {
+                                                       currency: Currency::Bitcoin,
+                                                       raw_amount: Some(25),
+                                                       si_prefix: Some(SiPrefix::Milli)
+                                               },
+                                               data: RawDataPart {
+                                                       timestamp: PositiveTimestamp::from_unix_timestamp(1496314658).unwrap(),
+                                                       tagged_fields: vec ! [
+                                                               PaymentHash(Sha256(sha256::Hash::from_hex(
+                                                                       "0001020304050607080900010203040506070809000102030405060708090102"
+                                                               ).unwrap())).into(),
+                                                               Description(::Description::new("coffee beans".to_owned()).unwrap()).into(),
+                                                               PaymentSecret(::PaymentSecret([17; 32])).into(),
+                                                               RawTaggedField::UnknownSemantics(vec![5, 0, 20, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+                                                                                                                                                                                                                       0, 0, 0, 0, 1, 0, 16,
+                                                                                                                                                                                                                       0].check_base32().unwrap())],
+                                                                       }
+                                                               },
+                                       hash: [0xb1, 0x96, 0x46, 0xc3, 0xbc, 0x56, 0x76, 0x1d, 0x20, 0x65, 0x6e, 0x0e, 0x32,
+                                                                       0xec, 0xd2, 0x69, 0x27, 0xb7, 0x62, 0x6e, 0x2a, 0x8b, 0xe6, 0x97, 0x71, 0x9f,
+                                                                       0xf8, 0x7e, 0x44, 0x54, 0x55, 0xb9],
+                                       signature: Signature(RecoverableSignature::from_compact(
+                                                                               &[0xd7, 0x90, 0x4c, 0xc4, 0xb7, 0x4a, 0x22, 0x26, 0x9c, 0x68, 0xc1, 0xdf, 0x68,
+                                                                                       0xa9, 0x6c, 0x21, 0x4d, 0x65, 0x1b, 0x93, 0x76, 0xe9, 0xf1, 0x64, 0xd3, 0x60,
+                                                                                       0x4d, 0xa4, 0xb7, 0xde, 0xcc, 0xce, 0x0e, 0x82, 0xaa, 0xab, 0x4c, 0x85, 0xd3,
+                                                                                       0x58, 0xea, 0x14, 0xd0, 0xae, 0x34, 0x2d, 0xa3, 0x08, 0x12, 0xf9, 0x5d, 0x97,
+                                                                                       0x60, 0x82, 0xea, 0xac, 0x81, 0x39, 0x11, 0xda, 0xe0, 0x1a, 0xf3, 0xc1],
+                                                                               RecoveryId::from_i32(1).unwrap()
+                                                               ).unwrap()),
+                       })
+               )
+       }
+
+       #[test]
+       fn test_raw_signed_invoice_deserialization() {
+               use TaggedField::*;
+               use secp256k1::recovery::{RecoveryId, RecoverableSignature};
+               use {SignedRawInvoice, Signature, RawInvoice, RawHrp, RawDataPart, Currency, Sha256,
+                        PositiveTimestamp};
+
+               assert_eq!(
+                       "lnbc1pvjluezpp5qqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqypqdpl2pkx2ctnv5sxxmmw\
+                       wd5kgetjypeh2ursdae8g6twvus8g6rfwvs8qun0dfjkxaq8rkx3yf5tcsyz3d73gafnh3cax9rn449d9p5uxz9\
+                       ezhhypd0elx87sjle52x86fux2ypatgddc6k63n7erqz25le42c4u4ecky03ylcqca784w".parse(),
+                       Ok(SignedRawInvoice {
+                               raw_invoice: RawInvoice {
+                                       hrp: RawHrp {
+                                               currency: Currency::Bitcoin,
+                                               raw_amount: None,
+                                               si_prefix: None,
+                                       },
+                                       data: RawDataPart {
+                                       timestamp: PositiveTimestamp::from_unix_timestamp(1496314658).unwrap(),
+                                       tagged_fields: vec ! [
+                                               PaymentHash(Sha256(sha256::Hash::from_hex(
+                                                       "0001020304050607080900010203040506070809000102030405060708090102"
+                                               ).unwrap())).into(),
+                                               Description(
+                                                       ::Description::new(
+                                                               "Please consider supporting this project".to_owned()
+                                                       ).unwrap()
+                                               ).into(),
+                                       ],
+                                       },
+                                       },
+                               hash: [
+                                       0xc3, 0xd4, 0xe8, 0x3f, 0x64, 0x6f, 0xa7, 0x9a, 0x39, 0x3d, 0x75, 0x27,
+                                       0x7b, 0x1d, 0x85, 0x8d, 0xb1, 0xd1, 0xf7, 0xab, 0x71, 0x37, 0xdc, 0xb7,
+                                       0x83, 0x5d, 0xb2, 0xec, 0xd5, 0x18, 0xe1, 0xc9
+                               ],
+                               signature: Signature(RecoverableSignature::from_compact(
+                                       & [
+                                               0x38u8, 0xec, 0x68, 0x91, 0x34, 0x5e, 0x20, 0x41, 0x45, 0xbe, 0x8a,
+                                               0x3a, 0x99, 0xde, 0x38, 0xe9, 0x8a, 0x39, 0xd6, 0xa5, 0x69, 0x43,
+                                               0x4e, 0x18, 0x45, 0xc8, 0xaf, 0x72, 0x05, 0xaf, 0xcf, 0xcc, 0x7f,
+                                               0x42, 0x5f, 0xcd, 0x14, 0x63, 0xe9, 0x3c, 0x32, 0x88, 0x1e, 0xad,
+                                               0x0d, 0x6e, 0x35, 0x6d, 0x46, 0x7e, 0xc8, 0xc0, 0x25, 0x53, 0xf9,
+                                               0xaa, 0xb1, 0x5e, 0x57, 0x38, 0xb1, 0x1f, 0x12, 0x7f
+                                       ],
+                                       RecoveryId::from_i32(0).unwrap()
+                               ).unwrap()),
+                               }
+                       )
+               )
+       }
+}
diff --git a/lightning-invoice/src/lib.rs b/lightning-invoice/src/lib.rs
new file mode 100644 (file)
index 0000000..e9ca442
--- /dev/null
@@ -0,0 +1,1581 @@
+#![deny(missing_docs)]
+#![deny(non_upper_case_globals)]
+#![deny(non_camel_case_types)]
+#![deny(non_snake_case)]
+#![deny(unused_mut)]
+
+#![cfg_attr(feature = "strict", deny(warnings))]
+
+//! This crate provides data structures to represent
+//! [lightning BOLT11](https://github.com/lightningnetwork/lightning-rfc/blob/master/11-payment-encoding.md)
+//! invoices and functions to create, encode and decode these. If you just want to use the standard
+//! en-/decoding functionality this should get you started:
+//!
+//!   * For parsing use `str::parse::<Invoice>(&self)` (see the docs of `impl FromStr for Invoice`)
+//!   * For constructing invoices use the `InvoiceBuilder`
+//!   * For serializing invoices use the `Display`/`ToString` traits
+
+extern crate bech32;
+extern crate bitcoin_hashes;
+extern crate num_traits;
+extern crate secp256k1;
+
+use bech32::u5;
+use bitcoin_hashes::Hash;
+use bitcoin_hashes::sha256;
+
+use secp256k1::key::PublicKey;
+use secp256k1::{Message, Secp256k1};
+use secp256k1::recovery::RecoverableSignature;
+use std::ops::Deref;
+
+use std::iter::FilterMap;
+use std::slice::Iter;
+use std::time::{SystemTime, Duration, UNIX_EPOCH};
+use std::fmt::{Display, Formatter, self};
+
+mod de;
+mod ser;
+mod tb;
+
+pub use de::{ParseError, ParseOrSemanticError};
+
+// TODO: fix before 2037 (see rust PR #55527)
+/// Defines the maximum UNIX timestamp that can be represented as `SystemTime`. This is checked by
+/// one of the unit tests, please run them.
+const SYSTEM_TIME_MAX_UNIX_TIMESTAMP: u64 = std::i32::MAX as u64;
+
+/// Allow the expiry time to be up to one year. Since this reduces the range of possible timestamps
+/// it should be rather low as long as we still have to support 32bit time representations
+const MAX_EXPIRY_TIME: u64 = 60 * 60 * 24 * 356;
+
+/// This function is used as a static assert for the size of `SystemTime`. If the crate fails to
+/// compile due to it this indicates that your system uses unexpected bounds for `SystemTime`. You
+/// can remove this functions and run the test `test_system_time_bounds_assumptions`. In any case,
+/// please open an issue. If all tests pass you should be able to use this library safely by just
+/// removing this function till we patch it accordingly.
+fn __system_time_size_check() {
+       // Use 2 * sizeof(u64) as expected size since the expected underlying implementation is storing
+       // a `Duration` since `SystemTime::UNIX_EPOCH`.
+       unsafe { std::mem::transmute_copy::<SystemTime, [u8; 16]>(&UNIX_EPOCH); }
+}
+
+
+/// **Call this function on startup to ensure that all assumptions about the platform are valid.**
+///
+/// Unfortunately we have to make assumptions about the upper bounds of the `SystemTime` type on
+/// your platform which we can't fully verify at compile time and which isn't part of it's contract.
+/// To our best knowledge our assumptions hold for all platforms officially supported by rust, but
+/// since this check is fast we recommend to do it anyway.
+///
+/// If this function fails this is considered a bug. Please open an issue describing your
+/// platform and stating your current system time.
+///
+/// # Panics
+/// If the check fails this function panics. By calling this function on startup you ensure that
+/// this wont happen at an arbitrary later point in time.
+pub fn check_platform() {
+    // The upper and lower bounds of `SystemTime` are not part of its public contract and are
+    // platform specific. That's why we have to test if our assumptions regarding these bounds
+    // hold on the target platform.
+    //
+    // If this test fails on your platform, please don't use the library and open an issue
+    // instead so we can resolve the situation. Currently this library is tested on:
+    //   * Linux (64bit)
+    let fail_date = UNIX_EPOCH + Duration::from_secs(SYSTEM_TIME_MAX_UNIX_TIMESTAMP);
+    let year = Duration::from_secs(60 * 60 * 24 * 365);
+
+    // Make sure that the library will keep working for another year
+    assert!(fail_date.duration_since(SystemTime::now()).unwrap() > year);
+
+    let max_ts = PositiveTimestamp::from_unix_timestamp(
+        SYSTEM_TIME_MAX_UNIX_TIMESTAMP - MAX_EXPIRY_TIME
+    ).unwrap();
+    let max_exp = ::ExpiryTime::from_seconds(MAX_EXPIRY_TIME).unwrap();
+
+    assert_eq!(
+        (*max_ts.as_time() + *max_exp.as_duration()).duration_since(UNIX_EPOCH).unwrap().as_secs(),
+        SYSTEM_TIME_MAX_UNIX_TIMESTAMP
+    );
+}
+
+
+/// Builder for `Invoice`s. It's the most convenient and advised way to use this library. It ensures
+/// that only a semantically and syntactically correct Invoice can be built using it.
+///
+/// ```
+/// extern crate secp256k1;
+/// extern crate lightning_invoice;
+/// extern crate bitcoin_hashes;
+///
+/// use bitcoin_hashes::Hash;
+/// use bitcoin_hashes::sha256;
+///
+/// use secp256k1::Secp256k1;
+/// use secp256k1::key::SecretKey;
+///
+/// use lightning_invoice::{Currency, InvoiceBuilder};
+///
+/// # fn main() {
+/// let private_key = SecretKey::from_slice(
+///            &[
+///                    0xe1, 0x26, 0xf6, 0x8f, 0x7e, 0xaf, 0xcc, 0x8b, 0x74, 0xf5, 0x4d, 0x26, 0x9f,
+///                    0xe2, 0x06, 0xbe, 0x71, 0x50, 0x00, 0xf9, 0x4d, 0xac, 0x06, 0x7d, 0x1c, 0x04,
+///            0xa8, 0xca, 0x3b, 0x2d, 0xb7, 0x34
+///    ][..]
+///    ).unwrap();
+///
+/// let payment_hash = sha256::Hash::from_slice(&[0; 32][..]).unwrap();
+///
+/// let invoice = InvoiceBuilder::new(Currency::Bitcoin)
+///    .description("Coins pls!".into())
+///    .payment_hash(payment_hash)
+///    .current_timestamp()
+///    .build_signed(|hash| {
+///            Secp256k1::new().sign_recoverable(hash, &private_key)
+///    })
+///    .unwrap();
+///
+/// assert!(invoice.to_string().starts_with("lnbc1"));
+/// # }
+/// ```
+///
+/// # Type parameters
+/// The two parameters `D` and `H` signal if the builder already contains the correct amount of the
+/// given field:
+///  * `D`: exactly one `Description` or `DescriptionHash`
+///  * `H`: exactly one `PaymentHash`
+///  * `T`: the timestamp is set
+#[derive(Eq, PartialEq, Debug, Clone)]
+pub struct InvoiceBuilder<D: tb::Bool, H: tb::Bool, T: tb::Bool> {
+       currency: Currency,
+       amount: Option<u64>,
+       si_prefix: Option<SiPrefix>,
+       timestamp: Option<PositiveTimestamp>,
+       tagged_fields: Vec<TaggedField>,
+       error: Option<CreationError>,
+
+       phantom_d: std::marker::PhantomData<D>,
+       phantom_h: std::marker::PhantomData<H>,
+       phantom_t: std::marker::PhantomData<T>,
+}
+
+/// Represents a syntactically and semantically correct lightning BOLT11 invoice.
+///
+/// There are three ways to construct an `Invoice`:
+///  1. using `InvoiceBuilder`
+///  2. using `Invoice::from_signed(SignedRawInvoice)`
+///  3. using `str::parse::<Invoice>(&str)`
+#[derive(Eq, PartialEq, Debug, Clone)]
+pub struct Invoice {
+       signed_invoice: SignedRawInvoice,
+}
+
+/// Represents the description of an invoice which has to be either a directly included string or
+/// a hash of a description provided out of band.
+#[derive(Eq, PartialEq, Debug, Clone)]
+pub enum InvoiceDescription<'f> {
+       /// Reference to the directly supplied description in the invoice
+       Direct(&'f Description),
+
+       /// Reference to the description's hash included in the invoice
+       Hash(&'f Sha256),
+}
+
+/// Represents a signed `RawInvoice` with cached hash. The signature is not checked and may be
+/// invalid.
+///
+/// # Invariants
+/// The hash has to be either from the deserialized invoice or from the serialized `raw_invoice`.
+#[derive(Eq, PartialEq, Debug, Clone)]
+pub struct SignedRawInvoice {
+       /// The rawInvoice that the signature belongs to
+       raw_invoice: RawInvoice,
+
+       /// Hash of the `RawInvoice` that will be used to check the signature.
+       ///
+       /// * if the `SignedRawInvoice` was deserialized the hash is of from the original encoded form,
+       /// since it's not guaranteed that encoding it again will lead to the same result since integers
+       /// could have been encoded with leading zeroes etc.
+       /// * if the `SignedRawInvoice` was constructed manually the hash will be the calculated hash
+       /// from the `RawInvoice`
+       hash: [u8; 32],
+
+       /// signature of the payment request
+       signature: Signature,
+}
+
+/// Represents an syntactically correct Invoice for a payment on the lightning network,
+/// but without the signature information.
+/// De- and encoding should not lead to information loss but may lead to different hashes.
+///
+/// For methods without docs see the corresponding methods in `Invoice`.
+#[derive(Eq, PartialEq, Debug, Clone)]
+pub struct RawInvoice {
+       /// human readable part
+       pub hrp: RawHrp,
+
+       /// data part
+       pub data: RawDataPart,
+}
+
+/// Data of the `RawInvoice` that is encoded in the human readable part
+#[derive(Eq, PartialEq, Debug, Clone)]
+pub struct RawHrp {
+       /// The currency deferred from the 3rd and 4th character of the bech32 transaction
+       pub currency: Currency,
+
+       /// The amount that, multiplied by the SI prefix, has to be payed
+       pub raw_amount: Option<u64>,
+
+       /// SI prefix that gets multiplied with the `raw_amount`
+       pub si_prefix: Option<SiPrefix>,
+}
+
+/// Data of the `RawInvoice` that is encoded in the data part
+#[derive(Eq, PartialEq, Debug, Clone)]
+pub struct RawDataPart {
+       /// generation time of the invoice
+       pub timestamp: PositiveTimestamp,
+
+       /// tagged fields of the payment request
+       pub tagged_fields: Vec<RawTaggedField>,
+}
+
+/// A timestamp that refers to a date after 1 January 1970 which means its representation as UNIX
+/// timestamp is positive.
+///
+/// # Invariants
+/// The UNIX timestamp representing the stored time has to be positive and small enough so that
+/// a `EpiryTime` can be added to it without an overflow.
+#[derive(Eq, PartialEq, Debug, Clone)]
+pub struct PositiveTimestamp(SystemTime);
+
+/// SI prefixes for the human readable part
+#[derive(Eq, PartialEq, Debug, Clone, Copy)]
+pub enum SiPrefix {
+       /// 10^-3
+       Milli,
+       /// 10^-6
+       Micro,
+       /// 10^-9
+       Nano,
+       /// 10^-12
+       Pico,
+}
+
+impl SiPrefix {
+       /// Returns the multiplier to go from a BTC value to picoBTC implied by this SiPrefix.
+       /// This is effectively 10^12 * the prefix multiplier
+       pub fn multiplier(&self) -> u64 {
+               match *self {
+                       SiPrefix::Milli => 1_000_000_000,
+                       SiPrefix::Micro => 1_000_000,
+                       SiPrefix::Nano => 1_000,
+                       SiPrefix::Pico => 1,
+               }
+       }
+
+       /// Returns all enum variants of `SiPrefix` sorted in descending order of their associated
+       /// multiplier.
+       pub fn values_desc() -> &'static [SiPrefix] {
+               use SiPrefix::*;
+               static VALUES: [SiPrefix; 4] = [Milli, Micro, Nano, Pico];
+               &VALUES
+       }
+}
+
+/// Enum representing the crypto currencies (or networks) supported by this library
+#[derive(Eq, PartialEq, Debug, Clone)]
+pub enum Currency {
+       /// Bitcoin mainnet
+       Bitcoin,
+
+       /// Bitcoin testnet
+       BitcoinTestnet,
+
+       /// Bitcoin regtest
+       Regtest,
+
+       /// Bitcoin simnet/signet
+       Simnet,
+}
+
+/// Tagged field which may have an unknown tag
+#[derive(Eq, PartialEq, Debug, Clone)]
+pub enum RawTaggedField {
+       /// Parsed tagged field with known tag
+       KnownSemantics(TaggedField),
+       /// tagged field which was not parsed due to an unknown tag or undefined field semantics
+       UnknownSemantics(Vec<u5>),
+}
+
+/// Tagged field with known tag
+///
+/// For descriptions of the enum values please refer to the enclosed type's docs.
+#[allow(missing_docs)]
+#[derive(Eq, PartialEq, Debug, Clone)]
+pub enum TaggedField {
+       PaymentHash(Sha256),
+       Description(Description),
+       PayeePubKey(PayeePubKey),
+       DescriptionHash(Sha256),
+       ExpiryTime(ExpiryTime),
+       MinFinalCltvExpiry(MinFinalCltvExpiry),
+       Fallback(Fallback),
+       Route(Route),
+       PaymentSecret(PaymentSecret),
+}
+
+/// SHA-256 hash
+#[derive(Eq, PartialEq, Debug, Clone)]
+pub struct Sha256(pub sha256::Hash);
+
+/// Description string
+///
+/// # Invariants
+/// The description can be at most 639 __bytes__ long
+#[derive(Eq, PartialEq, Debug, Clone)]
+pub struct Description(String);
+
+/// Payee public key
+#[derive(Eq, PartialEq, Debug, Clone)]
+pub struct PayeePubKey(pub PublicKey);
+
+/// 256-bit payment secret
+#[derive(Eq, PartialEq, Debug, Clone)]
+pub struct PaymentSecret(pub [u8; 32]);
+
+/// Positive duration that defines when (relatively to the timestamp) in the future the invoice
+/// expires
+///
+/// # Invariants
+/// The number of seconds this expiry time represents has to be in the range
+/// `0...(SYSTEM_TIME_MAX_UNIX_TIMESTAMP - MAX_EXPIRY_TIME)` to avoid overflows when adding it to a
+/// timestamp
+#[derive(Eq, PartialEq, Debug, Clone)]
+pub struct ExpiryTime(Duration);
+
+/// `min_final_cltv_expiry` to use for the last HTLC in the route
+#[derive(Eq, PartialEq, Debug, Clone)]
+pub struct MinFinalCltvExpiry(pub u64);
+
+// TODO: better types instead onf byte arrays
+/// Fallback address in case no LN payment is possible
+#[allow(missing_docs)]
+#[derive(Eq, PartialEq, Debug, Clone)]
+pub enum Fallback {
+       SegWitProgram {
+               version: u5,
+               program: Vec<u8>,
+       },
+       PubKeyHash([u8; 20]),
+       ScriptHash([u8; 20]),
+}
+
+/// Recoverable signature
+#[derive(Eq, PartialEq, Debug, Clone)]
+pub struct Signature(pub RecoverableSignature);
+
+/// Private routing information
+///
+/// # Invariants
+/// The encoded route has to be <1024 5bit characters long (<=639 bytes or <=12 hops)
+///
+#[derive(Eq, PartialEq, Debug, Clone)]
+pub struct Route(Vec<RouteHop>);
+
+/// Node on a private route
+#[derive(Eq, PartialEq, Debug, Clone)]
+pub struct RouteHop {
+       /// Node's public key
+       pub pubkey: PublicKey,
+
+       /// Which channel of this node we would be using
+       pub short_channel_id: [u8; 8],
+
+       /// Fee charged by this node per transaction
+       pub fee_base_msat: u32,
+
+       /// Fee charged by this node proportional to the amount routed
+       pub fee_proportional_millionths: u32,
+
+       /// Delta substracted by this node from incoming cltv_expiry value
+       pub cltv_expiry_delta: u16,
+}
+
+/// Tag constants as specified in BOLT11
+#[allow(missing_docs)]
+pub mod constants {
+       pub const TAG_PAYMENT_HASH: u8 = 1;
+       pub const TAG_DESCRIPTION: u8 = 13;
+       pub const TAG_PAYEE_PUB_KEY: u8 = 19;
+       pub const TAG_DESCRIPTION_HASH: u8 = 23;
+       pub const TAG_EXPIRY_TIME: u8 = 6;
+       pub const TAG_MIN_FINAL_CLTV_EXPIRY: u8 = 24;
+       pub const TAG_FALLBACK: u8 = 9;
+       pub const TAG_ROUTE: u8 = 3;
+       pub const TAG_PAYMENT_SECRET: u8 = 16;
+}
+
+impl InvoiceBuilder<tb::False, tb::False, tb::False> {
+       /// Construct new, empty `InvoiceBuilder`. All necessary fields have to be filled first before
+       /// `InvoiceBuilder::build(self)` becomes available.
+       pub fn new(currrency: Currency) -> Self {
+               InvoiceBuilder {
+                       currency: currrency,
+                       amount: None,
+                       si_prefix: None,
+                       timestamp: None,
+                       tagged_fields: Vec::new(),
+                       error: None,
+
+                       phantom_d: std::marker::PhantomData,
+                       phantom_h: std::marker::PhantomData,
+                       phantom_t: std::marker::PhantomData,
+               }
+       }
+}
+
+impl<D: tb::Bool, H: tb::Bool, T: tb::Bool> InvoiceBuilder<D, H, T> {
+       /// Helper function to set the completeness flags.
+       fn set_flags<DN: tb::Bool, HN: tb::Bool, TN: tb::Bool>(self) -> InvoiceBuilder<DN, HN, TN> {
+               InvoiceBuilder::<DN, HN, TN> {
+                       currency: self.currency,
+                       amount: self.amount,
+                       si_prefix: self.si_prefix,
+                       timestamp: self.timestamp,
+                       tagged_fields: self.tagged_fields,
+                       error: self.error,
+
+                       phantom_d: std::marker::PhantomData,
+                       phantom_h: std::marker::PhantomData,
+                       phantom_t: std::marker::PhantomData,
+               }
+       }
+
+       /// Sets the amount in pico BTC. The optimal SI prefix is choosen automatically.
+       pub fn amount_pico_btc(mut self, amount: u64) -> Self {
+               let biggest_possible_si_prefix = SiPrefix::values_desc()
+                       .iter()
+                       .find(|prefix| amount % prefix.multiplier() == 0)
+                       .expect("Pico should always match");
+               self.amount = Some(amount / biggest_possible_si_prefix.multiplier());
+               self.si_prefix = Some(*biggest_possible_si_prefix);
+               self
+       }
+
+       /// Sets the payee's public key.
+       pub fn payee_pub_key(mut self, pub_key: PublicKey) -> Self {
+               self.tagged_fields.push(TaggedField::PayeePubKey(PayeePubKey(pub_key)));
+               self
+       }
+
+       /// Sets the payment secret
+       pub fn payment_secret(mut self, payment_secret: PaymentSecret) -> Self {
+               self.tagged_fields.push(TaggedField::PaymentSecret(payment_secret));
+               self
+       }
+
+       /// Sets the expiry time
+       pub fn expiry_time(mut self, expiry_time: Duration) -> Self {
+        match ExpiryTime::from_duration(expiry_time) {
+            Ok(t) => self.tagged_fields.push(TaggedField::ExpiryTime(t)),
+            Err(e) => self.error = Some(e),
+        };
+               self
+       }
+
+       /// Sets `min_final_cltv_expiry`.
+       pub fn min_final_cltv_expiry(mut self, min_final_cltv_expiry: u64) -> Self {
+               self.tagged_fields.push(TaggedField::MinFinalCltvExpiry(MinFinalCltvExpiry(min_final_cltv_expiry)));
+               self
+       }
+
+       /// Adds a fallback address.
+       pub fn fallback(mut self, fallback: Fallback) -> Self {
+               self.tagged_fields.push(TaggedField::Fallback(fallback));
+               self
+       }
+
+       /// Adds a private route.
+       pub fn route(mut self, route: Vec<RouteHop>) -> Self {
+               match Route::new(route) {
+                       Ok(r) => self.tagged_fields.push(TaggedField::Route(r)),
+                       Err(e) => self.error = Some(e),
+               }
+               self
+       }
+}
+
+impl<D: tb::Bool, H: tb::Bool> InvoiceBuilder<D, H, tb::True> {
+       /// Builds a `RawInvoice` if no `CreationError` occurred while construction any of the fields.
+       pub fn build_raw(self) -> Result<RawInvoice, CreationError> {
+
+               // If an error occurred at any time before, return it now
+               if let Some(e) = self.error {
+                       return Err(e);
+               }
+
+               let hrp = RawHrp {
+                       currency: self.currency,
+                       raw_amount: self.amount,
+                       si_prefix: self.si_prefix,
+               };
+
+               let timestamp = self.timestamp.expect("ensured to be Some(t) by type T");
+
+               let tagged_fields = self.tagged_fields.into_iter().map(|tf| {
+                       RawTaggedField::KnownSemantics(tf)
+               }).collect::<Vec<_>>();
+
+               let data = RawDataPart {
+                       timestamp: timestamp,
+                       tagged_fields: tagged_fields,
+               };
+
+               Ok(RawInvoice {
+                       hrp: hrp,
+                       data: data,
+               })
+       }
+}
+
+impl<H: tb::Bool, T: tb::Bool> InvoiceBuilder<tb::False, H, T> {
+       /// Set the description. This function is only available if no description (hash) was set.
+       pub fn description(mut self, description: String) -> InvoiceBuilder<tb::True, H, T> {
+               match Description::new(description) {
+                       Ok(d) => self.tagged_fields.push(TaggedField::Description(d)),
+                       Err(e) => self.error = Some(e),
+               }
+               self.set_flags()
+       }
+
+       /// Set the description hash. This function is only available if no description (hash) was set.
+       pub fn description_hash(mut self, description_hash: sha256::Hash) -> InvoiceBuilder<tb::True, H, T> {
+               self.tagged_fields.push(TaggedField::DescriptionHash(Sha256(description_hash)));
+               self.set_flags()
+       }
+}
+
+impl<D: tb::Bool, T: tb::Bool> InvoiceBuilder<D, tb::False, T> {
+       /// Set the payment hash. This function is only available if no payment hash was set.
+       pub fn payment_hash(mut self, hash: sha256::Hash) -> InvoiceBuilder<D, tb::True, T> {
+               self.tagged_fields.push(TaggedField::PaymentHash(Sha256(hash)));
+               self.set_flags()
+       }
+}
+
+impl<D: tb::Bool, H: tb::Bool> InvoiceBuilder<D, H, tb::False> {
+       /// Sets the timestamp.
+       pub fn timestamp(mut self, time: SystemTime) -> InvoiceBuilder<D, H, tb::True> {
+               match PositiveTimestamp::from_system_time(time) {
+                       Ok(t) => self.timestamp = Some(t),
+                       Err(e) => self.error = Some(e),
+               }
+
+               self.set_flags()
+       }
+
+       /// Sets the timestamp to the current UNIX timestamp.
+       pub fn current_timestamp(mut self) -> InvoiceBuilder<D, H, tb::True> {
+               let now = PositiveTimestamp::from_system_time(SystemTime::now());
+               self.timestamp = Some(now.expect("for the foreseeable future this shouldn't happen"));
+               self.set_flags()
+       }
+}
+
+impl InvoiceBuilder<tb::True, tb::True, tb::True> {
+       /// Builds and signs an invoice using the supplied `sign_function`. This function MAY NOT fail
+       /// and MUST produce a recoverable signature valid for the given hash and if applicable also for
+       /// the included payee public key.
+       pub fn build_signed<F>(self, sign_function: F) -> Result<Invoice, CreationError>
+               where F: FnOnce(&Message) -> RecoverableSignature
+       {
+               let invoice = self.try_build_signed::<_, ()>(|hash| {
+                       Ok(sign_function(hash))
+               });
+
+               match invoice {
+                       Ok(i) => Ok(i),
+                       Err(SignOrCreationError::CreationError(e)) => Err(e),
+                       Err(SignOrCreationError::SignError(())) => unreachable!(),
+               }
+       }
+
+       /// Builds and signs an invoice using the supplied `sign_function`. This function MAY fail with
+       /// an error of type `E` and MUST produce a recoverable signature valid for the given hash and
+       /// if applicable also for the included payee public key.
+       pub fn try_build_signed<F, E>(self, sign_function: F) -> Result<Invoice, SignOrCreationError<E>>
+               where F: FnOnce(&Message) -> Result<RecoverableSignature, E>
+       {
+               let raw = match self.build_raw() {
+                       Ok(r) => r,
+                       Err(e) => return Err(SignOrCreationError::CreationError(e)),
+               };
+
+               let signed = match raw.sign(sign_function) {
+                       Ok(s) => s,
+                       Err(e) => return Err(SignOrCreationError::SignError(e)),
+               };
+
+               let invoice = Invoice {
+                       signed_invoice: signed,
+               };
+
+               invoice.check_field_counts().expect("should be ensured by type signature of builder");
+
+               Ok(invoice)
+       }
+}
+
+
+impl SignedRawInvoice {
+       /// Disassembles the `SignedRawInvoice` into its three parts:
+       ///  1. raw invoice
+       ///  2. hash of the raw invoice
+       ///  3. signature
+       pub fn into_parts(self) -> (RawInvoice, [u8; 32], Signature) {
+               (self.raw_invoice, self.hash, self.signature)
+       }
+
+       /// The `RawInvoice` which was signed.
+       pub fn raw_invoice(&self) -> &RawInvoice {
+               &self.raw_invoice
+       }
+
+       /// The hash of the `RawInvoice` that was signed.
+       pub fn hash(&self) -> &[u8; 32] {
+               &self.hash
+       }
+
+       /// Signature for the invoice.
+       pub fn signature(&self) -> &Signature {
+               &self.signature
+       }
+
+       /// Recovers the public key used for signing the invoice from the recoverable signature.
+       pub fn recover_payee_pub_key(&self) -> Result<PayeePubKey, secp256k1::Error> {
+               let hash = Message::from_slice(&self.hash[..])
+                       .expect("Hash is 32 bytes long, same as MESSAGE_SIZE");
+
+               Ok(PayeePubKey(Secp256k1::new().recover(
+                       &hash,
+                       &self.signature
+               )?))
+       }
+
+       /// Checks if the signature is valid for the included payee public key or if none exists if it's
+       /// valid for the recovered signature (which should always be true?).
+       pub fn check_signature(&self) -> bool {
+               let included_pub_key = self.raw_invoice.payee_pub_key();
+
+               let mut recovered_pub_key = Option::None;
+               if recovered_pub_key.is_none() {
+                       let recovered = match self.recover_payee_pub_key() {
+                               Ok(pk) => pk,
+                               Err(_) => return false,
+                       };
+                       recovered_pub_key = Some(recovered);
+               }
+
+               let pub_key = included_pub_key.or_else(|| recovered_pub_key.as_ref())
+                       .expect("One is always present");
+
+               let hash = Message::from_slice(&self.hash[..])
+                       .expect("Hash is 32 bytes long, same as MESSAGE_SIZE");
+
+               let secp_context = Secp256k1::new();
+               let verification_result = secp_context.verify(
+                       &hash,
+                       &self.signature.to_standard(),
+                       pub_key
+               );
+
+               match verification_result {
+                       Ok(()) => true,
+                       Err(_) => false,
+               }
+       }
+}
+
+/// Finds the first element of an enum stream of a given variant and extracts one member of the
+/// variant. If no element was found `None` gets returned.
+///
+/// The following example would extract the first
+/// ```
+/// use Enum::*
+///
+/// enum Enum {
+///    A(u8),
+///    B(u16)
+/// }
+///
+/// let elements = vec![A(1), A(2), B(3), A(4)]
+///
+/// assert_eq!(find_extract!(elements.iter(), Enum::B(ref x), x), Some(3u16))
+/// ```
+macro_rules! find_extract {
+    ($iter:expr, $enm:pat, $enm_var:ident) => {
+       $iter.filter_map(|tf| match *tf {
+                       $enm => Some($enm_var),
+                       _ => None,
+               }).next()
+    };
+}
+
+#[allow(missing_docs)]
+impl RawInvoice {
+       /// Hash the HRP as bytes and signatureless data part.
+       fn hash_from_parts(hrp_bytes: &[u8], data_without_signature: &[u5]) -> [u8; 32] {
+               use bech32::FromBase32;
+
+               let mut preimage = Vec::<u8>::from(hrp_bytes);
+
+               let mut data_part = Vec::from(data_without_signature);
+               let overhang = (data_part.len() * 5) % 8;
+               if overhang > 0 {
+                       // add padding if data does not end at a byte boundary
+                       data_part.push(u5::try_from_u8(0).unwrap());
+
+                       // if overhang is in (1..3) we need to add u5(0) padding two times
+                       if overhang < 3 {
+                               data_part.push(u5::try_from_u8(0).unwrap());
+                       }
+               }
+
+               preimage.extend_from_slice(&Vec::<u8>::from_base32(&data_part)
+                       .expect("No padding error may occur due to appended zero above."));
+
+               let mut hash: [u8; 32] = Default::default();
+               hash.copy_from_slice(&sha256::Hash::hash(&preimage)[..]);
+               hash
+       }
+
+       /// Calculate the hash of the encoded `RawInvoice`
+       pub fn hash(&self) -> [u8; 32] {
+               use bech32::ToBase32;
+
+               RawInvoice::hash_from_parts(
+                       self.hrp.to_string().as_bytes(),
+                       &self.data.to_base32()
+               )
+       }
+
+       /// Signs the invoice using the supplied `sign_function`. This function MAY fail with an error
+       /// of type `E`. Since the signature of a `SignedRawInvoice` is not required to be valid there
+       /// are no constraints regarding the validity of the produced signature.
+       pub fn sign<F, E>(self, sign_method: F) -> Result<SignedRawInvoice, E>
+               where F: FnOnce(&Message) -> Result<RecoverableSignature, E>
+       {
+               let raw_hash = self.hash();
+               let hash = Message::from_slice(&raw_hash[..])
+                       .expect("Hash is 32 bytes long, same as MESSAGE_SIZE");
+               let signature = sign_method(&hash)?;
+
+               Ok(SignedRawInvoice {
+                       raw_invoice: self,
+                       hash: raw_hash,
+                       signature: Signature(signature),
+               })
+       }
+
+       /// Returns an iterator over all tagged fields with known semantics.
+       pub fn known_tagged_fields(&self)
+               -> FilterMap<Iter<RawTaggedField>, fn(&RawTaggedField) -> Option<&TaggedField>>
+       {
+               // For 1.14.0 compatibility: closures' types can't be written an fn()->() in the
+               // function's type signature.
+               // TODO: refactor once impl Trait is available
+               fn match_raw(raw: &RawTaggedField) -> Option<&TaggedField> {
+                       match *raw {
+                               RawTaggedField::KnownSemantics(ref tf) => Some(tf),
+                               _ => None,
+                       }
+               }
+
+               self.data.tagged_fields.iter().filter_map(match_raw )
+       }
+
+       pub fn payment_hash(&self) -> Option<&Sha256> {
+               find_extract!(self.known_tagged_fields(), TaggedField::PaymentHash(ref x), x)
+       }
+
+       pub fn description(&self) -> Option<&Description> {
+               find_extract!(self.known_tagged_fields(), TaggedField::Description(ref x), x)
+       }
+
+       pub fn payee_pub_key(&self) -> Option<&PayeePubKey> {
+               find_extract!(self.known_tagged_fields(), TaggedField::PayeePubKey(ref x), x)
+       }
+
+       pub fn description_hash(&self) -> Option<&Sha256> {
+               find_extract!(self.known_tagged_fields(), TaggedField::DescriptionHash(ref x), x)
+       }
+
+       pub fn expiry_time(&self) -> Option<&ExpiryTime> {
+               find_extract!(self.known_tagged_fields(), TaggedField::ExpiryTime(ref x), x)
+       }
+
+       pub fn min_final_cltv_expiry(&self) -> Option<&MinFinalCltvExpiry> {
+               find_extract!(self.known_tagged_fields(), TaggedField::MinFinalCltvExpiry(ref x), x)
+       }
+
+       pub fn payment_secret(&self) -> Option<&PaymentSecret> {
+               find_extract!(self.known_tagged_fields(), TaggedField::PaymentSecret(ref x), x)
+       }
+
+       pub fn fallbacks(&self) -> Vec<&Fallback> {
+               self.known_tagged_fields().filter_map(|tf| match tf {
+                       &TaggedField::Fallback(ref f) => Some(f),
+                       _ => None,
+               }).collect::<Vec<&Fallback>>()
+       }
+
+       pub fn routes(&self) -> Vec<&Route> {
+               self.known_tagged_fields().filter_map(|tf| match tf {
+                       &TaggedField::Route(ref r) => Some(r),
+                       _ => None,
+               }).collect::<Vec<&Route>>()
+       }
+
+       pub fn amount_pico_btc(&self) -> Option<u64> {
+               self.hrp.raw_amount.map(|v| {
+                       v * self.hrp.si_prefix.as_ref().map_or(1_000_000_000_000, |si| { si.multiplier() })
+               })
+       }
+
+       pub fn currency(&self) -> Currency {
+               self.hrp.currency.clone()
+       }
+}
+
+impl PositiveTimestamp {
+       /// Create a new `PositiveTimestamp` from a unix timestamp in the Range
+       /// `0...SYSTEM_TIME_MAX_UNIX_TIMESTAMP - MAX_EXPIRY_TIME`, otherwise return a
+       /// `CreationError::TimestampOutOfBounds`.
+       pub fn from_unix_timestamp(unix_seconds: u64) -> Result<Self, CreationError> {
+               if unix_seconds > SYSTEM_TIME_MAX_UNIX_TIMESTAMP - MAX_EXPIRY_TIME {
+                       Err(CreationError::TimestampOutOfBounds)
+               } else {
+                       Ok(PositiveTimestamp(UNIX_EPOCH + Duration::from_secs(unix_seconds)))
+               }
+       }
+
+       /// Create a new `PositiveTimestamp` from a `SystemTime` with a corresponding unix timestamp in
+       /// the Range `0...SYSTEM_TIME_MAX_UNIX_TIMESTAMP - MAX_EXPIRY_TIME`, otherwise return a
+       /// `CreationError::TimestampOutOfBounds`.
+       pub fn from_system_time(time: SystemTime) -> Result<Self, CreationError> {
+               if time
+                       .duration_since(UNIX_EPOCH)
+                       .map(|t| t.as_secs() <= SYSTEM_TIME_MAX_UNIX_TIMESTAMP - MAX_EXPIRY_TIME)
+                       .unwrap_or(true)
+                       {
+                               Ok(PositiveTimestamp(time))
+                       } else {
+                       Err(CreationError::TimestampOutOfBounds)
+               }
+       }
+
+       /// Returns the UNIX timestamp representing the stored time
+       pub fn as_unix_timestamp(&self) -> u64 {
+               self.0.duration_since(UNIX_EPOCH)
+                       .expect("ensured by type contract/constructors")
+                       .as_secs()
+       }
+
+       /// Returns a reference to the internal `SystemTime` time representation
+       pub fn as_time(&self) -> &SystemTime {
+               &self.0
+       }
+}
+
+impl Into<SystemTime> for PositiveTimestamp {
+       fn into(self) -> SystemTime {
+               self.0
+       }
+}
+
+impl Deref for PositiveTimestamp {
+       type Target = SystemTime;
+
+       fn deref(&self) -> &Self::Target {
+               &self.0
+       }
+}
+
+impl Invoice {
+       /// Transform the `Invoice` into it's unchecked version
+       pub fn into_signed_raw(self) -> SignedRawInvoice {
+               self.signed_invoice
+       }
+
+       /// Check that all mandatory fields are present
+       fn check_field_counts(&self) -> Result<(), SemanticError> {
+               // "A writer MUST include exactly one p field […]."
+               let payment_hash_cnt = self.tagged_fields().filter(|&tf| match *tf {
+                       TaggedField::PaymentHash(_) => true,
+                       _ => false,
+               }).count();
+               if payment_hash_cnt < 1 {
+                       return Err(SemanticError::NoPaymentHash);
+               } else if payment_hash_cnt > 1 {
+                       return Err(SemanticError::MultiplePaymentHashes);
+               }
+
+               // "A writer MUST include either exactly one d or exactly one h field."
+               let description_cnt = self.tagged_fields().filter(|&tf| match *tf {
+                       TaggedField::Description(_) | TaggedField::DescriptionHash(_) => true,
+                       _ => false,
+               }).count();
+               if  description_cnt < 1 {
+                       return Err(SemanticError::NoDescription);
+               } else if description_cnt > 1 {
+                       return  Err(SemanticError::MultipleDescriptions);
+               }
+
+               Ok(())
+       }
+
+       /// Check that the invoice is signed correctly and that key recovery works
+       pub fn check_signature(&self) -> Result<(), SemanticError> {
+               match self.signed_invoice.recover_payee_pub_key() {
+                       Err(secp256k1::Error::InvalidRecoveryId) =>
+                               return Err(SemanticError::InvalidRecoveryId),
+                       Err(_) => panic!("no other error may occur"),
+                       Ok(_) => {},
+               }
+
+               if !self.signed_invoice.check_signature() {
+                       return Err(SemanticError::InvalidSignature);
+               }
+
+               Ok(())
+       }
+
+       /// Constructs an `Invoice` from a `SignedInvoice` by checking all its invariants.
+       /// ```
+       /// use lightning_invoice::*;
+       ///
+       /// let invoice = "lnbc1pvjluezpp5qqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqypqdp\
+       ///     l2pkx2ctnv5sxxmmwwd5kgetjypeh2ursdae8g6twvus8g6rfwvs8qun0dfjkxaq8rkx3yf5tcsyz3d7\
+       ///     3gafnh3cax9rn449d9p5uxz9ezhhypd0elx87sjle52x86fux2ypatgddc6k63n7erqz25le42c4u4ec\
+       ///     ky03ylcqca784w";
+       ///
+       /// let signed = invoice.parse::<SignedRawInvoice>().unwrap();
+       ///
+       /// assert!(Invoice::from_signed(signed).is_ok());
+       /// ```
+       pub fn from_signed(signed_invoice: SignedRawInvoice) -> Result<Self, SemanticError> {
+               let invoice = Invoice {
+                       signed_invoice: signed_invoice,
+               };
+               invoice.check_field_counts()?;
+               invoice.check_signature()?;
+
+               Ok(invoice)
+       }
+
+       /// Returns the `Invoice`'s timestamp (should equal it's creation time)
+       pub fn timestamp(&self) -> &SystemTime {
+               self.signed_invoice.raw_invoice().data.timestamp.as_time()
+       }
+
+       /// Returns an iterator over all tagged fields of this Invoice.
+       pub fn tagged_fields(&self)
+               -> FilterMap<Iter<RawTaggedField>, fn(&RawTaggedField) -> Option<&TaggedField>> {
+               self.signed_invoice.raw_invoice().known_tagged_fields()
+       }
+
+       /// Returns the hash to which we will receive the preimage on completion of the payment
+       pub fn payment_hash(&self) -> &sha256::Hash {
+               &self.signed_invoice.payment_hash().expect("checked by constructor").0
+       }
+
+       /// Return the description or a hash of it for longer ones
+       pub fn description(&self) -> InvoiceDescription {
+               if let Some(ref direct) = self.signed_invoice.description() {
+                       return InvoiceDescription::Direct(direct);
+               } else if let Some(ref hash) = self.signed_invoice.description_hash() {
+                       return InvoiceDescription::Hash(hash);
+               }
+               unreachable!("ensured by constructor");
+       }
+
+       /// Get the payee's public key if one was included in the invoice
+       pub fn payee_pub_key(&self) -> Option<&PublicKey> {
+               self.signed_invoice.payee_pub_key().map(|x| &x.0)
+       }
+
+    /// Get the payment secret if one was included in the invoice
+    pub fn payment_secret(&self) -> Option<&PaymentSecret> {
+        self.signed_invoice.payment_secret()
+    }
+
+       /// Recover the payee's public key (only to be used if none was included in the invoice)
+       pub fn recover_payee_pub_key(&self) -> PublicKey {
+               self.signed_invoice.recover_payee_pub_key().expect("was checked by constructor").0
+       }
+
+       /// Returns the invoice's expiry time if present
+       pub fn expiry_time(&self) -> Duration {
+               self.signed_invoice.expiry_time()
+                       .map(|x| x.0)
+                       .unwrap_or(Duration::from_secs(3600))
+       }
+
+       /// Returns the invoice's `min_cltv_expiry` time if present
+       pub fn min_final_cltv_expiry(&self) -> Option<&u64> {
+               self.signed_invoice.min_final_cltv_expiry().map(|x| &x.0)
+       }
+
+       /// Returns a list of all fallback addresses
+       pub fn fallbacks(&self) -> Vec<&Fallback> {
+               self.signed_invoice.fallbacks()
+       }
+
+       /// Returns a list of all routes included in the invoice
+       pub fn routes(&self) -> Vec<&Route> {
+               self.signed_invoice.routes()
+       }
+
+       /// Returns the currency for which the invoice was issued
+       pub fn currency(&self) -> Currency {
+               self.signed_invoice.currency()
+       }
+
+       /// Returns the amount if specified in the invoice as pico <currency>.
+       pub fn amount_pico_btc(&self) -> Option<u64> {
+               self.signed_invoice.amount_pico_btc()
+       }
+}
+
+impl From<TaggedField> for RawTaggedField {
+       fn from(tf: TaggedField) -> Self {
+               RawTaggedField::KnownSemantics(tf)
+       }
+}
+
+impl TaggedField {
+       /// Numeric representation of the field's tag
+       pub fn tag(&self) -> u5 {
+               let tag = match *self {
+                       TaggedField::PaymentHash(_) => constants::TAG_PAYMENT_HASH,
+                       TaggedField::Description(_) => constants::TAG_DESCRIPTION,
+                       TaggedField::PayeePubKey(_) => constants::TAG_PAYEE_PUB_KEY,
+                       TaggedField::DescriptionHash(_) => constants::TAG_DESCRIPTION_HASH,
+                       TaggedField::ExpiryTime(_) => constants::TAG_EXPIRY_TIME,
+                       TaggedField::MinFinalCltvExpiry(_) => constants::TAG_MIN_FINAL_CLTV_EXPIRY,
+                       TaggedField::Fallback(_) => constants::TAG_FALLBACK,
+                       TaggedField::Route(_) => constants::TAG_ROUTE,
+                       TaggedField::PaymentSecret(_) => constants::TAG_PAYMENT_SECRET,
+               };
+
+               u5::try_from_u8(tag).expect("all tags defined are <32")
+       }
+}
+
+impl Description {
+
+       /// Creates a new `Description` if `description` is at most 1023 __bytes__ long,
+       /// returns `CreationError::DescriptionTooLong` otherwise
+       ///
+       /// Please note that single characters may use more than one byte due to UTF8 encoding.
+       pub fn new(description: String) -> Result<Description, CreationError> {
+               if description.len() > 639 {
+                       Err(CreationError::DescriptionTooLong)
+               } else {
+                       Ok(Description(description))
+               }
+       }
+
+       /// Returns the underlying description `String`
+       pub fn into_inner(self) -> String {
+               self.0
+       }
+}
+
+impl Into<String> for Description {
+       fn into(self) -> String {
+               self.into_inner()
+       }
+}
+
+impl Deref for Description {
+       type Target = str;
+
+       fn deref(&self) -> &str {
+               &self.0
+       }
+}
+
+impl From<PublicKey> for PayeePubKey {
+       fn from(pk: PublicKey) -> Self {
+               PayeePubKey(pk)
+       }
+}
+
+impl Deref for PayeePubKey {
+       type Target = PublicKey;
+
+       fn deref(&self) -> &PublicKey {
+               &self.0
+       }
+}
+
+impl ExpiryTime {
+       /// Construct an `ExpiryTime` from seconds. If there exists a `PositiveTimestamp` which would
+       /// overflow on adding the `EpiryTime` to it then this function will return a
+       /// `CreationError::ExpiryTimeOutOfBounds`.
+       pub fn from_seconds(seconds: u64) -> Result<ExpiryTime, CreationError> {
+               if seconds <= MAX_EXPIRY_TIME {
+                       Ok(ExpiryTime(Duration::from_secs(seconds)))
+               } else {
+                       Err(CreationError::ExpiryTimeOutOfBounds)
+               }
+       }
+
+       /// Construct an `ExpiryTime` from a `Duration`. If there exists a `PositiveTimestamp` which
+       /// would overflow on adding the `EpiryTime` to it then this function will return a
+       /// `CreationError::ExpiryTimeOutOfBounds`.
+       pub fn from_duration(duration: Duration) -> Result<ExpiryTime, CreationError> {
+               if duration.as_secs() <= MAX_EXPIRY_TIME {
+                       Ok(ExpiryTime(duration))
+               } else {
+                       Err(CreationError::ExpiryTimeOutOfBounds)
+               }
+       }
+
+       /// Returns the expiry time in seconds
+       pub fn as_seconds(&self) -> u64 {
+               self.0.as_secs()
+       }
+
+       /// Returns a reference to the underlying `Duration` (=expiry time)
+       pub fn as_duration(&self) -> &Duration {
+               &self.0
+       }
+}
+
+impl Route {
+       /// Create a new (partial) route from a list of hops
+       pub fn new(hops: Vec<RouteHop>) -> Result<Route, CreationError> {
+               if hops.len() <= 12 {
+                       Ok(Route(hops))
+               } else {
+                       Err(CreationError::RouteTooLong)
+               }
+       }
+
+       /// Returrn the underlying vector of hops
+       pub fn into_inner(self) -> Vec<RouteHop> {
+               self.0
+       }
+}
+
+impl Into<Vec<RouteHop>> for Route {
+       fn into(self) -> Vec<RouteHop> {
+               self.into_inner()
+       }
+}
+
+impl Deref for Route {
+       type Target = Vec<RouteHop>;
+
+       fn deref(&self) -> &Vec<RouteHop> {
+               &self.0
+       }
+}
+
+impl Deref for Signature {
+       type Target = RecoverableSignature;
+
+       fn deref(&self) -> &RecoverableSignature {
+               &self.0
+       }
+}
+
+impl Deref for SignedRawInvoice {
+       type Target = RawInvoice;
+
+       fn deref(&self) -> &RawInvoice {
+               &self.raw_invoice
+       }
+}
+
+/// Errors that may occur when constructing a new `RawInvoice` or `Invoice`
+#[derive(Eq, PartialEq, Debug, Clone)]
+pub enum CreationError {
+       /// The supplied description string was longer than 639 __bytes__ (see [`Description::new(…)`](./struct.Description.html#method.new))
+       DescriptionTooLong,
+
+       /// The specified route has too many hops and can't be encoded
+       RouteTooLong,
+
+       /// The unix timestamp of the supplied date is <0 or can't be represented as `SystemTime`
+       TimestampOutOfBounds,
+
+       /// The supplied expiry time could cause an overflow if added to a `PositiveTimestamp`
+       ExpiryTimeOutOfBounds,
+}
+
+impl Display for CreationError {
+       fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
+               match self {
+                       CreationError::DescriptionTooLong => f.write_str("The supplied description string was longer than 639 bytes"),
+                       CreationError::RouteTooLong => f.write_str("The specified route has too many hops and can't be encoded"),
+                       CreationError::TimestampOutOfBounds => f.write_str("The unix timestamp of the supplied date is <0 or can't be represented as `SystemTime`"),
+                       CreationError::ExpiryTimeOutOfBounds => f.write_str("The supplied expiry time could cause an overflow if added to a `PositiveTimestamp`"),
+               }
+       }
+}
+
+impl std::error::Error for CreationError { }
+
+/// Errors that may occur when converting a `RawInvoice` to an `Invoice`. They relate to the
+/// requirements sections in BOLT #11
+#[derive(Eq, PartialEq, Debug, Clone)]
+pub enum SemanticError {
+       /// The invoice is missing the mandatory payment hash
+       NoPaymentHash,
+
+       /// The invoice has multiple payment hashes which isn't allowed
+       MultiplePaymentHashes,
+
+       /// No description or description hash are part of the invoice
+       NoDescription,
+
+       /// The invoice contains multiple descriptions and/or description hashes which isn't allowed
+       MultipleDescriptions,
+
+       /// The recovery id doesn't fit the signature/pub key
+       InvalidRecoveryId,
+
+       /// The invoice's signature is invalid
+       InvalidSignature,
+}
+
+impl Display for SemanticError {
+       fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
+               match self {
+                       SemanticError::NoPaymentHash => f.write_str("The invoice is missing the mandatory payment hash"),
+                       SemanticError::MultiplePaymentHashes => f.write_str("The invoice has multiple payment hashes which isn't allowed"),
+                       SemanticError::NoDescription => f.write_str("No description or description hash are part of the invoice"),
+                       SemanticError::MultipleDescriptions => f.write_str("The invoice contains multiple descriptions and/or description hashes which isn't allowed"),
+                       SemanticError::InvalidRecoveryId => f.write_str("The recovery id doesn't fit the signature/pub key"),
+                       SemanticError::InvalidSignature => f.write_str("The invoice's signature is invalid"),
+               }
+       }
+}
+
+impl std::error::Error for SemanticError { }
+
+/// When signing using a fallible method either an user-supplied `SignError` or a `CreationError`
+/// may occur.
+#[derive(Eq, PartialEq, Debug, Clone)]
+pub enum SignOrCreationError<S> {
+       /// An error occurred during signing
+       SignError(S),
+
+       /// An error occurred while building the transaction
+       CreationError(CreationError),
+}
+
+impl<S> Display for SignOrCreationError<S> {
+       fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
+               match self {
+                       SignOrCreationError::SignError(_) => f.write_str("An error occurred during signing"),
+                       SignOrCreationError::CreationError(err) => err.fmt(f),
+               }
+       }
+}
+
+#[cfg(test)]
+mod test {
+       use bitcoin_hashes::hex::FromHex;
+       use bitcoin_hashes::sha256;
+
+       #[test]
+       fn test_system_time_bounds_assumptions() {
+               ::check_platform();
+
+        assert_eq!(
+            ::PositiveTimestamp::from_unix_timestamp(::SYSTEM_TIME_MAX_UNIX_TIMESTAMP + 1),
+            Err(::CreationError::TimestampOutOfBounds)
+        );
+
+        assert_eq!(
+            ::ExpiryTime::from_seconds(::MAX_EXPIRY_TIME + 1),
+            Err(::CreationError::ExpiryTimeOutOfBounds)
+        );
+       }
+
+       #[test]
+       fn test_calc_invoice_hash() {
+               use ::{RawInvoice, RawHrp, RawDataPart, Currency, PositiveTimestamp};
+               use ::TaggedField::*;
+
+               let invoice = RawInvoice {
+                       hrp: RawHrp {
+                               currency: Currency::Bitcoin,
+                               raw_amount: None,
+                               si_prefix: None,
+                       },
+                       data: RawDataPart {
+                               timestamp: PositiveTimestamp::from_unix_timestamp(1496314658).unwrap(),
+                               tagged_fields: vec![
+                                       PaymentHash(::Sha256(sha256::Hash::from_hex(
+                                               "0001020304050607080900010203040506070809000102030405060708090102"
+                                       ).unwrap())).into(),
+                                       Description(::Description::new(
+                                               "Please consider supporting this project".to_owned()
+                                       ).unwrap()).into(),
+                               ],
+                       },
+               };
+
+               let expected_hash = [
+                       0xc3, 0xd4, 0xe8, 0x3f, 0x64, 0x6f, 0xa7, 0x9a, 0x39, 0x3d, 0x75, 0x27, 0x7b, 0x1d,
+                       0x85, 0x8d, 0xb1, 0xd1, 0xf7, 0xab, 0x71, 0x37, 0xdc, 0xb7, 0x83, 0x5d, 0xb2, 0xec,
+                       0xd5, 0x18, 0xe1, 0xc9
+               ];
+
+               assert_eq!(invoice.hash(), expected_hash)
+       }
+
+       #[test]
+       fn test_check_signature() {
+               use TaggedField::*;
+               use secp256k1::Secp256k1;
+               use secp256k1::recovery::{RecoveryId, RecoverableSignature};
+               use secp256k1::key::{SecretKey, PublicKey};
+               use {SignedRawInvoice, Signature, RawInvoice, RawHrp, RawDataPart, Currency, Sha256,
+                        PositiveTimestamp};
+
+               let invoice = SignedRawInvoice {
+                       raw_invoice: RawInvoice {
+                               hrp: RawHrp {
+                                       currency: Currency::Bitcoin,
+                                       raw_amount: None,
+                                       si_prefix: None,
+                               },
+                               data: RawDataPart {
+                                       timestamp: PositiveTimestamp::from_unix_timestamp(1496314658).unwrap(),
+                                       tagged_fields: vec ! [
+                                               PaymentHash(Sha256(sha256::Hash::from_hex(
+                                                       "0001020304050607080900010203040506070809000102030405060708090102"
+                                               ).unwrap())).into(),
+                                               Description(
+                                                       ::Description::new(
+                                                               "Please consider supporting this project".to_owned()
+                                                       ).unwrap()
+                                               ).into(),
+                                       ],
+                               },
+                       },
+                       hash: [
+                               0xc3, 0xd4, 0xe8, 0x3f, 0x64, 0x6f, 0xa7, 0x9a, 0x39, 0x3d, 0x75, 0x27,
+                               0x7b, 0x1d, 0x85, 0x8d, 0xb1, 0xd1, 0xf7, 0xab, 0x71, 0x37, 0xdc, 0xb7,
+                               0x83, 0x5d, 0xb2, 0xec, 0xd5, 0x18, 0xe1, 0xc9
+                       ],
+                       signature: Signature(RecoverableSignature::from_compact(
+                               & [
+                                       0x38u8, 0xec, 0x68, 0x91, 0x34, 0x5e, 0x20, 0x41, 0x45, 0xbe, 0x8a,
+                                       0x3a, 0x99, 0xde, 0x38, 0xe9, 0x8a, 0x39, 0xd6, 0xa5, 0x69, 0x43,
+                                       0x4e, 0x18, 0x45, 0xc8, 0xaf, 0x72, 0x05, 0xaf, 0xcf, 0xcc, 0x7f,
+                                       0x42, 0x5f, 0xcd, 0x14, 0x63, 0xe9, 0x3c, 0x32, 0x88, 0x1e, 0xad,
+                                       0x0d, 0x6e, 0x35, 0x6d, 0x46, 0x7e, 0xc8, 0xc0, 0x25, 0x53, 0xf9,
+                                       0xaa, 0xb1, 0x5e, 0x57, 0x38, 0xb1, 0x1f, 0x12, 0x7f
+                               ],
+                               RecoveryId::from_i32(0).unwrap()
+                       ).unwrap()),
+               };
+
+               assert!(invoice.check_signature());
+
+               let private_key = SecretKey::from_slice(
+                       &[
+                               0xe1, 0x26, 0xf6, 0x8f, 0x7e, 0xaf, 0xcc, 0x8b, 0x74, 0xf5, 0x4d, 0x26, 0x9f, 0xe2,
+                               0x06, 0xbe, 0x71, 0x50, 0x00, 0xf9, 0x4d, 0xac, 0x06, 0x7d, 0x1c, 0x04, 0xa8, 0xca,
+                               0x3b, 0x2d, 0xb7, 0x34
+                       ][..]
+               ).unwrap();
+               let public_key = PublicKey::from_secret_key(&Secp256k1::new(), &private_key);
+
+               assert_eq!(invoice.recover_payee_pub_key(), Ok(::PayeePubKey(public_key)));
+
+               let (raw_invoice, _, _) = invoice.into_parts();
+               let new_signed = raw_invoice.sign::<_, ()>(|hash| {
+                       Ok(Secp256k1::new().sign_recoverable(hash, &private_key))
+               }).unwrap();
+
+               assert!(new_signed.check_signature());
+       }
+
+       #[test]
+       fn test_builder_amount() {
+               use ::*;
+
+               let builder = InvoiceBuilder::new(Currency::Bitcoin)
+                       .description("Test".into())
+                       .payment_hash(sha256::Hash::from_slice(&[0;32][..]).unwrap())
+                       .current_timestamp();
+
+               let invoice = builder.clone()
+                       .amount_pico_btc(15000)
+                       .build_raw()
+                       .unwrap();
+
+               assert_eq!(invoice.hrp.si_prefix, Some(SiPrefix::Nano));
+               assert_eq!(invoice.hrp.raw_amount, Some(15));
+
+
+               let invoice = builder.clone()
+                       .amount_pico_btc(1500)
+                       .build_raw()
+                       .unwrap();
+
+               assert_eq!(invoice.hrp.si_prefix, Some(SiPrefix::Pico));
+               assert_eq!(invoice.hrp.raw_amount, Some(1500));
+       }
+
+       #[test]
+       fn test_builder_fail() {
+               use ::*;
+               use std::iter::FromIterator;
+               use secp256k1::key::PublicKey;
+
+               let builder = InvoiceBuilder::new(Currency::Bitcoin)
+                       .payment_hash(sha256::Hash::from_slice(&[0;32][..]).unwrap())
+                       .current_timestamp();
+
+               let too_long_string = String::from_iter(
+                       (0..1024).map(|_| '?')
+               );
+
+               let long_desc_res = builder.clone()
+                       .description(too_long_string)
+                       .build_raw();
+               assert_eq!(long_desc_res, Err(CreationError::DescriptionTooLong));
+
+               let route_hop = RouteHop {
+                       pubkey: PublicKey::from_slice(
+                                       &[
+                                               0x03, 0x9e, 0x03, 0xa9, 0x01, 0xb8, 0x55, 0x34, 0xff, 0x1e, 0x92, 0xc4,
+                                               0x3c, 0x74, 0x43, 0x1f, 0x7c, 0xe7, 0x20, 0x46, 0x06, 0x0f, 0xcf, 0x7a,
+                                               0x95, 0xc3, 0x7e, 0x14, 0x8f, 0x78, 0xc7, 0x72, 0x55
+                                       ][..]
+                               ).unwrap(),
+                       short_channel_id: [0; 8],
+                       fee_base_msat: 0,
+                       fee_proportional_millionths: 0,
+                       cltv_expiry_delta: 0,
+               };
+               let too_long_route = vec![route_hop; 13];
+               let long_route_res = builder.clone()
+                       .description("Test".into())
+                       .route(too_long_route)
+                       .build_raw();
+               assert_eq!(long_route_res, Err(CreationError::RouteTooLong));
+
+               let sign_error_res = builder.clone()
+                       .description("Test".into())
+                       .try_build_signed(|_| {
+                               Err("ImaginaryError")
+                       });
+               assert_eq!(sign_error_res, Err(SignOrCreationError::SignError("ImaginaryError")));
+       }
+
+       #[test]
+       fn test_builder_ok() {
+               use ::*;
+               use secp256k1::Secp256k1;
+               use secp256k1::key::{SecretKey, PublicKey};
+               use std::time::{UNIX_EPOCH, Duration};
+
+               let secp_ctx = Secp256k1::new();
+
+               let private_key = SecretKey::from_slice(
+                       &[
+                               0xe1, 0x26, 0xf6, 0x8f, 0x7e, 0xaf, 0xcc, 0x8b, 0x74, 0xf5, 0x4d, 0x26, 0x9f, 0xe2,
+                               0x06, 0xbe, 0x71, 0x50, 0x00, 0xf9, 0x4d, 0xac, 0x06, 0x7d, 0x1c, 0x04, 0xa8, 0xca,
+                               0x3b, 0x2d, 0xb7, 0x34
+                       ][..]
+               ).unwrap();
+               let public_key = PublicKey::from_secret_key(&secp_ctx, &private_key);
+
+               let route_1 = vec![
+                       RouteHop {
+                               pubkey: public_key.clone(),
+                               short_channel_id: [123; 8],
+                               fee_base_msat: 2,
+                               fee_proportional_millionths: 1,
+                               cltv_expiry_delta: 145,
+                       },
+                       RouteHop {
+                               pubkey: public_key.clone(),
+                               short_channel_id: [42; 8],
+                               fee_base_msat: 3,
+                               fee_proportional_millionths: 2,
+                               cltv_expiry_delta: 146,
+                       }
+               ];
+
+               let route_2 = vec![
+                       RouteHop {
+                               pubkey: public_key.clone(),
+                               short_channel_id: [0; 8],
+                               fee_base_msat: 4,
+                               fee_proportional_millionths: 3,
+                               cltv_expiry_delta: 147,
+                       },
+                       RouteHop {
+                               pubkey: public_key.clone(),
+                               short_channel_id: [1; 8],
+                               fee_base_msat: 5,
+                               fee_proportional_millionths: 4,
+                               cltv_expiry_delta: 148,
+                       }
+               ];
+
+               let builder = InvoiceBuilder::new(Currency::BitcoinTestnet)
+                       .amount_pico_btc(123)
+                       .timestamp(UNIX_EPOCH + Duration::from_secs(1234567))
+                       .payee_pub_key(public_key.clone())
+                       .expiry_time(Duration::from_secs(54321))
+                       .min_final_cltv_expiry(144)
+                       .min_final_cltv_expiry(143)
+                       .fallback(Fallback::PubKeyHash([0;20]))
+                       .route(route_1.clone())
+                       .route(route_2.clone())
+                       .description_hash(sha256::Hash::from_slice(&[3;32][..]).unwrap())
+                       .payment_hash(sha256::Hash::from_slice(&[21;32][..]).unwrap());
+
+               let invoice = builder.clone().build_signed(|hash| {
+                       secp_ctx.sign_recoverable(hash, &private_key)
+               }).unwrap();
+
+               assert!(invoice.check_signature().is_ok());
+               assert_eq!(invoice.tagged_fields().count(), 9);
+
+               assert_eq!(invoice.amount_pico_btc(), Some(123));
+               assert_eq!(invoice.currency(), Currency::BitcoinTestnet);
+               assert_eq!(
+                       invoice.timestamp().duration_since(UNIX_EPOCH).unwrap().as_secs(),
+                       1234567
+               );
+               assert_eq!(invoice.payee_pub_key(), Some(&public_key));
+               assert_eq!(invoice.expiry_time(), Duration::from_secs(54321));
+               assert_eq!(invoice.min_final_cltv_expiry(), Some(&144));
+               assert_eq!(invoice.fallbacks(), vec![&Fallback::PubKeyHash([0;20])]);
+               assert_eq!(invoice.routes(), vec![&Route(route_1), &Route(route_2)]);
+               assert_eq!(
+                       invoice.description(),
+                       InvoiceDescription::Hash(&Sha256(sha256::Hash::from_slice(&[3;32][..]).unwrap()))
+               );
+               assert_eq!(invoice.payment_hash(), &sha256::Hash::from_slice(&[21;32][..]).unwrap());
+
+               let raw_invoice = builder.build_raw().unwrap();
+               assert_eq!(raw_invoice, *invoice.into_signed_raw().raw_invoice())
+       }
+}
diff --git a/lightning-invoice/src/ser.rs b/lightning-invoice/src/ser.rs
new file mode 100644 (file)
index 0000000..2b4332f
--- /dev/null
@@ -0,0 +1,514 @@
+use std::fmt;
+use std::fmt::{Display, Formatter};
+use bech32::{ToBase32, u5, WriteBase32, Base32Len};
+
+use ::*;
+
+/// Converts a stream of bytes written to it to base32. On finalization the according padding will
+/// be applied. That means the results of writing two data blocks with one or two `BytesToBase32`
+/// converters will differ.
+struct BytesToBase32<'a, W: WriteBase32 + 'a> {
+       /// Target for writing the resulting `u5`s resulting from the written bytes
+       writer: &'a mut W,
+       /// Holds all unwritten bits left over from last round. The bits are stored beginning from
+       /// the most significant bit. E.g. if buffer_bits=3, then the byte with bits a, b and c will
+       /// look as follows: [a, b, c, 0, 0, 0, 0, 0]
+       buffer: u8,
+       /// Amount of bits left over from last round, stored in buffer.
+       buffer_bits: u8,
+}
+
+impl<'a, W: WriteBase32> BytesToBase32<'a, W> {
+       /// Create a new bytes-to-base32 converter with `writer` as  a sink for the resulting base32
+       /// data.
+       pub fn new(writer: &'a mut W) -> BytesToBase32<'a, W> {
+               BytesToBase32 {
+                       writer,
+                       buffer: 0,
+                       buffer_bits: 0,
+               }
+       }
+
+       /// Add more bytes to the current conversion unit
+       pub fn append(&mut self, bytes: &[u8]) -> Result<(), W::Err> {
+               for b in bytes {
+                       self.append_u8(*b)?;
+               }
+               Ok(())
+       }
+
+       pub fn append_u8(&mut self, byte: u8) -> Result<(), W::Err> {
+               // Write first u5 if we have to write two u5s this round. That only happens if the
+               // buffer holds too many bits, so we don't have to combine buffer bits with new bits
+               // from this rounds byte.
+               if self.buffer_bits >= 5 {
+                       self.writer.write_u5(
+                               u5::try_from_u8((self.buffer & 0b11111000) >> 3 ).expect("<32")
+                       )?;
+                       self.buffer = self.buffer << 5;
+                       self.buffer_bits -= 5;
+               }
+
+               // Combine all bits from buffer with enough bits from this rounds byte so that they fill
+               // a u5. Save reamining bits from byte to buffer.
+               let from_buffer = self.buffer >> 3;
+               let from_byte = byte >> (3 + self.buffer_bits); // buffer_bits <= 4
+
+               self.writer.write_u5(u5::try_from_u8(from_buffer | from_byte).expect("<32"))?;
+               self.buffer = byte << (5 - self.buffer_bits);
+               self.buffer_bits = 3 + self.buffer_bits;
+
+               Ok(())
+       }
+
+       pub fn finalize(mut self) ->  Result<(), W::Err> {
+               self.inner_finalize()?;
+               std::mem::forget(self);
+               Ok(())
+       }
+
+       fn inner_finalize(&mut self) -> Result<(), W::Err>{
+               // There can be at most two u5s left in the buffer after processing all bytes, write them.
+               if self.buffer_bits >= 5 {
+                       self.writer.write_u5(
+                               u5::try_from_u8((self.buffer & 0b11111000) >> 3).expect("<32")
+                       )?;
+                       self.buffer = self.buffer << 5;
+                       self.buffer_bits -= 5;
+               }
+
+               if self.buffer_bits != 0 {
+                       self.writer.write_u5(u5::try_from_u8(self.buffer >> 3).expect("<32"))?;
+               }
+
+               Ok(())
+       }
+}
+
+impl<'a, W: WriteBase32> Drop for BytesToBase32<'a, W> {
+       fn drop(&mut self) {
+               self.inner_finalize()
+                       .expect("Unhandled error when finalizing conversion on drop. User finalize to handle.")
+       }
+}
+
+/// Calculates the base32 encoded size of a byte slice
+fn bytes_size_to_base32_size(byte_size: usize) -> usize {
+       let bits = byte_size * 8;
+       if bits % 5 == 0 {
+               // without padding bits
+               bits / 5
+       } else {
+               // with padding bits
+               bits / 5 + 1
+       }
+}
+
+impl Display for Invoice {
+       fn fmt(&self, f: &mut Formatter) -> Result<(), fmt::Error> {
+               self.signed_invoice.fmt(f)
+       }
+}
+
+impl Display for SignedRawInvoice {
+       fn fmt(&self, f: &mut Formatter) -> Result<(), fmt::Error> {
+               let hrp = self.raw_invoice.hrp.to_string();
+               let mut data  = self.raw_invoice.data.to_base32();
+               data.extend_from_slice(&self.signature.to_base32());
+
+               bech32::encode_to_fmt(f, &hrp, data).expect("HRP is valid")?;
+
+               Ok(())
+       }
+}
+
+impl Display for RawHrp {
+       fn fmt(&self, f: &mut Formatter) -> Result<(), fmt::Error> {
+               let amount = match self.raw_amount {
+                       Some(ref amt) => amt.to_string(),
+                       None => String::new(),
+               };
+
+               let si_prefix = match self.si_prefix {
+                       Some(ref si) => si.to_string(),
+                       None => String::new(),
+               };
+
+               write!(
+                       f,
+                       "ln{}{}{}",
+                       self.currency,
+                       amount,
+                       si_prefix
+               )
+       }
+}
+
+impl Display for Currency {
+       fn fmt(&self, f: &mut Formatter) -> Result<(), fmt::Error> {
+               let currency_code = match *self {
+                       Currency::Bitcoin => "bc",
+                       Currency::BitcoinTestnet => "tb",
+                       Currency::Regtest => "bcrt",
+                       Currency::Simnet => "sb",
+               };
+               write!(f, "{}", currency_code)
+       }
+}
+
+impl Display for SiPrefix {
+       fn fmt(&self, f: &mut Formatter) -> Result<(), fmt::Error> {
+               write!(f, "{}",
+                       match *self {
+                               SiPrefix::Milli => "m",
+                               SiPrefix::Micro => "u",
+                               SiPrefix::Nano => "n",
+                               SiPrefix::Pico => "p",
+                       }
+               )
+       }
+}
+
+fn encode_int_be_base32(int: u64) -> Vec<u5> {
+       let base = 32u64;
+
+       let mut out_vec = Vec::<u5>::new();
+
+       let mut rem_int = int;
+       while rem_int != 0 {
+               out_vec.push(u5::try_from_u8((rem_int % base) as u8).expect("always <32"));
+               rem_int /= base;
+       }
+
+       out_vec.reverse();
+       out_vec
+}
+
+fn encoded_int_be_base32_size(int: u64) -> usize {
+       for pos in (0..13).rev() {
+               if int & (0x1f << (5 * pos)) != 0 {
+                       return (pos + 1) as usize;
+               }
+       }
+       0usize
+}
+
+fn encode_int_be_base256<T: Into<u64>>(int: T) -> Vec<u8> {
+       let base = 256u64;
+
+       let mut out_vec = Vec::<u8>::new();
+
+       let mut rem_int: u64 = int.into();
+       while rem_int != 0 {
+               out_vec.push((rem_int % base) as u8);
+               rem_int /= base;
+       }
+
+       out_vec.reverse();
+       out_vec
+}
+
+/// Appends the default value of `T` to the front of the `in_vec` till it reaches the length
+/// `target_length`. If `in_vec` already is too lang `None` is returned.
+fn try_stretch<T>(mut in_vec: Vec<T>, target_len: usize) -> Option<Vec<T>>
+       where T: Default + Copy
+{
+       if in_vec.len() > target_len {
+               None
+       } else if in_vec.len() == target_len {
+               Some(in_vec)
+       } else {
+               let mut out_vec = Vec::<T>::with_capacity(target_len);
+               out_vec.append(&mut vec![T::default(); target_len - in_vec.len()]);
+               out_vec.append(&mut in_vec);
+               Some(out_vec)
+       }
+}
+
+impl ToBase32 for RawDataPart {
+       fn write_base32<W: WriteBase32>(&self, writer: &mut W) -> Result<(), <W as WriteBase32>::Err> {
+               // encode timestamp
+               self.timestamp.write_base32(writer)?;
+
+               // encode tagged fields
+               for tagged_field in self.tagged_fields.iter() {
+                       tagged_field.write_base32(writer)?;
+               }
+
+               Ok(())
+       }
+}
+
+impl ToBase32 for PositiveTimestamp {
+       fn write_base32<W: WriteBase32>(&self, writer: &mut W) -> Result<(), <W as WriteBase32>::Err> {
+               // FIXME: use writer for int encoding
+               writer.write(
+                       &try_stretch(encode_int_be_base32(self.as_unix_timestamp()), 7)
+                               .expect("Can't be longer due than 7 u5s due to timestamp bounds")
+               )
+       }
+}
+
+impl ToBase32 for RawTaggedField {
+       fn write_base32<W: WriteBase32>(&self, writer: &mut W) -> Result<(), <W as WriteBase32>::Err> {
+               match *self {
+                       RawTaggedField::UnknownSemantics(ref content) => {
+                               writer.write(content)
+                       },
+                       RawTaggedField::KnownSemantics(ref tagged_field) => {
+                               tagged_field.write_base32(writer)
+                       }
+               }
+       }
+}
+
+impl ToBase32 for Sha256 {
+       fn write_base32<W: WriteBase32>(&self, writer: &mut W) -> Result<(), <W as WriteBase32>::Err> {
+               (&self.0[..]).write_base32(writer)
+       }
+}
+impl Base32Len for Sha256 {
+       fn base32_len(&self) -> usize {
+               (&self.0[..]).base32_len()
+       }
+}
+
+impl ToBase32 for Description {
+       fn write_base32<W: WriteBase32>(&self, writer: &mut W) -> Result<(), <W as WriteBase32>::Err> {
+               self.as_bytes().write_base32(writer)
+       }
+}
+
+impl Base32Len for Description {
+       fn base32_len(&self) -> usize {
+               self.0.as_bytes().base32_len()
+       }
+}
+
+impl ToBase32 for PayeePubKey {
+       fn write_base32<W: WriteBase32>(&self, writer: &mut W) -> Result<(), <W as WriteBase32>::Err> {
+               (&self.serialize()[..]).write_base32(writer)
+       }
+}
+
+impl Base32Len for PayeePubKey {
+       fn base32_len(&self) -> usize {
+               bytes_size_to_base32_size(secp256k1::constants::PUBLIC_KEY_SIZE)
+       }
+}
+
+impl ToBase32 for PaymentSecret {
+       fn write_base32<W: WriteBase32>(&self, writer: &mut W) -> Result<(), <W as WriteBase32>::Err> {
+               (&self.0[..]).write_base32(writer)
+       }
+}
+
+impl Base32Len for PaymentSecret {
+       fn base32_len(&self) -> usize {
+               bytes_size_to_base32_size(32)
+       }
+}
+
+impl ToBase32 for ExpiryTime {
+       fn write_base32<W: WriteBase32>(&self, writer: &mut W) -> Result<(), <W as WriteBase32>::Err> {
+               writer.write(&encode_int_be_base32(self.as_seconds()))
+       }
+}
+
+impl Base32Len for ExpiryTime {
+       fn base32_len(&self) -> usize {
+               encoded_int_be_base32_size(self.0.as_secs())
+       }
+}
+
+impl ToBase32 for MinFinalCltvExpiry {
+       fn write_base32<W: WriteBase32>(&self, writer: &mut W) -> Result<(), <W as WriteBase32>::Err> {
+               writer.write(&encode_int_be_base32(self.0))
+       }
+}
+
+impl Base32Len for MinFinalCltvExpiry {
+       fn base32_len(&self) -> usize {
+               encoded_int_be_base32_size(self.0)
+       }
+}
+
+impl ToBase32 for Fallback {
+       fn write_base32<W: WriteBase32>(&self, writer: &mut W) -> Result<(), <W as WriteBase32>::Err> {
+               match *self {
+                       Fallback::SegWitProgram {version: v, program: ref p} => {
+                               writer.write_u5(v)?;
+                               p.write_base32(writer)
+                       },
+                       Fallback::PubKeyHash(ref hash) => {
+                               writer.write_u5(u5::try_from_u8(17).expect("17 < 32"))?;
+                               (&hash[..]).write_base32(writer)
+                       },
+                       Fallback::ScriptHash(ref hash) => {
+                               writer.write_u5(u5::try_from_u8(18).expect("18 < 32"))?;
+                               (&hash[..]).write_base32(writer)
+                       }
+               }
+       }
+}
+
+impl Base32Len for Fallback {
+       fn base32_len(&self) -> usize {
+               match *self {
+                       Fallback::SegWitProgram {program: ref p, ..} => {
+                               bytes_size_to_base32_size(p.len()) + 1
+                       },
+                       Fallback::PubKeyHash(_) | Fallback::ScriptHash(_) => {
+                               33
+                       },
+               }
+       }
+}
+
+impl ToBase32 for Route {
+       fn write_base32<W: WriteBase32>(&self, writer: &mut W) -> Result<(), <W as WriteBase32>::Err> {
+               let mut converter = BytesToBase32::new(writer);
+
+               for hop in self.iter() {
+                       converter.append(&hop.pubkey.serialize()[..])?;
+                       converter.append(&hop.short_channel_id[..])?;
+
+                       let fee_base_msat = try_stretch(
+                               encode_int_be_base256(hop.fee_base_msat),
+                               4
+                       ).expect("sizeof(u32) == 4");
+                       converter.append(&fee_base_msat)?;
+
+                       let fee_proportional_millionths = try_stretch(
+                               encode_int_be_base256(hop.fee_proportional_millionths),
+                               4
+                       ).expect("sizeof(u32) == 4");
+                       converter.append(&fee_proportional_millionths)?;
+
+                       let cltv_expiry_delta = try_stretch(
+                               encode_int_be_base256(hop.cltv_expiry_delta),
+                               2
+                       ).expect("sizeof(u16) == 2");
+                       converter.append(&cltv_expiry_delta)?;
+               }
+
+               converter.finalize()?;
+               Ok(())
+       }
+}
+
+impl Base32Len for Route {
+       fn base32_len(&self) -> usize {
+               bytes_size_to_base32_size(self.0.len() * 51)
+       }
+}
+
+impl ToBase32 for TaggedField {
+       fn write_base32<W: WriteBase32>(&self, writer: &mut W) -> Result<(), <W as WriteBase32>::Err> {
+               /// Writes a tagged field: tag, length and data. `tag` should be in `0..32` otherwise the
+               /// function will panic.
+               fn write_tagged_field<W, P>(writer: &mut W, tag: u8, payload: &P) -> Result<(), W::Err>
+                       where W: WriteBase32,
+                                 P: ToBase32 + Base32Len,
+               {
+                       let len = payload.base32_len();
+                       assert!(len < 1024, "Every tagged field data can be at most 1023 bytes long.");
+
+                       writer.write_u5(u5::try_from_u8(tag).expect("invalid tag, not in 0..32"))?;
+                       writer.write(&try_stretch(
+                               encode_int_be_base32(len as u64),
+                               2
+                       ).expect("Can't be longer than 2, see assert above."))?;
+                       payload.write_base32(writer)
+               }
+
+               match *self {
+                       TaggedField::PaymentHash(ref hash) => {
+                               write_tagged_field(writer, constants::TAG_PAYMENT_HASH, hash)
+                       },
+                       TaggedField::Description(ref description) => {
+                               write_tagged_field(writer, constants::TAG_DESCRIPTION, description)
+                       },
+                       TaggedField::PayeePubKey(ref pub_key) => {
+                               write_tagged_field(writer, constants::TAG_PAYEE_PUB_KEY, pub_key)
+                       },
+                       TaggedField::DescriptionHash(ref hash) => {
+                               write_tagged_field(writer, constants::TAG_DESCRIPTION_HASH, hash)
+                       },
+                       TaggedField::ExpiryTime(ref duration) => {
+                               write_tagged_field(writer, constants::TAG_EXPIRY_TIME, duration)
+                       },
+                       TaggedField::MinFinalCltvExpiry(ref expiry) => {
+                               write_tagged_field(writer, constants::TAG_MIN_FINAL_CLTV_EXPIRY, expiry)
+                       },
+                       TaggedField::Fallback(ref fallback_address) => {
+                               write_tagged_field(writer, constants::TAG_FALLBACK, fallback_address)
+                       },
+                       TaggedField::Route(ref route_hops) => {
+                               write_tagged_field(writer, constants::TAG_ROUTE, route_hops)
+                       },
+                       TaggedField::PaymentSecret(ref payment_secret) => {
+                                 write_tagged_field(writer, constants::TAG_PAYMENT_SECRET, payment_secret)
+                       },
+
+               }
+       }
+}
+
+impl ToBase32 for Signature {
+       fn write_base32<W: WriteBase32>(&self, writer: &mut W) -> Result<(), <W as WriteBase32>::Err> {
+               let mut converter = BytesToBase32::new(writer);
+               let (recovery_id, signature) = self.0.serialize_compact();
+               converter.append(&signature[..])?;
+               converter.append_u8(recovery_id.to_i32() as u8)?;
+               converter.finalize()
+       }
+}
+
+#[cfg(test)]
+mod test {
+       use bech32::CheckBase32;
+
+       #[test]
+       fn test_currency_code() {
+               use Currency;
+
+               assert_eq!("bc", Currency::Bitcoin.to_string());
+               assert_eq!("tb", Currency::BitcoinTestnet.to_string());
+               assert_eq!("bcrt", Currency::Regtest.to_string());
+               assert_eq!("sb", Currency::Simnet.to_string());
+       }
+
+       #[test]
+       fn test_raw_hrp() {
+               use ::{Currency, RawHrp, SiPrefix};
+
+               let hrp = RawHrp {
+                       currency: Currency::Bitcoin,
+                       raw_amount: Some(100),
+                       si_prefix: Some(SiPrefix::Micro),
+               };
+
+               assert_eq!(hrp.to_string(), "lnbc100u");
+       }
+
+       #[test]
+       fn test_encode_int_be_base32() {
+               use ser::encode_int_be_base32;
+
+               let input: u64 = 33764;
+               let expected_out = CheckBase32::check_base32(&[1, 0, 31, 4]).unwrap();
+
+               assert_eq!(expected_out, encode_int_be_base32(input));
+       }
+
+       #[test]
+       fn test_encode_int_be_base256() {
+               use ser::encode_int_be_base256;
+
+               let input: u64 = 16842530;
+               let expected_out = vec![1, 0, 255, 34];
+
+               assert_eq!(expected_out, encode_int_be_base256(input));
+       }
+}
diff --git a/lightning-invoice/src/tb.rs b/lightning-invoice/src/tb.rs
new file mode 100644 (file)
index 0000000..dde8a53
--- /dev/null
@@ -0,0 +1,10 @@
+pub trait Bool {}
+
+#[derive(Copy, Clone)]
+pub struct True {}
+
+#[derive(Copy, Clone)]
+pub struct False {}
+
+impl Bool for True {}
+impl Bool for False {}
\ No newline at end of file
diff --git a/lightning-invoice/tests/ser_de.rs b/lightning-invoice/tests/ser_de.rs
new file mode 100644 (file)
index 0000000..403f8f1
--- /dev/null
@@ -0,0 +1,148 @@
+extern crate bitcoin_hashes;
+extern crate lightning_invoice;
+extern crate secp256k1;
+
+use bitcoin_hashes::hex::FromHex;
+use bitcoin_hashes::sha256;
+use lightning_invoice::*;
+use secp256k1::Secp256k1;
+use secp256k1::key::SecretKey;
+use secp256k1::recovery::{RecoverableSignature, RecoveryId};
+use std::time::{Duration, UNIX_EPOCH};
+
+// TODO: add more of the examples from BOLT11 and generate ones causing SemanticErrors
+
+fn get_test_tuples() -> Vec<(String, SignedRawInvoice, Option<SemanticError>)> {
+       vec![
+               (
+                       "lnbc1pvjluezpp5qqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqypqdpl2pkx2ctnv5sxxmmw\
+                       wd5kgetjypeh2ursdae8g6twvus8g6rfwvs8qun0dfjkxaq8rkx3yf5tcsyz3d73gafnh3cax9rn449d9p5uxz9\
+                       ezhhypd0elx87sjle52x86fux2ypatgddc6k63n7erqz25le42c4u4ecky03ylcqca784w".to_owned(),
+                       InvoiceBuilder::new(Currency::Bitcoin)
+                               .timestamp(UNIX_EPOCH + Duration::from_secs(1496314658))
+                               .payment_hash(sha256::Hash::from_hex(
+                                               "0001020304050607080900010203040506070809000102030405060708090102"
+                               ).unwrap())
+                               .description("Please consider supporting this project".to_owned())
+                               .build_raw()
+                               .unwrap()
+                               .sign(|_| {
+                                       RecoverableSignature::from_compact(
+                                               & [
+                                                       0x38u8, 0xec, 0x68, 0x91, 0x34, 0x5e, 0x20, 0x41, 0x45, 0xbe, 0x8a,
+                                                       0x3a, 0x99, 0xde, 0x38, 0xe9, 0x8a, 0x39, 0xd6, 0xa5, 0x69, 0x43,
+                                                       0x4e, 0x18, 0x45, 0xc8, 0xaf, 0x72, 0x05, 0xaf, 0xcf, 0xcc, 0x7f,
+                                                       0x42, 0x5f, 0xcd, 0x14, 0x63, 0xe9, 0x3c, 0x32, 0x88, 0x1e, 0xad,
+                                                       0x0d, 0x6e, 0x35, 0x6d, 0x46, 0x7e, 0xc8, 0xc0, 0x25, 0x53, 0xf9,
+                                                       0xaa, 0xb1, 0x5e, 0x57, 0x38, 0xb1, 0x1f, 0x12, 0x7f
+                                               ],
+                                               RecoveryId::from_i32(0).unwrap()
+                                       )
+                               }).unwrap(),
+                       None
+               ),
+               (
+                       "lnbc2500u1pvjluezpp5qqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqypqdq5xysxxatsyp3\
+                       k7enxv4jsxqzpuaztrnwngzn3kdzw5hydlzf03qdgm2hdq27cqv3agm2awhz5se903vruatfhq77w3ls4evs3ch\
+                       9zw97j25emudupq63nyw24cg27h2rspfj9srp".to_owned(),
+                       InvoiceBuilder::new(Currency::Bitcoin)
+                               .amount_pico_btc(2500000000)
+                               .timestamp(UNIX_EPOCH + Duration::from_secs(1496314658))
+                               .payment_hash(sha256::Hash::from_hex(
+                                       "0001020304050607080900010203040506070809000102030405060708090102"
+                               ).unwrap())
+                               .description("1 cup coffee".to_owned())
+                               .expiry_time(Duration::from_secs(60))
+                               .build_raw()
+                               .unwrap()
+                               .sign(|_| {
+                                       RecoverableSignature::from_compact(
+                                               & [
+                                                       0xe8, 0x96, 0x39, 0xba, 0x68, 0x14, 0xe3, 0x66, 0x89, 0xd4, 0xb9, 0x1b,
+                                                       0xf1, 0x25, 0xf1, 0x03, 0x51, 0xb5, 0x5d, 0xa0, 0x57, 0xb0, 0x06, 0x47,
+                                                       0xa8, 0xda, 0xba, 0xeb, 0x8a, 0x90, 0xc9, 0x5f, 0x16, 0x0f, 0x9d, 0x5a,
+                                                       0x6e, 0x0f, 0x79, 0xd1, 0xfc, 0x2b, 0x96, 0x42, 0x38, 0xb9, 0x44, 0xe2,
+                                                       0xfa, 0x4a, 0xa6, 0x77, 0xc6, 0xf0, 0x20, 0xd4, 0x66, 0x47, 0x2a, 0xb8,
+                                                       0x42, 0xbd, 0x75, 0x0e
+                                               ],
+                                               RecoveryId::from_i32(1).unwrap()
+                                       )
+                               }).unwrap(),
+                       None
+               ),
+               (
+                       "lnbc20m1pvjluezpp5qqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqypqhp58yjmdan79s6qq\
+                       dhdzgynm4zwqd5d7xmw5fk98klysy043l2ahrqscc6gd6ql3jrc5yzme8v4ntcewwz5cnw92tz0pc8qcuufvq7k\
+                       hhr8wpald05e92xw006sq94mg8v2ndf4sefvf9sygkshp5zfem29trqq2yxxz7".to_owned(),
+                       InvoiceBuilder::new(Currency::Bitcoin)
+                               .amount_pico_btc(20000000000)
+                               .timestamp(UNIX_EPOCH + Duration::from_secs(1496314658))
+                               .payment_hash(sha256::Hash::from_hex(
+                                       "0001020304050607080900010203040506070809000102030405060708090102"
+                               ).unwrap())
+                               .description_hash(sha256::Hash::from_hex(
+                                       "3925b6f67e2c340036ed12093dd44e0368df1b6ea26c53dbe4811f58fd5db8c1"
+                               ).unwrap())
+                               .build_raw()
+                               .unwrap()
+                               .sign(|_| {
+                                       RecoverableSignature::from_compact(
+                                               & [
+                                                       0xc6, 0x34, 0x86, 0xe8, 0x1f, 0x8c, 0x87, 0x8a, 0x10, 0x5b, 0xc9, 0xd9,
+                                                       0x59, 0xaf, 0x19, 0x73, 0x85, 0x4c, 0x4d, 0xc5, 0x52, 0xc4, 0xf0, 0xe0,
+                                                       0xe0, 0xc7, 0x38, 0x96, 0x03, 0xd6, 0xbd, 0xc6, 0x77, 0x07, 0xbf, 0x6b,
+                                                       0xe9, 0x92, 0xa8, 0xce, 0x7b, 0xf5, 0x00, 0x16, 0xbb, 0x41, 0xd8, 0xa9,
+                                                       0xb5, 0x35, 0x86, 0x52, 0xc4, 0x96, 0x04, 0x45, 0xa1, 0x70, 0xd0, 0x49,
+                                                       0xce, 0xd4, 0x55, 0x8c
+                                               ],
+                                               RecoveryId::from_i32(0).unwrap()
+                                       )
+                               }).unwrap(),
+                       None
+               ),
+               (
+                       "lnbc20m1pvjluezpp5qqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqypqdq5vdhkven9v5sxyetpdeessp59g4z52329g4z52329g4z52329g4z52329g4z52329g4z52329g4q9gkzyrw8zhfxmrcxsx7hj40yejq6lkvn75l9yjmapjv94haz8x8jy2tvmgex8rnyqkj825csd2t64fu0p4ctad2cf4tgy5gh2fns6ygp6pnc3y".to_owned(),
+                       InvoiceBuilder::new(Currency::Bitcoin)
+                               .payment_hash(sha256::Hash::from_hex(
+                                       "0001020304050607080900010203040506070809000102030405060708090102"
+                               ).unwrap())
+                               .description("coffee beans".to_string())
+                               .amount_pico_btc(20000000000)
+                               .timestamp(UNIX_EPOCH + Duration::from_secs(1496314658))
+                               .payment_secret(PaymentSecret([42; 32]))
+                               .build_signed(|msg_hash| {
+                                       let privkey = SecretKey::from_slice(&[41; 32]).unwrap();
+                                       let secp_ctx = Secp256k1::new();
+                                       secp_ctx.sign_recoverable(msg_hash, &privkey)
+                               })
+                               .unwrap()
+                               .into_signed_raw(),
+                       None
+               )
+       ]
+}
+
+
+#[test]
+fn serialize() {
+       for (serialized, deserialized, _) in get_test_tuples() {
+               assert_eq!(deserialized.to_string(), serialized);
+       }
+}
+
+#[test]
+fn deserialize() {
+       for (serialized, deserialized, maybe_error) in get_test_tuples() {
+               let parsed = serialized.parse::<SignedRawInvoice>().unwrap();
+
+               assert_eq!(parsed, deserialized);
+
+               let validated = Invoice::from_signed(parsed);
+
+               if let Some(error) = maybe_error {
+                       assert_eq!(Err(error), validated);
+               } else {
+                       assert!(validated.is_ok());
+               }
+       }
+}
index 0a9821f14cfc86768a57f48b1e31af2e80c07868..a6e7242b0cb9c13e0eae26ea6bb43063870eb8c1 100644 (file)
@@ -8,6 +8,9 @@ description = """
 Utilities to manage Rust-Lightning channel data persistence and retrieval.
 """
 
+[features]
+unstable = ["lightning/unstable"]
+
 [dependencies]
 bitcoin = "0.26"
 lightning = { version = "0.0.13", path = "../lightning" }
index afcda803397597f12d0d3a597dcf7bff44b3c248..368945d1cfb489db52752a1e4ec6b4afedb0497d 100644 (file)
@@ -3,13 +3,17 @@
 #![deny(broken_intra_doc_links)]
 #![deny(missing_docs)]
 
+#![cfg_attr(all(test, feature = "unstable"), feature(test))]
+#[cfg(all(test, feature = "unstable"))] extern crate test;
+
 mod util;
 
 extern crate lightning;
 extern crate bitcoin;
 extern crate libc;
 
-use bitcoin::hashes::hex::ToHex;
+use bitcoin::{BlockHash, Txid};
+use bitcoin::hashes::hex::{FromHex, ToHex};
 use crate::util::DiskWriteable;
 use lightning::chain;
 use lightning::chain::chaininterface::{BroadcasterInterface, FeeEstimator};
@@ -19,21 +23,14 @@ use lightning::chain::keysinterface::{Sign, KeysInterface};
 use lightning::chain::transaction::OutPoint;
 use lightning::ln::channelmanager::ChannelManager;
 use lightning::util::logger::Logger;
-use lightning::util::ser::Writeable;
+use lightning::util::ser::{ReadableArgs, Writeable};
+use std::collections::HashMap;
 use std::fs;
-use std::io::Error;
-use std::path::PathBuf;
+use std::io::{Cursor, Error};
+use std::ops::Deref;
+use std::path::{Path, PathBuf};
 use std::sync::Arc;
 
-#[cfg(test)]
-use {
-       lightning::util::ser::ReadableArgs,
-       bitcoin::{BlockHash, Txid},
-       bitcoin::hashes::hex::FromHex,
-       std::collections::HashMap,
-       std::io::Cursor
-};
-
 /// FilesystemPersister persists channel data on disk, where each channel's
 /// data is stored in a file named after its funding outpoint.
 ///
@@ -105,42 +102,64 @@ impl FilesystemPersister {
                util::write_to_file(path, "manager".to_string(), manager)
        }
 
-       #[cfg(test)]
-       fn load_channel_data<Keys: KeysInterface>(&self, keys: &Keys) ->
-               Result<HashMap<OutPoint, ChannelMonitor<Keys::Signer>>, ChannelMonitorUpdateErr> {
-                       if let Err(_) = fs::create_dir_all(self.path_to_monitor_data()) {
-                               return Err(ChannelMonitorUpdateErr::PermanentFailure);
+       /// Read `ChannelMonitor`s from disk.
+       pub fn read_channelmonitors<Signer: Sign, K: Deref> (
+               &self, keys_manager: K
+       ) -> Result<HashMap<OutPoint, (BlockHash, ChannelMonitor<Signer>)>, std::io::Error>
+            where K::Target: KeysInterface<Signer=Signer> + Sized
+       {
+               let path = self.path_to_monitor_data();
+               if !Path::new(&path).exists() {
+                       return Ok(HashMap::new());
+               }
+               let mut outpoint_to_channelmonitor = HashMap::new();
+               for file_option in fs::read_dir(path).unwrap() {
+                       let file = file_option.unwrap();
+                       let owned_file_name = file.file_name();
+                       let filename = owned_file_name.to_str();
+                       if !filename.is_some() || !filename.unwrap().is_ascii() || filename.unwrap().len() < 65 {
+                               return Err(std::io::Error::new(
+                                       std::io::ErrorKind::InvalidData,
+                                       "Invalid ChannelMonitor file name",
+                               ));
                        }
-                       let mut res = HashMap::new();
-                       for file_option in fs::read_dir(self.path_to_monitor_data()).unwrap() {
-                               let file = file_option.unwrap();
-                               let owned_file_name = file.file_name();
-                               let filename = owned_file_name.to_str();
-                               if !filename.is_some() || !filename.unwrap().is_ascii() || filename.unwrap().len() < 65 {
-                                       return Err(ChannelMonitorUpdateErr::PermanentFailure);
-                               }
 
-                               let txid = Txid::from_hex(filename.unwrap().split_at(64).0);
-                               if txid.is_err() { return Err(ChannelMonitorUpdateErr::PermanentFailure); }
-
-                               let index = filename.unwrap().split_at(65).1.split('.').next().unwrap().parse();
-                               if index.is_err() { return Err(ChannelMonitorUpdateErr::PermanentFailure); }
+                       let txid = Txid::from_hex(filename.unwrap().split_at(64).0);
+                       if txid.is_err() {
+                               return Err(std::io::Error::new(
+                                       std::io::ErrorKind::InvalidData,
+                                       "Invalid tx ID in filename",
+                               ));
+                       }
 
-                               let contents = fs::read(&file.path());
-                               if contents.is_err() { return Err(ChannelMonitorUpdateErr::PermanentFailure); }
+                       let index = filename.unwrap().split_at(65).1.parse();
+                       if index.is_err() {
+                               return Err(std::io::Error::new(
+                                       std::io::ErrorKind::InvalidData,
+                                       "Invalid tx index in filename",
+                               ));
+                       }
 
-                               if let Ok((_, loaded_monitor)) =
-                                       <(BlockHash, ChannelMonitor<Keys::Signer>)>::read(&mut Cursor::new(&contents.unwrap()), keys) {
-                                               res.insert(OutPoint { txid: txid.unwrap(), index: index.unwrap() }, loaded_monitor);
-                                       } else {
-                                               return Err(ChannelMonitorUpdateErr::PermanentFailure);
-                                       }
+                       let contents = fs::read(&file.path())?;
+                       let mut buffer = Cursor::new(&contents);
+                       match <(BlockHash, ChannelMonitor<Signer>)>::read(&mut buffer, &*keys_manager) {
+                               Ok((blockhash, channel_monitor)) => {
+                                       outpoint_to_channelmonitor.insert(
+                                               OutPoint { txid: txid.unwrap(), index: index.unwrap() },
+                                               (blockhash, channel_monitor),
+                                       );
+                               }
+                               Err(e) => return Err(std::io::Error::new(
+                                       std::io::ErrorKind::InvalidData,
+                                       format!("Failed to deserialize ChannelMonitor: {}", e),
+                               ))
                        }
-                       Ok(res)
                }
+               Ok(outpoint_to_channelmonitor)
+       }
 }
 
-impl<ChannelSigner: Sign + Send + Sync> channelmonitor::Persist<ChannelSigner> for FilesystemPersister {
+impl<ChannelSigner: Sign> channelmonitor::Persist<ChannelSigner> for FilesystemPersister {
        fn persist_new_channel(&self, funding_txo: OutPoint, monitor: &ChannelMonitor<ChannelSigner>) -> Result<(), ChannelMonitorUpdateErr> {
                let filename = format!("{}_{}", funding_txo.txid.to_hex(), funding_txo.index);
                util::write_to_file(self.path_to_monitor_data(), filename, monitor)
@@ -207,22 +226,22 @@ mod tests {
 
                // Check that the persisted channel data is empty before any channels are
                // open.
-               let mut persisted_chan_data_0 = persister_0.load_channel_data(nodes[0].keys_manager).unwrap();
+               let mut persisted_chan_data_0 = persister_0.read_channelmonitors(nodes[0].keys_manager).unwrap();
                assert_eq!(persisted_chan_data_0.keys().len(), 0);
-               let mut persisted_chan_data_1 = persister_1.load_channel_data(nodes[1].keys_manager).unwrap();
+               let mut persisted_chan_data_1 = persister_1.read_channelmonitors(nodes[1].keys_manager).unwrap();
                assert_eq!(persisted_chan_data_1.keys().len(), 0);
 
                // Helper to make sure the channel is on the expected update ID.
                macro_rules! check_persisted_data {
                        ($expected_update_id: expr) => {
-                               persisted_chan_data_0 = persister_0.load_channel_data(nodes[0].keys_manager).unwrap();
+                               persisted_chan_data_0 = persister_0.read_channelmonitors(nodes[0].keys_manager).unwrap();
                                assert_eq!(persisted_chan_data_0.keys().len(), 1);
-                               for mon in persisted_chan_data_0.values() {
+                               for (_, mon) in persisted_chan_data_0.values() {
                                        assert_eq!(mon.get_latest_update_id(), $expected_update_id);
                                }
-                               persisted_chan_data_1 = persister_1.load_channel_data(nodes[1].keys_manager).unwrap();
+                               persisted_chan_data_1 = persister_1.read_channelmonitors(nodes[1].keys_manager).unwrap();
                                assert_eq!(persisted_chan_data_1.keys().len(), 1);
-                               for mon in persisted_chan_data_1.values() {
+                               for (_, mon) in persisted_chan_data_1.values() {
                                        assert_eq!(mon.get_latest_update_id(), $expected_update_id);
                                }
                        }
@@ -241,7 +260,7 @@ mod tests {
                // Force close because cooperative close doesn't result in any persisted
                // updates.
                nodes[0].node.force_close_channel(&nodes[0].node.list_channels()[0].channel_id).unwrap();
-               check_closed_broadcast!(nodes[0], false);
+               check_closed_broadcast!(nodes[0], true);
                check_added_monitors!(nodes[0], 1);
 
                let node_txn = nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap();
@@ -249,7 +268,7 @@ mod tests {
 
                let header = BlockHeader { version: 0x20000000, prev_blockhash: nodes[0].best_block_hash(), merkle_root: Default::default(), time: 42, bits: 42, nonce: 42 };
                connect_block(&nodes[1], &Block { header, txdata: vec![node_txn[0].clone(), node_txn[0].clone()]});
-               check_closed_broadcast!(nodes[1], false);
+               check_closed_broadcast!(nodes[1], true);
                check_added_monitors!(nodes[1], 1);
 
                // Make sure everything is persisted as expected after close.
@@ -330,3 +349,15 @@ mod tests {
                added_monitors.clear();
        }
 }
+
+#[cfg(all(test, feature = "unstable"))]
+pub mod bench {
+       use test::Bencher;
+
+       #[bench]
+       fn bench_sends(bench: &mut Bencher) {
+               let persister_a = super::FilesystemPersister::new("bench_filesystem_persister_a".to_string());
+               let persister_b = super::FilesystemPersister::new("bench_filesystem_persister_b".to_string());
+               lightning::ln::channelmanager::bench::bench_two_sends(bench, persister_a, persister_b);
+       }
+}
index eb17b469a0365df2de4464ef313cc243441b6767..0fd088019247e4e8e9adbea39adb99798e274dd4 100644 (file)
@@ -26,7 +26,7 @@
 use bitcoin::blockdata::block::{Block, BlockHeader};
 
 use chain;
-use chain::Filter;
+use chain::{Filter, WatchedOutput};
 use chain::chaininterface::{BroadcasterInterface, FeeEstimator};
 use chain::channelmonitor;
 use chain::channelmonitor::{ChannelMonitor, ChannelMonitorUpdate, ChannelMonitorUpdateErr, MonitorEvent, Persist};
@@ -82,18 +82,40 @@ where C::Target: chain::Filter,
        /// descendants of such transactions. It is not necessary to re-fetch the block to obtain
        /// updated `txdata`.
        pub fn block_connected(&self, header: &BlockHeader, txdata: &TransactionData, height: u32) {
+               let mut dependent_txdata = Vec::new();
                let monitors = self.monitors.read().unwrap();
                for monitor in monitors.values() {
                        let mut txn_outputs = monitor.block_connected(header, txdata, height, &*self.broadcaster, &*self.fee_estimator, &*self.logger);
 
+                       // Register any new outputs with the chain source for filtering, storing any dependent
+                       // transactions from within the block that previously had not been included in txdata.
                        if let Some(ref chain_source) = self.chain_source {
+                               let block_hash = header.block_hash();
                                for (txid, outputs) in txn_outputs.drain(..) {
                                        for (idx, output) in outputs.iter() {
-                                               chain_source.register_output(&OutPoint { txid, index: *idx as u16 }, &output.script_pubkey);
+                                               // Register any new outputs with the chain source for filtering and recurse
+                                               // if it indicates that there are dependent transactions within the block
+                                               // that had not been previously included in txdata.
+                                               let output = WatchedOutput {
+                                                       block_hash: Some(block_hash),
+                                                       outpoint: OutPoint { txid, index: *idx as u16 },
+                                                       script_pubkey: output.script_pubkey.clone(),
+                                               };
+                                               if let Some(tx) = chain_source.register_output(output) {
+                                                       dependent_txdata.push(tx);
+                                               }
                                        }
                                }
                        }
                }
+
+               // Recursively call for any dependent transactions that were identified by the chain source.
+               if !dependent_txdata.is_empty() {
+                       dependent_txdata.sort_unstable_by_key(|(index, _tx)| *index);
+                       dependent_txdata.dedup_by_key(|(index, _tx)| *index);
+                       let txdata: Vec<_> = dependent_txdata.iter().map(|(index, tx)| (*index, tx)).collect();
+                       self.block_connected(header, &txdata, height);
+               }
        }
 
        /// Dispatches to per-channel monitors, which are responsible for updating their on-chain view
@@ -245,3 +267,56 @@ impl<ChannelSigner: Sign, C: Deref, T: Deref, F: Deref, L: Deref, P: Deref> even
                pending_events
        }
 }
+
+#[cfg(test)]
+mod tests {
+       use ::{check_added_monitors, get_local_commitment_txn};
+       use ln::features::InitFeatures;
+       use ln::functional_test_utils::*;
+       use util::events::EventsProvider;
+       use util::events::MessageSendEventsProvider;
+       use util::test_utils::{OnRegisterOutput, TxOutReference};
+
+       /// Tests that in-block dependent transactions are processed by `block_connected` when not
+       /// included in `txdata` but returned by [`chain::Filter::register_output`]. For instance,
+       /// a (non-anchor) commitment transaction's HTLC output may be spent in the same block as the
+       /// commitment transaction itself. An Electrum client may filter the commitment transaction but
+       /// needs to return the HTLC transaction so it can be processed.
+       #[test]
+       fn connect_block_checks_dependent_transactions() {
+               let chanmon_cfgs = create_chanmon_cfgs(2);
+               let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
+               let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
+               let nodes = create_network(2, &node_cfgs, &node_chanmgrs);
+               let channel = create_announced_chan_between_nodes(
+                       &nodes, 0, 1, InitFeatures::known(), InitFeatures::known());
+
+               // Send a payment, saving nodes[0]'s revoked commitment and HTLC-Timeout transactions.
+               let (commitment_tx, htlc_tx) = {
+                       let payment_preimage = route_payment(&nodes[0], &vec!(&nodes[1])[..], 5_000_000).0;
+                       let mut txn = get_local_commitment_txn!(nodes[0], channel.2);
+                       claim_payment(&nodes[0], &vec!(&nodes[1])[..], payment_preimage, 5_000_000);
+
+                       assert_eq!(txn.len(), 2);
+                       (txn.remove(0), txn.remove(0))
+               };
+
+               // Set expectations on nodes[1]'s chain source to return dependent transactions.
+               let htlc_output = TxOutReference(commitment_tx.clone(), 0);
+               let to_local_output = TxOutReference(commitment_tx.clone(), 1);
+               let htlc_timeout_output = TxOutReference(htlc_tx.clone(), 0);
+               nodes[1].chain_source
+                       .expect(OnRegisterOutput { with: htlc_output, returns: Some((1, htlc_tx)) })
+                       .expect(OnRegisterOutput { with: to_local_output, returns: None })
+                       .expect(OnRegisterOutput { with: htlc_timeout_output, returns: None });
+
+               // Notify nodes[1] that nodes[0]'s revoked commitment transaction was mined. The chain
+               // source should return the dependent HTLC transaction when the HTLC output is registered.
+               mine_transaction(&nodes[1], &commitment_tx);
+
+               // Clean up so uninteresting assertions don't fail.
+               check_added_monitors!(nodes[1], 1);
+               nodes[1].node.get_and_clear_pending_msg_events();
+               nodes[1].node.get_and_clear_pending_events();
+       }
+}
index 28b6da4722a7b4e3820541b30c89047d1e8e5cfe..939337d7b42d572d64c7ab5b1a6224479f6c1ce9 100644 (file)
@@ -40,6 +40,7 @@ use ln::chan_utils::{CounterpartyCommitmentSecrets, HTLCOutputInCommitment, HTLC
 use ln::channelmanager::{HTLCSource, PaymentPreimage, PaymentHash};
 use ln::onchaintx::{OnchainTxHandler, InputDescriptors};
 use chain;
+use chain::WatchedOutput;
 use chain::chaininterface::{BroadcasterInterface, FeeEstimator};
 use chain::transaction::{OutPoint, TransactionData};
 use chain::keysinterface::{SpendableOutputDescriptor, StaticPaymentOutputDescriptor, DelayedPaymentOutputDescriptor, Sign, KeysInterface};
@@ -1174,7 +1175,11 @@ impl<Signer: Sign> ChannelMonitor<Signer> {
                for (txid, outputs) in lock.get_outputs_to_watch().iter() {
                        for (index, script_pubkey) in outputs.iter() {
                                assert!(*index <= u16::max_value() as u32);
-                               filter.register_output(&OutPoint { txid: *txid, index: *index as u16 }, script_pubkey);
+                               filter.register_output(WatchedOutput {
+                                       block_hash: None,
+                                       outpoint: OutPoint { txid: *txid, index: *index as u16 },
+                                       script_pubkey: script_pubkey.clone(),
+                               });
                        }
                }
        }
index 89557b43670202f700c38939c3ff5506bb44b63e..89306c6e3c2f1ca34c9a3a8d43f4908c5baabb4d 100644 (file)
@@ -226,11 +226,11 @@ impl Readable for SpendableOutputDescriptor {
 /// of LN security model, orthogonal of key management issues.
 // TODO: We should remove Clone by instead requesting a new Sign copy when we create
 // ChannelMonitors instead of expecting to clone the one out of the Channel into the monitors.
-pub trait Sign : Send+Clone + Writeable {
+pub trait BaseSign : Send {
        /// Gets the per-commitment point for a specific commitment number
        ///
        /// Note that the commitment number starts at (1 << 48) - 1 and counts backwards.
-       fn get_per_commitment_point<T: secp256k1::Signing + secp256k1::Verification>(&self, idx: u64, secp_ctx: &Secp256k1<T>) -> PublicKey;
+       fn get_per_commitment_point(&self, idx: u64, secp_ctx: &Secp256k1<secp256k1::All>) -> PublicKey;
        /// Gets the commitment secret for a specific commitment number as part of the revocation process
        ///
        /// An external signer implementation should error here if the commitment was already signed
@@ -253,7 +253,7 @@ pub trait Sign : Send+Clone + Writeable {
        /// Note that if signing fails or is rejected, the channel will be force-closed.
        //
        // TODO: Document the things someone using this interface should enforce before signing.
-       fn sign_counterparty_commitment<T: secp256k1::Signing + secp256k1::Verification>(&self, commitment_tx: &CommitmentTransaction, secp_ctx: &Secp256k1<T>) -> Result<(Signature, Vec<Signature>), ()>;
+       fn sign_counterparty_commitment(&self, commitment_tx: &CommitmentTransaction, secp_ctx: &Secp256k1<secp256k1::All>) -> Result<(Signature, Vec<Signature>), ()>;
 
        /// Create a signatures for a holder's commitment transaction and its claiming HTLC transactions.
        /// This will only ever be called with a non-revoked commitment_tx.  This will be called with the
@@ -269,14 +269,14 @@ pub trait Sign : Send+Clone + Writeable {
        //
        // TODO: Document the things someone using this interface should enforce before signing.
        // TODO: Key derivation failure should panic rather than Err
-       fn sign_holder_commitment_and_htlcs<T: secp256k1::Signing + secp256k1::Verification>(&self, commitment_tx: &HolderCommitmentTransaction, secp_ctx: &Secp256k1<T>) -> Result<(Signature, Vec<Signature>), ()>;
+       fn sign_holder_commitment_and_htlcs(&self, commitment_tx: &HolderCommitmentTransaction, secp_ctx: &Secp256k1<secp256k1::All>) -> Result<(Signature, Vec<Signature>), ()>;
 
        /// Same as sign_holder_commitment, but exists only for tests to get access to holder commitment
        /// transactions which will be broadcasted later, after the channel has moved on to a newer
        /// state. Thus, needs its own method as sign_holder_commitment may enforce that we only ever
        /// get called once.
        #[cfg(any(test,feature = "unsafe_revoked_tx_signing"))]
-       fn unsafe_sign_holder_commitment_and_htlcs<T: secp256k1::Signing + secp256k1::Verification>(&self, commitment_tx: &HolderCommitmentTransaction, secp_ctx: &Secp256k1<T>) -> Result<(Signature, Vec<Signature>), ()>;
+       fn unsafe_sign_holder_commitment_and_htlcs(&self, commitment_tx: &HolderCommitmentTransaction, secp_ctx: &Secp256k1<secp256k1::All>) -> Result<(Signature, Vec<Signature>), ()>;
 
        /// Create a signature for the given input in a transaction spending an HTLC or commitment
        /// transaction output when our counterparty broadcasts an old state.
@@ -296,7 +296,7 @@ pub trait Sign : Send+Clone + Writeable {
        /// htlc holds HTLC elements (hash, timelock) if the output being spent is a HTLC output, thus
        /// changing the format of the witness script (which is committed to in the BIP 143
        /// signatures).
-       fn sign_justice_transaction<T: secp256k1::Signing + secp256k1::Verification>(&self, justice_tx: &Transaction, input: usize, amount: u64, per_commitment_key: &SecretKey, htlc: &Option<HTLCOutputInCommitment>, secp_ctx: &Secp256k1<T>) -> Result<Signature, ()>;
+       fn sign_justice_transaction(&self, justice_tx: &Transaction, input: usize, amount: u64, per_commitment_key: &SecretKey, htlc: &Option<HTLCOutputInCommitment>, secp_ctx: &Secp256k1<secp256k1::All>) -> Result<Signature, ()>;
 
        /// Create a signature for a claiming transaction for a HTLC output on a counterparty's commitment
        /// transaction, either offered or received.
@@ -315,13 +315,13 @@ pub trait Sign : Send+Clone + Writeable {
        /// detected onchain. It has been generated by our counterparty and is used to derive
        /// channel state keys, which are then included in the witness script and committed to in the
        /// BIP 143 signature.
-       fn sign_counterparty_htlc_transaction<T: secp256k1::Signing + secp256k1::Verification>(&self, htlc_tx: &Transaction, input: usize, amount: u64, per_commitment_point: &PublicKey, htlc: &HTLCOutputInCommitment, secp_ctx: &Secp256k1<T>) -> Result<Signature, ()>;
+       fn sign_counterparty_htlc_transaction(&self, htlc_tx: &Transaction, input: usize, amount: u64, per_commitment_point: &PublicKey, htlc: &HTLCOutputInCommitment, secp_ctx: &Secp256k1<secp256k1::All>) -> Result<Signature, ()>;
 
        /// Create a signature for a (proposed) closing transaction.
        ///
        /// Note that, due to rounding, there may be one "missing" satoshi, and either party may have
        /// chosen to forgo their output as dust.
-       fn sign_closing_transaction<T: secp256k1::Signing>(&self, closing_tx: &Transaction, secp_ctx: &Secp256k1<T>) -> Result<Signature, ()>;
+       fn sign_closing_transaction(&self, closing_tx: &Transaction, secp_ctx: &Secp256k1<secp256k1::All>) -> Result<Signature, ()>;
 
        /// Signs a channel announcement message with our funding key, proving it comes from one
        /// of the channel participants.
@@ -329,7 +329,7 @@ pub trait Sign : Send+Clone + Writeable {
        /// Note that if this fails or is rejected, the channel will not be publicly announced and
        /// our counterparty may (though likely will not) close the channel on us for violating the
        /// protocol.
-       fn sign_channel_announcement<T: secp256k1::Signing>(&self, msg: &UnsignedChannelAnnouncement, secp_ctx: &Secp256k1<T>) -> Result<Signature, ()>;
+       fn sign_channel_announcement(&self, msg: &UnsignedChannelAnnouncement, secp_ctx: &Secp256k1<secp256k1::All>) -> Result<Signature, ()>;
 
        /// Set the counterparty static channel data, including basepoints,
        /// counterparty_selected/holder_selected_contest_delay and funding outpoint.
@@ -344,6 +344,14 @@ pub trait Sign : Send+Clone + Writeable {
        fn ready_channel(&mut self, channel_parameters: &ChannelTransactionParameters);
 }
 
+/// A cloneable signer.
+///
+/// Although we require signers to be cloneable, it may be useful for developers to be able to use
+/// signers in an un-sized way, for example as `dyn BaseSign`. Therefore we separate the Clone trait,
+/// which implies Sized, into this derived trait.
+pub trait Sign: BaseSign + Writeable + Clone {
+}
+
 /// A trait to describe an object which can get user secrets and key material.
 pub trait KeysInterface: Send + Sync {
        /// A type which implements Sign which will be returned by get_channel_signer.
@@ -549,8 +557,8 @@ impl InMemorySigner {
        }
 }
 
-impl Sign for InMemorySigner {
-       fn get_per_commitment_point<T: secp256k1::Signing + secp256k1::Verification>(&self, idx: u64, secp_ctx: &Secp256k1<T>) -> PublicKey {
+impl BaseSign for InMemorySigner {
+       fn get_per_commitment_point(&self, idx: u64, secp_ctx: &Secp256k1<secp256k1::All>) -> PublicKey {
                let commitment_secret = SecretKey::from_slice(&chan_utils::build_commitment_secret(&self.commitment_seed, idx)).unwrap();
                PublicKey::from_secret_key(secp_ctx, &commitment_secret)
        }
@@ -562,7 +570,7 @@ impl Sign for InMemorySigner {
        fn pubkeys(&self) -> &ChannelPublicKeys { &self.holder_channel_pubkeys }
        fn channel_keys_id(&self) -> [u8; 32] { self.channel_keys_id }
 
-       fn sign_counterparty_commitment<T: secp256k1::Signing + secp256k1::Verification>(&self, commitment_tx: &CommitmentTransaction, secp_ctx: &Secp256k1<T>) -> Result<(Signature, Vec<Signature>), ()> {
+       fn sign_counterparty_commitment(&self, commitment_tx: &CommitmentTransaction, secp_ctx: &Secp256k1<secp256k1::All>) -> Result<(Signature, Vec<Signature>), ()> {
                let trusted_tx = commitment_tx.trust();
                let keys = trusted_tx.keys();
 
@@ -588,7 +596,7 @@ impl Sign for InMemorySigner {
                Ok((commitment_sig, htlc_sigs))
        }
 
-       fn sign_holder_commitment_and_htlcs<T: secp256k1::Signing + secp256k1::Verification>(&self, commitment_tx: &HolderCommitmentTransaction, secp_ctx: &Secp256k1<T>) -> Result<(Signature, Vec<Signature>), ()> {
+       fn sign_holder_commitment_and_htlcs(&self, commitment_tx: &HolderCommitmentTransaction, secp_ctx: &Secp256k1<secp256k1::All>) -> Result<(Signature, Vec<Signature>), ()> {
                let funding_pubkey = PublicKey::from_secret_key(secp_ctx, &self.funding_key);
                let funding_redeemscript = make_funding_redeemscript(&funding_pubkey, &self.counterparty_pubkeys().funding_pubkey);
                let trusted_tx = commitment_tx.trust();
@@ -599,7 +607,7 @@ impl Sign for InMemorySigner {
        }
 
        #[cfg(any(test,feature = "unsafe_revoked_tx_signing"))]
-       fn unsafe_sign_holder_commitment_and_htlcs<T: secp256k1::Signing + secp256k1::Verification>(&self, commitment_tx: &HolderCommitmentTransaction, secp_ctx: &Secp256k1<T>) -> Result<(Signature, Vec<Signature>), ()> {
+       fn unsafe_sign_holder_commitment_and_htlcs(&self, commitment_tx: &HolderCommitmentTransaction, secp_ctx: &Secp256k1<secp256k1::All>) -> Result<(Signature, Vec<Signature>), ()> {
                let funding_pubkey = PublicKey::from_secret_key(secp_ctx, &self.funding_key);
                let funding_redeemscript = make_funding_redeemscript(&funding_pubkey, &self.counterparty_pubkeys().funding_pubkey);
                let trusted_tx = commitment_tx.trust();
@@ -609,7 +617,7 @@ impl Sign for InMemorySigner {
                Ok((sig, htlc_sigs))
        }
 
-       fn sign_justice_transaction<T: secp256k1::Signing + secp256k1::Verification>(&self, justice_tx: &Transaction, input: usize, amount: u64, per_commitment_key: &SecretKey, htlc: &Option<HTLCOutputInCommitment>, secp_ctx: &Secp256k1<T>) -> Result<Signature, ()> {
+       fn sign_justice_transaction(&self, justice_tx: &Transaction, input: usize, amount: u64, per_commitment_key: &SecretKey, htlc: &Option<HTLCOutputInCommitment>, secp_ctx: &Secp256k1<secp256k1::All>) -> Result<Signature, ()> {
                let revocation_key = match chan_utils::derive_private_revocation_key(&secp_ctx, &per_commitment_key, &self.revocation_base_key) {
                        Ok(revocation_key) => revocation_key,
                        Err(_) => return Err(())
@@ -641,7 +649,7 @@ impl Sign for InMemorySigner {
                return Ok(secp_ctx.sign(&sighash, &revocation_key))
        }
 
-       fn sign_counterparty_htlc_transaction<T: secp256k1::Signing + secp256k1::Verification>(&self, htlc_tx: &Transaction, input: usize, amount: u64, per_commitment_point: &PublicKey, htlc: &HTLCOutputInCommitment, secp_ctx: &Secp256k1<T>) -> Result<Signature, ()> {
+       fn sign_counterparty_htlc_transaction(&self, htlc_tx: &Transaction, input: usize, amount: u64, per_commitment_point: &PublicKey, htlc: &HTLCOutputInCommitment, secp_ctx: &Secp256k1<secp256k1::All>) -> Result<Signature, ()> {
                if let Ok(htlc_key) = chan_utils::derive_private_key(&secp_ctx, &per_commitment_point, &self.htlc_base_key) {
                        let witness_script = if let Ok(revocation_pubkey) = chan_utils::derive_public_revocation_key(&secp_ctx, &per_commitment_point, &self.pubkeys().revocation_basepoint) {
                                if let Ok(counterparty_htlcpubkey) = chan_utils::derive_public_key(&secp_ctx, &per_commitment_point, &self.counterparty_pubkeys().htlc_basepoint) {
@@ -657,7 +665,7 @@ impl Sign for InMemorySigner {
                Err(())
        }
 
-       fn sign_closing_transaction<T: secp256k1::Signing>(&self, closing_tx: &Transaction, secp_ctx: &Secp256k1<T>) -> Result<Signature, ()> {
+       fn sign_closing_transaction(&self, closing_tx: &Transaction, secp_ctx: &Secp256k1<secp256k1::All>) -> Result<Signature, ()> {
                if closing_tx.input.len() != 1 { return Err(()); }
                if closing_tx.input[0].witness.len() != 0 { return Err(()); }
                if closing_tx.output.len() > 2 { return Err(()); }
@@ -670,7 +678,7 @@ impl Sign for InMemorySigner {
                Ok(secp_ctx.sign(&sighash, &self.funding_key))
        }
 
-       fn sign_channel_announcement<T: secp256k1::Signing>(&self, msg: &UnsignedChannelAnnouncement, secp_ctx: &Secp256k1<T>) -> Result<Signature, ()> {
+       fn sign_channel_announcement(&self, msg: &UnsignedChannelAnnouncement, secp_ctx: &Secp256k1<secp256k1::All>) -> Result<Signature, ()> {
                let msghash = hash_to_message!(&Sha256dHash::hash(&msg.encode()[..])[..]);
                Ok(secp_ctx.sign(&msghash, &self.funding_key))
        }
@@ -682,6 +690,8 @@ impl Sign for InMemorySigner {
        }
 }
 
+impl Sign for InMemorySigner {}
+
 impl Writeable for InMemorySigner {
        fn write<W: Writer>(&self, writer: &mut W) -> Result<(), Error> {
                self.funding_key.write(writer)?;
@@ -738,7 +748,7 @@ impl Readable for InMemorySigner {
 /// Cooperative closes may use seed/2'
 /// The two close keys may be needed to claim on-chain funds!
 pub struct KeysManager {
-       secp_ctx: Secp256k1<secp256k1::SignOnly>,
+       secp_ctx: Secp256k1<secp256k1::All>,
        node_secret: SecretKey,
        destination_script: Script,
        shutdown_pubkey: PublicKey,
@@ -775,7 +785,7 @@ impl KeysManager {
        /// versions. Once the library is more fully supported, the docs will be updated to include a
        /// detailed description of the guarantee.
        pub fn new(seed: &[u8; 32], starting_time_secs: u64, starting_time_nanos: u32) -> Self {
-               let secp_ctx = Secp256k1::signing_only();
+               let secp_ctx = Secp256k1::new();
                // Note that when we aren't serializing the key, network doesn't matter
                match ExtendedPrivKey::new_master(Network::Testnet, seed) {
                        Ok(master_key) => {
@@ -1038,3 +1048,9 @@ impl KeysInterface for KeysManager {
                InMemorySigner::read(&mut std::io::Cursor::new(reader))
        }
 }
+
+// Ensure that BaseSign can have a vtable
+#[test]
+pub fn dyn_sign() {
+       let _signer: Box<dyn BaseSign>;
+}
index 7d410b9b71dd116cbe22d2858e9664ec6bc14381..18c7fd55d7b6643bbbc140a5208df42e39e42ea3 100644 (file)
@@ -11,7 +11,7 @@
 
 use bitcoin::blockdata::block::{Block, BlockHeader};
 use bitcoin::blockdata::script::Script;
-use bitcoin::blockdata::transaction::TxOut;
+use bitcoin::blockdata::transaction::{Transaction, TxOut};
 use bitcoin::hash_types::{BlockHash, Txid};
 
 use chain::channelmonitor::{ChannelMonitor, ChannelMonitorUpdate, ChannelMonitorUpdateErr, MonitorEvent};
@@ -129,9 +129,38 @@ pub trait Filter: Send + Sync {
        /// a spending condition.
        fn register_tx(&self, txid: &Txid, script_pubkey: &Script);
 
-       /// Registers interest in spends of a transaction output identified by `outpoint` having
-       /// `script_pubkey` as the spending condition.
-       fn register_output(&self, outpoint: &OutPoint, script_pubkey: &Script);
+       /// Registers interest in spends of a transaction output.
+       ///
+       /// Optionally, when `output.block_hash` is set, should return any transaction spending the
+       /// output that is found in the corresponding block along with its index.
+       ///
+       /// This return value is useful for Electrum clients in order to supply in-block descendant
+       /// transactions which otherwise were not included. This is not necessary for other clients if
+       /// such descendant transactions were already included (e.g., when a BIP 157 client provides the
+       /// full block).
+       fn register_output(&self, output: WatchedOutput) -> Option<(usize, Transaction)>;
+}
+
+/// A transaction output watched by a [`ChannelMonitor`] for spends on-chain.
+///
+/// Used to convey to a [`Filter`] such an output with a given spending condition. Any transaction
+/// spending the output must be given to [`ChannelMonitor::block_connected`] either directly or via
+/// the return value of [`Filter::register_output`].
+///
+/// If `block_hash` is `Some`, this indicates the output was created in the corresponding block and
+/// may have been spent there. See [`Filter::register_output`] for details.
+///
+/// [`ChannelMonitor`]: channelmonitor::ChannelMonitor
+/// [`ChannelMonitor::block_connected`]: channelmonitor::ChannelMonitor::block_connected
+pub struct WatchedOutput {
+       /// First block where the transaction output may have been spent.
+       pub block_hash: Option<BlockHash>,
+
+       /// Outpoint identifying the transaction output.
+       pub outpoint: OutPoint,
+
+       /// Spending condition of the transaction output.
+       pub script_pubkey: Script,
 }
 
 impl<T: Listen> Listen for std::ops::Deref<Target = T> {
index 2d764c8b71b1355200ca062087fcb22044b5c24b..9bf9470458f8310f005788769d781b74055fd835 100644 (file)
@@ -28,8 +28,8 @@
 #![allow(bare_trait_objects)]
 #![allow(ellipsis_inclusive_range_patterns)]
 
-#![cfg_attr(all(test, feature = "unstable"), feature(test))]
-#[cfg(all(test, feature = "unstable"))] extern crate test;
+#![cfg_attr(all(any(test, feature = "_test_utils"), feature = "unstable"), feature(test))]
+#[cfg(all(any(test, feature = "_test_utils"), feature = "unstable"))] extern crate test;
 
 extern crate bitcoin;
 #[cfg(any(test, feature = "_test_utils"))] extern crate hex;
index dc37e3f6c7421295691f34f1b821d1fe9cb2f1ea..a7cc5377a2532a14644619e623a64ed6a0278ad1 100644 (file)
@@ -241,7 +241,7 @@ fn do_test_simple_monitor_temporary_update_fail(disconnect: bool, persister_fail
        // ...and make sure we can force-close a frozen channel
        nodes[0].node.force_close_channel(&channel_id).unwrap();
        check_added_monitors!(nodes[0], 1);
-       check_closed_broadcast!(nodes[0], false);
+       check_closed_broadcast!(nodes[0], true);
 
        // TODO: Once we hit the chain with the failure transaction we should check that we get a
        // PaymentFailed event
index 174aecd3ba036f46dc9acdc95d2b287ef536d812..c13ac9b61ffc94b000f4204ea23baf8a14a0e6aa 100644 (file)
@@ -7,7 +7,6 @@
 // You may not use this file except in accordance with one or both of these
 // licenses.
 
-use bitcoin::blockdata::block::BlockHeader;
 use bitcoin::blockdata::script::{Script,Builder};
 use bitcoin::blockdata::transaction::{TxIn, TxOut, Transaction, SigHashType};
 use bitcoin::blockdata::opcodes;
@@ -376,13 +375,10 @@ pub(super) struct Channel<Signer: Sign> {
 
        last_sent_closing_fee: Option<(u32, u64, Signature)>, // (feerate, fee, holder_sig)
 
-       /// The hash of the block in which the funding transaction reached our CONF_TARGET. We use this
-       /// to detect unconfirmation after a serialize-unserialize roundtrip where we may not see a full
-       /// series of block_connected/block_disconnected calls. Obviously this is not a guarantee as we
-       /// could miss the funding_tx_confirmed_in block as well, but it serves as a useful fallback.
+       /// The hash of the block in which the funding transaction was included.
        funding_tx_confirmed_in: Option<BlockHash>,
+       funding_tx_confirmation_height: u64,
        short_channel_id: Option<u64>,
-       funding_tx_confirmations: u64,
 
        counterparty_dust_limit_satoshis: u64,
        #[cfg(test)]
@@ -441,10 +437,6 @@ struct CommitmentTxInfoCached {
 }
 
 pub const OUR_MAX_HTLCS: u16 = 50; //TODO
-/// Confirmation count threshold at which we close a channel. Ideally we'd keep the channel around
-/// on ice until the funding transaction gets more confirmations, but the LN protocol doesn't
-/// really allow for this, so instead we're stuck closing it out at that point.
-const UNCONF_THRESHOLD: u32 = 6;
 const SPENDING_INPUT_FOR_A_OUTPUT_WEIGHT: u64 = 79; // prevout: 36, nSequence: 4, script len: 1, witness lengths: (3+1)/4, sig: 73/4, if-selector: 1, redeemScript: (6 ops + 2*33 pubkeys + 1*2 delay)/4
 const B_OUTPUT_PLUS_SPENDING_INPUT_WEIGHT: u64 = 104; // prevout: 40, nSequence: 4, script len: 1, witness lengths: 3/4, sig: 73/4, pubkey: 33/4, output: 31 (TODO: Wrong? Useless?)
 
@@ -581,8 +573,8 @@ impl<Signer: Sign> Channel<Signer> {
                        last_sent_closing_fee: None,
 
                        funding_tx_confirmed_in: None,
+                       funding_tx_confirmation_height: 0,
                        short_channel_id: None,
-                       funding_tx_confirmations: 0,
 
                        feerate_per_kw: feerate,
                        counterparty_dust_limit_satoshis: 0,
@@ -819,8 +811,8 @@ impl<Signer: Sign> Channel<Signer> {
                        last_sent_closing_fee: None,
 
                        funding_tx_confirmed_in: None,
+                       funding_tx_confirmation_height: 0,
                        short_channel_id: None,
-                       funding_tx_confirmations: 0,
 
                        feerate_per_kw: msg.feerate_per_kw,
                        channel_value_satoshis: msg.funding_satoshis,
@@ -3512,26 +3504,140 @@ impl<Signer: Sign> Channel<Signer> {
                self.network_sync == UpdateStatus::DisabledMarked
        }
 
-       /// When we receive a new block, we (a) check whether the block contains the funding
-       /// transaction (which would start us counting blocks until we send the funding_signed), and
-       /// (b) check the height of the block against outbound holding cell HTLCs in case we need to
-       /// give up on them prematurely and time them out. Everything else (e.g. commitment
-       /// transaction broadcasts, channel closure detection, HTLC transaction broadcasting, etc) is
+       fn check_get_funding_locked(&mut self, height: u32) -> Option<msgs::FundingLocked> {
+               if self.funding_tx_confirmation_height == 0 {
+                       return None;
+               }
+
+               let funding_tx_confirmations = height as i64 - self.funding_tx_confirmation_height as i64 + 1;
+               if funding_tx_confirmations <= 0 {
+                       self.funding_tx_confirmation_height = 0;
+               }
+
+               if funding_tx_confirmations < self.minimum_depth as i64 {
+                       return None;
+               }
+
+               let non_shutdown_state = self.channel_state & (!MULTI_STATE_FLAGS);
+               let need_commitment_update = if non_shutdown_state == ChannelState::FundingSent as u32 {
+                       self.channel_state |= ChannelState::OurFundingLocked as u32;
+                       true
+               } else if non_shutdown_state == (ChannelState::FundingSent as u32 | ChannelState::TheirFundingLocked as u32) {
+                       self.channel_state = ChannelState::ChannelFunded as u32 | (self.channel_state & MULTI_STATE_FLAGS);
+                       self.update_time_counter += 1;
+                       true
+               } else if non_shutdown_state == (ChannelState::FundingSent as u32 | ChannelState::OurFundingLocked as u32) {
+                       // We got a reorg but not enough to trigger a force close, just ignore.
+                       false
+               } else if self.channel_state < ChannelState::ChannelFunded as u32 {
+                       panic!("Started confirming a channel in a state pre-FundingSent?: {}", self.channel_state);
+               } else {
+                       // We got a reorg but not enough to trigger a force close, just ignore.
+                       false
+               };
+
+               if need_commitment_update {
+                       if self.channel_state & (ChannelState::MonitorUpdateFailed as u32) == 0 {
+                               let next_per_commitment_point = self.holder_signer.get_per_commitment_point(self.cur_holder_commitment_transaction_number, &self.secp_ctx);
+                               return Some(msgs::FundingLocked {
+                                       channel_id: self.channel_id,
+                                       next_per_commitment_point,
+                               });
+                       } else {
+                               self.monitor_pending_funding_locked = true;
+                       }
+               }
+               None
+       }
+
+       /// When a transaction is confirmed, we check whether it is or spends the funding transaction
+       /// In the first case, we store the confirmation height and calculating the short channel id.
+       /// In the second, we simply return an Err indicating we need to be force-closed now.
+       pub fn transactions_confirmed<L: Deref>(&mut self, block_hash: &BlockHash, height: u32, txdata: &TransactionData, logger: &L)
+                       -> Result<Option<msgs::FundingLocked>, msgs::ErrorMessage> where L::Target: Logger {
+               let non_shutdown_state = self.channel_state & (!MULTI_STATE_FLAGS);
+               for &(index_in_block, tx) in txdata.iter() {
+                       if let Some(funding_txo) = self.get_funding_txo() {
+                               // If we haven't yet sent a funding_locked, but are in FundingSent (ignoring
+                               // whether they've sent a funding_locked or not), check if we should send one.
+                               if non_shutdown_state & !(ChannelState::TheirFundingLocked as u32) == ChannelState::FundingSent as u32 {
+                                       if tx.txid() == funding_txo.txid {
+                                               let txo_idx = funding_txo.index as usize;
+                                               if txo_idx >= tx.output.len() || tx.output[txo_idx].script_pubkey != self.get_funding_redeemscript().to_v0_p2wsh() ||
+                                                               tx.output[txo_idx].value != self.channel_value_satoshis {
+                                                       if self.is_outbound() {
+                                                               // If we generated the funding transaction and it doesn't match what it
+                                                               // should, the client is really broken and we should just panic and
+                                                               // tell them off. That said, because hash collisions happen with high
+                                                               // probability in fuzztarget mode, if we're fuzzing we just close the
+                                                               // channel and move on.
+                                                               #[cfg(not(feature = "fuzztarget"))]
+                                                               panic!("Client called ChannelManager::funding_transaction_generated with bogus transaction!");
+                                                       }
+                                                       self.channel_state = ChannelState::ShutdownComplete as u32;
+                                                       self.update_time_counter += 1;
+                                                       return Err(msgs::ErrorMessage {
+                                                               channel_id: self.channel_id(),
+                                                               data: "funding tx had wrong script/value or output index".to_owned()
+                                                       });
+                                               } else {
+                                                       if self.is_outbound() {
+                                                               for input in tx.input.iter() {
+                                                                       if input.witness.is_empty() {
+                                                                               // We generated a malleable funding transaction, implying we've
+                                                                               // just exposed ourselves to funds loss to our counterparty.
+                                                                               #[cfg(not(feature = "fuzztarget"))]
+                                                                               panic!("Client called ChannelManager::funding_transaction_generated with bogus transaction!");
+                                                                       }
+                                                               }
+                                                       }
+                                                       self.funding_tx_confirmation_height = height as u64;
+                                                       self.funding_tx_confirmed_in = Some(*block_hash);
+                                                       self.short_channel_id = match scid_from_parts(height as u64, index_in_block as u64, txo_idx as u64) {
+                                                               Ok(scid) => Some(scid),
+                                                               Err(_) => panic!("Block was bogus - either height was > 16 million, had > 16 million transactions, or had > 65k outputs"),
+                                                       }
+                                               }
+                                       }
+                                       // If we allow 1-conf funding, we may need to check for funding_locked here and
+                                       // send it immediately instead of waiting for an update_best_block call (which
+                                       // may have already happened for this block).
+                                       if let Some(funding_locked) = self.check_get_funding_locked(height) {
+                                               return Ok(Some(funding_locked));
+                                       }
+                               }
+                               for inp in tx.input.iter() {
+                                       if inp.previous_output == funding_txo.into_bitcoin_outpoint() {
+                                               log_trace!(logger, "Detected channel-closing tx {} spending {}:{}, closing channel {}", tx.txid(), inp.previous_output.txid, inp.previous_output.vout, log_bytes!(self.channel_id()));
+                                               return Err(msgs::ErrorMessage {
+                                                       channel_id: self.channel_id(),
+                                                       data: "Commitment or closing transaction was confirmed on chain.".to_owned()
+                                               });
+                                       }
+                               }
+                       }
+               }
+               Ok(None)
+       }
+
+       /// When a new block is connected, we check the height of the block against outbound holding
+       /// cell HTLCs in case we need to give up on them prematurely and time them out. Everything
+       /// else (e.g. commitment transaction broadcasts, HTLC transaction broadcasting, etc) is
        /// handled by the ChannelMonitor.
        ///
        /// If we return Err, the channel may have been closed, at which point the standard
        /// requirements apply - no calls may be made except those explicitly stated to be allowed
        /// post-shutdown.
-       /// Only returns an ErrorAction of DisconnectPeer, if Err.
        ///
        /// May return some HTLCs (and their payment_hash) which have timed out and should be failed
        /// back.
-       pub fn block_connected(&mut self, header: &BlockHeader, txdata: &TransactionData, height: u32) -> Result<(Option<msgs::FundingLocked>, Vec<(HTLCSource, PaymentHash)>), msgs::ErrorMessage> {
+       pub fn update_best_block(&mut self, height: u32, highest_header_time: u32) -> Result<(Option<msgs::FundingLocked>, Vec<(HTLCSource, PaymentHash)>), msgs::ErrorMessage> {
                let mut timed_out_htlcs = Vec::new();
+               let unforwarded_htlc_cltv_limit = height + HTLC_FAIL_BACK_BUFFER;
                self.holding_cell_htlc_updates.retain(|htlc_update| {
                        match htlc_update {
                                &HTLCUpdateAwaitingACK::AddHTLC { ref payment_hash, ref source, ref cltv_expiry, .. } => {
-                                       if *cltv_expiry <= height + HTLC_FAIL_BACK_BUFFER {
+                                       if *cltv_expiry <= unforwarded_htlc_cltv_limit {
                                                timed_out_htlcs.push((source.clone(), payment_hash.clone()));
                                                false
                                        } else { true }
@@ -3540,112 +3646,36 @@ impl<Signer: Sign> Channel<Signer> {
                        }
                });
 
-               if self.funding_tx_confirmations > 0 {
-                       self.funding_tx_confirmations += 1;
+               self.update_time_counter = cmp::max(self.update_time_counter, highest_header_time);
+
+               if let Some(funding_locked) = self.check_get_funding_locked(height) {
+                       return Ok((Some(funding_locked), timed_out_htlcs));
                }
 
                let non_shutdown_state = self.channel_state & (!MULTI_STATE_FLAGS);
-               if non_shutdown_state & !(ChannelState::TheirFundingLocked as u32) == ChannelState::FundingSent as u32 {
-                       for &(index_in_block, tx) in txdata.iter() {
-                               let funding_txo = self.get_funding_txo().unwrap();
-                               if tx.txid() == funding_txo.txid {
-                                       let txo_idx = funding_txo.index as usize;
-                                       if txo_idx >= tx.output.len() || tx.output[txo_idx].script_pubkey != self.get_funding_redeemscript().to_v0_p2wsh() ||
-                                                       tx.output[txo_idx].value != self.channel_value_satoshis {
-                                               if self.is_outbound() {
-                                                       // If we generated the funding transaction and it doesn't match what it
-                                                       // should, the client is really broken and we should just panic and
-                                                       // tell them off. That said, because hash collisions happen with high
-                                                       // probability in fuzztarget mode, if we're fuzzing we just close the
-                                                       // channel and move on.
-                                                       #[cfg(not(feature = "fuzztarget"))]
-                                                       panic!("Client called ChannelManager::funding_transaction_generated with bogus transaction!");
-                                               }
-                                               self.channel_state = ChannelState::ShutdownComplete as u32;
-                                               self.update_time_counter += 1;
-                                               return Err(msgs::ErrorMessage {
-                                                       channel_id: self.channel_id(),
-                                                       data: "funding tx had wrong script/value".to_owned()
-                                               });
-                                       } else {
-                                               if self.is_outbound() {
-                                                       for input in tx.input.iter() {
-                                                               if input.witness.is_empty() {
-                                                                       // We generated a malleable funding transaction, implying we've
-                                                                       // just exposed ourselves to funds loss to our counterparty.
-                                                                       #[cfg(not(feature = "fuzztarget"))]
-                                                                       panic!("Client called ChannelManager::funding_transaction_generated with bogus transaction!");
-                                                               }
-                                                       }
-                                               }
-                                               self.funding_tx_confirmations = 1;
-                                               self.short_channel_id = match scid_from_parts(height as u64, index_in_block as u64, txo_idx as u64) {
-                                                       Ok(scid) => Some(scid),
-                                                       Err(_) => panic!("Block was bogus - either height was > 16 million, had > 16 million transactions, or had > 65k outputs"),
-                                               }
-                                       }
-                               }
+               if non_shutdown_state >= ChannelState::ChannelFunded as u32 ||
+                  (non_shutdown_state & ChannelState::OurFundingLocked as u32) == ChannelState::OurFundingLocked as u32 {
+                       let mut funding_tx_confirmations = height as i64 - self.funding_tx_confirmation_height as i64 + 1;
+                       if self.funding_tx_confirmation_height == 0 {
+                               // Note that check_get_funding_locked may reset funding_tx_confirmation_height to
+                               // zero if it has been reorged out, however in either case, our state flags
+                               // indicate we've already sent a funding_locked
+                               funding_tx_confirmations = 0;
                        }
-               }
 
-               self.update_time_counter = cmp::max(self.update_time_counter, header.time);
-               if self.funding_tx_confirmations > 0 {
-                       if self.funding_tx_confirmations == self.minimum_depth as u64 {
-                               let need_commitment_update = if non_shutdown_state == ChannelState::FundingSent as u32 {
-                                       self.channel_state |= ChannelState::OurFundingLocked as u32;
-                                       true
-                               } else if non_shutdown_state == (ChannelState::FundingSent as u32 | ChannelState::TheirFundingLocked as u32) {
-                                       self.channel_state = ChannelState::ChannelFunded as u32 | (self.channel_state & MULTI_STATE_FLAGS);
-                                       self.update_time_counter += 1;
-                                       true
-                               } else if non_shutdown_state == (ChannelState::FundingSent as u32 | ChannelState::OurFundingLocked as u32) {
-                                       // We got a reorg but not enough to trigger a force close, just update
-                                       // funding_tx_confirmed_in and return.
-                                       false
-                               } else if self.channel_state < ChannelState::ChannelFunded as u32 {
-                                       panic!("Started confirming a channel in a state pre-FundingSent?: {}", self.channel_state);
-                               } else {
-                                       // We got a reorg but not enough to trigger a force close, just update
-                                       // funding_tx_confirmed_in and return.
-                                       false
-                               };
-                               self.funding_tx_confirmed_in = Some(header.block_hash());
-
-                               //TODO: Note that this must be a duplicate of the previous commitment point they sent us,
-                               //as otherwise we will have a commitment transaction that they can't revoke (well, kinda,
-                               //they can by sending two revoke_and_acks back-to-back, but not really). This appears to be
-                               //a protocol oversight, but I assume I'm just missing something.
-                               if need_commitment_update {
-                                       if self.channel_state & (ChannelState::MonitorUpdateFailed as u32) == 0 {
-                                               let next_per_commitment_point = self.holder_signer.get_per_commitment_point(self.cur_holder_commitment_transaction_number, &self.secp_ctx);
-                                               return Ok((Some(msgs::FundingLocked {
-                                                       channel_id: self.channel_id,
-                                                       next_per_commitment_point,
-                                               }), timed_out_htlcs));
-                                       } else {
-                                               self.monitor_pending_funding_locked = true;
-                                               return Ok((None, timed_out_htlcs));
-                                       }
-                               }
+                       // If we've sent funding_locked (or have both sent and received funding_locked), and
+                       // the funding transaction's confirmation count has dipped below minimum_depth / 2,
+                       // close the channel and hope we can get the latest state on chain (because presumably
+                       // the funding transaction is at least still in the mempool of most nodes).
+                       if funding_tx_confirmations < self.minimum_depth as i64 / 2 {
+                               return Err(msgs::ErrorMessage {
+                                       channel_id: self.channel_id(),
+                                       data: format!("Funding transaction was un-confirmed. Locked at {} confs, now have {} confs.", self.minimum_depth, funding_tx_confirmations),
+                               });
                        }
                }
-               Ok((None, timed_out_htlcs))
-       }
 
-       /// Called by channelmanager based on chain blocks being disconnected.
-       /// Returns true if we need to close the channel now due to funding transaction
-       /// unconfirmation/reorg.
-       pub fn block_disconnected(&mut self, header: &BlockHeader) -> bool {
-               if self.funding_tx_confirmations > 0 {
-                       self.funding_tx_confirmations -= 1;
-                       if self.funding_tx_confirmations == UNCONF_THRESHOLD as u64 {
-                               return true;
-                       }
-               }
-               if Some(header.block_hash()) == self.funding_tx_confirmed_in {
-                       self.funding_tx_confirmations = self.minimum_depth as u64 - 1;
-               }
-               false
+               Ok((None, timed_out_htlcs))
        }
 
        // Methods to get unprompted messages to send to the remote end (or where we already returned
@@ -4470,8 +4500,8 @@ impl<Signer: Sign> Writeable for Channel<Signer> {
                }
 
                self.funding_tx_confirmed_in.write(writer)?;
+               self.funding_tx_confirmation_height.write(writer)?;
                self.short_channel_id.write(writer)?;
-               self.funding_tx_confirmations.write(writer)?;
 
                self.counterparty_dust_limit_satoshis.write(writer)?;
                self.holder_dust_limit_satoshis.write(writer)?;
@@ -4641,8 +4671,8 @@ impl<'a, Signer: Sign, K: Deref> ReadableArgs<&'a K> for Channel<Signer>
                };
 
                let funding_tx_confirmed_in = Readable::read(reader)?;
+               let funding_tx_confirmation_height = Readable::read(reader)?;
                let short_channel_id = Readable::read(reader)?;
-               let funding_tx_confirmations = Readable::read(reader)?;
 
                let counterparty_dust_limit_satoshis = Readable::read(reader)?;
                let holder_dust_limit_satoshis = Readable::read(reader)?;
@@ -4723,8 +4753,8 @@ impl<'a, Signer: Sign, K: Deref> ReadableArgs<&'a K> for Channel<Signer>
                        last_sent_closing_fee,
 
                        funding_tx_confirmed_in,
+                       funding_tx_confirmation_height,
                        short_channel_id,
-                       funding_tx_confirmations,
 
                        counterparty_dust_limit_satoshis,
                        holder_dust_limit_satoshis,
@@ -4770,14 +4800,14 @@ mod tests {
        use bitcoin::hashes::hex::FromHex;
        use hex;
        use ln::channelmanager::{HTLCSource, PaymentPreimage, PaymentHash};
-       use ln::channel::{Channel,Sign,InboundHTLCOutput,OutboundHTLCOutput,InboundHTLCState,OutboundHTLCState,HTLCOutputInCommitment,HTLCCandidate,HTLCInitiator,TxCreationKeys};
+       use ln::channel::{Channel,InboundHTLCOutput,OutboundHTLCOutput,InboundHTLCState,OutboundHTLCState,HTLCOutputInCommitment,HTLCCandidate,HTLCInitiator,TxCreationKeys};
        use ln::channel::MAX_FUNDING_SATOSHIS;
        use ln::features::InitFeatures;
        use ln::msgs::{ChannelUpdate, DataLossProtect, DecodeError, OptionalField, UnsignedChannelUpdate};
        use ln::chan_utils;
        use ln::chan_utils::{ChannelPublicKeys, HolderCommitmentTransaction, CounterpartyChannelTransactionParameters, HTLC_SUCCESS_TX_WEIGHT, HTLC_TIMEOUT_TX_WEIGHT};
        use chain::chaininterface::{FeeEstimator,ConfirmationTarget};
-       use chain::keysinterface::{InMemorySigner, KeysInterface};
+       use chain::keysinterface::{InMemorySigner, KeysInterface, BaseSign};
        use chain::transaction::OutPoint;
        use util::config::UserConfig;
        use util::enforcing_trait_impls::EnforcingSigner;
index 2f7c17b58d75736dc688dbf8b586a72ea8634623..e87b221025f189ade4bff5ebde5af8908dfa62d9 100644 (file)
@@ -435,6 +435,7 @@ pub struct ChannelManager<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref,
        #[cfg(not(any(test, feature = "_test_utils")))]
        channel_state: Mutex<ChannelHolder<Signer>>,
        our_network_key: SecretKey,
+       our_network_pubkey: PublicKey,
 
        /// Used to track the last value sent in a node_announcement "timestamp" field. We ensure this
        /// value increases strictly since we don't assume access to a time source.
@@ -823,7 +824,6 @@ impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelMana
 
                        latest_block_height: AtomicUsize::new(params.latest_height),
                        last_block_hash: RwLock::new(params.latest_hash),
-                       secp_ctx,
 
                        channel_state: Mutex::new(ChannelHolder{
                                by_id: HashMap::new(),
@@ -833,6 +833,8 @@ impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelMana
                                pending_msg_events: Vec::new(),
                        }),
                        our_network_key: keys_manager.get_node_secret(),
+                       our_network_pubkey: PublicKey::from_secret_key(&secp_ctx, &keys_manager.get_node_secret()),
+                       secp_ctx,
 
                        last_node_announcement_serial: AtomicUsize::new(0),
 
@@ -1004,16 +1006,14 @@ impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelMana
                }
        }
 
-       fn force_close_channel_with_peer(&self, channel_id: &[u8; 32], peer_node_id: Option<&PublicKey>) -> Result<(), APIError> {
+       fn force_close_channel_with_peer(&self, channel_id: &[u8; 32], peer_node_id: Option<&PublicKey>) -> Result<PublicKey, APIError> {
                let mut chan = {
                        let mut channel_state_lock = self.channel_state.lock().unwrap();
                        let channel_state = &mut *channel_state_lock;
                        if let hash_map::Entry::Occupied(chan) = channel_state.by_id.entry(channel_id.clone()) {
                                if let Some(node_id) = peer_node_id {
                                        if chan.get().get_counterparty_node_id() != *node_id {
-                                               // Error or Ok here doesn't matter - the result is only exposed publicly
-                                               // when peer_node_id is None anyway.
-                                               return Ok(());
+                                               return Err(APIError::ChannelUnavailable{err: "No such channel".to_owned()});
                                        }
                                }
                                if let Some(short_id) = chan.get().get_short_channel_id() {
@@ -1033,14 +1033,27 @@ impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelMana
                        });
                }
 
-               Ok(())
+               Ok(chan.get_counterparty_node_id())
        }
 
        /// Force closes a channel, immediately broadcasting the latest local commitment transaction to
        /// the chain and rejecting new HTLCs on the given channel. Fails if channel_id is unknown to the manager.
        pub fn force_close_channel(&self, channel_id: &[u8; 32]) -> Result<(), APIError> {
                let _persistence_guard = PersistenceNotifierGuard::new(&self.total_consistency_lock, &self.persistence_notifier);
-               self.force_close_channel_with_peer(channel_id, None)
+               match self.force_close_channel_with_peer(channel_id, None) {
+                       Ok(counterparty_node_id) => {
+                               self.channel_state.lock().unwrap().pending_msg_events.push(
+                                       events::MessageSendEvent::HandleError {
+                                               node_id: counterparty_node_id,
+                                               action: msgs::ErrorAction::SendErrorMessage {
+                                                       msg: msgs::ErrorMessage { channel_id: *channel_id, data: "Channel force-closed".to_owned() }
+                                               },
+                                       }
+                               );
+                               Ok(())
+                       },
+                       Err(e) => Err(e)
+               }
        }
 
        /// Force close all channels, immediately broadcasting the latest local commitment transaction
@@ -2361,7 +2374,7 @@ impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelMana
 
        /// Gets the node_id held by this ChannelManager
        pub fn get_our_node_id(&self) -> PublicKey {
-               PublicKey::from_secret_key(&self.secp_ctx, &self.our_network_key)
+               self.our_network_pubkey.clone()
        }
 
        /// Restores a single, given channel to normal operation after a
@@ -3233,6 +3246,12 @@ impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelMana
                                                                        msg: update
                                                                });
                                                        }
+                                                       pending_msg_events.push(events::MessageSendEvent::HandleError {
+                                                               node_id: chan.get_counterparty_node_id(),
+                                                               action: msgs::ErrorAction::SendErrorMessage {
+                                                                       msg: msgs::ErrorMessage { channel_id: chan.channel_id(), data: "Channel force-closed".to_owned() }
+                                                               },
+                                                       });
                                                }
                                        },
                                }
@@ -3315,12 +3334,26 @@ where
        L::Target: Logger,
 {
        fn block_connected(&self, block: &Block, height: u32) {
+               assert_eq!(*self.last_block_hash.read().unwrap(), block.header.prev_blockhash,
+                       "Blocks must be connected in chain-order - the connected header must build on the last connected header");
+               assert_eq!(self.latest_block_height.load(Ordering::Acquire) as u64, height as u64 - 1,
+                       "Blocks must be connected in chain-order - the connected block height must be one greater than the previous height");
                let txdata: Vec<_> = block.txdata.iter().enumerate().collect();
-               ChannelManager::block_connected(self, &block.header, &txdata, height);
+               self.transactions_confirmed(&block.header, height, &txdata);
+               self.update_best_block(&block.header, height);
        }
 
-       fn block_disconnected(&self, header: &BlockHeader, _height: u32) {
-               ChannelManager::block_disconnected(self, header);
+       fn block_disconnected(&self, header: &BlockHeader, height: u32) {
+               assert_eq!(*self.last_block_hash.read().unwrap(), header.block_hash(),
+                       "Blocks must be disconnected in chain-order - the disconnected header must be the last connected header");
+
+               let _persistence_guard = PersistenceNotifierGuard::new(&self.total_consistency_lock, &self.persistence_notifier);
+               let new_height = self.latest_block_height.fetch_sub(1, Ordering::AcqRel) as u32 - 1;
+               assert_eq!(new_height, height - 1,
+                       "Blocks must be disconnected in chain-order - the disconnected block must have the correct height");
+               *self.last_block_hash.write().unwrap() = header.prev_blockhash;
+
+               self.do_chain_event(new_height, |channel| channel.update_best_block(new_height, header.time));
        }
 }
 
@@ -3331,22 +3364,11 @@ impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelMana
         F::Target: FeeEstimator,
         L::Target: Logger,
 {
-       /// Updates channel state based on transactions seen in a connected block.
-       pub fn block_connected(&self, header: &BlockHeader, txdata: &TransactionData, height: u32) {
+       fn do_chain_event<FN: Fn(&mut Channel<Signer>) -> Result<(Option<msgs::FundingLocked>, Vec<(HTLCSource, PaymentHash)>), msgs::ErrorMessage>>
+                       (&self, height: u32, f: FN) {
                // Note that we MUST NOT end up calling methods on self.chain_monitor here - we're called
                // during initialization prior to the chain_monitor being fully configured in some cases.
                // See the docs for `ChannelManagerReadArgs` for more.
-               let block_hash = header.block_hash();
-               log_trace!(self.logger, "Block {} at height {} connected", block_hash, height);
-
-               let _persistence_guard = PersistenceNotifierGuard::new(&self.total_consistency_lock, &self.persistence_notifier);
-
-               assert_eq!(*self.last_block_hash.read().unwrap(), header.prev_blockhash,
-                       "Blocks must be connected in chain-order - the connected header must build on the last connected header");
-               assert_eq!(self.latest_block_height.load(Ordering::Acquire) as u64, height as u64 - 1,
-                       "Blocks must be connected in chain-order - the connected header must build on the last connected header");
-               self.latest_block_height.store(height as usize, Ordering::Release);
-               *self.last_block_hash.write().unwrap() = block_hash;
 
                let mut failed_channels = Vec::new();
                let mut timed_out_htlcs = Vec::new();
@@ -3356,7 +3378,7 @@ impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelMana
                        let short_to_id = &mut channel_state.short_to_id;
                        let pending_msg_events = &mut channel_state.pending_msg_events;
                        channel_state.by_id.retain(|_, channel| {
-                               let res = channel.block_connected(header, txdata, height);
+                               let res = f(channel);
                                if let Ok((chan_res, mut timed_out_pending_htlcs)) = res {
                                        for (source, payment_hash) in timed_out_pending_htlcs.drain(..) {
                                                let chan_update = self.get_channel_update(&channel).map(|u| u.encode_with_len()).unwrap(); // Cannot add/recv HTLCs before we have a short_id so unwrap is safe
@@ -3382,32 +3404,23 @@ impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelMana
                                                short_to_id.insert(channel.get_short_channel_id().unwrap(), channel.channel_id());
                                        }
                                } else if let Err(e) = res {
+                                       if let Some(short_id) = channel.get_short_channel_id() {
+                                               short_to_id.remove(&short_id);
+                                       }
+                                       // It looks like our counterparty went on-chain or funding transaction was
+                                       // reorged out of the main chain. Close the channel.
+                                       failed_channels.push(channel.force_shutdown(true));
+                                       if let Ok(update) = self.get_channel_update(&channel) {
+                                               pending_msg_events.push(events::MessageSendEvent::BroadcastChannelUpdate {
+                                                       msg: update
+                                               });
+                                       }
                                        pending_msg_events.push(events::MessageSendEvent::HandleError {
                                                node_id: channel.get_counterparty_node_id(),
                                                action: msgs::ErrorAction::SendErrorMessage { msg: e },
                                        });
                                        return false;
                                }
-                               if let Some(funding_txo) = channel.get_funding_txo() {
-                                       for &(_, tx) in txdata.iter() {
-                                               for inp in tx.input.iter() {
-                                                       if inp.previous_output == funding_txo.into_bitcoin_outpoint() {
-                                                               log_trace!(self.logger, "Detected channel-closing tx {} spending {}:{}, closing channel {}", tx.txid(), inp.previous_output.txid, inp.previous_output.vout, log_bytes!(channel.channel_id()));
-                                                               if let Some(short_id) = channel.get_short_channel_id() {
-                                                                       short_to_id.remove(&short_id);
-                                                               }
-                                                               // It looks like our counterparty went on-chain. Close the channel.
-                                                               failed_channels.push(channel.force_shutdown(true));
-                                                               if let Ok(update) = self.get_channel_update(&channel) {
-                                                                       pending_msg_events.push(events::MessageSendEvent::BroadcastChannelUpdate {
-                                                                               msg: update
-                                                                       });
-                                                               }
-                                                               return false;
-                                                       }
-                                               }
-                                       }
-                               }
                                true
                        });
 
@@ -3436,6 +3449,64 @@ impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelMana
                for (source, payment_hash, reason) in timed_out_htlcs.drain(..) {
                        self.fail_htlc_backwards_internal(self.channel_state.lock().unwrap(), source, &payment_hash, reason);
                }
+       }
+
+       /// Updates channel state to take note of transactions which were confirmed in the given block
+       /// at the given height.
+       ///
+       /// Note that you must still call (or have called) [`update_best_block`] with the block
+       /// information which is included here.
+       ///
+       /// This method may be called before or after [`update_best_block`] for a given block's
+       /// transaction data and may be called multiple times with additional transaction data for a
+       /// given block.
+       ///
+       /// This method may be called for a previous block after an [`update_best_block`] call has
+       /// been made for a later block, however it must *not* be called with transaction data from a
+       /// block which is no longer in the best chain (ie where [`update_best_block`] has already
+       /// been informed about a blockchain reorganization which no longer includes the block which
+       /// corresponds to `header`).
+       ///
+       /// [`update_best_block`]: `Self::update_best_block`
+       pub fn transactions_confirmed(&self, header: &BlockHeader, height: u32, txdata: &TransactionData) {
+               // Note that we MUST NOT end up calling methods on self.chain_monitor here - we're called
+               // during initialization prior to the chain_monitor being fully configured in some cases.
+               // See the docs for `ChannelManagerReadArgs` for more.
+
+               let block_hash = header.block_hash();
+               log_trace!(self.logger, "{} transactions included in block {} at height {} provided", txdata.len(), block_hash, height);
+
+               let _persistence_guard = PersistenceNotifierGuard::new(&self.total_consistency_lock, &self.persistence_notifier);
+               self.do_chain_event(height, |channel| channel.transactions_confirmed(&block_hash, height, txdata, &self.logger).map(|a| (a, Vec::new())));
+       }
+
+       /// Updates channel state with the current best blockchain tip. You should attempt to call this
+       /// quickly after a new block becomes available, however if multiple new blocks become
+       /// available at the same time, only a single `update_best_block()` call needs to be made.
+       ///
+       /// This method should also be called immediately after any block disconnections, once at the
+       /// reorganization fork point, and once with the new chain tip. Calling this method at the
+       /// blockchain reorganization fork point ensures we learn when a funding transaction which was
+       /// previously confirmed is reorganized out of the blockchain, ensuring we do not continue to
+       /// accept payments which cannot be enforced on-chain.
+       ///
+       /// In both the block-connection and block-disconnection case, this method may be called either
+       /// once per block connected or disconnected, or simply at the fork point and new tip(s),
+       /// skipping any intermediary blocks.
+       pub fn update_best_block(&self, header: &BlockHeader, height: u32) {
+               // Note that we MUST NOT end up calling methods on self.chain_monitor here - we're called
+               // during initialization prior to the chain_monitor being fully configured in some cases.
+               // See the docs for `ChannelManagerReadArgs` for more.
+
+               let block_hash = header.block_hash();
+               log_trace!(self.logger, "New best block: {} at height {}", block_hash, height);
+
+               let _persistence_guard = PersistenceNotifierGuard::new(&self.total_consistency_lock, &self.persistence_notifier);
+
+               self.latest_block_height.store(height as usize, Ordering::Release);
+               *self.last_block_hash.write().unwrap() = block_hash;
+
+               self.do_chain_event(height, |channel| channel.update_best_block(height, header.time));
 
                loop {
                        // Update last_node_announcement_serial to be the max of its current value and the
@@ -3451,48 +3522,6 @@ impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelMana
                }
        }
 
-       /// Updates channel state based on a disconnected block.
-       ///
-       /// If necessary, the channel may be force-closed without letting the counterparty participate
-       /// in the shutdown.
-       pub fn block_disconnected(&self, header: &BlockHeader) {
-               // Note that we MUST NOT end up calling methods on self.chain_monitor here - we're called
-               // during initialization prior to the chain_monitor being fully configured in some cases.
-               // See the docs for `ChannelManagerReadArgs` for more.
-               let _persistence_guard = PersistenceNotifierGuard::new(&self.total_consistency_lock, &self.persistence_notifier);
-
-               assert_eq!(*self.last_block_hash.read().unwrap(), header.block_hash(),
-                       "Blocks must be disconnected in chain-order - the disconnected header must be the last connected header");
-               self.latest_block_height.fetch_sub(1, Ordering::AcqRel);
-               *self.last_block_hash.write().unwrap() = header.prev_blockhash;
-
-               let mut failed_channels = Vec::new();
-               {
-                       let mut channel_lock = self.channel_state.lock().unwrap();
-                       let channel_state = &mut *channel_lock;
-                       let short_to_id = &mut channel_state.short_to_id;
-                       let pending_msg_events = &mut channel_state.pending_msg_events;
-                       channel_state.by_id.retain(|_,  v| {
-                               if v.block_disconnected(header) {
-                                       if let Some(short_id) = v.get_short_channel_id() {
-                                               short_to_id.remove(&short_id);
-                                       }
-                                       failed_channels.push(v.force_shutdown(true));
-                                       if let Ok(update) = self.get_channel_update(&v) {
-                                               pending_msg_events.push(events::MessageSendEvent::BroadcastChannelUpdate {
-                                                       msg: update
-                                               });
-                                       }
-                                       false
-                               } else {
-                                       true
-                               }
-                       });
-               }
-
-               self.handle_init_event_channel_failures(failed_channels);
-       }
-
        /// Blocks until ChannelManager needs to be persisted or a timeout is reached. It returns a bool
        /// indicating whether persistence is necessary. Only one listener on
        /// `await_persistable_update` or `await_persistable_update_timeout` is guaranteed to be woken
@@ -4357,7 +4386,6 @@ impl<'a, Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref>
 
                        latest_block_height: AtomicUsize::new(latest_block_height as usize),
                        last_block_hash: RwLock::new(last_block_hash),
-                       secp_ctx,
 
                        channel_state: Mutex::new(ChannelHolder {
                                by_id,
@@ -4367,6 +4395,8 @@ impl<'a, Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref>
                                pending_msg_events: Vec::new(),
                        }),
                        our_network_key: args.keys_manager.get_node_secret(),
+                       our_network_pubkey: PublicKey::from_secret_key(&secp_ctx, &args.keys_manager.get_node_secret()),
+                       secp_ctx,
 
                        last_node_announcement_serial: AtomicUsize::new(last_node_announcement_serial as usize),
 
@@ -4443,3 +4473,154 @@ mod tests {
                }
        }
 }
+
+#[cfg(all(any(test, feature = "_test_utils"), feature = "unstable"))]
+pub mod bench {
+       use chain::Listen;
+       use chain::chainmonitor::ChainMonitor;
+       use chain::channelmonitor::Persist;
+       use chain::keysinterface::{KeysManager, InMemorySigner};
+       use chain::transaction::OutPoint;
+       use ln::channelmanager::{ChainParameters, ChannelManager, PaymentHash, PaymentPreimage};
+       use ln::features::InitFeatures;
+       use ln::functional_test_utils::*;
+       use ln::msgs::ChannelMessageHandler;
+       use routing::network_graph::NetworkGraph;
+       use routing::router::get_route;
+       use util::test_utils;
+       use util::config::UserConfig;
+       use util::events::{Event, EventsProvider, MessageSendEvent, MessageSendEventsProvider};
+
+       use bitcoin::hashes::Hash;
+       use bitcoin::hashes::sha256::Hash as Sha256;
+       use bitcoin::{Block, BlockHeader, Transaction, TxOut};
+
+       use std::sync::Mutex;
+
+       use test::Bencher;
+
+       struct NodeHolder<'a, P: Persist<InMemorySigner>> {
+               node: &'a ChannelManager<InMemorySigner,
+                       &'a ChainMonitor<InMemorySigner, &'a test_utils::TestChainSource,
+                               &'a test_utils::TestBroadcaster, &'a test_utils::TestFeeEstimator,
+                               &'a test_utils::TestLogger, &'a P>,
+                       &'a test_utils::TestBroadcaster, &'a KeysManager,
+                       &'a test_utils::TestFeeEstimator, &'a test_utils::TestLogger>
+       }
+
+       #[cfg(test)]
+       #[bench]
+       fn bench_sends(bench: &mut Bencher) {
+               bench_two_sends(bench, test_utils::TestPersister::new(), test_utils::TestPersister::new());
+       }
+
+       pub fn bench_two_sends<P: Persist<InMemorySigner>>(bench: &mut Bencher, persister_a: P, persister_b: P) {
+               // Do a simple benchmark of sending a payment back and forth between two nodes.
+               // Note that this is unrealistic as each payment send will require at least two fsync
+               // calls per node.
+               let network = bitcoin::Network::Testnet;
+               let genesis_hash = bitcoin::blockdata::constants::genesis_block(network).header.block_hash();
+
+               let tx_broadcaster = test_utils::TestBroadcaster{txn_broadcasted: Mutex::new(Vec::new())};
+               let fee_estimator = test_utils::TestFeeEstimator { sat_per_kw: 253 };
+
+               let mut config: UserConfig = Default::default();
+               config.own_channel_config.minimum_depth = 1;
+
+               let logger_a = test_utils::TestLogger::with_id("node a".to_owned());
+               let chain_monitor_a = ChainMonitor::new(None, &tx_broadcaster, &logger_a, &fee_estimator, &persister_a);
+               let seed_a = [1u8; 32];
+               let keys_manager_a = KeysManager::new(&seed_a, 42, 42);
+               let node_a = ChannelManager::new(&fee_estimator, &chain_monitor_a, &tx_broadcaster, &logger_a, &keys_manager_a, config.clone(), ChainParameters {
+                       network,
+                       latest_hash: genesis_hash,
+                       latest_height: 0,
+               });
+               let node_a_holder = NodeHolder { node: &node_a };
+
+               let logger_b = test_utils::TestLogger::with_id("node a".to_owned());
+               let chain_monitor_b = ChainMonitor::new(None, &tx_broadcaster, &logger_a, &fee_estimator, &persister_b);
+               let seed_b = [2u8; 32];
+               let keys_manager_b = KeysManager::new(&seed_b, 42, 42);
+               let node_b = ChannelManager::new(&fee_estimator, &chain_monitor_b, &tx_broadcaster, &logger_b, &keys_manager_b, config.clone(), ChainParameters {
+                       network,
+                       latest_hash: genesis_hash,
+                       latest_height: 0,
+               });
+               let node_b_holder = NodeHolder { node: &node_b };
+
+               node_a.create_channel(node_b.get_our_node_id(), 8_000_000, 100_000_000, 42, None).unwrap();
+               node_b.handle_open_channel(&node_a.get_our_node_id(), InitFeatures::known(), &get_event_msg!(node_a_holder, MessageSendEvent::SendOpenChannel, node_b.get_our_node_id()));
+               node_a.handle_accept_channel(&node_b.get_our_node_id(), InitFeatures::known(), &get_event_msg!(node_b_holder, MessageSendEvent::SendAcceptChannel, node_a.get_our_node_id()));
+
+               let tx;
+               if let Event::FundingGenerationReady { temporary_channel_id, output_script, .. } = get_event!(node_a_holder, Event::FundingGenerationReady) {
+                       tx = Transaction { version: 2, lock_time: 0, input: Vec::new(), output: vec![TxOut {
+                               value: 8_000_000, script_pubkey: output_script,
+                       }]};
+                       let funding_outpoint = OutPoint { txid: tx.txid(), index: 0 };
+                       node_a.funding_transaction_generated(&temporary_channel_id, funding_outpoint);
+               } else { panic!(); }
+
+               node_b.handle_funding_created(&node_a.get_our_node_id(), &get_event_msg!(node_a_holder, MessageSendEvent::SendFundingCreated, node_b.get_our_node_id()));
+               node_a.handle_funding_signed(&node_b.get_our_node_id(), &get_event_msg!(node_b_holder, MessageSendEvent::SendFundingSigned, node_a.get_our_node_id()));
+
+               get_event!(node_a_holder, Event::FundingBroadcastSafe);
+
+               let block = Block {
+                       header: BlockHeader { version: 0x20000000, prev_blockhash: genesis_hash, merkle_root: Default::default(), time: 42, bits: 42, nonce: 42 },
+                       txdata: vec![tx],
+               };
+               Listen::block_connected(&node_a, &block, 1);
+               Listen::block_connected(&node_b, &block, 1);
+
+               node_a.handle_funding_locked(&node_b.get_our_node_id(), &get_event_msg!(node_b_holder, MessageSendEvent::SendFundingLocked, node_a.get_our_node_id()));
+               node_b.handle_funding_locked(&node_a.get_our_node_id(), &get_event_msg!(node_a_holder, MessageSendEvent::SendFundingLocked, node_b.get_our_node_id()));
+
+               let dummy_graph = NetworkGraph::new(genesis_hash);
+
+               macro_rules! send_payment {
+                       ($node_a: expr, $node_b: expr) => {
+                               let usable_channels = $node_a.list_usable_channels();
+                               let route = get_route(&$node_a.get_our_node_id(), &dummy_graph, &$node_b.get_our_node_id(), None, Some(&usable_channels.iter().map(|r| r).collect::<Vec<_>>()), &[], 10_000, TEST_FINAL_CLTV, &logger_a).unwrap();
+
+                               let payment_preimage = PaymentPreimage([0; 32]);
+                               let payment_hash = PaymentHash(Sha256::hash(&payment_preimage.0[..]).into_inner());
+
+                               $node_a.send_payment(&route, payment_hash, &None).unwrap();
+                               let payment_event = SendEvent::from_event($node_a.get_and_clear_pending_msg_events().pop().unwrap());
+                               $node_b.handle_update_add_htlc(&$node_a.get_our_node_id(), &payment_event.msgs[0]);
+                               $node_b.handle_commitment_signed(&$node_a.get_our_node_id(), &payment_event.commitment_msg);
+                               let (raa, cs) = get_revoke_commit_msgs!(NodeHolder { node: &$node_b }, $node_a.get_our_node_id());
+                               $node_a.handle_revoke_and_ack(&$node_b.get_our_node_id(), &raa);
+                               $node_a.handle_commitment_signed(&$node_b.get_our_node_id(), &cs);
+                               $node_b.handle_revoke_and_ack(&$node_a.get_our_node_id(), &get_event_msg!(NodeHolder { node: &$node_a }, MessageSendEvent::SendRevokeAndACK, $node_b.get_our_node_id()));
+
+                               expect_pending_htlcs_forwardable!(NodeHolder { node: &$node_b });
+                               expect_payment_received!(NodeHolder { node: &$node_b }, payment_hash, 10_000);
+                               assert!($node_b.claim_funds(payment_preimage, &None, 10_000));
+
+                               match $node_b.get_and_clear_pending_msg_events().pop().unwrap() {
+                                       MessageSendEvent::UpdateHTLCs { node_id, updates } => {
+                                               assert_eq!(node_id, $node_a.get_our_node_id());
+                                               $node_a.handle_update_fulfill_htlc(&$node_b.get_our_node_id(), &updates.update_fulfill_htlcs[0]);
+                                               $node_a.handle_commitment_signed(&$node_b.get_our_node_id(), &updates.commitment_signed);
+                                       },
+                                       _ => panic!("Failed to generate claim event"),
+                               }
+
+                               let (raa, cs) = get_revoke_commit_msgs!(NodeHolder { node: &$node_a }, $node_b.get_our_node_id());
+                               $node_b.handle_revoke_and_ack(&$node_a.get_our_node_id(), &raa);
+                               $node_b.handle_commitment_signed(&$node_a.get_our_node_id(), &cs);
+                               $node_a.handle_revoke_and_ack(&$node_b.get_our_node_id(), &get_event_msg!(NodeHolder { node: &$node_b }, MessageSendEvent::SendRevokeAndACK, $node_a.get_our_node_id()));
+
+                               expect_payment_sent!(NodeHolder { node: &$node_a }, payment_preimage);
+                       }
+               }
+
+               bench.iter(|| {
+                       send_payment!(node_a, node_b);
+                       send_payment!(node_b, node_a);
+               });
+       }
+}
index 2ebdac5d031f4a00b0efe2df418744a3b54199b9..da3e19c04e09afdf85b5878e5322ddd84d2aff3f 100644 (file)
@@ -10,7 +10,7 @@
 //! A bunch of useful utilities for building networks of nodes and exchanging messages between
 //! nodes for functional tests.
 
-use chain::Watch;
+use chain::{Listen, Watch};
 use chain::channelmonitor::ChannelMonitor;
 use chain::transaction::OutPoint;
 use ln::channelmanager::{ChainParameters, ChannelManager, ChannelManagerReadArgs, RAACommitmentOrder, PaymentPreimage, PaymentHash, PaymentSecret, PaymentSendFailure};
@@ -60,21 +60,15 @@ pub fn mine_transaction<'a, 'b, 'c, 'd>(node: &'a Node<'b, 'c, 'd>, tx: &Transac
 /// Mine the given transaction at the given height, mining blocks as required to build to that
 /// height
 pub fn confirm_transaction_at<'a, 'b, 'c, 'd>(node: &'a Node<'b, 'c, 'd>, tx: &Transaction, conf_height: u32) {
-       let starting_block = node.best_block_info();
+       let first_connect_height = node.best_block_info().1 + 1;
+       assert!(first_connect_height <= conf_height);
+       if conf_height - first_connect_height >= 1 {
+               connect_blocks(node, conf_height - first_connect_height);
+       }
        let mut block = Block {
-               header: BlockHeader { version: 0x20000000, prev_blockhash: starting_block.0, merkle_root: Default::default(), time: 42, bits: 42, nonce: 42 },
+               header: BlockHeader { version: 0x20000000, prev_blockhash: node.best_block_hash(), merkle_root: Default::default(), time: 42, bits: 42, nonce: 42 },
                txdata: Vec::new(),
        };
-       let height = starting_block.1 + 1;
-       assert!(height <= conf_height);
-       for _ in height..conf_height {
-               connect_block(node, &block);
-               block = Block {
-                       header: BlockHeader { version: 0x20000000, prev_blockhash: block.header.block_hash(), merkle_root: Default::default(), time: 42, bits: 42, nonce: 42 },
-                       txdata: vec![],
-               };
-       }
-
        for _ in 0..*node.network_chan_count.borrow() { // Make sure we don't end up with channels at the same short id by offsetting by chan_count
                block.txdata.push(Transaction { version: 0, lock_time: 0, input: Vec::new(), output: Vec::new() });
        }
@@ -82,37 +76,94 @@ pub fn confirm_transaction_at<'a, 'b, 'c, 'd>(node: &'a Node<'b, 'c, 'd>, tx: &T
        connect_block(node, &block);
 }
 
+/// The possible ways we may notify a ChannelManager of a new block
+pub enum ConnectStyle {
+       /// Calls update_best_block first, detecting transactions in the block only after receiving the
+       /// header and height information.
+       BestBlockFirst,
+       /// The same as BestBlockFirst, however when we have multiple blocks to connect, we only
+       /// make a single update_best_block call.
+       BestBlockFirstSkippingBlocks,
+       /// Calls transactions_confirmed first, detecting transactions in the block before updating the
+       /// header and height information.
+       TransactionsFirst,
+       /// The same as TransactionsFirst, however when we have multiple blocks to connect, we only
+       /// make a single update_best_block call.
+       TransactionsFirstSkippingBlocks,
+       /// Provides the full block via the chain::Listen interface. In the current code this is
+       /// equivalent to TransactionsFirst with some additional assertions.
+       FullBlockViaListen,
+}
+
 pub fn connect_blocks<'a, 'b, 'c, 'd>(node: &'a Node<'b, 'c, 'd>, depth: u32) -> BlockHash {
+       let skip_intermediaries = match *node.connect_style.borrow() {
+               ConnectStyle::BestBlockFirstSkippingBlocks|ConnectStyle::TransactionsFirstSkippingBlocks => true,
+               _ => false,
+       };
+
        let mut block = Block {
                header: BlockHeader { version: 0x2000000, prev_blockhash: node.best_block_hash(), merkle_root: Default::default(), time: 42, bits: 42, nonce: 42 },
                txdata: vec![],
        };
-       connect_block(node, &block);
-       for _ in 2..depth + 1 {
+       assert!(depth >= 1);
+       for _ in 0..depth - 1 {
+               do_connect_block(node, &block, skip_intermediaries);
                block = Block {
                        header: BlockHeader { version: 0x20000000, prev_blockhash: block.header.block_hash(), merkle_root: Default::default(), time: 42, bits: 42, nonce: 42 },
                        txdata: vec![],
                };
-               connect_block(node, &block);
        }
+       connect_block(node, &block);
        block.header.block_hash()
 }
 
 pub fn connect_block<'a, 'b, 'c, 'd>(node: &'a Node<'b, 'c, 'd>, block: &Block) {
+       do_connect_block(node, block, false);
+}
+
+fn do_connect_block<'a, 'b, 'c, 'd>(node: &'a Node<'b, 'c, 'd>, block: &Block, skip_manager: bool) {
        let txdata: Vec<_> = block.txdata.iter().enumerate().collect();
        let height = node.best_block_info().1 + 1;
        node.chain_monitor.chain_monitor.block_connected(&block.header, &txdata, height);
-       node.node.block_connected(&block.header, &txdata, height);
+       if !skip_manager {
+               match *node.connect_style.borrow() {
+                       ConnectStyle::BestBlockFirst|ConnectStyle::BestBlockFirstSkippingBlocks => {
+                               node.node.update_best_block(&block.header, height);
+                               node.node.transactions_confirmed(&block.header, height, &block.txdata.iter().enumerate().collect::<Vec<_>>());
+                       },
+                       ConnectStyle::TransactionsFirst|ConnectStyle::TransactionsFirstSkippingBlocks => {
+                               node.node.transactions_confirmed(&block.header, height, &block.txdata.iter().enumerate().collect::<Vec<_>>());
+                               node.node.update_best_block(&block.header, height);
+                       },
+                       ConnectStyle::FullBlockViaListen => {
+                               Listen::block_connected(node.node, &block, height);
+                       }
+               }
+       }
        node.node.test_process_background_events();
        node.blocks.borrow_mut().push((block.header, height));
 }
 
 pub fn disconnect_blocks<'a, 'b, 'c, 'd>(node: &'a Node<'b, 'c, 'd>, count: u32) {
-       for _ in 0..count {
+       for i in 0..count {
                let orig_header = node.blocks.borrow_mut().pop().unwrap();
                assert!(orig_header.1 > 0); // Cannot disconnect genesis
+               let prev_header = node.blocks.borrow().last().unwrap().clone();
+
                node.chain_monitor.chain_monitor.block_disconnected(&orig_header.0, orig_header.1);
-               node.node.block_disconnected(&orig_header.0);
+               match *node.connect_style.borrow() {
+                       ConnectStyle::FullBlockViaListen => {
+                               Listen::block_disconnected(node.node, &orig_header.0, orig_header.1);
+                       },
+                       ConnectStyle::BestBlockFirstSkippingBlocks|ConnectStyle::TransactionsFirstSkippingBlocks => {
+                               if i == count - 1 {
+                                       node.node.update_best_block(&prev_header.0, prev_header.1);
+                               }
+                       },
+                       _ => {
+                               node.node.update_best_block(&prev_header.0, prev_header.1);
+                       },
+               }
        }
 }
 
@@ -152,6 +203,7 @@ pub struct Node<'a, 'b: 'a, 'c: 'b> {
        pub network_chan_count: Rc<RefCell<u32>>,
        pub logger: &'c test_utils::TestLogger,
        pub blocks: RefCell<Vec<(BlockHeader, u32)>>,
+       pub connect_style: Rc<RefCell<ConnectStyle>>,
 }
 impl<'a, 'b, 'c> Node<'a, 'b, 'c> {
        pub fn best_block_hash(&self) -> BlockHash {
@@ -313,6 +365,24 @@ macro_rules! get_event_msg {
        }
 }
 
+/// Get a specific event from the pending events queue.
+#[macro_export]
+macro_rules! get_event {
+       ($node: expr, $event_type: path) => {
+               {
+                       let mut events = $node.node.get_and_clear_pending_events();
+                       assert_eq!(events.len(), 1);
+                       let ev = events.pop().unwrap();
+                       match ev {
+                               $event_type { .. } => {
+                                       ev
+                               },
+                               _ => panic!("Unexpected event"),
+                       }
+               }
+       }
+}
+
 #[cfg(test)]
 macro_rules! get_htlc_update_msgs {
        ($node: expr, $node_id: expr) => {
@@ -341,7 +411,8 @@ macro_rules! get_feerate {
        }
 }
 
-#[cfg(test)]
+/// Returns any local commitment transactions for the channel.
+#[macro_export]
 macro_rules! get_local_commitment_txn {
        ($node: expr, $channel_id: expr) => {
                {
@@ -844,7 +915,7 @@ macro_rules! expect_pending_htlcs_forwardable {
        }}
 }
 
-#[cfg(test)]
+#[cfg(any(test, feature = "unstable"))]
 macro_rules! expect_payment_received {
        ($node: expr, $expected_payment_hash: expr, $expected_recv_value: expr) => {
                let events = $node.node.get_and_clear_pending_events();
@@ -1221,6 +1292,7 @@ pub fn create_network<'a, 'b: 'a, 'c: 'b>(node_count: usize, cfgs: &'b Vec<NodeC
        let mut nodes = Vec::new();
        let chan_count = Rc::new(RefCell::new(0));
        let payment_count = Rc::new(RefCell::new(0));
+       let connect_style = Rc::new(RefCell::new(ConnectStyle::FullBlockViaListen));
 
        for i in 0..node_count {
                let net_graph_msg_handler = NetGraphMsgHandler::new(cfgs[i].chain_source.genesis_hash, None, cfgs[i].logger);
@@ -1229,7 +1301,8 @@ pub fn create_network<'a, 'b: 'a, 'c: 'b>(node_count: usize, cfgs: &'b Vec<NodeC
                                 keys_manager: &cfgs[i].keys_manager, node: &chan_mgrs[i], net_graph_msg_handler,
                                 node_seed: cfgs[i].node_seed, network_chan_count: chan_count.clone(),
                                 network_payment_count: payment_count.clone(), logger: cfgs[i].logger,
-                                blocks: RefCell::new(vec![(genesis_block(Network::Testnet).header, 0)])
+                                blocks: RefCell::new(vec![(genesis_block(Network::Testnet).header, 0)]),
+                                connect_style: Rc::clone(&connect_style),
                })
        }
 
@@ -1342,22 +1415,36 @@ pub fn check_preimage_claim<'a, 'b, 'c>(node: &Node<'a, 'b, 'c>, prev_txn: &Vec<
 
 pub fn get_announce_close_broadcast_events<'a, 'b, 'c>(nodes: &Vec<Node<'a, 'b, 'c>>, a: usize, b: usize)  {
        let events_1 = nodes[a].node.get_and_clear_pending_msg_events();
-       assert_eq!(events_1.len(), 1);
+       assert_eq!(events_1.len(), 2);
        let as_update = match events_1[0] {
                MessageSendEvent::BroadcastChannelUpdate { ref msg } => {
                        msg.clone()
                },
                _ => panic!("Unexpected event"),
        };
+       match events_1[1] {
+               MessageSendEvent::HandleError { node_id, action: msgs::ErrorAction::SendErrorMessage { ref msg } } => {
+                       assert_eq!(node_id, nodes[b].node.get_our_node_id());
+                       assert_eq!(msg.data, "Commitment or closing transaction was confirmed on chain.");
+               },
+               _ => panic!("Unexpected event"),
+       }
 
        let events_2 = nodes[b].node.get_and_clear_pending_msg_events();
-       assert_eq!(events_2.len(), 1);
+       assert_eq!(events_2.len(), 2);
        let bs_update = match events_2[0] {
                MessageSendEvent::BroadcastChannelUpdate { ref msg } => {
                        msg.clone()
                },
                _ => panic!("Unexpected event"),
        };
+       match events_2[1] {
+               MessageSendEvent::HandleError { node_id, action: msgs::ErrorAction::SendErrorMessage { ref msg } } => {
+                       assert_eq!(node_id, nodes[a].node.get_our_node_id());
+                       assert_eq!(msg.data, "Commitment or closing transaction was confirmed on chain.");
+               },
+               _ => panic!("Unexpected event"),
+       }
 
        for node in nodes {
                node.net_graph_msg_handler.handle_channel_update(&as_update).unwrap();
index 7c198e4803b97c7a5f1b3dbc5dad3705cbeadd97..22c0af4070b1b2fe25444e631347f118914bc46a 100644 (file)
@@ -16,7 +16,7 @@ use chain::Watch;
 use chain::channelmonitor;
 use chain::channelmonitor::{ChannelMonitor, CLTV_CLAIM_BUFFER, LATENCY_GRACE_PERIOD_BLOCKS, ANTI_REORG_DELAY};
 use chain::transaction::OutPoint;
-use chain::keysinterface::{Sign, KeysInterface};
+use chain::keysinterface::{KeysInterface, BaseSign};
 use ln::channel::{COMMITMENT_TX_BASE_WEIGHT, COMMITMENT_TX_WEIGHT_PER_HTLC};
 use ln::channelmanager::{ChannelManager, ChannelManagerReadArgs, RAACommitmentOrder, PaymentPreimage, PaymentHash, PaymentSecret, PaymentSendFailure, BREAKDOWN_TIMEOUT};
 use ln::channel::{Channel, ChannelError};
@@ -394,8 +394,7 @@ fn test_multi_flight_update_fee() {
        check_added_monitors!(nodes[1], 1);
 }
 
-#[test]
-fn test_1_conf_open() {
+fn do_test_1_conf_open(connect_style: ConnectStyle) {
        // Previously, if the minium_depth config was set to 1, we'd never send a funding_locked. This
        // tests that we properly send one in that case.
        let mut alice_config = UserConfig::default();
@@ -409,7 +408,8 @@ fn test_1_conf_open() {
        let chanmon_cfgs = create_chanmon_cfgs(2);
        let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
        let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[Some(alice_config), Some(bob_config)]);
-       let nodes = create_network(2, &node_cfgs, &node_chanmgrs);
+       let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
+       *nodes[0].connect_style.borrow_mut() = connect_style;
 
        let tx = create_chan_between_nodes_with_value_init(&nodes[0], &nodes[1], 100000, 10001, InitFeatures::known(), InitFeatures::known());
        mine_transaction(&nodes[1], &tx);
@@ -425,6 +425,12 @@ fn test_1_conf_open() {
                node.net_graph_msg_handler.handle_channel_update(&bs_update).unwrap();
        }
 }
+#[test]
+fn test_1_conf_open() {
+       do_test_1_conf_open(ConnectStyle::BestBlockFirst);
+       do_test_1_conf_open(ConnectStyle::TransactionsFirst);
+       do_test_1_conf_open(ConnectStyle::FullBlockViaListen);
+}
 
 fn do_test_sanity_on_in_flight_opens(steps: u8) {
        // Previously, we had issues deserializing channels when we hadn't connected the first block
@@ -1501,10 +1507,14 @@ fn test_duplicate_htlc_different_direction_onchain() {
        check_spends!(htlc_pair.1, remote_txn[0]);
 
        let events = nodes[0].node.get_and_clear_pending_msg_events();
-       assert_eq!(events.len(), 2);
+       assert_eq!(events.len(), 3);
        for e in events {
                match e {
                        MessageSendEvent::BroadcastChannelUpdate { .. } => {},
+                       MessageSendEvent::HandleError { node_id, action: msgs::ErrorAction::SendErrorMessage { ref msg } } => {
+                               assert_eq!(node_id, nodes[1].node.get_our_node_id());
+                               assert_eq!(msg.data, "Commitment or closing transaction was confirmed on chain.");
+                       },
                        MessageSendEvent::UpdateHTLCs { ref node_id, updates: msgs::CommitmentUpdate { ref update_add_htlcs, ref update_fulfill_htlcs, ref update_fail_htlcs, ref update_fail_malformed_htlcs, .. } } => {
                                assert!(update_add_htlcs.is_empty());
                                assert!(update_fail_htlcs.is_empty());
@@ -2327,6 +2337,7 @@ fn channel_monitor_network_test() {
        // Simple case with no pending HTLCs:
        nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), true);
        check_added_monitors!(nodes[1], 1);
+       check_closed_broadcast!(nodes[1], false);
        {
                let mut node_txn = test_txn_broadcast(&nodes[1], &chan_1, None, HTLCType::NONE);
                assert_eq!(node_txn.len(), 1);
@@ -2334,7 +2345,7 @@ fn channel_monitor_network_test() {
                check_added_monitors!(nodes[0], 1);
                test_txn_broadcast(&nodes[0], &chan_1, None, HTLCType::NONE);
        }
-       get_announce_close_broadcast_events(&nodes, 0, 1);
+       check_closed_broadcast!(nodes[0], true);
        assert_eq!(nodes[0].node.list_channels().len(), 0);
        assert_eq!(nodes[1].node.list_channels().len(), 1);
 
@@ -2343,6 +2354,7 @@ fn channel_monitor_network_test() {
 
        // Simple case of one pending HTLC to HTLC-Timeout
        nodes[1].node.peer_disconnected(&nodes[2].node.get_our_node_id(), true);
+       check_closed_broadcast!(nodes[1], false);
        check_added_monitors!(nodes[1], 1);
        {
                let mut node_txn = test_txn_broadcast(&nodes[1], &chan_2, None, HTLCType::TIMEOUT);
@@ -2350,7 +2362,7 @@ fn channel_monitor_network_test() {
                check_added_monitors!(nodes[2], 1);
                test_txn_broadcast(&nodes[2], &chan_2, None, HTLCType::NONE);
        }
-       get_announce_close_broadcast_events(&nodes, 1, 2);
+       check_closed_broadcast!(nodes[2], true);
        assert_eq!(nodes[1].node.list_channels().len(), 0);
        assert_eq!(nodes[2].node.list_channels().len(), 1);
 
@@ -2378,6 +2390,7 @@ fn channel_monitor_network_test() {
        // HTLC-Timeout and a nodes[3] claim against it (+ its own announces)
        nodes[2].node.peer_disconnected(&nodes[3].node.get_our_node_id(), true);
        check_added_monitors!(nodes[2], 1);
+       check_closed_broadcast!(nodes[2], false);
        let node2_commitment_txid;
        {
                let node_txn = test_txn_broadcast(&nodes[2], &chan_3, None, HTLCType::TIMEOUT);
@@ -2389,7 +2402,7 @@ fn channel_monitor_network_test() {
                check_added_monitors!(nodes[3], 1);
                check_preimage_claim(&nodes[3], &node_txn);
        }
-       get_announce_close_broadcast_events(&nodes, 2, 3);
+       check_closed_broadcast!(nodes[3], true);
        assert_eq!(nodes[2].node.list_channels().len(), 0);
        assert_eq!(nodes[3].node.list_channels().len(), 1);
 
@@ -2405,13 +2418,19 @@ fn channel_monitor_network_test() {
        let (close_chan_update_1, close_chan_update_2) = {
                connect_blocks(&nodes[3], TEST_FINAL_CLTV + LATENCY_GRACE_PERIOD_BLOCKS + 1);
                let events = nodes[3].node.get_and_clear_pending_msg_events();
-               assert_eq!(events.len(), 1);
+               assert_eq!(events.len(), 2);
                let close_chan_update_1 = match events[0] {
                        MessageSendEvent::BroadcastChannelUpdate { ref msg } => {
                                msg.clone()
                        },
                        _ => panic!("Unexpected event"),
                };
+               match events[1] {
+                       MessageSendEvent::HandleError { action: ErrorAction::SendErrorMessage { .. }, node_id } => {
+                               assert_eq!(node_id, nodes[4].node.get_our_node_id());
+                       },
+                       _ => panic!("Unexpected event"),
+               }
                check_added_monitors!(nodes[3], 1);
 
                // Clear bumped claiming txn spending node 2 commitment tx. Bumped txn are generated after reaching some height timer.
@@ -2431,13 +2450,19 @@ fn channel_monitor_network_test() {
 
                connect_blocks(&nodes[4], TEST_FINAL_CLTV - CLTV_CLAIM_BUFFER + 2);
                let events = nodes[4].node.get_and_clear_pending_msg_events();
-               assert_eq!(events.len(), 1);
+               assert_eq!(events.len(), 2);
                let close_chan_update_2 = match events[0] {
                        MessageSendEvent::BroadcastChannelUpdate { ref msg } => {
                                msg.clone()
                        },
                        _ => panic!("Unexpected event"),
                };
+               match events[1] {
+                       MessageSendEvent::HandleError { action: ErrorAction::SendErrorMessage { .. }, node_id } => {
+                               assert_eq!(node_id, nodes[3].node.get_our_node_id());
+                       },
+                       _ => panic!("Unexpected event"),
+               }
                check_added_monitors!(nodes[4], 1);
                test_txn_broadcast(&nodes[4], &chan_4, None, HTLCType::SUCCESS);
 
@@ -2778,7 +2803,7 @@ fn test_htlc_on_chain_success() {
        assert_eq!(updates.update_fulfill_htlcs.len(), 1);
 
        mine_transaction(&nodes[2], &commitment_tx[0]);
-       check_closed_broadcast!(nodes[2], false);
+       check_closed_broadcast!(nodes[2], true);
        check_added_monitors!(nodes[2], 1);
        let node_txn = nodes[2].tx_broadcaster.txn_broadcasted.lock().unwrap().clone(); // ChannelManager : 3 (commitment tx, 2*htlc-success tx), ChannelMonitor : 2 (2 * HTLC-Success tx)
        assert_eq!(node_txn.len(), 5);
@@ -2811,12 +2836,17 @@ fn test_htlc_on_chain_success() {
                assert_eq!(added_monitors[1].0.txid, chan_1.3.txid());
                added_monitors.clear();
        }
-       assert_eq!(events.len(), 2);
+       assert_eq!(events.len(), 3);
        match events[0] {
                MessageSendEvent::BroadcastChannelUpdate { .. } => {},
                _ => panic!("Unexpected event"),
        }
        match events[1] {
+               MessageSendEvent::HandleError { action: ErrorAction::SendErrorMessage { .. }, node_id: _ } => {},
+               _ => panic!("Unexpected event"),
+       }
+
+       match events[2] {
                MessageSendEvent::UpdateHTLCs { ref node_id, updates: msgs::CommitmentUpdate { ref update_add_htlcs, ref update_fail_htlcs, ref update_fulfill_htlcs, ref update_fail_malformed_htlcs, .. } } => {
                        assert!(update_add_htlcs.is_empty());
                        assert!(update_fail_htlcs.is_empty());
@@ -2870,7 +2900,7 @@ fn test_htlc_on_chain_success() {
        let commitment_tx = get_local_commitment_txn!(nodes[0], chan_1.2);
        check_spends!(commitment_tx[0], chan_1.3);
        mine_transaction(&nodes[1], &commitment_tx[0]);
-       check_closed_broadcast!(nodes[1], false);
+       check_closed_broadcast!(nodes[1], true);
        check_added_monitors!(nodes[1], 1);
        let node_txn = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap().clone(); // ChannelManager : 3 (commitment tx + HTLC-Sucess * 2), ChannelMonitor : 1 (HTLC-Success)
        assert_eq!(node_txn.len(), 4);
@@ -2890,7 +2920,7 @@ fn test_htlc_on_chain_success() {
        // Verify that A's ChannelManager is able to extract preimage from preimage tx and generate PaymentSent
        let mut header = BlockHeader { version: 0x20000000, prev_blockhash: nodes[0].best_block_hash(), merkle_root: Default::default(), time: 42, bits: 42, nonce: 42};
        connect_block(&nodes[0], &Block { header, txdata: vec![commitment_tx[0].clone(), node_txn[0].clone()] });
-       check_closed_broadcast!(nodes[0], false);
+       check_closed_broadcast!(nodes[0], true);
        check_added_monitors!(nodes[0], 1);
        let events = nodes[0].node.get_and_clear_pending_events();
        assert_eq!(events.len(), 2);
@@ -2957,7 +2987,7 @@ fn test_htlc_on_chain_timeout() {
                _ => panic!("Unexpected event"),
        };
        mine_transaction(&nodes[2], &commitment_tx[0]);
-       check_closed_broadcast!(nodes[2], false);
+       check_closed_broadcast!(nodes[2], true);
        check_added_monitors!(nodes[2], 1);
        let node_txn = nodes[2].tx_broadcaster.txn_broadcasted.lock().unwrap().clone(); // ChannelManager : 1 (commitment tx)
        assert_eq!(node_txn.len(), 1);
@@ -2989,7 +3019,7 @@ fn test_htlc_on_chain_timeout() {
 
        mine_transaction(&nodes[1], &timeout_tx);
        check_added_monitors!(nodes[1], 1);
-       check_closed_broadcast!(nodes[1], false);
+       check_closed_broadcast!(nodes[1], true);
        {
                // B will rebroadcast a fee-bumped timeout transaction here.
                let node_txn = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
@@ -3026,7 +3056,7 @@ fn test_htlc_on_chain_timeout() {
 
        mine_transaction(&nodes[0], &commitment_tx[0]);
 
-       check_closed_broadcast!(nodes[0], false);
+       check_closed_broadcast!(nodes[0], true);
        check_added_monitors!(nodes[0], 1);
        let node_txn = nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().clone(); // ChannelManager : 2 (commitment tx, HTLC-Timeout tx), ChannelMonitor : 1 timeout tx
        assert_eq!(node_txn.len(), 3);
@@ -3063,7 +3093,7 @@ fn test_simple_commitment_revoked_fail_backward() {
        mine_transaction(&nodes[1], &revoked_local_txn[0]);
        connect_blocks(&nodes[1], ANTI_REORG_DELAY - 1);
        check_added_monitors!(nodes[1], 1);
-       check_closed_broadcast!(nodes[1], false);
+       check_closed_broadcast!(nodes[1], true);
 
        expect_pending_htlcs_forwardable!(nodes[1]);
        check_added_monitors!(nodes[1], 1);
@@ -3234,11 +3264,18 @@ fn do_test_commitment_revoked_fail_backward_exhaustive(deliver_bs_raa: bool, use
        check_added_monitors!(nodes[1], 1);
 
        let events = nodes[1].node.get_and_clear_pending_msg_events();
-       assert_eq!(events.len(), if deliver_bs_raa { 3 } else { 2 });
+       assert_eq!(events.len(), if deliver_bs_raa { 4 } else { 3 });
        match events[if deliver_bs_raa { 1 } else { 0 }] {
                MessageSendEvent::BroadcastChannelUpdate { msg: msgs::ChannelUpdate { .. } } => {},
                _ => panic!("Unexpected event"),
        }
+       match events[if deliver_bs_raa { 2 } else { 1 }] {
+               MessageSendEvent::HandleError { action: ErrorAction::SendErrorMessage { msg: msgs::ErrorMessage { channel_id, ref data } }, node_id: _ } => {
+                       assert_eq!(channel_id, chan_2.2);
+                       assert_eq!(data.as_str(), "Commitment or closing transaction was confirmed on chain.");
+               },
+               _ => panic!("Unexpected event"),
+       }
        if deliver_bs_raa {
                match events[0] {
                        MessageSendEvent::UpdateHTLCs { ref node_id, updates: msgs::CommitmentUpdate { ref update_add_htlcs, ref update_fail_htlcs, ref update_fulfill_htlcs, ref update_fail_malformed_htlcs, .. } } => {
@@ -3251,7 +3288,7 @@ fn do_test_commitment_revoked_fail_backward_exhaustive(deliver_bs_raa: bool, use
                        _ => panic!("Unexpected event"),
                }
        }
-       match events[if deliver_bs_raa { 2 } else { 1 }] {
+       match events[if deliver_bs_raa { 3 } else { 2 }] {
                MessageSendEvent::UpdateHTLCs { ref node_id, updates: msgs::CommitmentUpdate { ref update_add_htlcs, ref update_fail_htlcs, ref update_fulfill_htlcs, ref update_fail_malformed_htlcs, ref commitment_signed, .. } } => {
                        assert!(update_add_htlcs.is_empty());
                        assert_eq!(update_fail_htlcs.len(), 3);
@@ -3400,7 +3437,7 @@ fn test_htlc_ignore_latest_remote_commitment() {
 
        route_payment(&nodes[0], &[&nodes[1]], 10000000);
        nodes[0].node.force_close_channel(&nodes[0].node.list_channels()[0].channel_id).unwrap();
-       check_closed_broadcast!(nodes[0], false);
+       check_closed_broadcast!(nodes[0], true);
        check_added_monitors!(nodes[0], 1);
 
        let node_txn = nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap();
@@ -3408,7 +3445,7 @@ fn test_htlc_ignore_latest_remote_commitment() {
 
        let mut header = BlockHeader { version: 0x20000000, prev_blockhash: nodes[1].best_block_hash(), merkle_root: Default::default(), time: 42, bits: 42, nonce: 42 };
        connect_block(&nodes[1], &Block { header, txdata: vec![node_txn[0].clone(), node_txn[1].clone()]});
-       check_closed_broadcast!(nodes[1], false);
+       check_closed_broadcast!(nodes[1], true);
        check_added_monitors!(nodes[1], 1);
 
        // Duplicate the connect_block call since this may happen due to other listeners
@@ -3462,7 +3499,7 @@ fn test_force_close_fail_back() {
        // transaction and ensure nodes[1] doesn't fail-backwards (this was originally a bug!).
 
        nodes[2].node.force_close_channel(&payment_event.commitment_msg.channel_id).unwrap();
-       check_closed_broadcast!(nodes[2], false);
+       check_closed_broadcast!(nodes[2], true);
        check_added_monitors!(nodes[2], 1);
        let tx = {
                let mut node_txn = nodes[2].tx_broadcaster.txn_broadcasted.lock().unwrap();
@@ -3476,7 +3513,7 @@ fn test_force_close_fail_back() {
        mine_transaction(&nodes[1], &tx);
 
        // Note no UpdateHTLCs event here from nodes[1] to nodes[0]!
-       check_closed_broadcast!(nodes[1], false);
+       check_closed_broadcast!(nodes[1], true);
        check_added_monitors!(nodes[1], 1);
 
        // Now check that if we add the preimage to ChannelMonitor it broadcasts our HTLC-Success..
@@ -4646,7 +4683,7 @@ fn test_claim_sizeable_push_msat() {
 
        let chan = create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 100000, 99000000, InitFeatures::known(), InitFeatures::known());
        nodes[1].node.force_close_channel(&chan.2).unwrap();
-       check_closed_broadcast!(nodes[1], false);
+       check_closed_broadcast!(nodes[1], true);
        check_added_monitors!(nodes[1], 1);
        let node_txn = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap();
        assert_eq!(node_txn.len(), 1);
@@ -4672,7 +4709,7 @@ fn test_claim_on_remote_sizeable_push_msat() {
 
        let chan = create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 100000, 99000000, InitFeatures::known(), InitFeatures::known());
        nodes[0].node.force_close_channel(&chan.2).unwrap();
-       check_closed_broadcast!(nodes[0], false);
+       check_closed_broadcast!(nodes[0], true);
        check_added_monitors!(nodes[0], 1);
 
        let node_txn = nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap();
@@ -4681,7 +4718,7 @@ fn test_claim_on_remote_sizeable_push_msat() {
        assert_eq!(node_txn[0].output.len(), 2); // We can't force trimming of to_remote output as channel_reserve_satoshis block us to do so at channel opening
 
        mine_transaction(&nodes[1], &node_txn[0]);
-       check_closed_broadcast!(nodes[1], false);
+       check_closed_broadcast!(nodes[1], true);
        check_added_monitors!(nodes[1], 1);
        connect_blocks(&nodes[1], ANTI_REORG_DELAY - 1);
 
@@ -4708,7 +4745,7 @@ fn test_claim_on_remote_revoked_sizeable_push_msat() {
 
        claim_payment(&nodes[0], &vec!(&nodes[1])[..], payment_preimage, 3_000_000);
        mine_transaction(&nodes[1], &revoked_local_txn[0]);
-       check_closed_broadcast!(nodes[1], false);
+       check_closed_broadcast!(nodes[1], true);
        check_added_monitors!(nodes[1], 1);
 
        let node_txn = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap();
@@ -4834,7 +4871,7 @@ fn test_static_spendable_outputs_justice_tx_revoked_commitment_tx() {
        claim_payment(&nodes[0], &vec!(&nodes[1])[..], payment_preimage, 3_000_000);
 
        mine_transaction(&nodes[1], &revoked_local_txn[0]);
-       check_closed_broadcast!(nodes[1], false);
+       check_closed_broadcast!(nodes[1], true);
        check_added_monitors!(nodes[1], 1);
 
        let node_txn = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap();
@@ -4870,7 +4907,7 @@ fn test_static_spendable_outputs_justice_tx_revoked_htlc_timeout_tx() {
 
        // A will generate HTLC-Timeout from revoked commitment tx
        mine_transaction(&nodes[0], &revoked_local_txn[0]);
-       check_closed_broadcast!(nodes[0], false);
+       check_closed_broadcast!(nodes[0], true);
        check_added_monitors!(nodes[0], 1);
 
        let revoked_htlc_txn = nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap();
@@ -4883,7 +4920,7 @@ fn test_static_spendable_outputs_justice_tx_revoked_htlc_timeout_tx() {
        // B will generate justice tx from A's revoked commitment/HTLC tx
        let header = BlockHeader { version: 0x20000000, prev_blockhash: nodes[1].best_block_hash(), merkle_root: Default::default(), time: 42, bits: 42, nonce: 42 };
        connect_block(&nodes[1], &Block { header, txdata: vec![revoked_local_txn[0].clone(), revoked_htlc_txn[0].clone()] });
-       check_closed_broadcast!(nodes[1], false);
+       check_closed_broadcast!(nodes[1], true);
        check_added_monitors!(nodes[1], 1);
 
        let node_txn = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap();
@@ -4939,7 +4976,7 @@ fn test_static_spendable_outputs_justice_tx_revoked_htlc_success_tx() {
 
        // B will generate HTLC-Success from revoked commitment tx
        mine_transaction(&nodes[1], &revoked_local_txn[0]);
-       check_closed_broadcast!(nodes[1], false);
+       check_closed_broadcast!(nodes[1], true);
        check_added_monitors!(nodes[1], 1);
        let revoked_htlc_txn = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap();
 
@@ -4955,7 +4992,7 @@ fn test_static_spendable_outputs_justice_tx_revoked_htlc_success_tx() {
        // A will generate justice tx from B's revoked commitment/HTLC tx
        let header = BlockHeader { version: 0x20000000, prev_blockhash: nodes[0].best_block_hash(), merkle_root: Default::default(), time: 42, bits: 42, nonce: 42 };
        connect_block(&nodes[0], &Block { header, txdata: vec![revoked_local_txn[0].clone(), revoked_htlc_txn[0].clone()] });
-       check_closed_broadcast!(nodes[0], false);
+       check_closed_broadcast!(nodes[0], true);
        check_added_monitors!(nodes[0], 1);
 
        let node_txn = nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap();
@@ -5029,7 +5066,7 @@ fn test_onchain_to_onchain_claim() {
        assert!(updates.update_fail_malformed_htlcs.is_empty());
 
        mine_transaction(&nodes[2], &commitment_tx[0]);
-       check_closed_broadcast!(nodes[2], false);
+       check_closed_broadcast!(nodes[2], true);
        check_added_monitors!(nodes[2], 1);
 
        let c_txn = nodes[2].tx_broadcaster.txn_broadcasted.lock().unwrap().clone(); // ChannelManager : 2 (commitment tx, HTLC-Success tx), ChannelMonitor : 1 (HTLC-Success tx)
@@ -5063,12 +5100,17 @@ fn test_onchain_to_onchain_claim() {
        }
        check_added_monitors!(nodes[1], 1);
        let msg_events = nodes[1].node.get_and_clear_pending_msg_events();
+       assert_eq!(msg_events.len(), 3);
        check_added_monitors!(nodes[1], 1);
        match msg_events[0] {
-               MessageSendEvent::BroadcastChannelUpdate {  .. } => {},
+               MessageSendEvent::BroadcastChannelUpdate { .. } => {},
                _ => panic!("Unexpected event"),
        }
        match msg_events[1] {
+               MessageSendEvent::HandleError { action: ErrorAction::SendErrorMessage { .. }, node_id: _ } => {},
+               _ => panic!("Unexpected event"),
+       }
+       match msg_events[2] {
                MessageSendEvent::UpdateHTLCs { ref node_id, updates: msgs::CommitmentUpdate { ref update_add_htlcs, ref update_fulfill_htlcs, ref update_fail_htlcs, ref update_fail_malformed_htlcs, .. } } => {
                        assert!(update_add_htlcs.is_empty());
                        assert!(update_fail_htlcs.is_empty());
@@ -5091,7 +5133,7 @@ fn test_onchain_to_onchain_claim() {
        assert!(b_txn[0].output[0].script_pubkey.is_v0_p2wpkh()); // direct payment
        assert_eq!(b_txn[0].lock_time, 0); // Success tx
 
-       check_closed_broadcast!(nodes[1], false);
+       check_closed_broadcast!(nodes[1], true);
        check_added_monitors!(nodes[1], 1);
 }
 
@@ -5116,7 +5158,7 @@ fn test_duplicate_payment_hash_one_failure_one_success() {
        check_spends!(commitment_txn[0], chan_2.3);
 
        mine_transaction(&nodes[1], &commitment_txn[0]);
-       check_closed_broadcast!(nodes[1], false);
+       check_closed_broadcast!(nodes[1], true);
        check_added_monitors!(nodes[1], 1);
 
        let htlc_timeout_tx;
@@ -5396,7 +5438,7 @@ fn do_test_fail_backwards_unrevoked_remote_announce(deliver_last_raa: bool, anno
                mine_transaction(&nodes[2], &ds_prev_commitment_tx[0]);
        }
        connect_blocks(&nodes[2], ANTI_REORG_DELAY - 1);
-       check_closed_broadcast!(nodes[2], false);
+       check_closed_broadcast!(nodes[2], true);
        expect_pending_htlcs_forwardable!(nodes[2]);
        check_added_monitors!(nodes[2], 3);
 
@@ -5533,7 +5575,7 @@ fn test_dynamic_spendable_outputs_local_htlc_timeout_tx() {
 
        // Timeout HTLC on A's chain and so it can generate a HTLC-Timeout tx
        mine_transaction(&nodes[0], &local_txn[0]);
-       check_closed_broadcast!(nodes[0], false);
+       check_closed_broadcast!(nodes[0], true);
        check_added_monitors!(nodes[0], 1);
 
        let htlc_timeout = {
@@ -5601,7 +5643,7 @@ fn test_key_derivation_params() {
 
        // Timeout HTLC on A's chain and so it can generate a HTLC-Timeout tx
        mine_transaction(&nodes[0], &local_txn_1[0]);
-       check_closed_broadcast!(nodes[0], false);
+       check_closed_broadcast!(nodes[0], true);
        check_added_monitors!(nodes[0], 1);
 
        let htlc_timeout = {
@@ -5693,7 +5735,7 @@ fn do_htlc_claim_local_commitment_only(use_dust: bool) {
                block.header.prev_blockhash = block.block_hash();
        }
        test_txn_broadcast(&nodes[1], &chan, None, if use_dust { HTLCType::NONE } else { HTLCType::SUCCESS });
-       check_closed_broadcast!(nodes[1], false);
+       check_closed_broadcast!(nodes[1], true);
        check_added_monitors!(nodes[1], 1);
 }
 
@@ -5725,7 +5767,7 @@ fn do_htlc_claim_current_remote_commitment_only(use_dust: bool) {
                header.prev_blockhash = header.block_hash();
        }
        test_txn_broadcast(&nodes[0], &chan, None, HTLCType::NONE);
-       check_closed_broadcast!(nodes[0], false);
+       check_closed_broadcast!(nodes[0], true);
        check_added_monitors!(nodes[0], 1);
 }
 
@@ -5773,7 +5815,7 @@ fn do_htlc_claim_previous_remote_commitment_only(use_dust: bool, check_revoke_no
        }
        if !check_revoke_no_close {
                test_txn_broadcast(&nodes[0], &chan, None, HTLCType::NONE);
-               check_closed_broadcast!(nodes[0], false);
+               check_closed_broadcast!(nodes[0], true);
                check_added_monitors!(nodes[0], 1);
        } else {
                expect_payment_failed!(nodes[0], our_payment_hash, true);
@@ -6955,7 +6997,7 @@ fn do_test_failure_delay_dust_htlc_local_commitment(announce_latest: bool) {
                mine_transaction(&nodes[0], &as_prev_commitment_tx[0]);
        }
 
-       check_closed_broadcast!(nodes[0], false);
+       check_closed_broadcast!(nodes[0], true);
        check_added_monitors!(nodes[0], 1);
 
        assert_eq!(nodes[0].node.get_and_clear_pending_events().len(), 0);
@@ -7017,7 +7059,7 @@ fn do_test_sweep_outbound_htlc_failure_update(revoked: bool, local: bool) {
        if local {
                // We fail dust-HTLC 1 by broadcast of local commitment tx
                mine_transaction(&nodes[0], &as_commitment_tx[0]);
-               check_closed_broadcast!(nodes[0], false);
+               check_closed_broadcast!(nodes[0], true);
                check_added_monitors!(nodes[0], 1);
                assert_eq!(nodes[0].node.get_and_clear_pending_events().len(), 0);
                timeout_tx.push(nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap()[0].clone());
@@ -7032,7 +7074,7 @@ fn do_test_sweep_outbound_htlc_failure_update(revoked: bool, local: bool) {
        } else {
                // We fail dust-HTLC 1 by broadcast of remote commitment tx. If revoked, fail also non-dust HTLC
                mine_transaction(&nodes[0], &bs_commitment_tx[0]);
-               check_closed_broadcast!(nodes[0], false);
+               check_closed_broadcast!(nodes[0], true);
                check_added_monitors!(nodes[0], 1);
                assert_eq!(nodes[0].node.get_and_clear_pending_events().len(), 0);
                timeout_tx.push(nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap()[0].clone());
@@ -7707,7 +7749,7 @@ fn test_bump_penalty_txn_on_revoked_htlcs() {
        let header = BlockHeader { version: 0x20000000, prev_blockhash: nodes[1].best_block_hash(), merkle_root: Default::default(), time: 42, bits: 42, nonce: 42 };
        // B will generate both revoked HTLC-timeout/HTLC-preimage txn from revoked commitment tx
        connect_block(&nodes[1], &Block { header, txdata: vec![revoked_local_txn[0].clone()] });
-       check_closed_broadcast!(nodes[1], false);
+       check_closed_broadcast!(nodes[1], true);
        check_added_monitors!(nodes[1], 1);
 
        let revoked_htlc_txn = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap();
@@ -7839,7 +7881,7 @@ fn test_bump_penalty_txn_on_revoked_htlcs() {
                assert_eq!(node_txn.len(), 0);
                node_txn.clear();
        }
-       check_closed_broadcast!(nodes[0], false);
+       check_closed_broadcast!(nodes[0], true);
        check_added_monitors!(nodes[0], 1);
 }
 
@@ -8013,7 +8055,7 @@ fn test_bump_txn_sanitize_tracking_maps() {
        assert_eq!(nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().len(), 0);
 
        mine_transaction(&nodes[0], &revoked_local_txn[0]);
-       check_closed_broadcast!(nodes[0], false);
+       check_closed_broadcast!(nodes[0], true);
        check_added_monitors!(nodes[0], 1);
        let penalty_txn = {
                let mut node_txn = nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap();
@@ -8376,7 +8418,7 @@ fn test_htlc_no_detection() {
        // We deliberately connect the local tx twice as this should provoke a failure calling
        // this test before #653 fix.
        chain::Listen::block_connected(&nodes[0].chain_monitor.chain_monitor, &Block { header, txdata: vec![local_txn[0].clone()] }, nodes[0].best_block_info().1 + 1);
-       check_closed_broadcast!(nodes[0], false);
+       check_closed_broadcast!(nodes[0], true);
        check_added_monitors!(nodes[0], 1);
 
        let htlc_timeout = {
@@ -8434,7 +8476,7 @@ fn do_test_onchain_htlc_settlement_after_close(broadcast_alice: bool, go_onchain
        let mut force_closing_node = 0; // Alice force-closes
        if !broadcast_alice { force_closing_node = 1; } // Bob force-closes
        nodes[force_closing_node].node.force_close_channel(&chan_ab.2).unwrap();
-       check_closed_broadcast!(nodes[force_closing_node], false);
+       check_closed_broadcast!(nodes[force_closing_node], true);
        check_added_monitors!(nodes[force_closing_node], 1);
        if go_onchain_before_fulfill {
                let txn_to_broadcast = match broadcast_alice {
@@ -8445,7 +8487,7 @@ fn do_test_onchain_htlc_settlement_after_close(broadcast_alice: bool, go_onchain
                connect_block(&nodes[1], &Block { header, txdata: vec![txn_to_broadcast[0].clone()]});
                let mut bob_txn = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap();
                if broadcast_alice {
-                       check_closed_broadcast!(nodes[1], false);
+                       check_closed_broadcast!(nodes[1], true);
                        check_added_monitors!(nodes[1], 1);
                }
                assert_eq!(bob_txn.len(), 1);
@@ -8524,7 +8566,7 @@ fn do_test_onchain_htlc_settlement_after_close(broadcast_alice: bool, go_onchain
                connect_block(&nodes[1], &Block { header, txdata: vec![txn_to_broadcast[0].clone()]});
                // If Bob was the one to force-close, he will have already passed these checks earlier.
                if broadcast_alice {
-                       check_closed_broadcast!(nodes[1], false);
+                       check_closed_broadcast!(nodes[1], true);
                        check_added_monitors!(nodes[1], 1);
                }
                let mut bob_txn = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap();
index 1afcb3530fe16b995a4671c563da98d8e8e8d115..3827cea84e0a3d983c8946bf9703fa5f9ff6826c 100644 (file)
 //! you want to learn things about the network topology (eg get a route for sending a payment),
 //! call into your NetGraphMsgHandler.
 
+#[cfg(any(test, feature = "_test_utils"))]
+#[macro_use]
+pub mod functional_test_utils;
+
 pub mod channelmanager;
 pub mod msgs;
 pub mod peer_handler;
@@ -38,9 +42,6 @@ mod wire;
 // without the node parameter being mut. This is incorrect, and thus newer rustcs will complain
 // about an unnecessary mut. Thus, we silence the unused_mut warning in two test modules below.
 
-#[cfg(any(test, feature = "_test_utils"))]
-#[macro_use]
-pub mod functional_test_utils;
 #[cfg(test)]
 #[allow(unused_mut)]
 mod functional_tests;
index 1c45da14a382c78aef88acda8a1a3d56ea0c1dca..8feef1697af40ad33032afe23b7bf5f77e8a6b21 100644 (file)
@@ -449,10 +449,10 @@ pub(super) fn process_onion_failure<T: secp256k1::Signing, L: Deref>(secp_ctx: &
 
                                                let (description, title) = errors::get_onion_error_description(error_code);
                                                if debug_field_size > 0 && err_packet.failuremsg.len() >= 4 + debug_field_size {
-                                                       log_warn!(logger, "Onion Error[{}({:#x}) {}({})] {}", title, error_code, debug_field, log_bytes!(&err_packet.failuremsg[4..4+debug_field_size]), description);
+                                                       log_warn!(logger, "Onion Error[from {}: {}({:#x}) {}({})] {}", route_hop.pubkey, title, error_code, debug_field, log_bytes!(&err_packet.failuremsg[4..4+debug_field_size]), description);
                                                }
                                                else {
-                                                       log_warn!(logger, "Onion Error[{}({:#x})] {}", title, error_code, description);
+                                                       log_warn!(logger, "Onion Error[from {}: {}({:#x})] {}", route_hop.pubkey, title, error_code, description);
                                                }
                                        } else {
                                                // Useless packet that we can't use but it passed HMAC, so it
index 46400641bc178ba21d6e41366f62fe0cbaaef4f9..c9573a67c13244cc73e1298ca43a9e1d66537e33 100644 (file)
@@ -76,7 +76,7 @@ fn do_test_onchain_htlc_reorg(local_commitment: bool, claim: bool) {
                // Give node 2 node 1's transactions and get its response (claiming the HTLC instead).
                connect_block(&nodes[2], &Block { header, txdata: node_1_commitment_txn.clone() });
                check_added_monitors!(nodes[2], 1);
-               check_closed_broadcast!(nodes[2], false); // We should get a BroadcastChannelUpdate (and *only* a BroadcstChannelUpdate)
+               check_closed_broadcast!(nodes[2], true); // We should get a BroadcastChannelUpdate (and *only* a BroadcstChannelUpdate)
                let node_2_commitment_txn = nodes[2].tx_broadcaster.txn_broadcasted.lock().unwrap();
                assert_eq!(node_2_commitment_txn.len(), 3); // ChannelMonitor: 1 offered HTLC-Claim, ChannelManger: 1 local commitment tx, 1 Received HTLC-Claim
                assert_eq!(node_2_commitment_txn[1].output.len(), 2); // to-remote and Received HTLC (to-self is dust)
@@ -116,7 +116,7 @@ fn do_test_onchain_htlc_reorg(local_commitment: bool, claim: bool) {
                node_2_commitment_txn
        };
        check_added_monitors!(nodes[1], 1);
-       check_closed_broadcast!(nodes[1], false); // We should get a BroadcastChannelUpdate (and *only* a BroadcstChannelUpdate)
+       check_closed_broadcast!(nodes[1], true); // We should get a BroadcastChannelUpdate (and *only* a BroadcstChannelUpdate)
        // Connect ANTI_REORG_DELAY - 2 blocks, giving us a confirmation count of ANTI_REORG_DELAY - 1.
        connect_blocks(&nodes[1], ANTI_REORG_DELAY - 2);
        check_added_monitors!(nodes[1], 0);
@@ -184,7 +184,7 @@ fn test_onchain_htlc_timeout_delay_remote_commitment() {
        do_test_onchain_htlc_reorg(false, false);
 }
 
-fn do_test_unconf_chan(reload_node: bool, reorg_after_reload: bool) {
+fn do_test_unconf_chan(reload_node: bool, reorg_after_reload: bool, connect_style: ConnectStyle) {
        // After creating a chan between nodes, we disconnect all blocks previously seen to force a
        // channel close on nodes[0] side. We also use this to provide very basic testing of logic
        // around freeing background events which store monitor updates during block_[dis]connected.
@@ -195,6 +195,8 @@ fn do_test_unconf_chan(reload_node: bool, reorg_after_reload: bool) {
        let new_chain_monitor: test_utils::TestChainMonitor;
        let nodes_0_deserialized: ChannelManager<EnforcingSigner, &test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestLogger>;
        let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
+       *nodes[0].connect_style.borrow_mut() = connect_style;
+
        let chan_id = create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::known(), InitFeatures::known()).2;
 
        let channel_state = nodes[0].node.channel_state.lock().unwrap();
@@ -204,7 +206,7 @@ fn do_test_unconf_chan(reload_node: bool, reorg_after_reload: bool) {
 
        if !reorg_after_reload {
                disconnect_all_blocks(&nodes[0]);
-               check_closed_broadcast!(nodes[0], false);
+               check_closed_broadcast!(nodes[0], true);
                {
                        let channel_state = nodes[0].node.channel_state.lock().unwrap();
                        assert_eq!(channel_state.by_id.len(), 0);
@@ -256,7 +258,7 @@ fn do_test_unconf_chan(reload_node: bool, reorg_after_reload: bool) {
 
        if reorg_after_reload {
                disconnect_all_blocks(&nodes[0]);
-               check_closed_broadcast!(nodes[0], false);
+               check_closed_broadcast!(nodes[0], true);
                {
                        let channel_state = nodes[0].node.channel_state.lock().unwrap();
                        assert_eq!(channel_state.by_id.len(), 0);
@@ -272,10 +274,18 @@ fn do_test_unconf_chan(reload_node: bool, reorg_after_reload: bool) {
 
 #[test]
 fn test_unconf_chan() {
-       do_test_unconf_chan(true, true);
-       do_test_unconf_chan(false, true);
-       do_test_unconf_chan(true, false);
-       do_test_unconf_chan(false, false);
+       do_test_unconf_chan(true, true, ConnectStyle::BestBlockFirstSkippingBlocks);
+       do_test_unconf_chan(false, true, ConnectStyle::BestBlockFirstSkippingBlocks);
+       do_test_unconf_chan(true, false, ConnectStyle::BestBlockFirstSkippingBlocks);
+       do_test_unconf_chan(false, false, ConnectStyle::BestBlockFirstSkippingBlocks);
+}
+
+#[test]
+fn test_unconf_chan_via_listen() {
+       do_test_unconf_chan(true, true, ConnectStyle::FullBlockViaListen);
+       do_test_unconf_chan(false, true, ConnectStyle::FullBlockViaListen);
+       do_test_unconf_chan(true, false, ConnectStyle::FullBlockViaListen);
+       do_test_unconf_chan(false, false, ConnectStyle::FullBlockViaListen);
 }
 
 #[test]
@@ -311,7 +321,7 @@ fn test_set_outpoints_partial_claiming() {
 
        // Connect blocks on node A commitment transaction
        mine_transaction(&nodes[0], &remote_txn[0]);
-       check_closed_broadcast!(nodes[0], false);
+       check_closed_broadcast!(nodes[0], true);
        check_added_monitors!(nodes[0], 1);
        // Verify node A broadcast tx claiming both HTLCs
        {
@@ -328,7 +338,7 @@ fn test_set_outpoints_partial_claiming() {
 
        // Connect blocks on node B
        connect_blocks(&nodes[1], 135);
-       check_closed_broadcast!(nodes[1], false);
+       check_closed_broadcast!(nodes[1], true);
        check_added_monitors!(nodes[1], 1);
        // Verify node B broadcast 2 HTLC-timeout txn
        let partial_claim_tx = {
index c9d7cc253217e28578ad0b26e58cef4665f7dbbe..a74f0cf53d9a650a43517c2c155744f1b8935934 100644 (file)
@@ -143,13 +143,17 @@ struct RouteGraphNode {
        // - how much is needed for a path being constructed
        // - how much value can channels following this node (up to the destination) can contribute,
        //   considering their capacity and fees
-       value_contribution_msat: u64
+       value_contribution_msat: u64,
+       /// The effective htlc_minimum_msat at this hop. If a later hop on the path had a higher HTLC
+       /// minimum, we use it, plus the fees required at each earlier hop to meet it.
+       path_htlc_minimum_msat: u64,
 }
 
 impl cmp::Ord for RouteGraphNode {
        fn cmp(&self, other: &RouteGraphNode) -> cmp::Ordering {
-               other.lowest_fee_to_peer_through_node.cmp(&self.lowest_fee_to_peer_through_node)
-                       .then_with(|| other.pubkey.serialize().cmp(&self.pubkey.serialize()))
+               let other_score = cmp::max(other.lowest_fee_to_peer_through_node, other.path_htlc_minimum_msat);
+               let self_score = cmp::max(self.lowest_fee_to_peer_through_node, self.path_htlc_minimum_msat);
+               other_score.cmp(&self_score).then_with(|| other.pubkey.serialize().cmp(&self.pubkey.serialize()))
        }
 }
 
@@ -171,10 +175,15 @@ struct DummyDirectionalChannelInfo {
 /// Fee values should be updated only in the context of the whole path, see update_value_and_recompute_fees.
 /// These fee values are useful to choose hops as we traverse the graph "payee-to-payer".
 #[derive(Clone)]
-struct PathBuildingHop {
-       /// Hop-specific details unrelated to the path during the routing phase,
-       /// but rather relevant to the LN graph.
-       route_hop: RouteHop,
+struct PathBuildingHop<'a> {
+       // The RouteHint fields which will eventually be used if this hop is used in a final Route.
+       // Note that node_features is calculated separately after our initial graph walk.
+       pubkey: PublicKey,
+       short_channel_id: u64,
+       channel_features: &'a ChannelFeatures,
+       fee_msat: u64,
+       cltv_expiry_delta: u32,
+
        /// Minimal fees required to route to the source node of the current hop via any of its inbound channels.
        src_lowest_inbound_fees: RoutingFees,
        /// Fees of the channel used in this hop.
@@ -193,20 +202,34 @@ struct PathBuildingHop {
        /// we don't fall below the minimum. Should not be updated manually and
        /// generally should not be accessed.
        htlc_minimum_msat: u64,
+       /// A mirror of the same field in RouteGraphNode. Note that this is only used during the graph
+       /// walk and may be invalid thereafter.
+       path_htlc_minimum_msat: u64,
+       /// If we've already processed a node as the best node, we shouldn't process it again. Normally
+       /// we'd just ignore it if we did as all channels would have a higher new fee, but because we
+       /// may decrease the amounts in use as we walk the graph, the actual calculated fee may
+       /// decrease as well. Thus, we have to explicitly track which nodes have been processed and
+       /// avoid processing them again.
+       was_processed: bool,
+       #[cfg(any(test, feature = "fuzztarget"))]
+       // In tests, we apply further sanity checks on cases where we skip nodes we already processed
+       // to ensure it is specifically in cases where the fee has gone down because of a decrease in
+       // value_contribution_msat, which requires tracking it here. See comments below where it is
+       // used for more info.
+       value_contribution_msat: u64,
 }
 
 // Instantiated with a list of hops with correct data in them collected during path finding,
 // an instance of this struct should be further modified only via given methods.
 #[derive(Clone)]
-struct PaymentPath {
-       hops: Vec<PathBuildingHop>,
+struct PaymentPath<'a> {
+       hops: Vec<(PathBuildingHop<'a>, NodeFeatures)>,
 }
 
-impl PaymentPath {
-
+impl<'a> PaymentPath<'a> {
        // TODO: Add a value_msat field to PaymentPath and use it instead of this function.
        fn get_value_msat(&self) -> u64 {
-               self.hops.last().unwrap().route_hop.fee_msat
+               self.hops.last().unwrap().0.fee_msat
        }
 
        fn get_total_fee_paid_msat(&self) -> u64 {
@@ -215,9 +238,9 @@ impl PaymentPath {
                }
                let mut result = 0;
                // Can't use next_hops_fee_msat because it gets outdated.
-               for (i, hop) in self.hops.iter().enumerate() {
+               for (i, (hop, _)) in self.hops.iter().enumerate() {
                        if i != self.hops.len() - 1 {
-                               result += hop.route_hop.fee_msat;
+                               result += hop.fee_msat;
                        }
                }
                return result;
@@ -226,16 +249,14 @@ impl PaymentPath {
        // If the amount transferred by the path is updated, the fees should be adjusted. Any other way
        // to change fees may result in an inconsistency.
        //
-       // Sometimes we call this function right after constructing a path which has inconsistent
-       // (in terms of reaching htlc_minimum_msat), so that this function puts the fees in order.
-       // In that case we call it on the "same" amount we initially allocated for this path, and which
-       // could have been reduced on the way. In that case, there is also a risk of exceeding
-       // available_liquidity inside this function, because the function is unaware of this bound.
-       // In our specific recomputation cases where we never increase the value the risk is pretty low.
-       // This function, however, does not support arbitrarily increasing the value being transferred,
-       // and the exception will be triggered.
+       // Sometimes we call this function right after constructing a path which is inconsistent in
+       // that it the value being transferred has decreased while we were doing path finding, leading
+       // to the fees being paid not lining up with the actual limits.
+       //
+       // Note that this function is not aware of the available_liquidity limit, and thus does not
+       // support increasing the value being transferred.
        fn update_value_and_recompute_fees(&mut self, value_msat: u64) {
-               assert!(value_msat <= self.hops.last().unwrap().route_hop.fee_msat);
+               assert!(value_msat <= self.hops.last().unwrap().0.fee_msat);
 
                let mut total_fee_paid_msat = 0 as u64;
                for i in (0..self.hops.len()).rev() {
@@ -246,10 +267,10 @@ impl PaymentPath {
                        // htlc_minimum_msat of the current channel. Last hop is handled separately.
                        let mut cur_hop_fees_msat = 0;
                        if !last_hop {
-                               cur_hop_fees_msat = self.hops.get(i + 1).unwrap().hop_use_fee_msat;
+                               cur_hop_fees_msat = self.hops.get(i + 1).unwrap().0.hop_use_fee_msat;
                        }
 
-                       let mut cur_hop = self.hops.get_mut(i).unwrap();
+                       let mut cur_hop = &mut self.hops.get_mut(i).unwrap().0;
                        cur_hop.next_hops_fee_msat = total_fee_paid_msat;
                        // Overpay in fees if we can't save these funds due to htlc_minimum_msat.
                        // We try to account for htlc_minimum_msat in scoring (add_entry!), so that nodes don't
@@ -273,11 +294,11 @@ impl PaymentPath {
                        if last_hop {
                                // Final hop is a special case: it usually has just value_msat (by design), but also
                                // it still could overpay for the htlc_minimum_msat.
-                               cur_hop.route_hop.fee_msat = cur_hop_transferred_amount_msat;
+                               cur_hop.fee_msat = cur_hop_transferred_amount_msat;
                        } else {
                                // Propagate updated fees for the use of the channels to one hop back, where they
                                // will be actually paid (fee_msat). The last hop is handled above separately.
-                               cur_hop.route_hop.fee_msat = cur_hop_fees_msat;
+                               cur_hop.fee_msat = cur_hop_fees_msat;
                        }
 
                        // Fee for the use of the current hop which will be deducted on the previous hop.
@@ -372,8 +393,43 @@ pub fn get_route<L: Deref>(our_node_id: &PublicKey, network: &NetworkGraph, paye
        // 8. Choose the best route by the lowest total fee.
 
        // As for the actual search algorithm,
-       // we do a payee-to-payer Dijkstra's sorting by each node's distance from the payee
-       // plus the minimum per-HTLC fee to get from it to another node (aka "shitty A*").
+       // we do a payee-to-payer pseudo-Dijkstra's sorting by each node's distance from the payee
+       // plus the minimum per-HTLC fee to get from it to another node (aka "shitty pseudo-A*").
+       //
+       // We are not a faithful Dijkstra's implementation because we can change values which impact
+       // earlier nodes while processing later nodes. Specifically, if we reach a channel with a lower
+       // liquidity limit (via htlc_maximum_msat, on-chain capacity or assumed liquidity limits) then
+       // the value we are currently attempting to send over a path, we simply reduce the value being
+       // sent along the path for any hops after that channel. This may imply that later fees (which
+       // we've already tabulated) are lower because a smaller value is passing through the channels
+       // (and the proportional fee is thus lower). There isn't a trivial way to recalculate the
+       // channels which were selected earlier (and which may still be used for other paths without a
+       // lower liquidity limit), so we simply accept that some liquidity-limited paths may be
+       // de-preferenced.
+       //
+       // One potentially problematic case for this algorithm would be if there are many
+       // liquidity-limited paths which are liquidity-limited near the destination (ie early in our
+       // graph walking), we may never find a path which is not liquidity-limited and has lower
+       // proportional fee (and only lower absolute fee when considering the ultimate value sent).
+       // Because we only consider paths with at least 5% of the total value being sent, the damage
+       // from such a case should be limited, however this could be further reduced in the future by
+       // calculating fees on the amount we wish to route over a path, ie ignoring the liquidity
+       // limits for the purposes of fee calculation.
+       //
+       // Alternatively, we could store more detailed path information in the heap (targets, below)
+       // and index the best-path map (dist, below) by node *and* HTLC limits, however that would blow
+       // up the runtime significantly both algorithmically (as we'd traverse nodes multiple times)
+       // and practically (as we would need to store dynamically-allocated path information in heap
+       // objects, increasing malloc traffic and indirect memory access significantly). Further, the
+       // results of such an algorithm would likely be biased towards lower-value paths.
+       //
+       // Further, we could return to a faithful Dijkstra's algorithm by rejecting paths with limits
+       // outside of our current search value, running a path search more times to gather candidate
+       // paths at different values. While this may be acceptable, further path searches may increase
+       // runtime for little gain. Specifically, the current algorithm rather efficiently explores the
+       // graph for candidate paths, calculating the maximum value which can realistically be sent at
+       // the same time, remaining generic across different payment values.
+       //
        // TODO: There are a few tweaks we could do, including possibly pre-calculating more stuff
        // to use as the A* heuristic beyond just the cost to get one node further than the current
        // one.
@@ -388,17 +444,6 @@ pub fn get_route<L: Deref>(our_node_id: &PublicKey, network: &NetworkGraph, paye
                }
        };
 
-       let mut targets = BinaryHeap::new(); //TODO: Do we care about switching to eg Fibbonaci heap?
-       let mut dist = HashMap::with_capacity(network.get_nodes().len());
-
-       // When arranging a route, we select multiple paths so that we can make a multi-path payment.
-       // Don't stop searching for paths when we think they're
-       // sufficient to transfer a given value aggregately.
-       // Search for higher value, so that we collect many more paths,
-       // and then select the best combination among them.
-       const ROUTE_CAPACITY_PROVISION_FACTOR: u64 = 3;
-       let recommended_value_msat = final_value_msat * ROUTE_CAPACITY_PROVISION_FACTOR as u64;
-
        // Allow MPP only if we have a features set from somewhere that indicates the payee supports
        // it. If the payee supports it they're supposed to include it in the invoice, so that should
        // work reliably.
@@ -414,25 +459,50 @@ pub fn get_route<L: Deref>(our_node_id: &PublicKey, network: &NetworkGraph, paye
        // Prepare the data we'll use for payee-to-payer search by
        // inserting first hops suggested by the caller as targets.
        // Our search will then attempt to reach them while traversing from the payee node.
-       let mut first_hop_targets = HashMap::with_capacity(if first_hops.is_some() { first_hops.as_ref().unwrap().len() } else { 0 });
+       let mut first_hop_targets: HashMap<_, (_, ChannelFeatures, _, NodeFeatures)> =
+               HashMap::with_capacity(if first_hops.is_some() { first_hops.as_ref().unwrap().len() } else { 0 });
        if let Some(hops) = first_hops {
                for chan in hops {
                        let short_channel_id = chan.short_channel_id.expect("first_hops should be filled in with usable channels, not pending ones");
                        if chan.remote_network_id == *our_node_id {
                                return Err(LightningError{err: "First hop cannot have our_node_id as a destination.".to_owned(), action: ErrorAction::IgnoreError});
                        }
-                       first_hop_targets.insert(chan.remote_network_id, (short_channel_id, chan.counterparty_features.clone(), chan.outbound_capacity_msat));
+                       first_hop_targets.insert(chan.remote_network_id, (short_channel_id, chan.counterparty_features.to_context(), chan.outbound_capacity_msat, chan.counterparty_features.to_context()));
                }
                if first_hop_targets.is_empty() {
                        return Err(LightningError{err: "Cannot route when there are no outbound routes away from us".to_owned(), action: ErrorAction::IgnoreError});
                }
        }
 
+       let empty_channel_features = ChannelFeatures::empty();
+
+       // The main heap containing all candidate next-hops sorted by their score (max(A* fee,
+       // htlc_minimum)). Ideally this would be a heap which allowed cheap score reduction instead of
+       // adding duplicate entries when we find a better path to a given node.
+       let mut targets = BinaryHeap::new();
+
+       // Map from node_id to information about the best current path to that node, including feerate
+       // information.
+       let mut dist = HashMap::with_capacity(network.get_nodes().len());
+
+       // During routing, if we ignore a path due to an htlc_minimum_msat limit, we set this,
+       // indicating that we may wish to try again with a higher value, potentially paying to meet an
+       // htlc_minimum with extra fees while still finding a cheaper path.
+       let mut hit_minimum_limit;
+
+       // When arranging a route, we select multiple paths so that we can make a multi-path payment.
+       // We start with a path_value of the exact amount we want, and if that generates a route we may
+       // return it immediately. Otherwise, we don't stop searching for paths until we have 3x the
+       // amount we want in total across paths, selecting the best subset at the end.
+       const ROUTE_CAPACITY_PROVISION_FACTOR: u64 = 3;
+       let recommended_value_msat = final_value_msat * ROUTE_CAPACITY_PROVISION_FACTOR as u64;
+       let mut path_value_msat = final_value_msat;
+
        // We don't want multiple paths (as per MPP) share liquidity of the same channels.
        // This map allows paths to be aware of the channel use by other paths in the same call.
        // This would help to make a better path finding decisions and not "overbook" channels.
        // It is unaware of the directions (except for `outbound_capacity_msat` in `first_hops`).
-       let mut bookkeeped_channels_liquidity_available_msat = HashMap::new();
+       let mut bookkeeped_channels_liquidity_available_msat = HashMap::with_capacity(network.get_nodes().len());
 
        // Keeping track of how much value we already collected across other paths. Helps to decide:
        // - how much a new path should be transferring (upper bound);
@@ -448,7 +518,7 @@ pub fn get_route<L: Deref>(our_node_id: &PublicKey, network: &NetworkGraph, paye
                // $next_hops_fee_msat represents the fees paid for using all the channel *after* this one,
                // since that value has to be transferred over this channel.
                ( $chan_id: expr, $src_node_id: expr, $dest_node_id: expr, $directional_info: expr, $capacity_sats: expr, $chan_features: expr, $next_hops_fee_msat: expr,
-                  $next_hops_value_contribution: expr ) => {
+                  $next_hops_value_contribution: expr, $next_hops_path_htlc_minimum_msat: expr ) => {
                        // Channels to self should not be used. This is more of belt-and-suspenders, because in
                        // practice these cases should be caught earlier:
                        // - for regular channels at channel announcement (TODO)
@@ -515,18 +585,27 @@ pub fn get_route<L: Deref>(our_node_id: &PublicKey, network: &NetworkGraph, paye
                                                // Can't overflow due to how the values were computed right above.
                                                None => unreachable!(),
                                        };
+                                       #[allow(unused_comparisons)] // $next_hops_path_htlc_minimum_msat is 0 in some calls so rustc complains
+                                       let over_path_minimum_msat = amount_to_transfer_over_msat >= $directional_info.htlc_minimum_msat &&
+                                               amount_to_transfer_over_msat >= $next_hops_path_htlc_minimum_msat;
 
                                        // If HTLC minimum is larger than the amount we're going to transfer, we shouldn't
                                        // bother considering this channel.
                                        // Since we're choosing amount_to_transfer_over_msat as maximum possible, it can
                                        // be only reduced later (not increased), so this channel should just be skipped
                                        // as not sufficient.
-                                       // TODO: Explore simply adding fee to hit htlc_minimum_msat
-                                       if contributes_sufficient_value && amount_to_transfer_over_msat >= $directional_info.htlc_minimum_msat {
+                                       if !over_path_minimum_msat {
+                                               hit_minimum_limit = true;
+                                       } else if contributes_sufficient_value {
                                                // Note that low contribution here (limited by available_liquidity_msat)
                                                // might violate htlc_minimum_msat on the hops which are next along the
                                                // payment path (upstream to the payee). To avoid that, we recompute path
                                                // path fees knowing the final path contribution after constructing it.
+                                               let path_htlc_minimum_msat = match compute_fees($next_hops_path_htlc_minimum_msat, $directional_info.fees)
+                                                               .map(|fee_msat| fee_msat.checked_add($next_hops_path_htlc_minimum_msat)) {
+                                                       Some(Some(value_msat)) => cmp::max(value_msat, $directional_info.htlc_minimum_msat),
+                                                       _ => u64::max_value()
+                                               };
                                                let hm_entry = dist.entry(&$src_node_id);
                                                let old_entry = hm_entry.or_insert_with(|| {
                                                        // If there was previously no known way to access
@@ -541,14 +620,11 @@ pub fn get_route<L: Deref>(our_node_id: &PublicKey, network: &NetworkGraph, paye
                                                                fee_proportional_millionths = fees.proportional_millionths;
                                                        }
                                                        PathBuildingHop {
-                                                               route_hop: RouteHop {
-                                                                       pubkey: $dest_node_id.clone(),
-                                                                       node_features: NodeFeatures::empty(),
-                                                                       short_channel_id: 0,
-                                                                       channel_features: $chan_features.clone(),
-                                                                       fee_msat: 0,
-                                                                       cltv_expiry_delta: 0,
-                                                               },
+                                                               pubkey: $dest_node_id.clone(),
+                                                               short_channel_id: 0,
+                                                               channel_features: $chan_features,
+                                                               fee_msat: 0,
+                                                               cltv_expiry_delta: 0,
                                                                src_lowest_inbound_fees: RoutingFees {
                                                                        base_msat: fee_base_msat,
                                                                        proportional_millionths: fee_proportional_millionths,
@@ -558,92 +634,128 @@ pub fn get_route<L: Deref>(our_node_id: &PublicKey, network: &NetworkGraph, paye
                                                                hop_use_fee_msat: u64::max_value(),
                                                                total_fee_msat: u64::max_value(),
                                                                htlc_minimum_msat: $directional_info.htlc_minimum_msat,
+                                                               path_htlc_minimum_msat,
+                                                               was_processed: false,
+                                                               #[cfg(any(test, feature = "fuzztarget"))]
+                                                               value_contribution_msat,
                                                        }
                                                });
 
-                                               let mut hop_use_fee_msat = 0;
-                                               let mut total_fee_msat = $next_hops_fee_msat;
-
-                                               // Ignore hop_use_fee_msat for channel-from-us as we assume all channels-from-us
-                                               // will have the same effective-fee
-                                               if $src_node_id != *our_node_id {
-                                                       match compute_fees(amount_to_transfer_over_msat, $directional_info.fees) {
-                                                               // max_value means we'll always fail
-                                                               // the old_entry.total_fee_msat > total_fee_msat check
-                                                               None => total_fee_msat = u64::max_value(),
-                                                               Some(fee_msat) => {
-                                                                       hop_use_fee_msat = fee_msat;
-                                                                       total_fee_msat += hop_use_fee_msat;
-                                                                       if let Some(prev_hop_fee_msat) = compute_fees(total_fee_msat + amount_to_transfer_over_msat,
-                                                                                                                                                               old_entry.src_lowest_inbound_fees) {
-                                                                               if let Some(incremented_total_fee_msat) = total_fee_msat.checked_add(prev_hop_fee_msat) {
-                                                                                       total_fee_msat = incremented_total_fee_msat;
-                                                                               }
-                                                                               else {
-                                                                                       // max_value means we'll always fail
-                                                                                       // the old_entry.total_fee_msat > total_fee_msat check
-                                                                                       total_fee_msat = u64::max_value();
-                                                                               }
-                                                                       } else {
-                                                                               // max_value means we'll always fail
-                                                                               // the old_entry.total_fee_msat > total_fee_msat check
-                                                                               total_fee_msat = u64::max_value();
+                                               #[allow(unused_mut)] // We only use the mut in cfg(test)
+                                               let mut should_process = !old_entry.was_processed;
+                                               #[cfg(any(test, feature = "fuzztarget"))]
+                                               {
+                                                       // In test/fuzzing builds, we do extra checks to make sure the skipping
+                                                       // of already-seen nodes only happens in cases we expect (see below).
+                                                       if !should_process { should_process = true; }
+                                               }
+
+                                               if should_process {
+                                                       let mut hop_use_fee_msat = 0;
+                                                       let mut total_fee_msat = $next_hops_fee_msat;
+
+                                                       // Ignore hop_use_fee_msat for channel-from-us as we assume all channels-from-us
+                                                       // will have the same effective-fee
+                                                       if $src_node_id != *our_node_id {
+                                                               match compute_fees(amount_to_transfer_over_msat, $directional_info.fees) {
+                                                                       // max_value means we'll always fail
+                                                                       // the old_entry.total_fee_msat > total_fee_msat check
+                                                                       None => total_fee_msat = u64::max_value(),
+                                                                       Some(fee_msat) => {
+                                                                               hop_use_fee_msat = fee_msat;
+                                                                               total_fee_msat += hop_use_fee_msat;
+                                                                               // When calculating the lowest inbound fees to a node, we
+                                                                               // calculate fees here not based on the actual value we think
+                                                                               // will flow over this channel, but on the minimum value that
+                                                                               // we'll accept flowing over it. The minimum accepted value
+                                                                               // is a constant through each path collection run, ensuring
+                                                                               // consistent basis. Otherwise we may later find a
+                                                                               // different path to the source node that is more expensive,
+                                                                               // but which we consider to be cheaper because we are capacity
+                                                                               // constrained and the relative fee becomes lower.
+                                                                               match compute_fees(minimal_value_contribution_msat, old_entry.src_lowest_inbound_fees)
+                                                                                               .map(|a| a.checked_add(total_fee_msat)) {
+                                                                                       Some(Some(v)) => {
+                                                                                               total_fee_msat = v;
+                                                                                       },
+                                                                                       _ => {
+                                                                                               total_fee_msat = u64::max_value();
+                                                                                       }
+                                                                               };
                                                                        }
                                                                }
                                                        }
-                                               }
-
-                                               let new_graph_node = RouteGraphNode {
-                                                       pubkey: $src_node_id,
-                                                       lowest_fee_to_peer_through_node: total_fee_msat,
-                                                       lowest_fee_to_node: $next_hops_fee_msat as u64 + hop_use_fee_msat,
-                                                       value_contribution_msat: value_contribution_msat,
-                                               };
-
-                                               // Update the way of reaching $src_node_id with the given $chan_id (from $dest_node_id),
-                                               // if this way is cheaper than the already known
-                                               // (considering the cost to "reach" this channel from the route destination,
-                                               // the cost of using this channel,
-                                               // and the cost of routing to the source node of this channel).
-                                               // Also, consider that htlc_minimum_msat_difference, because we might end up
-                                               // paying it. Consider the following exploit:
-                                               // we use 2 paths to transfer 1.5 BTC. One of them is 0-fee normal 1 BTC path,
-                                               // and for the other one we picked a 1sat-fee path with htlc_minimum_msat of
-                                               // 1 BTC. Now, since the latter is more expensive, we gonna try to cut it
-                                               // by 0.5 BTC, but then match htlc_minimum_msat by paying a fee of 0.5 BTC
-                                               // to this channel.
-                                               // TODO: this scoring could be smarter (e.g. 0.5*htlc_minimum_msat here).
-                                               let mut old_cost = old_entry.total_fee_msat;
-                                               if let Some(increased_old_cost) = old_cost.checked_add(old_entry.htlc_minimum_msat) {
-                                                       old_cost = increased_old_cost;
-                                               } else {
-                                                       old_cost = u64::max_value();
-                                               }
-
-                                               let mut new_cost = total_fee_msat;
-                                               if let Some(increased_new_cost) = new_cost.checked_add($directional_info.htlc_minimum_msat) {
-                                                       new_cost = increased_new_cost;
-                                               } else {
-                                                       new_cost = u64::max_value();
-                                               }
 
-                                               if new_cost < old_cost {
-                                                       targets.push(new_graph_node);
-                                                       old_entry.next_hops_fee_msat = $next_hops_fee_msat;
-                                                       old_entry.hop_use_fee_msat = hop_use_fee_msat;
-                                                       old_entry.total_fee_msat = total_fee_msat;
-                                                       old_entry.route_hop = RouteHop {
-                                                               pubkey: $dest_node_id.clone(),
-                                                               node_features: NodeFeatures::empty(),
-                                                               short_channel_id: $chan_id.clone(),
-                                                               channel_features: $chan_features.clone(),
-                                                               fee_msat: 0, // This value will be later filled with hop_use_fee_msat of the following channel
-                                                               cltv_expiry_delta: $directional_info.cltv_expiry_delta as u32,
+                                                       let new_graph_node = RouteGraphNode {
+                                                               pubkey: $src_node_id,
+                                                               lowest_fee_to_peer_through_node: total_fee_msat,
+                                                               lowest_fee_to_node: $next_hops_fee_msat as u64 + hop_use_fee_msat,
+                                                               value_contribution_msat: value_contribution_msat,
+                                                               path_htlc_minimum_msat,
                                                        };
-                                                       old_entry.channel_fees = $directional_info.fees;
-                                                       // It's probably fine to replace the old entry, because the new one
-                                                       // passed the htlc_minimum-related checks above.
-                                                       old_entry.htlc_minimum_msat = $directional_info.htlc_minimum_msat;
+
+                                                       // Update the way of reaching $src_node_id with the given $chan_id (from $dest_node_id),
+                                                       // if this way is cheaper than the already known
+                                                       // (considering the cost to "reach" this channel from the route destination,
+                                                       // the cost of using this channel,
+                                                       // and the cost of routing to the source node of this channel).
+                                                       // Also, consider that htlc_minimum_msat_difference, because we might end up
+                                                       // paying it. Consider the following exploit:
+                                                       // we use 2 paths to transfer 1.5 BTC. One of them is 0-fee normal 1 BTC path,
+                                                       // and for the other one we picked a 1sat-fee path with htlc_minimum_msat of
+                                                       // 1 BTC. Now, since the latter is more expensive, we gonna try to cut it
+                                                       // by 0.5 BTC, but then match htlc_minimum_msat by paying a fee of 0.5 BTC
+                                                       // to this channel.
+                                                       // Ideally the scoring could be smarter (e.g. 0.5*htlc_minimum_msat here),
+                                                       // but it may require additional tracking - we don't want to double-count
+                                                       // the fees included in $next_hops_path_htlc_minimum_msat, but also
+                                                       // can't use something that may decrease on future hops.
+                                                       let old_cost = cmp::max(old_entry.total_fee_msat, old_entry.path_htlc_minimum_msat);
+                                                       let new_cost = cmp::max(total_fee_msat, path_htlc_minimum_msat);
+
+                                                       if !old_entry.was_processed && new_cost < old_cost {
+                                                               targets.push(new_graph_node);
+                                                               old_entry.next_hops_fee_msat = $next_hops_fee_msat;
+                                                               old_entry.hop_use_fee_msat = hop_use_fee_msat;
+                                                               old_entry.total_fee_msat = total_fee_msat;
+                                                               old_entry.pubkey = $dest_node_id.clone();
+                                                               old_entry.short_channel_id = $chan_id.clone();
+                                                               old_entry.channel_features = $chan_features;
+                                                               old_entry.fee_msat = 0; // This value will be later filled with hop_use_fee_msat of the following channel
+                                                               old_entry.cltv_expiry_delta = $directional_info.cltv_expiry_delta as u32;
+                                                               old_entry.channel_fees = $directional_info.fees;
+                                                               old_entry.htlc_minimum_msat = $directional_info.htlc_minimum_msat;
+                                                               old_entry.path_htlc_minimum_msat = path_htlc_minimum_msat;
+                                                               #[cfg(any(test, feature = "fuzztarget"))]
+                                                               {
+                                                                       old_entry.value_contribution_msat = value_contribution_msat;
+                                                               }
+                                                       } else if old_entry.was_processed && new_cost < old_cost {
+                                                               #[cfg(any(test, feature = "fuzztarget"))]
+                                                               {
+                                                                       // If we're skipping processing a node which was previously
+                                                                       // processed even though we found another path to it with a
+                                                                       // cheaper fee, check that it was because the second path we
+                                                                       // found (which we are processing now) has a lower value
+                                                                       // contribution due to an HTLC minimum limit.
+                                                                       //
+                                                                       // e.g. take a graph with two paths from node 1 to node 2, one
+                                                                       // through channel A, and one through channel B. Channel A and
+                                                                       // B are both in the to-process heap, with their scores set by
+                                                                       // a higher htlc_minimum than fee.
+                                                                       // Channel A is processed first, and the channels onwards from
+                                                                       // node 1 are added to the to-process heap. Thereafter, we pop
+                                                                       // Channel B off of the heap, note that it has a much more
+                                                                       // restrictive htlc_maximum_msat, and recalculate the fees for
+                                                                       // all of node 1's channels using the new, reduced, amount.
+                                                                       //
+                                                                       // This would be bogus - we'd be selecting a higher-fee path
+                                                                       // with a lower htlc_maximum_msat instead of the one we'd
+                                                                       // already decided to use.
+                                                                       debug_assert!(path_htlc_minimum_msat < old_entry.path_htlc_minimum_msat);
+                                                                       debug_assert!(value_contribution_msat < old_entry.value_contribution_msat);
+                                                               }
+                                                       }
                                                }
                                        }
                                }
@@ -651,47 +763,60 @@ pub fn get_route<L: Deref>(our_node_id: &PublicKey, network: &NetworkGraph, paye
                };
        }
 
+       let empty_node_features = NodeFeatures::empty();
        // Find ways (channels with destination) to reach a given node and store them
        // in the corresponding data structures (routing graph etc).
        // $fee_to_target_msat represents how much it costs to reach to this node from the payee,
        // meaning how much will be paid in fees after this node (to the best of our knowledge).
        // This data can later be helpful to optimize routing (pay lower fees).
        macro_rules! add_entries_to_cheapest_to_target_node {
-               ( $node: expr, $node_id: expr, $fee_to_target_msat: expr, $next_hops_value_contribution: expr ) => {
-                       if first_hops.is_some() {
-                               if let Some(&(ref first_hop, ref features, ref outbound_capacity_msat)) = first_hop_targets.get(&$node_id) {
-                                       add_entry!(first_hop, *our_node_id, $node_id, dummy_directional_info, Some(outbound_capacity_msat / 1000), features.to_context(), $fee_to_target_msat, $next_hops_value_contribution);
+               ( $node: expr, $node_id: expr, $fee_to_target_msat: expr, $next_hops_value_contribution: expr, $next_hops_path_htlc_minimum_msat: expr ) => {
+                       let skip_node = if let Some(elem) = dist.get_mut($node_id) {
+                               let was_processed = elem.was_processed;
+                               elem.was_processed = true;
+                               was_processed
+                       } else {
+                               // Entries are added to dist in add_entry!() when there is a channel from a node.
+                               // Because there are no channels from payee, it will not have a dist entry at this point.
+                               // If we're processing any other node, it is always be the result of a channel from it.
+                               assert_eq!($node_id, payee);
+                               false
+                       };
+
+                       if !skip_node {
+                               if first_hops.is_some() {
+                                       if let Some(&(ref first_hop, ref features, ref outbound_capacity_msat, _)) = first_hop_targets.get(&$node_id) {
+                                               add_entry!(first_hop, *our_node_id, $node_id, dummy_directional_info, Some(outbound_capacity_msat / 1000), features, $fee_to_target_msat, $next_hops_value_contribution, $next_hops_path_htlc_minimum_msat);
+                                       }
                                }
-                       }
 
-                       let features;
-                       if let Some(node_info) = $node.announcement_info.as_ref() {
-                               features = node_info.features.clone();
-                       } else {
-                               features = NodeFeatures::empty();
-                       }
+                               let features = if let Some(node_info) = $node.announcement_info.as_ref() {
+                                       &node_info.features
+                               } else {
+                                       &empty_node_features
+                               };
 
-                       if !features.requires_unknown_bits() {
-                               for chan_id in $node.channels.iter() {
-                                       let chan = network.get_channels().get(chan_id).unwrap();
-                                       if !chan.features.requires_unknown_bits() {
-                                               if chan.node_one == *$node_id {
-                                                       // ie $node is one, ie next hop in A* is two, via the two_to_one channel
-                                                       if first_hops.is_none() || chan.node_two != *our_node_id {
-                                                               if let Some(two_to_one) = chan.two_to_one.as_ref() {
-                                                                       if two_to_one.enabled {
-                                                                               add_entry!(chan_id, chan.node_two, chan.node_one, two_to_one, chan.capacity_sats, chan.features, $fee_to_target_msat, $next_hops_value_contribution);
+                               if !features.requires_unknown_bits() {
+                                       for chan_id in $node.channels.iter() {
+                                               let chan = network.get_channels().get(chan_id).unwrap();
+                                               if !chan.features.requires_unknown_bits() {
+                                                       if chan.node_one == *$node_id {
+                                                               // ie $node is one, ie next hop in A* is two, via the two_to_one channel
+                                                               if first_hops.is_none() || chan.node_two != *our_node_id {
+                                                                       if let Some(two_to_one) = chan.two_to_one.as_ref() {
+                                                                               if two_to_one.enabled {
+                                                                                       add_entry!(chan_id, chan.node_two, chan.node_one, two_to_one, chan.capacity_sats, &chan.features, $fee_to_target_msat, $next_hops_value_contribution, $next_hops_path_htlc_minimum_msat);
+                                                                               }
                                                                        }
                                                                }
-                                                       }
-                                               } else {
-                                                       if first_hops.is_none() || chan.node_one != *our_node_id {
-                                                               if let Some(one_to_two) = chan.one_to_two.as_ref() {
-                                                                       if one_to_two.enabled {
-                                                                               add_entry!(chan_id, chan.node_one, chan.node_two, one_to_two, chan.capacity_sats, chan.features, $fee_to_target_msat, $next_hops_value_contribution);
+                                                       } else {
+                                                               if first_hops.is_none() || chan.node_one != *our_node_id {
+                                                                       if let Some(one_to_two) = chan.one_to_two.as_ref() {
+                                                                               if one_to_two.enabled {
+                                                                                       add_entry!(chan_id, chan.node_one, chan.node_two, one_to_two, chan.capacity_sats, &chan.features, $fee_to_target_msat, $next_hops_value_contribution, $next_hops_path_htlc_minimum_msat);
+                                                                               }
                                                                        }
                                                                }
-
                                                        }
                                                }
                                        }
@@ -709,12 +834,13 @@ pub fn get_route<L: Deref>(our_node_id: &PublicKey, network: &NetworkGraph, paye
                // the further iterations of path finding. Also don't erase first_hop_targets.
                targets.clear();
                dist.clear();
+               hit_minimum_limit = false;
 
                // If first hop is a private channel and the only way to reach the payee, this is the only
                // place where it could be added.
                if first_hops.is_some() {
-                       if let Some(&(ref first_hop, ref features, ref outbound_capacity_msat)) = first_hop_targets.get(&payee) {
-                               add_entry!(first_hop, *our_node_id, payee, dummy_directional_info, Some(outbound_capacity_msat / 1000), features.to_context(), 0, recommended_value_msat);
+                       if let Some(&(ref first_hop, ref features, ref outbound_capacity_msat, _)) = first_hop_targets.get(&payee) {
+                               add_entry!(first_hop, *our_node_id, payee, dummy_directional_info, Some(outbound_capacity_msat / 1000), features, 0, path_value_msat, 0);
                        }
                }
 
@@ -727,7 +853,7 @@ pub fn get_route<L: Deref>(our_node_id: &PublicKey, network: &NetworkGraph, paye
                        // If not, targets.pop() will not even let us enter the loop in step 2.
                        None => {},
                        Some(node) => {
-                               add_entries_to_cheapest_to_target_node!(node, payee, 0, recommended_value_msat);
+                               add_entries_to_cheapest_to_target_node!(node, payee, 0, path_value_msat, 0);
                        },
                }
 
@@ -737,7 +863,7 @@ pub fn get_route<L: Deref>(our_node_id: &PublicKey, network: &NetworkGraph, paye
                // it matters only if the fees are exactly the same.
                for hop in last_hops.iter() {
                        let have_hop_src_in_graph =
-                               if let Some(&(ref first_hop, ref features, ref outbound_capacity_msat)) = first_hop_targets.get(&hop.src_node_id) {
+                               if let Some(&(ref first_hop, ref features, ref outbound_capacity_msat, _)) = first_hop_targets.get(&hop.src_node_id) {
                                        // If this hop connects to a node with which we have a direct channel, ignore
                                        // the network graph and add both the hop and our direct channel to
                                        // the candidate set.
@@ -746,7 +872,7 @@ pub fn get_route<L: Deref>(our_node_id: &PublicKey, network: &NetworkGraph, paye
                                        // bit lazy here. In the future, we should pull them out via our
                                        // ChannelManager, but there's no reason to waste the space until we
                                        // need them.
-                                       add_entry!(first_hop, *our_node_id , hop.src_node_id, dummy_directional_info, Some(outbound_capacity_msat / 1000), features.to_context(), 0, recommended_value_msat);
+                                       add_entry!(first_hop, *our_node_id , hop.src_node_id, dummy_directional_info, Some(outbound_capacity_msat / 1000), features, 0, path_value_msat, 0);
                                        true
                                } else {
                                        // In any other case, only add the hop if the source is in the regular network
@@ -766,7 +892,7 @@ pub fn get_route<L: Deref>(our_node_id: &PublicKey, network: &NetworkGraph, paye
                                        htlc_maximum_msat: hop.htlc_maximum_msat,
                                        fees: hop.fees,
                                };
-                               add_entry!(hop.short_channel_id, hop.src_node_id, payee, directional_info, None::<u64>, ChannelFeatures::empty(), 0, recommended_value_msat);
+                               add_entry!(hop.short_channel_id, hop.src_node_id, payee, directional_info, None::<u64>, &empty_channel_features, 0, path_value_msat, 0);
                        }
                }
 
@@ -783,40 +909,40 @@ pub fn get_route<L: Deref>(our_node_id: &PublicKey, network: &NetworkGraph, paye
                // Both these cases (and other cases except reaching recommended_value_msat) mean that
                // paths_collection will be stopped because found_new_path==false.
                // This is not necessarily a routing failure.
-               'path_construction: while let Some(RouteGraphNode { pubkey, lowest_fee_to_node, value_contribution_msat, .. }) = targets.pop() {
+               'path_construction: while let Some(RouteGraphNode { pubkey, lowest_fee_to_node, value_contribution_msat, path_htlc_minimum_msat, .. }) = targets.pop() {
 
                        // Since we're going payee-to-payer, hitting our node as a target means we should stop
                        // traversing the graph and arrange the path out of what we found.
                        if pubkey == *our_node_id {
                                let mut new_entry = dist.remove(&our_node_id).unwrap();
-                               let mut ordered_hops = vec!(new_entry.clone());
+                               let mut ordered_hops = vec!((new_entry.clone(), NodeFeatures::empty()));
 
                                'path_walk: loop {
-                                       if let Some(&(_, ref features, _)) = first_hop_targets.get(&ordered_hops.last().unwrap().route_hop.pubkey) {
-                                               ordered_hops.last_mut().unwrap().route_hop.node_features = features.to_context();
-                                       } else if let Some(node) = network.get_nodes().get(&ordered_hops.last().unwrap().route_hop.pubkey) {
+                                       if let Some(&(_, _, _, ref features)) = first_hop_targets.get(&ordered_hops.last().unwrap().0.pubkey) {
+                                               ordered_hops.last_mut().unwrap().1 = features.clone();
+                                       } else if let Some(node) = network.get_nodes().get(&ordered_hops.last().unwrap().0.pubkey) {
                                                if let Some(node_info) = node.announcement_info.as_ref() {
-                                                       ordered_hops.last_mut().unwrap().route_hop.node_features = node_info.features.clone();
+                                                       ordered_hops.last_mut().unwrap().1 = node_info.features.clone();
                                                } else {
-                                                       ordered_hops.last_mut().unwrap().route_hop.node_features = NodeFeatures::empty();
+                                                       ordered_hops.last_mut().unwrap().1 = NodeFeatures::empty();
                                                }
                                        } else {
                                                // We should be able to fill in features for everything except the last
                                                // hop, if the last hop was provided via a BOLT 11 invoice (though we
                                                // should be able to extend it further as BOLT 11 does have feature
                                                // flags for the last hop node itself).
-                                               assert!(ordered_hops.last().unwrap().route_hop.pubkey == *payee);
+                                               assert!(ordered_hops.last().unwrap().0.pubkey == *payee);
                                        }
 
                                        // Means we succesfully traversed from the payer to the payee, now
                                        // save this path for the payment route. Also, update the liquidity
                                        // remaining on the used hops, so that we take them into account
                                        // while looking for more paths.
-                                       if ordered_hops.last().unwrap().route_hop.pubkey == *payee {
+                                       if ordered_hops.last().unwrap().0.pubkey == *payee {
                                                break 'path_walk;
                                        }
 
-                                       new_entry = match dist.remove(&ordered_hops.last().unwrap().route_hop.pubkey) {
+                                       new_entry = match dist.remove(&ordered_hops.last().unwrap().0.pubkey) {
                                                Some(payment_hop) => payment_hop,
                                                // We can't arrive at None because, if we ever add an entry to targets,
                                                // we also fill in the entry in dist (see add_entry!).
@@ -825,13 +951,13 @@ pub fn get_route<L: Deref>(our_node_id: &PublicKey, network: &NetworkGraph, paye
                                        // We "propagate" the fees one hop backward (topologically) here,
                                        // so that fees paid for a HTLC forwarding on the current channel are
                                        // associated with the previous channel (where they will be subtracted).
-                                       ordered_hops.last_mut().unwrap().route_hop.fee_msat = new_entry.hop_use_fee_msat;
-                                       ordered_hops.last_mut().unwrap().route_hop.cltv_expiry_delta = new_entry.route_hop.cltv_expiry_delta;
-                                       ordered_hops.push(new_entry.clone());
+                                       ordered_hops.last_mut().unwrap().0.fee_msat = new_entry.hop_use_fee_msat;
+                                       ordered_hops.last_mut().unwrap().0.cltv_expiry_delta = new_entry.cltv_expiry_delta;
+                                       ordered_hops.push((new_entry.clone(), NodeFeatures::empty()));
                                }
-                               ordered_hops.last_mut().unwrap().route_hop.fee_msat = value_contribution_msat;
-                               ordered_hops.last_mut().unwrap().hop_use_fee_msat = 0;
-                               ordered_hops.last_mut().unwrap().route_hop.cltv_expiry_delta = final_cltv;
+                               ordered_hops.last_mut().unwrap().0.fee_msat = value_contribution_msat;
+                               ordered_hops.last_mut().unwrap().0.hop_use_fee_msat = 0;
+                               ordered_hops.last_mut().unwrap().0.cltv_expiry_delta = final_cltv;
 
                                let mut payment_path = PaymentPath {hops: ordered_hops};
 
@@ -840,7 +966,7 @@ pub fn get_route<L: Deref>(our_node_id: &PublicKey, network: &NetworkGraph, paye
                                // on some channels we already passed (assuming dest->source direction). Here, we
                                // recompute the fees again, so that if that's the case, we match the currently
                                // underpaid htlc_minimum_msat with fees.
-                               payment_path.update_value_and_recompute_fees(value_contribution_msat);
+                               payment_path.update_value_and_recompute_fees(cmp::min(value_contribution_msat, final_value_msat));
 
                                // Since a path allows to transfer as much value as
                                // the smallest channel it has ("bottleneck"), we should recompute
@@ -851,18 +977,28 @@ pub fn get_route<L: Deref>(our_node_id: &PublicKey, network: &NetworkGraph, paye
                                // might have been computed considering a larger value.
                                // Remember that we used these channels so that we don't rely
                                // on the same liquidity in future paths.
-                               for payment_hop in payment_path.hops.iter() {
-                                       let channel_liquidity_available_msat = bookkeeped_channels_liquidity_available_msat.get_mut(&payment_hop.route_hop.short_channel_id).unwrap();
+                               let mut prevented_redundant_path_selection = false;
+                               for (payment_hop, _) in payment_path.hops.iter() {
+                                       let channel_liquidity_available_msat = bookkeeped_channels_liquidity_available_msat.get_mut(&payment_hop.short_channel_id).unwrap();
                                        let mut spent_on_hop_msat = value_contribution_msat;
                                        let next_hops_fee_msat = payment_hop.next_hops_fee_msat;
                                        spent_on_hop_msat += next_hops_fee_msat;
-                                       if *channel_liquidity_available_msat < spent_on_hop_msat {
-                                               // This should not happen because we do recompute fees right before,
-                                               // trying to avoid cases when a hop is not usable due to the fee situation.
-                                               break 'path_construction;
+                                       if spent_on_hop_msat == *channel_liquidity_available_msat {
+                                               // If this path used all of this channel's available liquidity, we know
+                                               // this path will not be selected again in the next loop iteration.
+                                               prevented_redundant_path_selection = true;
                                        }
                                        *channel_liquidity_available_msat -= spent_on_hop_msat;
                                }
+                               if !prevented_redundant_path_selection {
+                                       // If we weren't capped by hitting a liquidity limit on a channel in the path,
+                                       // we'll probably end up picking the same path again on the next iteration.
+                                       // Decrease the available liquidity of a hop in the middle of the path.
+                                       let victim_liquidity = bookkeeped_channels_liquidity_available_msat.get_mut(
+                                               &payment_path.hops[(payment_path.hops.len() - 1) / 2].0.short_channel_id).unwrap();
+                                       *victim_liquidity = 0;
+                               }
+
                                // Track the total amount all our collected paths allow to send so that we:
                                // - know when to stop looking for more paths
                                // - know which of the hops are useless considering how much more sats we need
@@ -874,13 +1010,18 @@ pub fn get_route<L: Deref>(our_node_id: &PublicKey, network: &NetworkGraph, paye
                                break 'path_construction;
                        }
 
+                       // If we found a path back to the payee, we shouldn't try to process it again. This is
+                       // the equivalent of the `elem.was_processed` check in
+                       // add_entries_to_cheapest_to_target_node!() (see comment there for more info).
+                       if pubkey == *payee { continue 'path_construction; }
+
                        // Otherwise, since the current target node is not us,
                        // keep "unrolling" the payment graph from payee to payer by
                        // finding a way to reach the current target from the payer side.
                        match network.get_nodes().get(&pubkey) {
                                None => {},
                                Some(node) => {
-                                       add_entries_to_cheapest_to_target_node!(node, &pubkey, lowest_fee_to_node, value_contribution_msat);
+                                       add_entries_to_cheapest_to_target_node!(node, &pubkey, lowest_fee_to_node, value_contribution_msat, path_htlc_minimum_msat);
                                },
                        }
                }
@@ -891,12 +1032,22 @@ pub fn get_route<L: Deref>(our_node_id: &PublicKey, network: &NetworkGraph, paye
                }
 
                // Step (3).
-               // Stop either when recommended value is reached,
-               // or if during last iteration no new path was found.
-               // In the latter case, making another path finding attempt could not help,
-               // because we deterministically terminate the search due to low liquidity.
+               // Stop either when the recommended value is reached or if no new path was found in this
+               // iteration.
+               // In the latter case, making another path finding attempt won't help,
+               // because we deterministically terminated the search due to low liquidity.
                if already_collected_value_msat >= recommended_value_msat || !found_new_path {
                        break 'paths_collection;
+               } else if found_new_path && already_collected_value_msat == final_value_msat && payment_paths.len() == 1 {
+                       // Further, if this was our first walk of the graph, and we weren't limited by an
+                       // htlc_minimum_msat, return immediately because this path should suffice. If we were
+                       // limited by an htlc_minimum_msat value, find another path with a higher value,
+                       // potentially allowing us to pay fees to meet the htlc_minimum on the new path while
+                       // still keeping a lower total fee than this path.
+                       if !hit_minimum_limit {
+                               break 'paths_collection;
+                       }
+                       path_value_msat = recommended_value_msat;
                }
        }
 
@@ -969,7 +1120,7 @@ pub fn get_route<L: Deref>(our_node_id: &PublicKey, network: &NetworkGraph, paye
                                // Now, substract the overpaid value from the most-expensive path.
                                // TODO: this could also be optimized by also sorting by feerate_per_sat_routed,
                                // so that the sender pays less fees overall. And also htlc_minimum_msat.
-                               cur_route.sort_by_key(|path| { path.hops.iter().map(|hop| hop.channel_fees.proportional_millionths as u64).sum::<u64>() });
+                               cur_route.sort_by_key(|path| { path.hops.iter().map(|hop| hop.0.channel_fees.proportional_millionths as u64).sum::<u64>() });
                                let expensive_payment_path = cur_route.first_mut().unwrap();
                                // We already dropped all the small channels above, meaning all the
                                // remaining channels are larger than remaining overpaid_value_msat.
@@ -987,7 +1138,16 @@ pub fn get_route<L: Deref>(our_node_id: &PublicKey, network: &NetworkGraph, paye
        drawn_routes.sort_by_key(|paths| paths.iter().map(|path| path.get_total_fee_paid_msat()).sum::<u64>());
        let mut selected_paths = Vec::<Vec<RouteHop>>::new();
        for payment_path in drawn_routes.first().unwrap() {
-               selected_paths.push(payment_path.hops.iter().map(|payment_hop| payment_hop.route_hop.clone()).collect());
+               selected_paths.push(payment_path.hops.iter().map(|(payment_hop, node_features)| {
+                       RouteHop {
+                               pubkey: payment_hop.pubkey,
+                               node_features: node_features.clone(),
+                               short_channel_id: payment_hop.short_channel_id,
+                               channel_features: payment_hop.channel_features.clone(),
+                               fee_msat: payment_hop.fee_msat,
+                               cltv_expiry_delta: payment_hop.cltv_expiry_delta,
+                       }
+               }).collect());
        }
 
        if let Some(features) = &payee_features {
@@ -3400,6 +3560,361 @@ mod tests {
                        assert_eq!(total_amount_paid_msat, 90_000);
                }
        }
+
+       #[test]
+       fn min_criteria_consistency() {
+               // Test that we don't use an inconsistent metric between updating and walking nodes during
+               // our Dijkstra's pass. In the initial version of MPP, the "best source" for a given node
+               // was updated with a different criterion from the heap sorting, resulting in loops in
+               // calculated paths. We test for that specific case here.
+
+               // We construct a network that looks like this:
+               //
+               //            node2 -1(3)2- node3
+               //              2          2
+               //               (2)     (4)
+               //                  1   1
+               //    node1 -1(5)2- node4 -1(1)2- node6
+               //    2
+               //   (6)
+               //        1
+               // our_node
+               //
+               // We create a loop on the side of our real path - our destination is node 6, with a
+               // previous hop of node 4. From 4, the cheapest previous path is channel 2 from node 2,
+               // followed by node 3 over channel 3. Thereafter, the cheapest next-hop is back to node 4
+               // (this time over channel 4). Channel 4 has 0 htlc_minimum_msat whereas channel 1 (the
+               // other channel with a previous-hop of node 4) has a high (but irrelevant to the overall
+               // payment) htlc_minimum_msat. In the original algorithm, this resulted in node4's
+               // "previous hop" being set to node 3, creating a loop in the path.
+               let secp_ctx = Secp256k1::new();
+               let logger = Arc::new(test_utils::TestLogger::new());
+               let net_graph_msg_handler = NetGraphMsgHandler::new(genesis_block(Network::Testnet).header.block_hash(), None, Arc::clone(&logger));
+               let (our_privkey, our_id, privkeys, nodes) = get_nodes(&secp_ctx);
+
+               add_channel(&net_graph_msg_handler, &secp_ctx, &our_privkey, &privkeys[1], ChannelFeatures::from_le_bytes(id_to_feature_flags(6)), 6);
+               update_channel(&net_graph_msg_handler, &secp_ctx, &our_privkey, UnsignedChannelUpdate {
+                       chain_hash: genesis_block(Network::Testnet).header.block_hash(),
+                       short_channel_id: 6,
+                       timestamp: 1,
+                       flags: 0,
+                       cltv_expiry_delta: (6 << 8) | 0,
+                       htlc_minimum_msat: 0,
+                       htlc_maximum_msat: OptionalField::Absent,
+                       fee_base_msat: 0,
+                       fee_proportional_millionths: 0,
+                       excess_data: Vec::new()
+               });
+               add_or_update_node(&net_graph_msg_handler, &secp_ctx, &privkeys[1], NodeFeatures::from_le_bytes(id_to_feature_flags(1)), 0);
+
+               add_channel(&net_graph_msg_handler, &secp_ctx, &privkeys[1], &privkeys[4], ChannelFeatures::from_le_bytes(id_to_feature_flags(5)), 5);
+               update_channel(&net_graph_msg_handler, &secp_ctx, &privkeys[1], UnsignedChannelUpdate {
+                       chain_hash: genesis_block(Network::Testnet).header.block_hash(),
+                       short_channel_id: 5,
+                       timestamp: 1,
+                       flags: 0,
+                       cltv_expiry_delta: (5 << 8) | 0,
+                       htlc_minimum_msat: 0,
+                       htlc_maximum_msat: OptionalField::Absent,
+                       fee_base_msat: 100,
+                       fee_proportional_millionths: 0,
+                       excess_data: Vec::new()
+               });
+               add_or_update_node(&net_graph_msg_handler, &secp_ctx, &privkeys[4], NodeFeatures::from_le_bytes(id_to_feature_flags(4)), 0);
+
+               add_channel(&net_graph_msg_handler, &secp_ctx, &privkeys[4], &privkeys[3], ChannelFeatures::from_le_bytes(id_to_feature_flags(4)), 4);
+               update_channel(&net_graph_msg_handler, &secp_ctx, &privkeys[4], UnsignedChannelUpdate {
+                       chain_hash: genesis_block(Network::Testnet).header.block_hash(),
+                       short_channel_id: 4,
+                       timestamp: 1,
+                       flags: 0,
+                       cltv_expiry_delta: (4 << 8) | 0,
+                       htlc_minimum_msat: 0,
+                       htlc_maximum_msat: OptionalField::Absent,
+                       fee_base_msat: 0,
+                       fee_proportional_millionths: 0,
+                       excess_data: Vec::new()
+               });
+               add_or_update_node(&net_graph_msg_handler, &secp_ctx, &privkeys[3], NodeFeatures::from_le_bytes(id_to_feature_flags(3)), 0);
+
+               add_channel(&net_graph_msg_handler, &secp_ctx, &privkeys[3], &privkeys[2], ChannelFeatures::from_le_bytes(id_to_feature_flags(3)), 3);
+               update_channel(&net_graph_msg_handler, &secp_ctx, &privkeys[3], UnsignedChannelUpdate {
+                       chain_hash: genesis_block(Network::Testnet).header.block_hash(),
+                       short_channel_id: 3,
+                       timestamp: 1,
+                       flags: 0,
+                       cltv_expiry_delta: (3 << 8) | 0,
+                       htlc_minimum_msat: 0,
+                       htlc_maximum_msat: OptionalField::Absent,
+                       fee_base_msat: 0,
+                       fee_proportional_millionths: 0,
+                       excess_data: Vec::new()
+               });
+               add_or_update_node(&net_graph_msg_handler, &secp_ctx, &privkeys[2], NodeFeatures::from_le_bytes(id_to_feature_flags(2)), 0);
+
+               add_channel(&net_graph_msg_handler, &secp_ctx, &privkeys[2], &privkeys[4], ChannelFeatures::from_le_bytes(id_to_feature_flags(2)), 2);
+               update_channel(&net_graph_msg_handler, &secp_ctx, &privkeys[2], UnsignedChannelUpdate {
+                       chain_hash: genesis_block(Network::Testnet).header.block_hash(),
+                       short_channel_id: 2,
+                       timestamp: 1,
+                       flags: 0,
+                       cltv_expiry_delta: (2 << 8) | 0,
+                       htlc_minimum_msat: 0,
+                       htlc_maximum_msat: OptionalField::Absent,
+                       fee_base_msat: 0,
+                       fee_proportional_millionths: 0,
+                       excess_data: Vec::new()
+               });
+
+               add_channel(&net_graph_msg_handler, &secp_ctx, &privkeys[4], &privkeys[6], ChannelFeatures::from_le_bytes(id_to_feature_flags(1)), 1);
+               update_channel(&net_graph_msg_handler, &secp_ctx, &privkeys[4], UnsignedChannelUpdate {
+                       chain_hash: genesis_block(Network::Testnet).header.block_hash(),
+                       short_channel_id: 1,
+                       timestamp: 1,
+                       flags: 0,
+                       cltv_expiry_delta: (1 << 8) | 0,
+                       htlc_minimum_msat: 100,
+                       htlc_maximum_msat: OptionalField::Absent,
+                       fee_base_msat: 0,
+                       fee_proportional_millionths: 0,
+                       excess_data: Vec::new()
+               });
+               add_or_update_node(&net_graph_msg_handler, &secp_ctx, &privkeys[6], NodeFeatures::from_le_bytes(id_to_feature_flags(6)), 0);
+
+               {
+                       // Now ensure the route flows simply over nodes 1 and 4 to 6.
+                       let route = get_route(&our_id, &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[6], None, None, &Vec::new(), 10_000, 42, Arc::clone(&logger)).unwrap();
+                       assert_eq!(route.paths.len(), 1);
+                       assert_eq!(route.paths[0].len(), 3);
+
+                       assert_eq!(route.paths[0][0].pubkey, nodes[1]);
+                       assert_eq!(route.paths[0][0].short_channel_id, 6);
+                       assert_eq!(route.paths[0][0].fee_msat, 100);
+                       assert_eq!(route.paths[0][0].cltv_expiry_delta, (5 << 8) | 0);
+                       assert_eq!(route.paths[0][0].node_features.le_flags(), &id_to_feature_flags(1));
+                       assert_eq!(route.paths[0][0].channel_features.le_flags(), &id_to_feature_flags(6));
+
+                       assert_eq!(route.paths[0][1].pubkey, nodes[4]);
+                       assert_eq!(route.paths[0][1].short_channel_id, 5);
+                       assert_eq!(route.paths[0][1].fee_msat, 0);
+                       assert_eq!(route.paths[0][1].cltv_expiry_delta, (1 << 8) | 0);
+                       assert_eq!(route.paths[0][1].node_features.le_flags(), &id_to_feature_flags(4));
+                       assert_eq!(route.paths[0][1].channel_features.le_flags(), &id_to_feature_flags(5));
+
+                       assert_eq!(route.paths[0][2].pubkey, nodes[6]);
+                       assert_eq!(route.paths[0][2].short_channel_id, 1);
+                       assert_eq!(route.paths[0][2].fee_msat, 10_000);
+                       assert_eq!(route.paths[0][2].cltv_expiry_delta, 42);
+                       assert_eq!(route.paths[0][2].node_features.le_flags(), &id_to_feature_flags(6));
+                       assert_eq!(route.paths[0][2].channel_features.le_flags(), &id_to_feature_flags(1));
+               }
+       }
+
+
+       #[test]
+       fn exact_fee_liquidity_limit() {
+               // Test that if, while walking the graph, we find a hop that has exactly enough liquidity
+               // for us, including later hop fees, we take it. In the first version of our MPP algorithm
+               // we calculated fees on a higher value, resulting in us ignoring such paths.
+               let (secp_ctx, net_graph_msg_handler, _, logger) = build_graph();
+               let (our_privkey, our_id, _, nodes) = get_nodes(&secp_ctx);
+
+               // We modify the graph to set the htlc_maximum of channel 2 to below the value we wish to
+               // send.
+               update_channel(&net_graph_msg_handler, &secp_ctx, &our_privkey, UnsignedChannelUpdate {
+                       chain_hash: genesis_block(Network::Testnet).header.block_hash(),
+                       short_channel_id: 2,
+                       timestamp: 2,
+                       flags: 0,
+                       cltv_expiry_delta: 0,
+                       htlc_minimum_msat: 0,
+                       htlc_maximum_msat: OptionalField::Present(85_000),
+                       fee_base_msat: 0,
+                       fee_proportional_millionths: 0,
+                       excess_data: Vec::new()
+               });
+
+               update_channel(&net_graph_msg_handler, &secp_ctx, &our_privkey, UnsignedChannelUpdate {
+                       chain_hash: genesis_block(Network::Testnet).header.block_hash(),
+                       short_channel_id: 12,
+                       timestamp: 2,
+                       flags: 0,
+                       cltv_expiry_delta: (4 << 8) | 1,
+                       htlc_minimum_msat: 0,
+                       htlc_maximum_msat: OptionalField::Present(270_000),
+                       fee_base_msat: 0,
+                       fee_proportional_millionths: 1000000,
+                       excess_data: Vec::new()
+               });
+
+               {
+                       // Now, attempt to route 90 sats, which is exactly 90 sats at the last hop, plus the
+                       // 200% fee charged channel 13 in the 1-to-2 direction.
+                       let route = get_route(&our_id, &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[2], None, None, &Vec::new(), 90_000, 42, Arc::clone(&logger)).unwrap();
+                       assert_eq!(route.paths.len(), 1);
+                       assert_eq!(route.paths[0].len(), 2);
+
+                       assert_eq!(route.paths[0][0].pubkey, nodes[7]);
+                       assert_eq!(route.paths[0][0].short_channel_id, 12);
+                       assert_eq!(route.paths[0][0].fee_msat, 90_000*2);
+                       assert_eq!(route.paths[0][0].cltv_expiry_delta, (13 << 8) | 1);
+                       assert_eq!(route.paths[0][0].node_features.le_flags(), &id_to_feature_flags(8));
+                       assert_eq!(route.paths[0][0].channel_features.le_flags(), &id_to_feature_flags(12));
+
+                       assert_eq!(route.paths[0][1].pubkey, nodes[2]);
+                       assert_eq!(route.paths[0][1].short_channel_id, 13);
+                       assert_eq!(route.paths[0][1].fee_msat, 90_000);
+                       assert_eq!(route.paths[0][1].cltv_expiry_delta, 42);
+                       assert_eq!(route.paths[0][1].node_features.le_flags(), &id_to_feature_flags(3));
+                       assert_eq!(route.paths[0][1].channel_features.le_flags(), &id_to_feature_flags(13));
+               }
+       }
+
+       #[test]
+       fn htlc_max_reduction_below_min() {
+               // Test that if, while walking the graph, we reduce the value being sent to meet an
+               // htlc_maximum_msat, we don't end up undershooting a later htlc_minimum_msat. In the
+               // initial version of MPP we'd accept such routes but reject them while recalculating fees,
+               // resulting in us thinking there is no possible path, even if other paths exist.
+               let (secp_ctx, net_graph_msg_handler, _, logger) = build_graph();
+               let (our_privkey, our_id, privkeys, nodes) = get_nodes(&secp_ctx);
+
+               // We modify the graph to set the htlc_minimum of channel 2 and 4 as needed - channel 2
+               // gets an htlc_maximum_msat of 80_000 and channel 4 an htlc_minimum_msat of 90_000. We
+               // then try to send 90_000.
+               update_channel(&net_graph_msg_handler, &secp_ctx, &our_privkey, UnsignedChannelUpdate {
+                       chain_hash: genesis_block(Network::Testnet).header.block_hash(),
+                       short_channel_id: 2,
+                       timestamp: 2,
+                       flags: 0,
+                       cltv_expiry_delta: 0,
+                       htlc_minimum_msat: 0,
+                       htlc_maximum_msat: OptionalField::Present(80_000),
+                       fee_base_msat: 0,
+                       fee_proportional_millionths: 0,
+                       excess_data: Vec::new()
+               });
+               update_channel(&net_graph_msg_handler, &secp_ctx, &privkeys[1], UnsignedChannelUpdate {
+                       chain_hash: genesis_block(Network::Testnet).header.block_hash(),
+                       short_channel_id: 4,
+                       timestamp: 2,
+                       flags: 0,
+                       cltv_expiry_delta: (4 << 8) | 1,
+                       htlc_minimum_msat: 90_000,
+                       htlc_maximum_msat: OptionalField::Absent,
+                       fee_base_msat: 0,
+                       fee_proportional_millionths: 0,
+                       excess_data: Vec::new()
+               });
+
+               {
+                       // Now, attempt to route 90 sats, hitting the htlc_minimum on channel 4, but
+                       // overshooting the htlc_maximum on channel 2. Thus, we should pick the (absurdly
+                       // expensive) channels 12-13 path.
+                       let route = get_route(&our_id, &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[2], Some(InvoiceFeatures::known()), None, &Vec::new(), 90_000, 42, Arc::clone(&logger)).unwrap();
+                       assert_eq!(route.paths.len(), 1);
+                       assert_eq!(route.paths[0].len(), 2);
+
+                       assert_eq!(route.paths[0][0].pubkey, nodes[7]);
+                       assert_eq!(route.paths[0][0].short_channel_id, 12);
+                       assert_eq!(route.paths[0][0].fee_msat, 90_000*2);
+                       assert_eq!(route.paths[0][0].cltv_expiry_delta, (13 << 8) | 1);
+                       assert_eq!(route.paths[0][0].node_features.le_flags(), &id_to_feature_flags(8));
+                       assert_eq!(route.paths[0][0].channel_features.le_flags(), &id_to_feature_flags(12));
+
+                       assert_eq!(route.paths[0][1].pubkey, nodes[2]);
+                       assert_eq!(route.paths[0][1].short_channel_id, 13);
+                       assert_eq!(route.paths[0][1].fee_msat, 90_000);
+                       assert_eq!(route.paths[0][1].cltv_expiry_delta, 42);
+                       assert_eq!(route.paths[0][1].node_features.le_flags(), InvoiceFeatures::known().le_flags());
+                       assert_eq!(route.paths[0][1].channel_features.le_flags(), &id_to_feature_flags(13));
+               }
+       }
+
+       use std::fs::File;
+       use util::ser::Readable;
+       /// Tries to open a network graph file, or panics with a URL to fetch it.
+       pub(super) fn get_route_file() -> Result<std::fs::File, std::io::Error> {
+               let res = File::open("net_graph-2021-02-12.bin") // By default we're run in RL/lightning
+                       .or_else(|_| File::open("lightning/net_graph-2021-02-12.bin")) // We may be run manually in RL/
+                       .or_else(|_| { // Fall back to guessing based on the binary location
+                               // path is likely something like .../rust-lightning/target/debug/deps/lightning-...
+                               let mut path = std::env::current_exe().unwrap();
+                               path.pop(); // lightning-...
+                               path.pop(); // deps
+                               path.pop(); // debug
+                               path.pop(); // target
+                               path.push("lightning");
+                               path.push("net_graph-2021-02-12.bin");
+                               eprintln!("{}", path.to_str().unwrap());
+                               File::open(path)
+                       });
+               #[cfg(require_route_graph_test)]
+               return Ok(res.expect("Didn't have route graph and was configured to require it"));
+               res
+       }
+
+       pub(super) fn random_init_seed() -> u64 {
+               // Because the default HashMap in std pulls OS randomness, we can use it as a (bad) RNG.
+               use std::hash::{BuildHasher, Hasher};
+               let seed = std::collections::hash_map::RandomState::new().build_hasher().finish();
+               println!("Using seed of {}", seed);
+               seed
+       }
+
+       #[test]
+       fn generate_routes() {
+               let mut d = match get_route_file() {
+                       Ok(f) => f,
+                       Err(_) => {
+                               eprintln!("Please fetch https://bitcoin.ninja/ldk-net_graph-879e309c128-2020-02-12.bin and place it at lightning/net_graph-2021-02-12.bin");
+                               return;
+                       },
+               };
+               let graph = NetworkGraph::read(&mut d).unwrap();
+
+               // First, get 100 (source, destination) pairs for which route-getting actually succeeds...
+               let mut seed = random_init_seed() as usize;
+               'load_endpoints: for _ in 0..10 {
+                       loop {
+                               seed = seed.overflowing_mul(0xdeadbeef).0;
+                               let src = graph.get_nodes().keys().skip(seed % graph.get_nodes().len()).next().unwrap();
+                               seed = seed.overflowing_mul(0xdeadbeef).0;
+                               let dst = graph.get_nodes().keys().skip(seed % graph.get_nodes().len()).next().unwrap();
+                               let amt = seed as u64 % 200_000_000;
+                               if get_route(src, &graph, dst, None, None, &[], amt, 42, &test_utils::TestLogger::new()).is_ok() {
+                                       continue 'load_endpoints;
+                               }
+                       }
+               }
+       }
+
+       #[test]
+       fn generate_routes_mpp() {
+               let mut d = match get_route_file() {
+                       Ok(f) => f,
+                       Err(_) => {
+                               eprintln!("Please fetch https://bitcoin.ninja/ldk-net_graph-879e309c128-2020-02-12.bin and place it at lightning/net_graph-2021-02-12.bin");
+                               return;
+                       },
+               };
+               let graph = NetworkGraph::read(&mut d).unwrap();
+
+               // First, get 100 (source, destination) pairs for which route-getting actually succeeds...
+               let mut seed = random_init_seed() as usize;
+               'load_endpoints: for _ in 0..10 {
+                       loop {
+                               seed = seed.overflowing_mul(0xdeadbeef).0;
+                               let src = graph.get_nodes().keys().skip(seed % graph.get_nodes().len()).next().unwrap();
+                               seed = seed.overflowing_mul(0xdeadbeef).0;
+                               let dst = graph.get_nodes().keys().skip(seed % graph.get_nodes().len()).next().unwrap();
+                               let amt = seed as u64 % 200_000_000;
+                               if get_route(src, &graph, dst, Some(InvoiceFeatures::known()), None, &[], amt, 42, &test_utils::TestLogger::new()).is_ok() {
+                                       continue 'load_endpoints;
+                               }
+                       }
+               }
+       }
 }
 
 #[cfg(all(test, feature = "unstable"))]
@@ -3417,7 +3932,8 @@ mod benches {
 
        #[bench]
        fn generate_routes(bench: &mut Bencher) {
-               let mut d = File::open("net_graph-2021-02-12.bin").expect("Please fetch https://bitcoin.ninja/ldk-net_graph-879e309c128-2020-02-12.bin and place it at lightning/net_graph-2021-02-12.bin");
+               let mut d = tests::get_route_file()
+                       .expect("Please fetch https://bitcoin.ninja/ldk-net_graph-879e309c128-2020-02-12.bin and place it at lightning/net_graph-2021-02-12.bin");
                let graph = NetworkGraph::read(&mut d).unwrap();
 
                // First, get 100 (source, destination) pairs for which route-getting actually succeeds...
@@ -3448,7 +3964,8 @@ mod benches {
 
        #[bench]
        fn generate_mpp_routes(bench: &mut Bencher) {
-               let mut d = File::open("net_graph-2021-02-12.bin").expect("Please fetch https://bitcoin.ninja/ldk-net_graph-879e309c128-2020-02-12.bin and place it at lightning/net_graph-2021-02-12.bin");
+               let mut d = tests::get_route_file()
+                       .expect("Please fetch https://bitcoin.ninja/ldk-net_graph-879e309c128-2020-02-12.bin and place it at lightning/net_graph-2021-02-12.bin");
                let graph = NetworkGraph::read(&mut d).unwrap();
 
                // First, get 100 (source, destination) pairs for which route-getting actually succeeds...
index 2e9e793bdd806ec5205ece7f7b493df312d5ec31..8b0152d834e44972995805f1b6f79aff36d8d07c 100644 (file)
@@ -9,7 +9,7 @@
 
 use ln::chan_utils::{HTLCOutputInCommitment, ChannelPublicKeys, HolderCommitmentTransaction, CommitmentTransaction, ChannelTransactionParameters, TrustedCommitmentTransaction};
 use ln::{chan_utils, msgs};
-use chain::keysinterface::{Sign, InMemorySigner};
+use chain::keysinterface::{Sign, InMemorySigner, BaseSign};
 
 use std::cmp;
 use std::sync::{Mutex, Arc};
@@ -74,8 +74,8 @@ impl EnforcingSigner {
        }
 }
 
-impl Sign for EnforcingSigner {
-       fn get_per_commitment_point<T: secp256k1::Signing + secp256k1::Verification>(&self, idx: u64, secp_ctx: &Secp256k1<T>) -> PublicKey {
+impl BaseSign for EnforcingSigner {
+       fn get_per_commitment_point(&self, idx: u64, secp_ctx: &Secp256k1<secp256k1::All>) -> PublicKey {
                self.inner.get_per_commitment_point(idx, secp_ctx)
        }
 
@@ -91,7 +91,7 @@ impl Sign for EnforcingSigner {
        fn pubkeys(&self) -> &ChannelPublicKeys { self.inner.pubkeys() }
        fn channel_keys_id(&self) -> [u8; 32] { self.inner.channel_keys_id() }
 
-       fn sign_counterparty_commitment<T: secp256k1::Signing + secp256k1::Verification>(&self, commitment_tx: &CommitmentTransaction, secp_ctx: &Secp256k1<T>) -> Result<(Signature, Vec<Signature>), ()> {
+       fn sign_counterparty_commitment(&self, commitment_tx: &CommitmentTransaction, secp_ctx: &Secp256k1<secp256k1::All>) -> Result<(Signature, Vec<Signature>), ()> {
                self.verify_counterparty_commitment_tx(commitment_tx, secp_ctx);
 
                {
@@ -107,7 +107,7 @@ impl Sign for EnforcingSigner {
                Ok(self.inner.sign_counterparty_commitment(commitment_tx, secp_ctx).unwrap())
        }
 
-       fn sign_holder_commitment_and_htlcs<T: secp256k1::Signing + secp256k1::Verification>(&self, commitment_tx: &HolderCommitmentTransaction, secp_ctx: &Secp256k1<T>) -> Result<(Signature, Vec<Signature>), ()> {
+       fn sign_holder_commitment_and_htlcs(&self, commitment_tx: &HolderCommitmentTransaction, secp_ctx: &Secp256k1<secp256k1::All>) -> Result<(Signature, Vec<Signature>), ()> {
                let trusted_tx = self.verify_holder_commitment_tx(commitment_tx, secp_ctx);
                let commitment_txid = trusted_tx.txid();
                let holder_csv = self.inner.counterparty_selected_contest_delay();
@@ -136,23 +136,23 @@ impl Sign for EnforcingSigner {
        }
 
        #[cfg(any(test,feature = "unsafe_revoked_tx_signing"))]
-       fn unsafe_sign_holder_commitment_and_htlcs<T: secp256k1::Signing + secp256k1::Verification>(&self, commitment_tx: &HolderCommitmentTransaction, secp_ctx: &Secp256k1<T>) -> Result<(Signature, Vec<Signature>), ()> {
+       fn unsafe_sign_holder_commitment_and_htlcs(&self, commitment_tx: &HolderCommitmentTransaction, secp_ctx: &Secp256k1<secp256k1::All>) -> Result<(Signature, Vec<Signature>), ()> {
                Ok(self.inner.unsafe_sign_holder_commitment_and_htlcs(commitment_tx, secp_ctx).unwrap())
        }
 
-       fn sign_justice_transaction<T: secp256k1::Signing + secp256k1::Verification>(&self, justice_tx: &Transaction, input: usize, amount: u64, per_commitment_key: &SecretKey, htlc: &Option<HTLCOutputInCommitment>, secp_ctx: &Secp256k1<T>) -> Result<Signature, ()> {
+       fn sign_justice_transaction(&self, justice_tx: &Transaction, input: usize, amount: u64, per_commitment_key: &SecretKey, htlc: &Option<HTLCOutputInCommitment>, secp_ctx: &Secp256k1<secp256k1::All>) -> Result<Signature, ()> {
                Ok(self.inner.sign_justice_transaction(justice_tx, input, amount, per_commitment_key, htlc, secp_ctx).unwrap())
        }
 
-       fn sign_counterparty_htlc_transaction<T: secp256k1::Signing + secp256k1::Verification>(&self, htlc_tx: &Transaction, input: usize, amount: u64, per_commitment_point: &PublicKey, htlc: &HTLCOutputInCommitment, secp_ctx: &Secp256k1<T>) -> Result<Signature, ()> {
+       fn sign_counterparty_htlc_transaction(&self, htlc_tx: &Transaction, input: usize, amount: u64, per_commitment_point: &PublicKey, htlc: &HTLCOutputInCommitment, secp_ctx: &Secp256k1<secp256k1::All>) -> Result<Signature, ()> {
                Ok(self.inner.sign_counterparty_htlc_transaction(htlc_tx, input, amount, per_commitment_point, htlc, secp_ctx).unwrap())
        }
 
-       fn sign_closing_transaction<T: secp256k1::Signing>(&self, closing_tx: &Transaction, secp_ctx: &Secp256k1<T>) -> Result<Signature, ()> {
+       fn sign_closing_transaction(&self, closing_tx: &Transaction, secp_ctx: &Secp256k1<secp256k1::All>) -> Result<Signature, ()> {
                Ok(self.inner.sign_closing_transaction(closing_tx, secp_ctx).unwrap())
        }
 
-       fn sign_channel_announcement<T: secp256k1::Signing>(&self, msg: &msgs::UnsignedChannelAnnouncement, secp_ctx: &Secp256k1<T>) -> Result<Signature, ()> {
+       fn sign_channel_announcement(&self, msg: &msgs::UnsignedChannelAnnouncement, secp_ctx: &Secp256k1<secp256k1::All>) -> Result<Signature, ()> {
                self.inner.sign_channel_announcement(msg, secp_ctx)
        }
 
@@ -161,6 +161,7 @@ impl Sign for EnforcingSigner {
        }
 }
 
+impl Sign for EnforcingSigner {}
 
 impl Writeable for EnforcingSigner {
        fn write<W: Writer>(&self, writer: &mut W) -> Result<(), Error> {
index a0ccf4f816ea991e31a019351018d43a95633324..153f2f28eed11ffe7e55536c98c191714edb6443 100644 (file)
@@ -8,6 +8,7 @@
 // licenses.
 
 use chain;
+use chain::WatchedOutput;
 use chain::chaininterface;
 use chain::chaininterface::ConfirmationTarget;
 use chain::chainmonitor;
@@ -38,7 +39,7 @@ use std::time::Duration;
 use std::sync::{Mutex, Arc};
 use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
 use std::{cmp, mem};
-use std::collections::{HashMap, HashSet};
+use std::collections::{HashMap, HashSet, VecDeque};
 use chain::keysinterface::InMemorySigner;
 
 pub struct TestVecWriter(pub Vec<u8>);
@@ -185,12 +186,12 @@ impl TestPersister {
                *self.update_ret.lock().unwrap() = ret;
        }
 }
-impl channelmonitor::Persist<EnforcingSigner> for TestPersister {
-       fn persist_new_channel(&self, _funding_txo: OutPoint, _data: &channelmonitor::ChannelMonitor<EnforcingSigner>) -> Result<(), channelmonitor::ChannelMonitorUpdateErr> {
+impl<Signer: keysinterface::Sign> channelmonitor::Persist<Signer> for TestPersister {
+       fn persist_new_channel(&self, _funding_txo: OutPoint, _data: &channelmonitor::ChannelMonitor<Signer>) -> Result<(), channelmonitor::ChannelMonitorUpdateErr> {
                self.update_ret.lock().unwrap().clone()
        }
 
-       fn update_persisted_channel(&self, _funding_txo: OutPoint, _update: &channelmonitor::ChannelMonitorUpdate, _data: &channelmonitor::ChannelMonitor<EnforcingSigner>) -> Result<(), channelmonitor::ChannelMonitorUpdateErr> {
+       fn update_persisted_channel(&self, _funding_txo: OutPoint, _update: &channelmonitor::ChannelMonitorUpdate, _data: &channelmonitor::ChannelMonitor<Signer>) -> Result<(), channelmonitor::ChannelMonitorUpdateErr> {
                self.update_ret.lock().unwrap().clone()
        }
 }
@@ -517,6 +518,7 @@ pub struct TestChainSource {
        pub utxo_ret: Mutex<Result<TxOut, chain::AccessError>>,
        pub watched_txn: Mutex<HashSet<(Txid, Script)>>,
        pub watched_outputs: Mutex<HashSet<(OutPoint, Script)>>,
+       expectations: Mutex<Option<VecDeque<OnRegisterOutput>>>,
 }
 
 impl TestChainSource {
@@ -527,8 +529,17 @@ impl TestChainSource {
                        utxo_ret: Mutex::new(Ok(TxOut { value: u64::max_value(), script_pubkey })),
                        watched_txn: Mutex::new(HashSet::new()),
                        watched_outputs: Mutex::new(HashSet::new()),
+                       expectations: Mutex::new(None),
                }
        }
+
+       /// Sets an expectation that [`chain::Filter::register_output`] is called.
+       pub fn expect(&self, expectation: OnRegisterOutput) -> &Self {
+               self.expectations.lock().unwrap()
+                       .get_or_insert_with(|| VecDeque::new())
+                       .push_back(expectation);
+               self
+       }
 }
 
 impl chain::Access for TestChainSource {
@@ -546,7 +557,72 @@ impl chain::Filter for TestChainSource {
                self.watched_txn.lock().unwrap().insert((*txid, script_pubkey.clone()));
        }
 
-       fn register_output(&self, outpoint: &OutPoint, script_pubkey: &Script) {
-               self.watched_outputs.lock().unwrap().insert((*outpoint, script_pubkey.clone()));
+       fn register_output(&self, output: WatchedOutput) -> Option<(usize, Transaction)> {
+               let dependent_tx = match &mut *self.expectations.lock().unwrap() {
+                       None => None,
+                       Some(expectations) => match expectations.pop_front() {
+                               None => {
+                                       panic!("Unexpected register_output: {:?}",
+                                               (output.outpoint, output.script_pubkey));
+                               },
+                               Some(expectation) => {
+                                       assert_eq!(output.outpoint, expectation.outpoint());
+                                       assert_eq!(&output.script_pubkey, expectation.script_pubkey());
+                                       expectation.returns
+                               },
+                       },
+               };
+
+               self.watched_outputs.lock().unwrap().insert((output.outpoint, output.script_pubkey));
+               dependent_tx
+       }
+}
+
+impl Drop for TestChainSource {
+       fn drop(&mut self) {
+               if std::thread::panicking() {
+                       return;
+               }
+
+               if let Some(expectations) = &*self.expectations.lock().unwrap() {
+                       if !expectations.is_empty() {
+                               panic!("Unsatisfied expectations: {:?}", expectations);
+                       }
+               }
+       }
+}
+
+/// An expectation that [`chain::Filter::register_output`] was called with a transaction output and
+/// returns an optional dependent transaction that spends the output in the same block.
+pub struct OnRegisterOutput {
+       /// The transaction output to register.
+       pub with: TxOutReference,
+
+       /// A dependent transaction spending the output along with its position in the block.
+       pub returns: Option<(usize, Transaction)>,
+}
+
+/// A transaction output as identified by an index into a transaction's output list.
+pub struct TxOutReference(pub Transaction, pub usize);
+
+impl OnRegisterOutput {
+       fn outpoint(&self) -> OutPoint {
+               let txid = self.with.0.txid();
+               let index = self.with.1 as u16;
+               OutPoint { txid, index }
+       }
+
+       fn script_pubkey(&self) -> &Script {
+               let index = self.with.1;
+               &self.with.0.output[index].script_pubkey
+       }
+}
+
+impl std::fmt::Debug for OnRegisterOutput {
+       fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+               f.debug_struct("OnRegisterOutput")
+                       .field("outpoint", &self.outpoint())
+                       .field("script_pubkey", self.script_pubkey())
+                       .finish()
        }
 }