From: Matt Corallo Date: Sat, 29 Apr 2023 22:32:57 +0000 (+0000) Subject: Implement the `UtxoSource` interface for REST/RPC clients X-Git-Tag: v0.0.117-alpha1~39^2~3 X-Git-Url: http://git.bitcoin.ninja/index.cgi?a=commitdiff_plain;h=01857b51a1e798ef344af845351b1c05c1c8a677;p=rust-lightning Implement the `UtxoSource` interface for REST/RPC clients In LDK, we expect users operating nodes on the public network to implement the `UtxoSource` interface in order to validate the gossip they receive from the network. Sadly, because the DoS attack of flooding a node's gossip store isn't a common issue, and because we do not provide an implementation off-the-shelf to make doing so easily, many of our downstream users do not have a `UtxoSource` implementation. In order to change that, here we implement an async `UtxoSource` in the `lightning-block-sync` crate, providing one for users who sync the chain from Bitcoin Core's RPC or REST interfaces. --- diff --git a/lightning-block-sync/src/convert.rs b/lightning-block-sync/src/convert.rs index d6294e1d2..bf9e95776 100644 --- a/lightning-block-sync/src/convert.rs +++ b/lightning-block-sync/src/convert.rs @@ -13,8 +13,14 @@ use serde_json; use std::convert::From; use std::convert::TryFrom; use std::convert::TryInto; +use std::str::FromStr; use bitcoin::hashes::Hash; +impl TryInto for JsonResponse { + type Error = std::io::Error; + fn try_into(self) -> Result { Ok(self.0) } +} + /// Conversion from `std::io::Error` into `BlockSourceError`. impl From for BlockSourceError { fn from(e: std::io::Error) -> BlockSourceError { @@ -38,6 +44,17 @@ impl TryInto for BinaryResponse { } } +/// Parses binary data as a block hash. +impl TryInto for BinaryResponse { + type Error = std::io::Error; + + fn try_into(self) -> std::io::Result { + BlockHash::from_slice(&self.0).map_err(|_| + std::io::Error::new(std::io::ErrorKind::InvalidData, "bad block hash length") + ) + } +} + /// Converts a JSON value into block header data. The JSON value may be an object representing a /// block header or an array of such objects. In the latter case, the first object is converted. impl TryInto for JsonResponse { @@ -226,6 +243,46 @@ impl TryInto for JsonResponse { } } +impl TryInto for JsonResponse { + type Error = std::io::Error; + + fn try_into(self) -> std::io::Result { + match self.0.as_str() { + None => Err(std::io::Error::new(std::io::ErrorKind::InvalidData, "expected JSON string")), + Some(hex_data) if hex_data.len() != 64 => + Err(std::io::Error::new(std::io::ErrorKind::InvalidData, "invalid hash length")), + Some(hex_data) => BlockHash::from_str(hex_data) + .map_err(|_| std::io::Error::new(std::io::ErrorKind::InvalidData, "invalid hex data")), + } + } +} + +/// The REST `getutxos` endpoint retuns a whole pile of data we don't care about and one bit we do +/// - whether the `hit bitmap` field had any entries. Thus we condense the result down into only +/// that. +pub(crate) struct GetUtxosResponse { + pub(crate) hit_bitmap_nonempty: bool +} + +impl TryInto for JsonResponse { + type Error = std::io::Error; + + fn try_into(self) -> std::io::Result { + let bitmap_str = + self.0.as_object().ok_or(std::io::Error::new(std::io::ErrorKind::InvalidData, "expected an object"))? + .get("bitmap").ok_or(std::io::Error::new(std::io::ErrorKind::InvalidData, "missing bitmap field"))? + .as_str().ok_or(std::io::Error::new(std::io::ErrorKind::InvalidData, "bitmap should be an str"))?; + let mut hit_bitmap_nonempty = false; + for c in bitmap_str.chars() { + if c < '0' || c > '9' { + return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, "invalid byte")); + } + if c > '0' { hit_bitmap_nonempty = true; } + } + Ok(GetUtxosResponse { hit_bitmap_nonempty }) + } +} + #[cfg(test)] pub(crate) mod tests { use super::*; diff --git a/lightning-block-sync/src/gossip.rs b/lightning-block-sync/src/gossip.rs new file mode 100644 index 000000000..3eb7ad4ae --- /dev/null +++ b/lightning-block-sync/src/gossip.rs @@ -0,0 +1,201 @@ +//! When fetching gossip from peers, lightning nodes need to validate that gossip against the +//! current UTXO set. This module defines an implementation of the LDK API required to do so +//! against a [`BlockSource`] which implements a few additional methods for accessing the UTXO set. + +use crate::{AsyncBlockSourceResult, BlockData, BlockSource}; + +use bitcoin::blockdata::transaction::{TxOut, OutPoint}; +use bitcoin::hash_types::BlockHash; + +use lightning::sign::NodeSigner; + +use lightning::ln::peer_handler::{CustomMessageHandler, PeerManager, SocketDescriptor}; +use lightning::ln::msgs::{ChannelMessageHandler, OnionMessageHandler}; + +use lightning::routing::gossip::{NetworkGraph, P2PGossipSync}; +use lightning::routing::utxo::{UtxoFuture, UtxoLookup, UtxoResult, UtxoLookupError}; + +use lightning::util::logger::Logger; + +use std::sync::Arc; +use std::future::Future; +use std::ops::Deref; + +/// A trait which extends [`BlockSource`] and can be queried to fetch the block at a given height +/// as well as whether a given output is unspent (i.e. a member of the current UTXO set). +/// +/// Note that while this is implementable for a [`BlockSource`] which returns filtered block data +/// (i.e. [`BlockData::HeaderOnly`] for [`BlockSource::get_block`] requests), such an +/// implementation will reject all gossip as it is not fully able to verify the UTXOs referenced. +/// +/// For efficiency, an implementation may consider caching some set of blocks, as many redundant +/// calls may be made. +pub trait UtxoSource : BlockSource + 'static { + /// Fetches the block hash of the block at the given height. + /// + /// This will, in turn, be passed to to [`BlockSource::get_block`] to fetch the block needed + /// for gossip validation. + fn get_block_hash_by_height<'a>(&'a self, block_height: u32) -> AsyncBlockSourceResult<'a, BlockHash>; + + /// Returns true if the given output has *not* been spent, i.e. is a member of the current UTXO + /// set. + fn is_output_unspent<'a>(&'a self, outpoint: OutPoint) -> AsyncBlockSourceResult<'a, bool>; +} + +/// A generic trait which is able to spawn futures in the background. +/// +/// If the `tokio` feature is enabled, this is implemented on `TokioSpawner` struct which +/// delegates to `tokio::spawn()`. +pub trait FutureSpawner : Send + Sync + 'static { + /// Spawns the given future as a background task. + /// + /// This method MUST NOT block on the given future immediately. + fn spawn + Send + 'static>(&self, future: T); +} + +#[cfg(feature = "tokio")] +/// A trivial [`FutureSpawner`] which delegates to `tokio::spawn`. +pub struct TokioSpawner; +#[cfg(feature = "tokio")] +impl FutureSpawner for TokioSpawner { + fn spawn + Send + 'static>(&self, future: T) { + tokio::spawn(future); + } +} + +/// A struct which wraps a [`UtxoSource`] and a few LDK objects and implements the LDK +/// [`UtxoLookup`] trait. +/// +/// Note that if you're using this against a Bitcoin Core REST or RPC server, you likely wish to +/// increase the `rpcworkqueue` setting in Bitcoin Core as LDK attempts to parallelize requests (a +/// value of 1024 should more than suffice), and ensure you have sufficient file descriptors +/// available on both Bitcoin Core and your LDK application for each request to hold its own +/// connection. +pub struct GossipVerifier where + Blocks::Target: UtxoSource, + L::Target: Logger, + CM::Target: ChannelMessageHandler, + OM::Target: OnionMessageHandler, + CMH::Target: CustomMessageHandler, + NS::Target: NodeSigner, +{ + source: Blocks, + peer_manager: Arc>, Self, L>>, OM, L, CMH, NS>>, + gossiper: Arc>, Self, L>>, + spawn: S, +} + +impl GossipVerifier where + Blocks::Target: UtxoSource, + L::Target: Logger, + CM::Target: ChannelMessageHandler, + OM::Target: OnionMessageHandler, + CMH::Target: CustomMessageHandler, + NS::Target: NodeSigner, +{ + /// Constructs a new [`GossipVerifier`]. + /// + /// This is expected to be given to a [`P2PGossipSync`] (initially constructed with `None` for + /// the UTXO lookup) via [`P2PGossipSync::add_utxo_lookup`]. + pub fn new(source: Blocks, spawn: S, gossiper: Arc>, Self, L>>, peer_manager: Arc>, Self, L>>, OM, L, CMH, NS>>) -> Self { + Self { source, spawn, gossiper, peer_manager } + } + + async fn retrieve_utxo(source: Blocks, short_channel_id: u64) -> Result { + let block_height = (short_channel_id >> 5 * 8) as u32; // block height is most significant three bytes + let transaction_index = ((short_channel_id >> 2 * 8) & 0xffffff) as u32; + let output_index = (short_channel_id & 0xffff) as u16; + + let block_hash = source.get_block_hash_by_height(block_height).await + .map_err(|_| UtxoLookupError::UnknownTx)?; + let block_data = source.get_block(&block_hash).await + .map_err(|_| UtxoLookupError::UnknownTx)?; + let mut block = match block_data { + BlockData::HeaderOnly(_) => return Err(UtxoLookupError::UnknownTx), + BlockData::FullBlock(block) => block, + }; + if transaction_index as usize >= block.txdata.len() { + return Err(UtxoLookupError::UnknownTx); + } + let mut transaction = block.txdata.swap_remove(transaction_index as usize); + if output_index as usize >= transaction.output.len() { + return Err(UtxoLookupError::UnknownTx); + } + let outpoint_unspent = + source.is_output_unspent(OutPoint::new(transaction.txid(), output_index.into())).await + .map_err(|_| UtxoLookupError::UnknownTx)?; + if outpoint_unspent { + Ok(transaction.output.swap_remove(output_index as usize)) + } else { + Err(UtxoLookupError::UnknownTx) + } + } +} + +impl Deref for GossipVerifier where + Blocks::Target: UtxoSource, + L::Target: Logger, + CM::Target: ChannelMessageHandler, + OM::Target: OnionMessageHandler, + CMH::Target: CustomMessageHandler, + NS::Target: NodeSigner, +{ + type Target = Self; + fn deref(&self) -> &Self { self } +} + + +impl UtxoLookup for GossipVerifier where + Blocks::Target: UtxoSource, + L::Target: Logger, + CM::Target: ChannelMessageHandler, + OM::Target: OnionMessageHandler, + CMH::Target: CustomMessageHandler, + NS::Target: NodeSigner, +{ + fn get_utxo(&self, _genesis_hash: &BlockHash, short_channel_id: u64) -> UtxoResult { + let res = UtxoFuture::new(); + let fut = res.clone(); + let source = self.source.clone(); + let gossiper = Arc::clone(&self.gossiper); + let pm = Arc::clone(&self.peer_manager); + self.spawn.spawn(async move { + let res = Self::retrieve_utxo(source, short_channel_id).await; + fut.resolve(gossiper.network_graph(), &*gossiper, res); + pm.process_events(); + }); + UtxoResult::Async(res) + } +} diff --git a/lightning-block-sync/src/lib.rs b/lightning-block-sync/src/lib.rs index 5c7c0dee8..3561a1b5d 100644 --- a/lightning-block-sync/src/lib.rs +++ b/lightning-block-sync/src/lib.rs @@ -28,6 +28,8 @@ pub mod http; pub mod init; pub mod poll; +pub mod gossip; + #[cfg(feature = "rest-client")] pub mod rest; diff --git a/lightning-block-sync/src/rest.rs b/lightning-block-sync/src/rest.rs index 430089301..5690da12e 100644 --- a/lightning-block-sync/src/rest.rs +++ b/lightning-block-sync/src/rest.rs @@ -3,7 +3,10 @@ use crate::{BlockData, BlockHeaderData, BlockSource, AsyncBlockSourceResult}; use crate::http::{BinaryResponse, HttpEndpoint, HttpClient, JsonResponse}; +use crate::gossip::UtxoSource; +use crate::convert::GetUtxosResponse; +use bitcoin::OutPoint; use bitcoin::hash_types::BlockHash; use bitcoin::hashes::hex::ToHex; @@ -60,11 +63,30 @@ impl BlockSource for RestClient { } } +impl UtxoSource for RestClient { + fn get_block_hash_by_height<'a>(&'a self, block_height: u32) -> AsyncBlockSourceResult<'a, BlockHash> { + Box::pin(async move { + let resource_path = format!("blockhashbyheight/{}.bin", block_height); + Ok(self.request_resource::(&resource_path).await?) + }) + } + + fn is_output_unspent<'a>(&'a self, outpoint: OutPoint) -> AsyncBlockSourceResult<'a, bool> { + Box::pin(async move { + let resource_path = format!("getutxos/{}-{}.json", outpoint.txid.to_hex(), outpoint.vout); + let utxo_result = + self.request_resource::(&resource_path).await?; + Ok(utxo_result.hit_bitmap_nonempty) + }) + } +} + #[cfg(test)] mod tests { use super::*; use crate::http::BinaryResponse; use crate::http::client_tests::{HttpServer, MessageBody}; + use bitcoin::hashes::Hash; /// Parses binary data as a string-encoded `u32`. impl TryInto for BinaryResponse { @@ -113,4 +135,32 @@ mod tests { Ok(n) => assert_eq!(n, 42), } } + + #[tokio::test] + async fn parses_negative_getutxos() { + let server = HttpServer::responding_with_ok(MessageBody::Content( + // A real response contains a few more fields, but we actually only look at the + // "bitmap" field, so this should suffice for testing + "{\"chainHeight\": 1, \"bitmap\":\"0\",\"utxos\":[]}" + )); + let client = RestClient::new(server.endpoint()).unwrap(); + + let outpoint = OutPoint::new(bitcoin::Txid::from_inner([0; 32]), 0); + let unspent_output = client.is_output_unspent(outpoint).await.unwrap(); + assert_eq!(unspent_output, false); + } + + #[tokio::test] + async fn parses_positive_getutxos() { + let server = HttpServer::responding_with_ok(MessageBody::Content( + // A real response contains lots more data, but we actually only look at the "bitmap" + // field, so this should suffice for testing + "{\"chainHeight\": 1, \"bitmap\":\"1\",\"utxos\":[]}" + )); + let client = RestClient::new(server.endpoint()).unwrap(); + + let outpoint = OutPoint::new(bitcoin::Txid::from_inner([0; 32]), 0); + let unspent_output = client.is_output_unspent(outpoint).await.unwrap(); + assert_eq!(unspent_output, true); + } } diff --git a/lightning-block-sync/src/rpc.rs b/lightning-block-sync/src/rpc.rs index c778279ae..0ad94040a 100644 --- a/lightning-block-sync/src/rpc.rs +++ b/lightning-block-sync/src/rpc.rs @@ -3,9 +3,11 @@ use crate::{BlockData, BlockHeaderData, BlockSource, AsyncBlockSourceResult}; use crate::http::{HttpClient, HttpEndpoint, HttpError, JsonResponse}; +use crate::gossip::UtxoSource; use bitcoin::hash_types::BlockHash; use bitcoin::hashes::hex::ToHex; +use bitcoin::OutPoint; use std::sync::Mutex; @@ -138,11 +140,33 @@ impl BlockSource for RpcClient { } } +impl UtxoSource for RpcClient { + fn get_block_hash_by_height<'a>(&'a self, block_height: u32) -> AsyncBlockSourceResult<'a, BlockHash> { + Box::pin(async move { + let height_param = serde_json::json!(block_height); + Ok(self.call_method("getblockhash", &[height_param]).await?) + }) + } + + fn is_output_unspent<'a>(&'a self, outpoint: OutPoint) -> AsyncBlockSourceResult<'a, bool> { + Box::pin(async move { + let txid_param = serde_json::json!(outpoint.txid.to_hex()); + let vout_param = serde_json::json!(outpoint.vout); + let include_mempool = serde_json::json!(false); + let utxo_opt: serde_json::Value = self.call_method( + "gettxout", &[txid_param, vout_param, include_mempool]).await?; + Ok(!utxo_opt.is_null()) + }) + } +} + #[cfg(test)] mod tests { use super::*; use crate::http::client_tests::{HttpServer, MessageBody}; + use bitcoin::hashes::Hash; + /// Credentials encoded in base64. const CREDENTIALS: &'static str = "dXNlcjpwYXNzd29yZA=="; @@ -245,4 +269,24 @@ mod tests { Ok(count) => assert_eq!(count, 654470), } } + + #[tokio::test] + async fn fails_to_fetch_spent_utxo() { + let response = serde_json::json!({ "result": null }); + let server = HttpServer::responding_with_ok(MessageBody::Content(response)); + let client = RpcClient::new(CREDENTIALS, server.endpoint()).unwrap(); + let outpoint = OutPoint::new(bitcoin::Txid::from_inner([0; 32]), 0); + let unspent_output = client.is_output_unspent(outpoint).await.unwrap(); + assert_eq!(unspent_output, false); + } + + #[tokio::test] + async fn fetches_utxo() { + let response = serde_json::json!({ "result": {"bestblock": 1, "confirmations": 42}}); + let server = HttpServer::responding_with_ok(MessageBody::Content(response)); + let client = RpcClient::new(CREDENTIALS, server.endpoint()).unwrap(); + let outpoint = OutPoint::new(bitcoin::Txid::from_inner([0; 32]), 0); + let unspent_output = client.is_output_unspent(outpoint).await.unwrap(); + assert_eq!(unspent_output, true); + } }