From: Jeffrey Czyz Date: Tue, 12 Jan 2021 07:33:42 +0000 (-0800) Subject: Implement BlockSource for REST and RPC clients X-Git-Tag: v0.0.13~36^2 X-Git-Url: http://git.bitcoin.ninja/index.cgi?a=commitdiff_plain;h=85fdfaaa9c4421d8fbeb89202508ea7477730eb0;p=rust-lightning Implement BlockSource for REST and RPC clients Interprets HTTP responses as either binary or JSON format, which are then converted to the appropriate data types. --- diff --git a/lightning-block-sync/Cargo.toml b/lightning-block-sync/Cargo.toml index f2032b05c..aec6d1404 100644 --- a/lightning-block-sync/Cargo.toml +++ b/lightning-block-sync/Cargo.toml @@ -9,13 +9,14 @@ Utilities to fetch the chain data from a block source and feed them into Rust Li """ [features] -rest-client = [ "serde_json", "chunked_transfer" ] -rpc-client = [ "serde_json", "chunked_transfer" ] +rest-client = [ "serde", "serde_json", "chunked_transfer" ] +rpc-client = [ "serde", "serde_json", "chunked_transfer" ] [dependencies] bitcoin = "0.24" lightning = { version = "0.0.12", path = "../lightning" } tokio = { version = "1.0", features = [ "io-util", "net" ], optional = true } +serde = { version = "1.0", features = ["derive"], optional = true } serde_json = { version = "1.0", optional = true } chunked_transfer = { version = "1.4", optional = true } futures = { version = "0.3" } diff --git a/lightning-block-sync/src/convert.rs b/lightning-block-sync/src/convert.rs new file mode 100644 index 000000000..37b2c4323 --- /dev/null +++ b/lightning-block-sync/src/convert.rs @@ -0,0 +1,472 @@ +use crate::{BlockHeaderData, BlockSourceError}; +use crate::http::{BinaryResponse, JsonResponse}; +use crate::utils::hex_to_uint256; + +use bitcoin::blockdata::block::{Block, BlockHeader}; +use bitcoin::consensus::encode; +use bitcoin::hash_types::{BlockHash, TxMerkleNode}; +use bitcoin::hashes::hex::{ToHex, FromHex}; + +use serde::Deserialize; + +use serde_json; + +use std::convert::From; +use std::convert::TryFrom; +use std::convert::TryInto; + +/// Conversion from `std::io::Error` into `BlockSourceError`. +impl From for BlockSourceError { + fn from(e: std::io::Error) -> BlockSourceError { + match e.kind() { + std::io::ErrorKind::InvalidData => BlockSourceError::persistent(e), + std::io::ErrorKind::InvalidInput => BlockSourceError::persistent(e), + _ => BlockSourceError::transient(e), + } + } +} + +/// Parses binary data as a block. +impl TryInto for BinaryResponse { + type Error = std::io::Error; + + fn try_into(self) -> std::io::Result { + match encode::deserialize(&self.0) { + Err(_) => return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, "invalid block data")), + Ok(block) => Ok(block), + } + } +} + +/// Converts a JSON value into block header data. The JSON value may be an object representing a +/// block header or an array of such objects. In the latter case, the first object is converted. +impl TryInto for JsonResponse { + type Error = std::io::Error; + + fn try_into(self) -> std::io::Result { + let mut header = match self.0 { + serde_json::Value::Array(mut array) if !array.is_empty() => array.drain(..).next().unwrap(), + serde_json::Value::Object(_) => self.0, + _ => return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, "unexpected JSON type")), + }; + + if !header.is_object() { + return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, "expected JSON object")); + } + + // Add an empty previousblockhash for the genesis block. + if let None = header.get("previousblockhash") { + let hash: BlockHash = Default::default(); + header.as_object_mut().unwrap().insert("previousblockhash".to_string(), serde_json::json!(hash.to_hex())); + } + + match serde_json::from_value::(header) { + Err(_) => Err(std::io::Error::new(std::io::ErrorKind::InvalidData, "invalid header response")), + Ok(response) => match response.try_into() { + Err(_) => Err(std::io::Error::new(std::io::ErrorKind::InvalidData, "invalid header data")), + Ok(header) => Ok(header), + }, + } + } +} + +/// Response data from `getblockheader` RPC and `headers` REST requests. +#[derive(Deserialize)] +struct GetHeaderResponse { + pub version: i32, + pub merkleroot: String, + pub time: u32, + pub nonce: u32, + pub bits: String, + pub previousblockhash: String, + + pub chainwork: String, + pub height: u32, +} + +/// Converts from `GetHeaderResponse` to `BlockHeaderData`. +impl TryFrom for BlockHeaderData { + type Error = bitcoin::hashes::hex::Error; + + fn try_from(response: GetHeaderResponse) -> Result { + Ok(BlockHeaderData { + header: BlockHeader { + version: response.version, + prev_blockhash: BlockHash::from_hex(&response.previousblockhash)?, + merkle_root: TxMerkleNode::from_hex(&response.merkleroot)?, + time: response.time, + bits: u32::from_be_bytes(<[u8; 4]>::from_hex(&response.bits)?), + nonce: response.nonce, + }, + chainwork: hex_to_uint256(&response.chainwork)?, + height: response.height, + }) + } +} + + +/// Converts a JSON value into a block. Assumes the block is hex-encoded in a JSON string. +impl TryInto for JsonResponse { + type Error = std::io::Error; + + fn try_into(self) -> std::io::Result { + match self.0.as_str() { + None => Err(std::io::Error::new(std::io::ErrorKind::InvalidData, "expected JSON string")), + Some(hex_data) => match Vec::::from_hex(hex_data) { + Err(_) => Err(std::io::Error::new(std::io::ErrorKind::InvalidData, "invalid hex data")), + Ok(block_data) => match encode::deserialize(&block_data) { + Err(_) => Err(std::io::Error::new(std::io::ErrorKind::InvalidData, "invalid block data")), + Ok(block) => Ok(block), + }, + }, + } + } +} + +/// Converts a JSON value into the best block hash and optional height. +impl TryInto<(BlockHash, Option)> for JsonResponse { + type Error = std::io::Error; + + fn try_into(self) -> std::io::Result<(BlockHash, Option)> { + if !self.0.is_object() { + return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, "expected JSON object")); + } + + let hash = match &self.0["bestblockhash"] { + serde_json::Value::String(hex_data) => match BlockHash::from_hex(&hex_data) { + Err(_) => return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, "invalid hex data")), + Ok(block_hash) => block_hash, + }, + _ => return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, "expected JSON string")), + }; + + let height = match &self.0["blocks"] { + serde_json::Value::Null => None, + serde_json::Value::Number(height) => match height.as_u64() { + None => return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, "invalid height")), + Some(height) => match height.try_into() { + Err(_) => return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, "invalid height")), + Ok(height) => Some(height), + } + }, + _ => return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, "expected JSON number")), + }; + + Ok((hash, height)) + } +} + +#[cfg(test)] +pub(crate) mod tests { + use super::*; + use bitcoin::blockdata::constants::genesis_block; + use bitcoin::consensus::encode; + use bitcoin::network::constants::Network; + + /// Converts from `BlockHeaderData` into a `GetHeaderResponse` JSON value. + impl From for serde_json::Value { + fn from(data: BlockHeaderData) -> Self { + let BlockHeaderData { chainwork, height, header } = data; + serde_json::json!({ + "chainwork": chainwork.to_string()["0x".len()..], + "height": height, + "version": header.version, + "merkleroot": header.merkle_root.to_hex(), + "time": header.time, + "nonce": header.nonce, + "bits": header.bits.to_hex(), + "previousblockhash": header.prev_blockhash.to_hex(), + }) + } + } + + #[test] + fn into_block_header_from_json_response_with_unexpected_type() { + let response = JsonResponse(serde_json::json!(42)); + match TryInto::::try_into(response) { + Err(e) => { + assert_eq!(e.kind(), std::io::ErrorKind::InvalidData); + assert_eq!(e.get_ref().unwrap().to_string(), "unexpected JSON type"); + }, + Ok(_) => panic!("Expected error"), + } + } + + #[test] + fn into_block_header_from_json_response_with_unexpected_header_type() { + let response = JsonResponse(serde_json::json!([42])); + match TryInto::::try_into(response) { + Err(e) => { + assert_eq!(e.kind(), std::io::ErrorKind::InvalidData); + assert_eq!(e.get_ref().unwrap().to_string(), "expected JSON object"); + }, + Ok(_) => panic!("Expected error"), + } + } + + #[test] + fn into_block_header_from_json_response_with_invalid_header_response() { + let block = genesis_block(Network::Bitcoin); + let mut response = JsonResponse(BlockHeaderData { + chainwork: block.header.work(), + height: 0, + header: block.header + }.into()); + response.0["chainwork"].take(); + + match TryInto::::try_into(response) { + Err(e) => { + assert_eq!(e.kind(), std::io::ErrorKind::InvalidData); + assert_eq!(e.get_ref().unwrap().to_string(), "invalid header response"); + }, + Ok(_) => panic!("Expected error"), + } + } + + #[test] + fn into_block_header_from_json_response_with_invalid_header_data() { + let block = genesis_block(Network::Bitcoin); + let mut response = JsonResponse(BlockHeaderData { + chainwork: block.header.work(), + height: 0, + header: block.header + }.into()); + response.0["chainwork"] = serde_json::json!("foobar"); + + match TryInto::::try_into(response) { + Err(e) => { + assert_eq!(e.kind(), std::io::ErrorKind::InvalidData); + assert_eq!(e.get_ref().unwrap().to_string(), "invalid header data"); + }, + Ok(_) => panic!("Expected error"), + } + } + + #[test] + fn into_block_header_from_json_response_with_valid_header() { + let block = genesis_block(Network::Bitcoin); + let response = JsonResponse(BlockHeaderData { + chainwork: block.header.work(), + height: 0, + header: block.header + }.into()); + + match TryInto::::try_into(response) { + Err(e) => panic!("Unexpected error: {:?}", e), + Ok(data) => { + assert_eq!(data.chainwork, block.header.work()); + assert_eq!(data.height, 0); + assert_eq!(data.header, block.header); + }, + } + } + + #[test] + fn into_block_header_from_json_response_with_valid_header_array() { + let genesis_block = genesis_block(Network::Bitcoin); + let best_block_header = BlockHeader { + prev_blockhash: genesis_block.block_hash(), + ..genesis_block.header + }; + let chainwork = genesis_block.header.work() + best_block_header.work(); + let response = JsonResponse(serde_json::json!([ + serde_json::Value::from(BlockHeaderData { + chainwork, height: 1, header: best_block_header, + }), + serde_json::Value::from(BlockHeaderData { + chainwork: genesis_block.header.work(), height: 0, header: genesis_block.header, + }), + ])); + + match TryInto::::try_into(response) { + Err(e) => panic!("Unexpected error: {:?}", e), + Ok(data) => { + assert_eq!(data.chainwork, chainwork); + assert_eq!(data.height, 1); + assert_eq!(data.header, best_block_header); + }, + } + } + + #[test] + fn into_block_header_from_json_response_without_previous_block_hash() { + let block = genesis_block(Network::Bitcoin); + let mut response = JsonResponse(BlockHeaderData { + chainwork: block.header.work(), + height: 0, + header: block.header + }.into()); + response.0.as_object_mut().unwrap().remove("previousblockhash"); + + match TryInto::::try_into(response) { + Err(e) => panic!("Unexpected error: {:?}", e), + Ok(BlockHeaderData { chainwork: _, height: _, header }) => { + assert_eq!(header, block.header); + }, + } + } + + #[test] + fn into_block_from_invalid_binary_response() { + let response = BinaryResponse(b"foo".to_vec()); + match TryInto::::try_into(response) { + Err(_) => {}, + Ok(_) => panic!("Expected error"), + } + } + + #[test] + fn into_block_from_valid_binary_response() { + let genesis_block = genesis_block(Network::Bitcoin); + let response = BinaryResponse(encode::serialize(&genesis_block)); + match TryInto::::try_into(response) { + Err(e) => panic!("Unexpected error: {:?}", e), + Ok(block) => assert_eq!(block, genesis_block), + } + } + + #[test] + fn into_block_from_json_response_with_unexpected_type() { + let response = JsonResponse(serde_json::json!({ "result": "foo" })); + match TryInto::::try_into(response) { + Err(e) => { + assert_eq!(e.kind(), std::io::ErrorKind::InvalidData); + assert_eq!(e.get_ref().unwrap().to_string(), "expected JSON string"); + }, + Ok(_) => panic!("Expected error"), + } + } + + #[test] + fn into_block_from_json_response_with_invalid_hex_data() { + let response = JsonResponse(serde_json::json!("foobar")); + match TryInto::::try_into(response) { + Err(e) => { + assert_eq!(e.kind(), std::io::ErrorKind::InvalidData); + assert_eq!(e.get_ref().unwrap().to_string(), "invalid hex data"); + }, + Ok(_) => panic!("Expected error"), + } + } + + #[test] + fn into_block_from_json_response_with_invalid_block_data() { + let response = JsonResponse(serde_json::json!("abcd")); + match TryInto::::try_into(response) { + Err(e) => { + assert_eq!(e.kind(), std::io::ErrorKind::InvalidData); + assert_eq!(e.get_ref().unwrap().to_string(), "invalid block data"); + }, + Ok(_) => panic!("Expected error"), + } + } + + #[test] + fn into_block_from_json_response_with_valid_block_data() { + let genesis_block = genesis_block(Network::Bitcoin); + let response = JsonResponse(serde_json::json!(encode::serialize_hex(&genesis_block))); + match TryInto::::try_into(response) { + Err(e) => panic!("Unexpected error: {:?}", e), + Ok(block) => assert_eq!(block, genesis_block), + } + } + + #[test] + fn into_block_hash_from_json_response_with_unexpected_type() { + let response = JsonResponse(serde_json::json!("foo")); + match TryInto::<(BlockHash, Option)>::try_into(response) { + Err(e) => { + assert_eq!(e.kind(), std::io::ErrorKind::InvalidData); + assert_eq!(e.get_ref().unwrap().to_string(), "expected JSON object"); + }, + Ok(_) => panic!("Expected error"), + } + } + + #[test] + fn into_block_hash_from_json_response_with_unexpected_bestblockhash_type() { + let response = JsonResponse(serde_json::json!({ "bestblockhash": 42 })); + match TryInto::<(BlockHash, Option)>::try_into(response) { + Err(e) => { + assert_eq!(e.kind(), std::io::ErrorKind::InvalidData); + assert_eq!(e.get_ref().unwrap().to_string(), "expected JSON string"); + }, + Ok(_) => panic!("Expected error"), + } + } + + #[test] + fn into_block_hash_from_json_response_with_invalid_hex_data() { + let response = JsonResponse(serde_json::json!({ "bestblockhash": "foobar"} )); + match TryInto::<(BlockHash, Option)>::try_into(response) { + Err(e) => { + assert_eq!(e.kind(), std::io::ErrorKind::InvalidData); + assert_eq!(e.get_ref().unwrap().to_string(), "invalid hex data"); + }, + Ok(_) => panic!("Expected error"), + } + } + + #[test] + fn into_block_hash_from_json_response_without_height() { + let block = genesis_block(Network::Bitcoin); + let response = JsonResponse(serde_json::json!({ + "bestblockhash": block.block_hash().to_hex(), + })); + match TryInto::<(BlockHash, Option)>::try_into(response) { + Err(e) => panic!("Unexpected error: {:?}", e), + Ok((hash, height)) => { + assert_eq!(hash, block.block_hash()); + assert!(height.is_none()); + }, + } + } + + #[test] + fn into_block_hash_from_json_response_with_unexpected_blocks_type() { + let block = genesis_block(Network::Bitcoin); + let response = JsonResponse(serde_json::json!({ + "bestblockhash": block.block_hash().to_hex(), + "blocks": "foo", + })); + match TryInto::<(BlockHash, Option)>::try_into(response) { + Err(e) => { + assert_eq!(e.kind(), std::io::ErrorKind::InvalidData); + assert_eq!(e.get_ref().unwrap().to_string(), "expected JSON number"); + }, + Ok(_) => panic!("Expected error"), + } + } + + #[test] + fn into_block_hash_from_json_response_with_invalid_height() { + let block = genesis_block(Network::Bitcoin); + let response = JsonResponse(serde_json::json!({ + "bestblockhash": block.block_hash().to_hex(), + "blocks": std::u64::MAX, + })); + match TryInto::<(BlockHash, Option)>::try_into(response) { + Err(e) => { + assert_eq!(e.kind(), std::io::ErrorKind::InvalidData); + assert_eq!(e.get_ref().unwrap().to_string(), "invalid height"); + }, + Ok(_) => panic!("Expected error"), + } + } + + #[test] + fn into_block_hash_from_json_response_with_height() { + let block = genesis_block(Network::Bitcoin); + let response = JsonResponse(serde_json::json!({ + "bestblockhash": block.block_hash().to_hex(), + "blocks": 1, + })); + match TryInto::<(BlockHash, Option)>::try_into(response) { + Err(e) => panic!("Unexpected error: {:?}", e), + Ok((hash, height)) => { + assert_eq!(hash, block.block_hash()); + assert_eq!(height.unwrap(), 1); + }, + } + } +} diff --git a/lightning-block-sync/src/lib.rs b/lightning-block-sync/src/lib.rs index 4f004b8f2..58f77bdca 100644 --- a/lightning-block-sync/src/lib.rs +++ b/lightning-block-sync/src/lib.rs @@ -20,6 +20,12 @@ pub mod rest; #[cfg(feature = "rpc-client")] pub mod rpc; +#[cfg(any(feature = "rest-client", feature = "rpc-client"))] +mod convert; + +#[cfg(any(feature = "rest-client", feature = "rpc-client"))] +mod utils; + use bitcoin::blockdata::block::{Block, BlockHeader}; use bitcoin::hash_types::BlockHash; use bitcoin::util::uint::Uint256; diff --git a/lightning-block-sync/src/rest.rs b/lightning-block-sync/src/rest.rs index 3ccd5d482..3c2e76e23 100644 --- a/lightning-block-sync/src/rest.rs +++ b/lightning-block-sync/src/rest.rs @@ -1,4 +1,9 @@ -use crate::http::{HttpEndpoint, HttpClient}; +use crate::{BlockHeaderData, BlockSource, AsyncBlockSourceResult}; +use crate::http::{BinaryResponse, HttpEndpoint, HttpClient, JsonResponse}; + +use bitcoin::blockdata::block::Block; +use bitcoin::hash_types::BlockHash; +use bitcoin::hashes::hex::ToHex; use std::convert::TryFrom; use std::convert::TryInto; @@ -11,6 +16,8 @@ pub struct RestClient { impl RestClient { /// Creates a new REST client connected to the given endpoint. + /// + /// The endpoint should contain the REST path component (e.g., http://127.0.0.1:8332/rest). pub fn new(endpoint: HttpEndpoint) -> std::io::Result { let client = HttpClient::connect(&endpoint)?; Ok(Self { endpoint, client }) @@ -25,6 +32,28 @@ impl RestClient { } } +impl BlockSource for RestClient { + fn get_header<'a>(&'a mut self, header_hash: &'a BlockHash, _height: Option) -> AsyncBlockSourceResult<'a, BlockHeaderData> { + Box::pin(async move { + let resource_path = format!("headers/1/{}.json", header_hash.to_hex()); + Ok(self.request_resource::(&resource_path).await?) + }) + } + + fn get_block<'a>(&'a mut self, header_hash: &'a BlockHash) -> AsyncBlockSourceResult<'a, Block> { + Box::pin(async move { + let resource_path = format!("block/{}.bin", header_hash.to_hex()); + Ok(self.request_resource::(&resource_path).await?) + }) + } + + fn get_best_block<'a>(&'a mut self) -> AsyncBlockSourceResult<'a, (BlockHash, Option)> { + Box::pin(async move { + Ok(self.request_resource::("chaininfo.json").await?) + }) + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/lightning-block-sync/src/rpc.rs b/lightning-block-sync/src/rpc.rs index 60ea1a360..34cbd2e02 100644 --- a/lightning-block-sync/src/rpc.rs +++ b/lightning-block-sync/src/rpc.rs @@ -1,5 +1,10 @@ +use crate::{BlockHeaderData, BlockSource, AsyncBlockSourceResult}; use crate::http::{HttpClient, HttpEndpoint, JsonResponse}; +use bitcoin::blockdata::block::Block; +use bitcoin::hash_types::BlockHash; +use bitcoin::hashes::hex::ToHex; + use serde_json; use std::convert::TryFrom; @@ -61,6 +66,29 @@ impl RpcClient { } } +impl BlockSource for RpcClient { + fn get_header<'a>(&'a mut self, header_hash: &'a BlockHash, _height: Option) -> AsyncBlockSourceResult<'a, BlockHeaderData> { + Box::pin(async move { + let header_hash = serde_json::json!(header_hash.to_hex()); + Ok(self.call_method("getblockheader", &[header_hash]).await?) + }) + } + + fn get_block<'a>(&'a mut self, header_hash: &'a BlockHash) -> AsyncBlockSourceResult<'a, Block> { + Box::pin(async move { + let header_hash = serde_json::json!(header_hash.to_hex()); + let verbosity = serde_json::json!(0); + Ok(self.call_method("getblock", &[header_hash, verbosity]).await?) + }) + } + + fn get_best_block<'a>(&'a mut self) -> AsyncBlockSourceResult<'a, (BlockHash, Option)> { + Box::pin(async move { + Ok(self.call_method("getblockchaininfo", &[]).await?) + }) + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/lightning-block-sync/src/utils.rs b/lightning-block-sync/src/utils.rs new file mode 100644 index 000000000..96a2e5788 --- /dev/null +++ b/lightning-block-sync/src/utils.rs @@ -0,0 +1,54 @@ +use bitcoin::hashes::hex::FromHex; +use bitcoin::util::uint::Uint256; + +pub fn hex_to_uint256(hex: &str) -> Result { + let bytes = <[u8; 32]>::from_hex(hex)?; + Ok(Uint256::from_be_bytes(bytes)) +} + +#[cfg(test)] +mod tests { + use super::*; + use bitcoin::util::uint::Uint256; + + #[test] + fn hex_to_uint256_empty_str() { + assert!(hex_to_uint256("").is_err()); + } + + #[test] + fn hex_to_uint256_too_short_str() { + let hex = String::from_utf8(vec![b'0'; 32]).unwrap(); + assert_eq!(hex_to_uint256(&hex), Err(bitcoin::hashes::hex::Error::InvalidLength(64, 32))); + } + + #[test] + fn hex_to_uint256_too_long_str() { + let hex = String::from_utf8(vec![b'0'; 128]).unwrap(); + assert_eq!(hex_to_uint256(&hex), Err(bitcoin::hashes::hex::Error::InvalidLength(64, 128))); + } + + #[test] + fn hex_to_uint256_odd_length_str() { + let hex = String::from_utf8(vec![b'0'; 65]).unwrap(); + assert_eq!(hex_to_uint256(&hex), Err(bitcoin::hashes::hex::Error::OddLengthString(65))); + } + + #[test] + fn hex_to_uint256_invalid_char() { + let hex = String::from_utf8(vec![b'G'; 64]).unwrap(); + assert_eq!(hex_to_uint256(&hex), Err(bitcoin::hashes::hex::Error::InvalidChar(b'G'))); + } + + #[test] + fn hex_to_uint256_lowercase_str() { + let hex: String = std::iter::repeat("0123456789abcdef").take(4).collect(); + assert_eq!(hex_to_uint256(&hex).unwrap(), Uint256([0x0123456789abcdefu64; 4])); + } + + #[test] + fn hex_to_uint256_uppercase_str() { + let hex: String = std::iter::repeat("0123456789ABCDEF").take(4).collect(); + assert_eq!(hex_to_uint256(&hex).unwrap(), Uint256([0x0123456789abcdefu64; 4])); + } +}