From: Jeffrey Czyz Date: Fri, 24 Jun 2022 21:27:42 +0000 (-0500) Subject: Serialization macro for TLV streams X-Git-Url: http://git.bitcoin.ninja/?a=commitdiff_plain;h=16606afd7fef76ff546daafe6c018c77e3475203;p=rust-lightning Serialization macro for TLV streams BOLT 12's offer message is encoded as a TLV stream (i.e., a sequence of TLV records). impl_writeable_tlv_based can't be used because it writes the overall length of the struct, whereas TLV streams only include the length of each TLV record. Add a `tlv_stream` macro for defining structs used in encoding. TLV records containing a single variable-length type should not encode the types length in the value since it is redundant. Add a wrapper type that can be used within a TLV stream to support the correct behavior during serialization and de-serialization. --- diff --git a/lightning/src/util/ser.rs b/lightning/src/util/ser.rs index 8aafeda13..c3f7c19c3 100644 --- a/lightning/src/util/ser.rs +++ b/lightning/src/util/ser.rs @@ -452,6 +452,12 @@ macro_rules! impl_writeable_primitive { } } } + impl From<$val_type> for HighZeroBytesDroppedBigSize<$val_type> { + fn from(val: $val_type) -> Self { Self(val) } + } + impl From> for $val_type { + fn from(val: HighZeroBytesDroppedBigSize<$val_type>) -> Self { val.0 } + } } } @@ -524,6 +530,50 @@ impl_array!(PUBLIC_KEY_SIZE); // for PublicKey impl_array!(COMPACT_SIGNATURE_SIZE); // for Signature impl_array!(1300); // for OnionPacket.hop_data +/// For variable-length values within TLV record where the length is encoded as part of the record. +/// Used to prevent encoding the length twice. +#[derive(Clone)] +pub(crate) struct WithoutLength(pub T); + +impl Writeable for WithoutLength<&String> { + #[inline] + fn write(&self, w: &mut W) -> Result<(), io::Error> { + w.write_all(self.0.as_bytes()) + } +} +impl Readable for WithoutLength { + #[inline] + fn read(r: &mut R) -> Result { + let v: VecReadWrapper = Readable::read(r)?; + Ok(Self(String::from_utf8(v.0).map_err(|_| DecodeError::InvalidValue)?)) + } +} +impl<'a> From<&'a String> for WithoutLength<&'a String> { + fn from(s: &'a String) -> Self { Self(s) } +} +impl From> for String { + fn from(s: WithoutLength) -> Self { s.0 } +} + +impl Writeable for WithoutLength<&Vec> { + #[inline] + fn write(&self, w: &mut W) -> Result<(), io::Error> { + VecWriteWrapper(self.0).write(w) + } +} +impl Readable for WithoutLength> { + #[inline] + fn read(r: &mut R) -> Result { + Ok(Self( as Readable>::read(r)?.0)) + } +} +impl<'a, T> From<&'a Vec> for WithoutLength<&'a Vec> { + fn from(v: &'a Vec) -> Self { Self(v) } +} +impl From>> for Vec { + fn from(s: WithoutLength>) -> Self { s.0 } +} + // HashMap impl Writeable for HashMap where K: Writeable + Eq + Hash, diff --git a/lightning/src/util/ser_macros.rs b/lightning/src/util/ser_macros.rs index 94990fcb8..374585483 100644 --- a/lightning/src/util/ser_macros.rs +++ b/lightning/src/util/ser_macros.rs @@ -26,6 +26,14 @@ macro_rules! encode_tlv { field.write($stream)?; } }; + ($stream: expr, $type: expr, $field: expr, (tlv_record, $fieldty:ident$(<$gen:ident>)?)) => { + if let Some(field) = $field { + let field: encoded_tlv_record_ref_type!($fieldty$(<$gen>)?) = From::from(field); + BigSize($type).write($stream)?; + BigSize(field.serialized_length() as u64).write($stream)?; + field.write($stream)?; + } + }; } macro_rules! encode_tlv_stream { @@ -121,6 +129,9 @@ macro_rules! check_tlv_order { ($last_seen_type: expr, $typ: expr, $type: expr, $field: ident, (option: $trait: ident $(, $read_arg: expr)?)) => {{ // no-op }}; + ($last_seen_type: expr, $typ: expr, $type: expr, $field: ident, (tlv_record, $fieldty:ident$(<$gen:ident>)?)) => {{ + // no-op + }}; } macro_rules! check_missing_tlv { @@ -150,6 +161,9 @@ macro_rules! check_missing_tlv { ($last_seen_type: expr, $type: expr, $field: ident, (option: $trait: ident $(, $read_arg: expr)?)) => {{ // no-op }}; + ($last_seen_type: expr, $type: expr, $field: ident, (tlv_record, $fieldty:ident$(<$gen:ident>)?)) => {{ + // no-op + }}; } macro_rules! decode_tlv { @@ -172,6 +186,13 @@ macro_rules! decode_tlv { ($reader: expr, $field: ident, (option: $trait: ident $(, $read_arg: expr)?)) => {{ $field = Some($trait::read(&mut $reader $(, $read_arg)*)?); }}; + ($reader: expr, $field: ident, (tlv_record, $fieldty:ident$(<$gen:ident>)?)) => {{ + $field = { + let field: encoded_tlv_record_type!($fieldty$(<$gen>)?) = + ser::Readable::read(&mut $reader)?; + Some(field.into()) + }; + }}; } macro_rules! decode_tlv_stream { @@ -360,6 +381,9 @@ macro_rules! init_tlv_based_struct_field { ($field: ident, vec_type) => { $field.unwrap() }; + ($field: ident, tlv_record) => { + $field + }; } macro_rules! init_tlv_field_var { @@ -375,6 +399,9 @@ macro_rules! init_tlv_field_var { ($field: ident, option) => { let mut $field = None; }; + ($field: ident, tlv_record) => { + let mut $field = None; + }; } /// Implements Readable/Writeable for a struct storing it as a set of TLVs @@ -427,6 +454,139 @@ macro_rules! impl_writeable_tlv_based { } } +/// Defines a struct for a TLV stream and a similar struct using references for non-primitive types, +/// implementing [`Readable`] for the former and [`Writeable`] for the latter. Useful as an +/// intermediary format when reading or writing a type encoded as a TLV stream. Note that each field +/// representing a TLV record has its type wrapped with an [`Option`]. +/// +/// [`Readable`]: crate::util::ser::Readable +/// [`Writeable`]: crate::util::ser::Writeable +macro_rules! tlv_stream { + ($name:ident, $nameref:ident, { + $(($type:expr, $field:ident : $fieldty:ident$(<$gen:ident>)?)),* $(,)* + }) => { + #[derive(Debug)] + struct $name { + $( + $field: Option<$fieldty$(<$gen>)?>, + )* + } + + pub(crate) struct $nameref<'a> { + $( + pub(crate) $field: Option)?)>, + )* + } + + impl<'a> ::util::ser::Writeable for $nameref<'a> { + fn write(&self, writer: &mut W) -> Result<(), $crate::io::Error> { + encode_tlv_stream!(writer, { + $(($type, self.$field, (tlv_record, $fieldty$(<$gen>)?))),* + }); + Ok(()) + } + } + + impl ::util::ser::Readable for $name { + fn read(reader: &mut R) -> Result { + $( + init_tlv_field_var!($field, tlv_record); + )* + decode_tlv_stream!(reader, { + $(($type, $field, (tlv_record, $fieldty$(<$gen>)?))),* + }); + + Ok(Self { + $( + $field: init_tlv_based_struct_field!($field, tlv_record) + ),* + }) + } + } + } +} + +macro_rules! tlv_record_ref_type { + (u8) => { + u8 + }; + (u16) => { + u16 + }; + (u32) => { + u32 + }; + (u64) => { + u64 + }; + (char) => { + char + }; + (String) => { + &'a crate::prelude::String + }; + (Vec<$type:ty>) => { + &'a crate::prelude::Vec<$type> + }; + ($type:ident$(<$gen:ident>)?) => { + &'a $type$(<$gen>)? + }; +} + +macro_rules! encoded_tlv_record_type { + (u8) => { + u8 + }; + (u16) => { + ::util::ser::HighZeroBytesDroppedBigSize + }; + (u32) => { + ::util::ser::HighZeroBytesDroppedBigSize + }; + (u64) => { + ::util::ser::HighZeroBytesDroppedBigSize + }; + (char) => { + char + }; + (String) => { + ::util::ser::WithoutLength + }; + (Vec<$type:ty>) => { + ::util::ser::WithoutLength> + }; + ($type:ident$(<$gen:ident>)?) => { + $type$(<$gen>)? + }; +} + +macro_rules! encoded_tlv_record_ref_type { + (u8) => { + u8 + }; + (u16) => { + ::util::ser::HighZeroBytesDroppedBigSize + }; + (u32) => { + ::util::ser::HighZeroBytesDroppedBigSize + }; + (u64) => { + ::util::ser::HighZeroBytesDroppedBigSize + }; + (char) => { + char + }; + (String) => { + ::util::ser::WithoutLength<&crate::prelude::String> + }; + (Vec<$type:ty>) => { + ::util::ser::WithoutLength<&crate::prelude::Vec<$type>> + }; + ($type:ident$(<$gen:ident>)?) => { + &$type$(<$gen>)? + }; +} + macro_rules! _impl_writeable_tlv_based_enum_common { ($st: ident, $(($variant_id: expr, $variant_name: ident) => {$(($type: expr, $field: ident, $fieldty: tt)),* $(,)*}