Serialization macro for TLV streams

BOLT 12's offer message is encoded as a TLV stream (i.e., a sequence of
TLV records). impl_writeable_tlv_based can't be used because it writes
the overall length of the struct, whereas TLV streams only include the
length of each TLV record. Add a `tlv_stream` macro for defining structs
used in encoding.

TLV records containing a single variable-length type should not encode
the types length in the value since it is redundant. Add a wrapper type
that can be used within a TLV stream to support the correct behavior
during serialization and de-serialization.

diff --git a/lightning/src/util/ser.rs b/lightning/src/util/ser.rs
index 8aafeda13d6373c20035177370a96cdbc21b8fc1..c3f7c19c3dbd07c3024c74a16ebcd998b3d99cd9 100644 (file)
--- a/lightning/src/util/ser.rs
+++ b/lightning/src/util/ser.rs
@@ -452,6 +452,12 @@ macro_rules! impl_writeable_primitive {
                                }
                        }
                }
+               impl From<$val_type> for HighZeroBytesDroppedBigSize<$val_type> {
+                       fn from(val: $val_type) -> Self { Self(val) }
+               }
+               impl From<HighZeroBytesDroppedBigSize<$val_type>> for $val_type {
+                       fn from(val: HighZeroBytesDroppedBigSize<$val_type>) -> Self { val.0 }
+               }
        }
 }
 
@@ -524,6 +530,50 @@ impl_array!(PUBLIC_KEY_SIZE); // for PublicKey
 impl_array!(COMPACT_SIGNATURE_SIZE); // for Signature
 impl_array!(1300); // for OnionPacket.hop_data
 
+/// For variable-length values within TLV record where the length is encoded as part of the record.
+/// Used to prevent encoding the length twice.
+#[derive(Clone)]
+pub(crate) struct WithoutLength<T>(pub T);
+
+impl Writeable for WithoutLength<&String> {
+       #[inline]
+       fn write<W: Writer>(&self, w: &mut W) -> Result<(), io::Error> {
+               w.write_all(self.0.as_bytes())
+       }
+}
+impl Readable for WithoutLength<String> {
+       #[inline]
+       fn read<R: Read>(r: &mut R) -> Result<Self, DecodeError> {
+               let v: VecReadWrapper<u8> = Readable::read(r)?;
+               Ok(Self(String::from_utf8(v.0).map_err(|_| DecodeError::InvalidValue)?))
+       }
+}
+impl<'a> From<&'a String> for WithoutLength<&'a String> {
+       fn from(s: &'a String) -> Self { Self(s) }
+}
+impl From<WithoutLength<String>> for String {
+       fn from(s: WithoutLength<String>) -> Self { s.0 }
+}
+
+impl<T: Writeable> Writeable for WithoutLength<&Vec<T>> {
+       #[inline]
+       fn write<W: Writer>(&self, w: &mut W) -> Result<(), io::Error> {
+               VecWriteWrapper(self.0).write(w)
+       }
+}
+impl<T: Readable> Readable for WithoutLength<Vec<T>> {
+       #[inline]
+       fn read<R: Read>(r: &mut R) -> Result<Self, DecodeError> {
+               Ok(Self(<VecReadWrapper<T> as Readable>::read(r)?.0))
+       }
+}
+impl<'a, T> From<&'a Vec<T>> for WithoutLength<&'a Vec<T>> {
+       fn from(v: &'a Vec<T>) -> Self { Self(v) }
+}
+impl<T> From<WithoutLength<Vec<T>>> for Vec<T> {
+       fn from(s: WithoutLength<Vec<T>>) -> Self { s.0 }
+}
+
 // HashMap
 impl<K, V> Writeable for HashMap<K, V>
        where K: Writeable + Eq + Hash,

diff --git a/lightning/src/util/ser_macros.rs b/lightning/src/util/ser_macros.rs
index 94990fcb8a17c8056dfd01d00b18db78d58e466f..374585483615218b8018c784c8fa006d2c28ee46 100644 (file)
--- a/lightning/src/util/ser_macros.rs
+++ b/lightning/src/util/ser_macros.rs
@@ -26,6 +26,14 @@ macro_rules! encode_tlv {
                        field.write($stream)?;
                }
        };
+       ($stream: expr, $type: expr, $field: expr, (tlv_record, $fieldty:ident$(<$gen:ident>)?)) => {
+               if let Some(field) = $field {
+                       let field: encoded_tlv_record_ref_type!($fieldty$(<$gen>)?) = From::from(field);
+                       BigSize($type).write($stream)?;
+                       BigSize(field.serialized_length() as u64).write($stream)?;
+                       field.write($stream)?;
+               }
+       };
 }
 
 macro_rules! encode_tlv_stream {
@@ -121,6 +129,9 @@ macro_rules! check_tlv_order {
        ($last_seen_type: expr, $typ: expr, $type: expr, $field: ident, (option: $trait: ident $(, $read_arg: expr)?)) => {{
                // no-op
        }};
+       ($last_seen_type: expr, $typ: expr, $type: expr, $field: ident, (tlv_record, $fieldty:ident$(<$gen:ident>)?)) => {{
+               // no-op
+       }};
 }
 
 macro_rules! check_missing_tlv {
@@ -150,6 +161,9 @@ macro_rules! check_missing_tlv {
        ($last_seen_type: expr, $type: expr, $field: ident, (option: $trait: ident $(, $read_arg: expr)?)) => {{
                // no-op
        }};
+       ($last_seen_type: expr, $type: expr, $field: ident, (tlv_record, $fieldty:ident$(<$gen:ident>)?)) => {{
+               // no-op
+       }};
 }
 
 macro_rules! decode_tlv {
@@ -172,6 +186,13 @@ macro_rules! decode_tlv {
        ($reader: expr, $field: ident, (option: $trait: ident $(, $read_arg: expr)?)) => {{
                $field = Some($trait::read(&mut $reader $(, $read_arg)*)?);
        }};
+       ($reader: expr, $field: ident, (tlv_record, $fieldty:ident$(<$gen:ident>)?)) => {{
+               $field = {
+                       let field: encoded_tlv_record_type!($fieldty$(<$gen>)?) =
+                               ser::Readable::read(&mut $reader)?;
+                       Some(field.into())
+               };
+       }};
 }
 
 macro_rules! decode_tlv_stream {
@@ -360,6 +381,9 @@ macro_rules! init_tlv_based_struct_field {
        ($field: ident, vec_type) => {
                $field.unwrap()
        };
+       ($field: ident, tlv_record) => {
+               $field
+       };
 }
 
 macro_rules! init_tlv_field_var {
@@ -375,6 +399,9 @@ macro_rules! init_tlv_field_var {
        ($field: ident, option) => {
                let mut $field = None;
        };
+       ($field: ident, tlv_record) => {
+               let mut $field = None;
+       };
 }
 
 /// Implements Readable/Writeable for a struct storing it as a set of TLVs
@@ -427,6 +454,139 @@ macro_rules! impl_writeable_tlv_based {
        }
 }
 
+/// Defines a struct for a TLV stream and a similar struct using references for non-primitive types,
+/// implementing [`Readable`] for the former and [`Writeable`] for the latter. Useful as an
+/// intermediary format when reading or writing a type encoded as a TLV stream. Note that each field
+/// representing a TLV record has its type wrapped with an [`Option`].
+///
+/// [`Readable`]: crate::util::ser::Readable
+/// [`Writeable`]: crate::util::ser::Writeable
+macro_rules! tlv_stream {
+       ($name:ident, $nameref:ident, {
+               $(($type:expr, $field:ident : $fieldty:ident$(<$gen:ident>)?)),* $(,)*
+       }) => {
+               #[derive(Debug)]
+               struct $name {
+                       $(
+                               $field: Option<$fieldty$(<$gen>)?>,
+                       )*
+               }
+
+               pub(crate) struct $nameref<'a> {
+                       $(
+                               pub(crate) $field: Option<tlv_record_ref_type!($fieldty$(<$gen>)?)>,
+                       )*
+               }
+
+               impl<'a> ::util::ser::Writeable for $nameref<'a> {
+                       fn write<W: ::util::ser::Writer>(&self, writer: &mut W) -> Result<(), $crate::io::Error> {
+                               encode_tlv_stream!(writer, {
+                                       $(($type, self.$field, (tlv_record, $fieldty$(<$gen>)?))),*
+                               });
+                               Ok(())
+                       }
+               }
+
+               impl ::util::ser::Readable for $name {
+                       fn read<R: $crate::io::Read>(reader: &mut R) -> Result<Self, ::ln::msgs::DecodeError> {
+                               $(
+                                       init_tlv_field_var!($field, tlv_record);
+                               )*
+                               decode_tlv_stream!(reader, {
+                                       $(($type, $field, (tlv_record, $fieldty$(<$gen>)?))),*
+                               });
+
+                               Ok(Self {
+                                       $(
+                                               $field: init_tlv_based_struct_field!($field, tlv_record)
+                                       ),*
+                               })
+                       }
+               }
+       }
+}
+
+macro_rules! tlv_record_ref_type {
+       (u8) => {
+               u8
+       };
+       (u16) => {
+               u16
+       };
+       (u32) => {
+               u32
+       };
+       (u64) => {
+               u64
+       };
+       (char) => {
+               char
+       };
+       (String) => {
+               &'a crate::prelude::String
+       };
+       (Vec<$type:ty>) => {
+               &'a crate::prelude::Vec<$type>
+       };
+       ($type:ident$(<$gen:ident>)?) => {
+               &'a $type$(<$gen>)?
+       };
+}
+
+macro_rules! encoded_tlv_record_type {
+       (u8) => {
+               u8
+       };
+       (u16) => {
+               ::util::ser::HighZeroBytesDroppedBigSize<u16>
+       };
+       (u32) => {
+               ::util::ser::HighZeroBytesDroppedBigSize<u32>
+       };
+       (u64) => {
+               ::util::ser::HighZeroBytesDroppedBigSize<u64>
+       };
+       (char) => {
+               char
+       };
+       (String) => {
+               ::util::ser::WithoutLength<crate::prelude::String>
+       };
+       (Vec<$type:ty>) => {
+               ::util::ser::WithoutLength<crate::prelude::Vec<$type>>
+       };
+       ($type:ident$(<$gen:ident>)?) => {
+               $type$(<$gen>)?
+       };
+}
+
+macro_rules! encoded_tlv_record_ref_type {
+       (u8) => {
+               u8
+       };
+       (u16) => {
+               ::util::ser::HighZeroBytesDroppedBigSize<u16>
+       };
+       (u32) => {
+               ::util::ser::HighZeroBytesDroppedBigSize<u32>
+       };
+       (u64) => {
+               ::util::ser::HighZeroBytesDroppedBigSize<u64>
+       };
+       (char) => {
+               char
+       };
+       (String) => {
+               ::util::ser::WithoutLength<&crate::prelude::String>
+       };
+       (Vec<$type:ty>) => {
+               ::util::ser::WithoutLength<&crate::prelude::Vec<$type>>
+       };
+       ($type:ident$(<$gen:ident>)?) => {
+               &$type$(<$gen>)?
+       };
+}
+
 macro_rules! _impl_writeable_tlv_based_enum_common {
        ($st: ident, $(($variant_id: expr, $variant_name: ident) =>
                {$(($type: expr, $field: ident, $fieldty: tt)),* $(,)*}