Merge pull request #2133 from tnull/2023-03-pin-syn
[rust-lightning] / lightning / src / ln / msgs.rs
index 94136a9ee65cb16fb66d292ea0f5e4bcf714b9b8..cb857ed5ee6004d9e98d96c935c253f835d30d2e 100644 (file)
 //! For a normal node you probably don't need to use anything here, however, if you wish to split a
 //! node into an internet-facing route/message socket handling daemon and a separate daemon (or
 //! server entirely) which handles only channel-related messages you may wish to implement
-//! ChannelMessageHandler yourself and use it to re-serialize messages and pass them across
+//! [`ChannelMessageHandler`] yourself and use it to re-serialize messages and pass them across
 //! daemons/servers.
 //!
 //! Note that if you go with such an architecture (instead of passing raw socket events to a
 //! non-internet-facing system) you trust the frontend internet-facing system to not lie about the
-//! source node_id of the message, however this does allow you to significantly reduce bandwidth
+//! source `node_id` of the message, however this does allow you to significantly reduce bandwidth
 //! between the systems as routing messages can represent a significant chunk of bandwidth usage
 //! (especially for non-channel-publicly-announcing nodes). As an alternate design which avoids
 //! this issue, if you have sufficient bidirectional bandwidth between your systems, you may send
 //! raw socket events into your non-internet-facing system and then send routing events back to
 //! track the network on the less-secure system.
 
-use bitcoin::secp256k1::key::PublicKey;
-use bitcoin::secp256k1::Signature;
+use bitcoin::secp256k1::PublicKey;
+use bitcoin::secp256k1::ecdsa::Signature;
 use bitcoin::secp256k1;
 use bitcoin::blockdata::script::Script;
 use bitcoin::hash_types::{Txid, BlockHash};
 
-use ln::features::{ChannelFeatures, InitFeatures, NodeFeatures};
+use crate::ln::features::{ChannelFeatures, ChannelTypeFeatures, InitFeatures, NodeFeatures};
+use crate::ln::onion_utils;
+use crate::onion_message;
 
-use core::{cmp, fmt};
+use crate::prelude::*;
+use core::fmt;
 use core::fmt::Debug;
-use std::io::Read;
+use crate::io::{self, Read};
+use crate::io_extras::read_to_end;
 
-use util::events::MessageSendEventsProvider;
-use util::ser::{Readable, Writeable, Writer, FixedLengthReader, HighZeroBytesDroppedVarInt};
+use crate::events::{MessageSendEventsProvider, OnionMessageProvider};
+use crate::util::logger;
+use crate::util::ser::{LengthReadable, Readable, ReadableArgs, Writeable, Writer, FixedLengthReader, HighZeroBytesDroppedBigSize, Hostname};
 
-use ln::{PaymentPreimage, PaymentHash, PaymentSecret};
+use crate::ln::{PaymentPreimage, PaymentHash, PaymentSecret};
+
+use crate::routing::gossip::NodeId;
 
 /// 21 million * 10^8 * 1000
 pub(crate) const MAX_VALUE_MSAT: u64 = 21_000_000_0000_0000_000;
 
 /// An error in decoding a message or struct.
-#[derive(Clone, Debug)]
+#[derive(Clone, Debug, PartialEq, Eq)]
 pub enum DecodeError {
        /// A version byte specified something we don't know how to handle.
-       /// Includes unknown realm byte in an OnionHopData packet
+       ///
+       /// Includes unknown realm byte in an onion hop data packet.
        UnknownVersion,
-       /// Unknown feature mandating we fail to parse message (eg TLV with an even, unknown type)
+       /// Unknown feature mandating we fail to parse message (e.g., TLV with an even, unknown type)
        UnknownRequiredFeature,
-       /// Value was invalid, eg a byte which was supposed to be a bool was something other than a 0
+       /// Value was invalid.
+       ///
+       /// For example, a byte which was supposed to be a bool was something other than a 0
        /// or 1, a public key/private key/signature was invalid, text wasn't UTF-8, TLV was
-       /// syntactically incorrect, etc
+       /// syntactically incorrect, etc.
        InvalidValue,
-       /// Buffer too short
+       /// The buffer to be read was too short.
        ShortRead,
-       /// A length descriptor in the packet didn't describe the later data correctly
+       /// A length descriptor in the packet didn't describe the later data correctly.
        BadLengthDescriptor,
-       /// Error from std::io
-       Io(/// (C-not exported) as ErrorKind doesn't have a reasonable mapping
-        ::std::io::ErrorKind),
+       /// Error from [`std::io`].
+       Io(io::ErrorKind),
        /// The message included zlib-compressed values, which we don't support.
        UnsupportedCompression,
 }
 
-/// An init message to be sent or received from a peer
-#[derive(Clone, Debug, PartialEq)]
+/// An [`init`] message to be sent to or received from a peer.
+///
+/// [`init`]: https://github.com/lightning/bolts/blob/master/01-messaging.md#the-init-message
+#[derive(Clone, Debug, PartialEq, Eq)]
 pub struct Init {
-       /// The relevant features which the sender supports
+       /// The relevant features which the sender supports.
        pub features: InitFeatures,
+       /// The receipient's network address.
+       ///
+       /// This adds the option to report a remote IP address back to a connecting peer using the init
+       /// message. A node can decide to use that information to discover a potential update to its
+       /// public IPv4 address (NAT) and use that for a [`NodeAnnouncement`] update message containing
+       /// the new address.
+       pub remote_network_address: Option<NetAddress>,
 }
 
-/// An error message to be sent or received from a peer
-#[derive(Clone, Debug, PartialEq)]
+/// An [`error`] message to be sent to or received from a peer.
+///
+/// [`error`]: https://github.com/lightning/bolts/blob/master/01-messaging.md#the-error-and-warning-messages
+#[derive(Clone, Debug, PartialEq, Eq)]
 pub struct ErrorMessage {
-       /// The channel ID involved in the error
+       /// The channel ID involved in the error.
+       ///
+       /// All-0s indicates a general error unrelated to a specific channel, after which all channels
+       /// with the sending peer should be closed.
        pub channel_id: [u8; 32],
        /// A possibly human-readable error description.
-       /// The string should be sanitized before it is used (e.g. emitted to logs
-       /// or printed to stdout).  Otherwise, a well crafted error message may trigger a security
-       /// vulnerability in the terminal emulator or the logging subsystem.
+       ///
+       /// The string should be sanitized before it is used (e.g., emitted to logs or printed to
+       /// `stdout`). Otherwise, a well crafted error message may trigger a security vulnerability in
+       /// the terminal emulator or the logging subsystem.
+       pub data: String,
+}
+
+/// A [`warning`] message to be sent to or received from a peer.
+///
+/// [`warning`]: https://github.com/lightning/bolts/blob/master/01-messaging.md#the-error-and-warning-messages
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct WarningMessage {
+       /// The channel ID involved in the warning.
+       ///
+       /// All-0s indicates a warning unrelated to a specific channel.
+       pub channel_id: [u8; 32],
+       /// A possibly human-readable warning description.
+       ///
+       /// The string should be sanitized before it is used (e.g. emitted to logs or printed to
+       /// stdout). Otherwise, a well crafted error message may trigger a security vulnerability in
+       /// the terminal emulator or the logging subsystem.
        pub data: String,
 }
 
-/// A ping message to be sent or received from a peer
-#[derive(Clone, Debug, PartialEq)]
+/// A [`ping`] message to be sent to or received from a peer.
+///
+/// [`ping`]: https://github.com/lightning/bolts/blob/master/01-messaging.md#the-ping-and-pong-messages
+#[derive(Clone, Debug, PartialEq, Eq)]
 pub struct Ping {
-       /// The desired response length
+       /// The desired response length.
        pub ponglen: u16,
        /// The ping packet size.
+       ///
        /// This field is not sent on the wire. byteslen zeros are sent.
        pub byteslen: u16,
 }
 
-/// A pong message to be sent or received from a peer
-#[derive(Clone, Debug, PartialEq)]
+/// A [`pong`] message to be sent to or received from a peer.
+///
+/// [`pong`]: https://github.com/lightning/bolts/blob/master/01-messaging.md#the-ping-and-pong-messages
+#[derive(Clone, Debug, PartialEq, Eq)]
 pub struct Pong {
        /// The pong packet size.
+       ///
        /// This field is not sent on the wire. byteslen zeros are sent.
        pub byteslen: u16,
 }
 
-/// An open_channel message to be sent or received from a peer
-#[derive(Clone, Debug, PartialEq)]
+/// An [`open_channel`] message to be sent to or received from a peer.
+///
+/// [`open_channel`]: https://github.com/lightning/bolts/blob/master/02-peer-protocol.md#the-open_channel-message
+#[derive(Clone, Debug, PartialEq, Eq)]
 pub struct OpenChannel {
        /// The genesis hash of the blockchain where the channel is to be opened
        pub chain_hash: BlockHash,
@@ -123,9 +172,11 @@ pub struct OpenChannel {
        pub channel_reserve_satoshis: u64,
        /// The minimum HTLC size incoming to sender, in milli-satoshi
        pub htlc_minimum_msat: u64,
-       /// The feerate per 1000-weight of sender generated transactions, until updated by update_fee
+       /// The feerate per 1000-weight of sender generated transactions, until updated by
+       /// [`UpdateFee`]
        pub feerate_per_kw: u32,
-       /// The number of blocks which the counterparty will have to wait to claim on-chain funds if they broadcast a commitment transaction
+       /// The number of blocks which the counterparty will have to wait to claim on-chain funds if
+       /// they broadcast a commitment transaction
        pub to_self_delay: u16,
        /// The maximum number of inbound HTLCs towards sender
        pub max_accepted_htlcs: u16,
@@ -141,14 +192,21 @@ pub struct OpenChannel {
        pub htlc_basepoint: PublicKey,
        /// The first to-be-broadcast-by-sender transaction's per commitment point
        pub first_per_commitment_point: PublicKey,
-       /// Channel flags
+       /// The channel flags to be used
        pub channel_flags: u8,
-       /// Optionally, a request to pre-set the to-sender output's scriptPubkey for when we collaboratively close
+       /// Optionally, a request to pre-set the to-sender output's `scriptPubkey` for when we collaboratively close
        pub shutdown_scriptpubkey: OptionalField<Script>,
+       /// The channel type that this channel will represent
+       ///
+       /// If this is `None`, we derive the channel type from the intersection of our
+       /// feature bits with our counterparty's feature bits from the [`Init`] message.
+       pub channel_type: Option<ChannelTypeFeatures>,
 }
 
-/// An accept_channel message to be sent or received from a peer
-#[derive(Clone, Debug, PartialEq)]
+/// An [`accept_channel`] message to be sent to or received from a peer.
+///
+/// [`accept_channel`]: https://github.com/lightning/bolts/blob/master/02-peer-protocol.md#the-accept_channel-message
+#[derive(Clone, Debug, PartialEq, Eq)]
 pub struct AcceptChannel {
        /// A temporary channel ID, until the funding outpoint is announced
        pub temporary_channel_id: [u8; 32],
@@ -180,10 +238,18 @@ pub struct AcceptChannel {
        pub first_per_commitment_point: PublicKey,
        /// Optionally, a request to pre-set the to-sender output's scriptPubkey for when we collaboratively close
        pub shutdown_scriptpubkey: OptionalField<Script>,
+       /// The channel type that this channel will represent.
+       ///
+       /// If this is `None`, we derive the channel type from the intersection of
+       /// our feature bits with our counterparty's feature bits from the [`Init`] message.
+       /// This is required to match the equivalent field in [`OpenChannel::channel_type`].
+       pub channel_type: Option<ChannelTypeFeatures>,
 }
 
-/// A funding_created message to be sent or received from a peer
-#[derive(Clone, Debug, PartialEq)]
+/// A [`funding_created`] message to be sent to or received from a peer.
+///
+/// [`funding_created`]: https://github.com/lightning/bolts/blob/master/02-peer-protocol.md#the-funding_created-message
+#[derive(Clone, Debug, PartialEq, Eq)]
 pub struct FundingCreated {
        /// A temporary channel ID, until the funding is established
        pub temporary_channel_id: [u8; 32],
@@ -191,40 +257,68 @@ pub struct FundingCreated {
        pub funding_txid: Txid,
        /// The specific output index funding this channel
        pub funding_output_index: u16,
-       /// The signature of the channel initiator (funder) on the funding transaction
+       /// The signature of the channel initiator (funder) on the initial commitment transaction
        pub signature: Signature,
 }
 
-/// A funding_signed message to be sent or received from a peer
-#[derive(Clone, Debug, PartialEq)]
+/// A [`funding_signed`] message to be sent to or received from a peer.
+///
+/// [`funding_signed`]: https://github.com/lightning/bolts/blob/master/02-peer-protocol.md#the-funding_signed-message
+#[derive(Clone, Debug, PartialEq, Eq)]
 pub struct FundingSigned {
        /// The channel ID
        pub channel_id: [u8; 32],
-       /// The signature of the channel acceptor (fundee) on the funding transaction
+       /// The signature of the channel acceptor (fundee) on the initial commitment transaction
        pub signature: Signature,
 }
 
-/// A funding_locked message to be sent or received from a peer
-#[derive(Clone, Debug, PartialEq)]
-pub struct FundingLocked {
+/// A [`channel_ready`] message to be sent to or received from a peer.
+///
+/// [`channel_ready`]: https://github.com/lightning/bolts/blob/master/02-peer-protocol.md#the-channel_ready-message
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct ChannelReady {
        /// The channel ID
        pub channel_id: [u8; 32],
        /// The per-commitment point of the second commitment transaction
        pub next_per_commitment_point: PublicKey,
+       /// If set, provides a `short_channel_id` alias for this channel.
+       ///
+       /// The sender will accept payments to be forwarded over this SCID and forward them to this
+       /// messages' recipient.
+       pub short_channel_id_alias: Option<u64>,
 }
 
-/// A shutdown message to be sent or received from a peer
-#[derive(Clone, Debug, PartialEq)]
+/// A [`shutdown`] message to be sent to or received from a peer.
+///
+/// [`shutdown`]: https://github.com/lightning/bolts/blob/master/02-peer-protocol.md#closing-initiation-shutdown
+#[derive(Clone, Debug, PartialEq, Eq)]
 pub struct Shutdown {
        /// The channel ID
        pub channel_id: [u8; 32],
        /// The destination of this peer's funds on closing.
-       /// Must be in one of these forms: p2pkh, p2sh, p2wpkh, p2wsh.
+       ///
+       /// Must be in one of these forms: P2PKH, P2SH, P2WPKH, P2WSH, P2TR.
        pub scriptpubkey: Script,
 }
 
-/// A closing_signed message to be sent or received from a peer
-#[derive(Clone, Debug, PartialEq)]
+/// The minimum and maximum fees which the sender is willing to place on the closing transaction.
+///
+/// This is provided in [`ClosingSigned`] by both sides to indicate the fee range they are willing
+/// to use.
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct ClosingSignedFeeRange {
+       /// The minimum absolute fee, in satoshis, which the sender is willing to place on the closing
+       /// transaction.
+       pub min_fee_satoshis: u64,
+       /// The maximum absolute fee, in satoshis, which the sender is willing to place on the closing
+       /// transaction.
+       pub max_fee_satoshis: u64,
+}
+
+/// A [`closing_signed`] message to be sent to or received from a peer.
+///
+/// [`closing_signed`]: https://github.com/lightning/bolts/blob/master/02-peer-protocol.md#closing-negotiation-closing_signed
+#[derive(Clone, Debug, PartialEq, Eq)]
 pub struct ClosingSigned {
        /// The channel ID
        pub channel_id: [u8; 32],
@@ -232,10 +326,15 @@ pub struct ClosingSigned {
        pub fee_satoshis: u64,
        /// A signature on the closing transaction
        pub signature: Signature,
+       /// The minimum and maximum fees which the sender is willing to accept, provided only by new
+       /// nodes.
+       pub fee_range: Option<ClosingSignedFeeRange>,
 }
 
-/// An update_add_htlc message to be sent or received from a peer
-#[derive(Clone, Debug, PartialEq)]
+/// An [`update_add_htlc`] message to be sent to or received from a peer.
+///
+/// [`update_add_htlc`]: https://github.com/lightning/bolts/blob/master/02-peer-protocol.md#adding-an-htlc-update_add_htlc
+#[derive(Clone, Debug, PartialEq, Eq)]
 pub struct UpdateAddHTLC {
        /// The channel ID
        pub channel_id: [u8; 32],
@@ -250,8 +349,20 @@ pub struct UpdateAddHTLC {
        pub(crate) onion_routing_packet: OnionPacket,
 }
 
-/// An update_fulfill_htlc message to be sent or received from a peer
-#[derive(Clone, Debug, PartialEq)]
+ /// An onion message to be sent to or received from a peer.
+ ///
+ // TODO: update with link to OM when they are merged into the BOLTs
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct OnionMessage {
+       /// Used in decrypting the onion packet's payload.
+       pub blinding_point: PublicKey,
+       pub(crate) onion_routing_packet: onion_message::Packet,
+}
+
+/// An [`update_fulfill_htlc`] message to be sent to or received from a peer.
+///
+/// [`update_fulfill_htlc`]: https://github.com/lightning/bolts/blob/master/02-peer-protocol.md#removing-an-htlc-update_fulfill_htlc-update_fail_htlc-and-update_fail_malformed_htlc
+#[derive(Clone, Debug, PartialEq, Eq)]
 pub struct UpdateFulfillHTLC {
        /// The channel ID
        pub channel_id: [u8; 32],
@@ -261,8 +372,10 @@ pub struct UpdateFulfillHTLC {
        pub payment_preimage: PaymentPreimage,
 }
 
-/// An update_fail_htlc message to be sent or received from a peer
-#[derive(Clone, Debug, PartialEq)]
+/// An [`update_fail_htlc`] message to be sent to or received from a peer.
+///
+/// [`update_fail_htlc`]: https://github.com/lightning/bolts/blob/master/02-peer-protocol.md#removing-an-htlc-update_fulfill_htlc-update_fail_htlc-and-update_fail_malformed_htlc
+#[derive(Clone, Debug, PartialEq, Eq)]
 pub struct UpdateFailHTLC {
        /// The channel ID
        pub channel_id: [u8; 32],
@@ -271,8 +384,10 @@ pub struct UpdateFailHTLC {
        pub(crate) reason: OnionErrorPacket,
 }
 
-/// An update_fail_malformed_htlc message to be sent or received from a peer
-#[derive(Clone, Debug, PartialEq)]
+/// An [`update_fail_malformed_htlc`] message to be sent to or received from a peer.
+///
+/// [`update_fail_malformed_htlc`]: https://github.com/lightning/bolts/blob/master/02-peer-protocol.md#removing-an-htlc-update_fulfill_htlc-update_fail_htlc-and-update_fail_malformed_htlc
+#[derive(Clone, Debug, PartialEq, Eq)]
 pub struct UpdateFailMalformedHTLC {
        /// The channel ID
        pub channel_id: [u8; 32],
@@ -283,8 +398,10 @@ pub struct UpdateFailMalformedHTLC {
        pub failure_code: u16,
 }
 
-/// A commitment_signed message to be sent or received from a peer
-#[derive(Clone, Debug, PartialEq)]
+/// A [`commitment_signed`] message to be sent to or received from a peer.
+///
+/// [`commitment_signed`]: https://github.com/lightning/bolts/blob/master/02-peer-protocol.md#committing-updates-so-far-commitment_signed
+#[derive(Clone, Debug, PartialEq, Eq)]
 pub struct CommitmentSigned {
        /// The channel ID
        pub channel_id: [u8; 32],
@@ -294,8 +411,10 @@ pub struct CommitmentSigned {
        pub htlc_signatures: Vec<Signature>,
 }
 
-/// A revoke_and_ack message to be sent or received from a peer
-#[derive(Clone, Debug, PartialEq)]
+/// A [`revoke_and_ack`] message to be sent to or received from a peer.
+///
+/// [`revoke_and_ack`]: https://github.com/lightning/bolts/blob/master/02-peer-protocol.md#completing-the-transition-to-the-updated-state-revoke_and_ack
+#[derive(Clone, Debug, PartialEq, Eq)]
 pub struct RevokeAndACK {
        /// The channel ID
        pub channel_id: [u8; 32],
@@ -305,8 +424,10 @@ pub struct RevokeAndACK {
        pub next_per_commitment_point: PublicKey,
 }
 
-/// An update_fee message to be sent or received from a peer
-#[derive(Clone, Debug, PartialEq)]
+/// An [`update_fee`] message to be sent to or received from a peer
+///
+/// [`update_fee`]: https://github.com/lightning/bolts/blob/master/02-peer-protocol.md#updating-fees-update_fee
+#[derive(Clone, Debug, PartialEq, Eq)]
 pub struct UpdateFee {
        /// The channel ID
        pub channel_id: [u8; 32],
@@ -314,10 +435,11 @@ pub struct UpdateFee {
        pub feerate_per_kw: u32,
 }
 
-#[derive(Clone, Debug, PartialEq)]
+#[derive(Clone, Debug, PartialEq, Eq)]
 /// Proof that the sender knows the per-commitment secret of the previous commitment transaction.
+///
 /// This is used to convince the recipient that the channel is at a certain commitment
-/// number even if they lost that data due to a local failure.  Of course, the peer may lie
+/// number even if they lost that data due to a local failure. Of course, the peer may lie
 /// and even later commitments may have been revoked.
 pub struct DataLossProtect {
        /// Proof that the sender knows the per-commitment secret of a specific commitment transaction
@@ -327,8 +449,10 @@ pub struct DataLossProtect {
        pub my_current_per_commitment_point: PublicKey,
 }
 
-/// A channel_reestablish message to be sent or received from a peer
-#[derive(Clone, Debug, PartialEq)]
+/// A [`channel_reestablish`] message to be sent to or received from a peer.
+///
+/// [`channel_reestablish`]: https://github.com/lightning/bolts/blob/master/02-peer-protocol.md#message-retransmission
+#[derive(Clone, Debug, PartialEq, Eq)]
 pub struct ChannelReestablish {
        /// The channel ID
        pub channel_id: [u8; 32],
@@ -340,8 +464,10 @@ pub struct ChannelReestablish {
        pub data_loss_protect: OptionalField<DataLossProtect>,
 }
 
-/// An announcement_signatures message to be sent or received from a peer
-#[derive(Clone, Debug, PartialEq)]
+/// An [`announcement_signatures`] message to be sent to or received from a peer.
+///
+/// [`announcement_signatures`]: https://github.com/lightning/bolts/blob/master/07-routing-gossip.md#the-announcement_signatures-message
+#[derive(Clone, Debug, PartialEq, Eq)]
 pub struct AnnouncementSignatures {
        /// The channel ID
        pub channel_id: [u8; 32],
@@ -353,8 +479,8 @@ pub struct AnnouncementSignatures {
        pub bitcoin_signature: Signature,
 }
 
-/// An address which can be used to connect to a remote peer
-#[derive(Clone, Debug, PartialEq)]
+/// An address which can be used to connect to a remote peer.
+#[derive(Clone, Debug, PartialEq, Eq)]
 pub enum NetAddress {
        /// An IPv4 address/port on which the peer is listening.
        IPv4 {
@@ -371,14 +497,13 @@ pub enum NetAddress {
                port: u16,
        },
        /// An old-style Tor onion address/port on which the peer is listening.
-       OnionV2 {
-               /// The bytes (usually encoded in base32 with ".onion" appended)
-               addr: [u8; 10],
-               /// The port on which the node is listening
-               port: u16,
-       },
+       ///
+       /// This field is deprecated and the Tor network generally no longer supports V2 Onion
+       /// addresses. Thus, the details are not parsed here.
+       OnionV2([u8; 12]),
        /// A new-style Tor onion address/port on which the peer is listening.
-       /// To create the human-readable "hostname", concatenate ed25519_pubkey, checksum, and version,
+       ///
+       /// To create the human-readable "hostname", concatenate the ED25519 pubkey, checksum, and version,
        /// wrap as base32 and append ".onion".
        OnionV3 {
                /// The ed25519 long-term public key of the peer
@@ -390,16 +515,24 @@ pub enum NetAddress {
                /// The port on which the node is listening
                port: u16,
        },
+       /// A hostname/port on which the peer is listening.
+       Hostname {
+               /// The hostname on which the node is listening.
+               hostname: Hostname,
+               /// The port on which the node is listening.
+               port: u16,
+       },
 }
 impl NetAddress {
-       /// Gets the ID of this address type. Addresses in node_announcement messages should be sorted
+       /// Gets the ID of this address type. Addresses in [`NodeAnnouncement`] messages should be sorted
        /// by this.
        pub(crate) fn get_id(&self) -> u8 {
                match self {
                        &NetAddress::IPv4 {..} => { 1 },
                        &NetAddress::IPv6 {..} => { 2 },
-                       &NetAddress::OnionV2 {..} => { 3 },
+                       &NetAddress::OnionV2(_) => { 3 },
                        &NetAddress::OnionV3 {..} => { 4 },
+                       &NetAddress::Hostname {..} => { 5 },
                }
        }
 
@@ -408,17 +541,21 @@ impl NetAddress {
                match self {
                        &NetAddress::IPv4 { .. } => { 6 },
                        &NetAddress::IPv6 { .. } => { 18 },
-                       &NetAddress::OnionV2 { .. } => { 12 },
+                       &NetAddress::OnionV2(_) => { 12 },
                        &NetAddress::OnionV3 { .. } => { 37 },
+                       // Consists of 1-byte hostname length, hostname bytes, and 2-byte port.
+                       &NetAddress::Hostname { ref hostname, .. } => { u16::from(hostname.len()) + 3 },
                }
        }
 
-       /// The maximum length of any address descriptor, not including the 1-byte type
-       pub(crate) const MAX_LEN: u16 = 37;
+       /// The maximum length of any address descriptor, not including the 1-byte type.
+       /// This maximum length is reached by a hostname address descriptor:
+       /// a hostname with a maximum length of 255, its 1-byte length and a 2-byte port.
+       pub(crate) const MAX_LEN: u16 = 258;
 }
 
 impl Writeable for NetAddress {
-       fn write<W: Writer>(&self, writer: &mut W) -> Result<(), ::std::io::Error> {
+       fn write<W: Writer>(&self, writer: &mut W) -> Result<(), io::Error> {
                match self {
                        &NetAddress::IPv4 { ref addr, ref port } => {
                                1u8.write(writer)?;
@@ -430,10 +567,9 @@ impl Writeable for NetAddress {
                                addr.write(writer)?;
                                port.write(writer)?;
                        },
-                       &NetAddress::OnionV2 { ref addr, ref port } => {
+                       &NetAddress::OnionV2(bytes) => {
                                3u8.write(writer)?;
-                               addr.write(writer)?;
-                               port.write(writer)?;
+                               bytes.write(writer)?;
                        },
                        &NetAddress::OnionV3 { ref ed25519_pubkey, ref checksum, ref version, ref port } => {
                                4u8.write(writer)?;
@@ -441,7 +577,12 @@ impl Writeable for NetAddress {
                                checksum.write(writer)?;
                                version.write(writer)?;
                                port.write(writer)?;
-                       }
+                       },
+                       &NetAddress::Hostname { ref hostname, ref port } => {
+                               5u8.write(writer)?;
+                               hostname.write(writer)?;
+                               port.write(writer)?;
+                       },
                }
                Ok(())
        }
@@ -463,12 +604,7 @@ impl Readable for Result<NetAddress, u8> {
                                        port: Readable::read(reader)?,
                                }))
                        },
-                       3 => {
-                               Ok(Ok(NetAddress::OnionV2 {
-                                       addr: Readable::read(reader)?,
-                                       port: Readable::read(reader)?,
-                               }))
-                       },
+                       3 => Ok(Ok(NetAddress::OnionV2(Readable::read(reader)?))),
                        4 => {
                                Ok(Ok(NetAddress::OnionV3 {
                                        ed25519_pubkey: Readable::read(reader)?,
@@ -477,33 +613,74 @@ impl Readable for Result<NetAddress, u8> {
                                        port: Readable::read(reader)?,
                                }))
                        },
+                       5 => {
+                               Ok(Ok(NetAddress::Hostname {
+                                       hostname: Readable::read(reader)?,
+                                       port: Readable::read(reader)?,
+                               }))
+                       },
                        _ => return Ok(Err(byte)),
                }
        }
 }
 
-/// The unsigned part of a node_announcement
-#[derive(Clone, Debug, PartialEq)]
+impl Readable for NetAddress {
+       fn read<R: Read>(reader: &mut R) -> Result<NetAddress, DecodeError> {
+               match Readable::read(reader) {
+                       Ok(Ok(res)) => Ok(res),
+                       Ok(Err(_)) => Err(DecodeError::UnknownVersion),
+                       Err(e) => Err(e),
+               }
+       }
+}
+
+/// Represents the set of gossip messages that require a signature from a node's identity key.
+pub enum UnsignedGossipMessage<'a> {
+       /// An unsigned channel announcement.
+       ChannelAnnouncement(&'a UnsignedChannelAnnouncement),
+       /// An unsigned channel update.
+       ChannelUpdate(&'a UnsignedChannelUpdate),
+       /// An unsigned node announcement.
+       NodeAnnouncement(&'a UnsignedNodeAnnouncement)
+}
+
+impl<'a> Writeable for UnsignedGossipMessage<'a> {
+       fn write<W: Writer>(&self, writer: &mut W) -> Result<(), io::Error> {
+               match self {
+                       UnsignedGossipMessage::ChannelAnnouncement(ref msg) => msg.write(writer),
+                       UnsignedGossipMessage::ChannelUpdate(ref msg) => msg.write(writer),
+                       UnsignedGossipMessage::NodeAnnouncement(ref msg) => msg.write(writer),
+               }
+       }
+}
+
+/// The unsigned part of a [`node_announcement`] message.
+///
+/// [`node_announcement`]: https://github.com/lightning/bolts/blob/master/07-routing-gossip.md#the-node_announcement-message
+#[derive(Clone, Debug, PartialEq, Eq)]
 pub struct UnsignedNodeAnnouncement {
        /// The advertised features
        pub features: NodeFeatures,
        /// A strictly monotonic announcement counter, with gaps allowed
        pub timestamp: u32,
-       /// The node_id this announcement originated from (don't rebroadcast the node_announcement back
+       /// The `node_id` this announcement originated from (don't rebroadcast the `node_announcement` back
        /// to this node).
-       pub node_id: PublicKey,
+       pub node_id: NodeId,
        /// An RGB color for UI purposes
        pub rgb: [u8; 3],
-       /// An alias, for UI purposes.  This should be sanitized before use.  There is no guarantee
-       /// of uniqueness.
+       /// An alias, for UI purposes.
+       ///
+       /// This should be sanitized before use. There is no guarantee of uniqueness.
        pub alias: [u8; 32],
        /// List of addresses on which this node is reachable
        pub addresses: Vec<NetAddress>,
        pub(crate) excess_address_data: Vec<u8>,
        pub(crate) excess_data: Vec<u8>,
 }
-#[derive(Clone, Debug, PartialEq)]
-/// A node_announcement message to be sent or received from a peer
+#[derive(Clone, Debug, PartialEq, Eq)]
+/// A [`node_announcement`] message to be sent to or received from a peer.
+///
+/// [`node_announcement`]: https://github.com/lightning/bolts/blob/master/07-routing-gossip.md#the-node_announcement-message
 pub struct NodeAnnouncement {
        /// The signature by the node key
        pub signature: Signature,
@@ -511,8 +688,10 @@ pub struct NodeAnnouncement {
        pub contents: UnsignedNodeAnnouncement,
 }
 
-/// The unsigned part of a channel_announcement
-#[derive(Clone, Debug, PartialEq)]
+/// The unsigned part of a [`channel_announcement`] message.
+///
+/// [`channel_announcement`]: https://github.com/lightning/bolts/blob/master/07-routing-gossip.md#the-channel_announcement-message
+#[derive(Clone, Debug, PartialEq, Eq)]
 pub struct UnsignedChannelAnnouncement {
        /// The advertised channel features
        pub features: ChannelFeatures,
@@ -520,18 +699,20 @@ pub struct UnsignedChannelAnnouncement {
        pub chain_hash: BlockHash,
        /// The short channel ID
        pub short_channel_id: u64,
-       /// One of the two node_ids which are endpoints of this channel
-       pub node_id_1: PublicKey,
-       /// The other of the two node_ids which are endpoints of this channel
-       pub node_id_2: PublicKey,
+       /// One of the two `node_id`s which are endpoints of this channel
+       pub node_id_1: NodeId,
+       /// The other of the two `node_id`s which are endpoints of this channel
+       pub node_id_2: NodeId,
        /// The funding key for the first node
-       pub bitcoin_key_1: PublicKey,
+       pub bitcoin_key_1: NodeId,
        /// The funding key for the second node
-       pub bitcoin_key_2: PublicKey,
+       pub bitcoin_key_2: NodeId,
        pub(crate) excess_data: Vec<u8>,
 }
-/// A channel_announcement message to be sent or received from a peer
-#[derive(Clone, Debug, PartialEq)]
+/// A [`channel_announcement`] message to be sent to or received from a peer.
+///
+/// [`channel_announcement`]: https://github.com/lightning/bolts/blob/master/07-routing-gossip.md#the-channel_announcement-message
+#[derive(Clone, Debug, PartialEq, Eq)]
 pub struct ChannelAnnouncement {
        /// Authentication of the announcement by the first public node
        pub node_signature_1: Signature,
@@ -545,8 +726,10 @@ pub struct ChannelAnnouncement {
        pub contents: UnsignedChannelAnnouncement,
 }
 
-/// The unsigned part of a channel_update
-#[derive(Clone, Debug, PartialEq)]
+/// The unsigned part of a [`channel_update`] message.
+///
+/// [`channel_update`]: https://github.com/lightning/bolts/blob/master/07-routing-gossip.md#the-channel_update-message
+#[derive(Clone, Debug, PartialEq, Eq)]
 pub struct UnsignedChannelUpdate {
        /// The genesis hash of the blockchain where the channel is to be opened
        pub chain_hash: BlockHash,
@@ -558,25 +741,33 @@ pub struct UnsignedChannelUpdate {
        pub flags: u8,
        /// The number of blocks such that if:
        /// `incoming_htlc.cltv_expiry < outgoing_htlc.cltv_expiry + cltv_expiry_delta`
-       /// then we need to fail the HTLC backwards. When forwarding an HTLC, cltv_expiry_delta determines
-       /// the outgoing HTLC's minimum cltv_expiry value -- so, if an incoming HTLC comes in with a
-       /// cltv_expiry of 100000, and the node we're forwarding to has a cltv_expiry_delta value of 10,
-       /// then we'll check that the outgoing HTLC's cltv_expiry value is at least 100010 before
+       /// then we need to fail the HTLC backwards. When forwarding an HTLC, `cltv_expiry_delta` determines
+       /// the outgoing HTLC's minimum `cltv_expiry` value -- so, if an incoming HTLC comes in with a
+       /// `cltv_expiry` of 100000, and the node we're forwarding to has a `cltv_expiry_delta` value of 10,
+       /// then we'll check that the outgoing HTLC's `cltv_expiry` value is at least 100010 before
        /// forwarding. Note that the HTLC sender is the one who originally sets this value when
        /// constructing the route.
        pub cltv_expiry_delta: u16,
        /// The minimum HTLC size incoming to sender, in milli-satoshi
        pub htlc_minimum_msat: u64,
-       /// Optionally, the maximum HTLC value incoming to sender, in milli-satoshi
-       pub htlc_maximum_msat: OptionalField<u64>,
+       /// The maximum HTLC value incoming to sender, in milli-satoshi.
+       ///
+       /// This used to be optional.
+       pub htlc_maximum_msat: u64,
        /// The base HTLC fee charged by sender, in milli-satoshi
        pub fee_base_msat: u32,
        /// The amount to fee multiplier, in micro-satoshi
        pub fee_proportional_millionths: u32,
-       pub(crate) excess_data: Vec<u8>,
+       /// Excess data which was signed as a part of the message which we do not (yet) understand how
+       /// to decode.
+       ///
+       /// This is stored to ensure forward-compatibility as new fields are added to the lightning gossip protocol.
+       pub excess_data: Vec<u8>,
 }
-/// A channel_update message to be sent or received from a peer
-#[derive(Clone, Debug, PartialEq)]
+/// A [`channel_update`] message to be sent to or received from a peer.
+///
+/// [`channel_update`]: https://github.com/lightning/bolts/blob/master/07-routing-gossip.md#the-channel_update-message
+#[derive(Clone, Debug, PartialEq, Eq)]
 pub struct ChannelUpdate {
        /// A signature of the channel update
        pub signature: Signature,
@@ -584,11 +775,13 @@ pub struct ChannelUpdate {
        pub contents: UnsignedChannelUpdate,
 }
 
-/// A query_channel_range message is used to query a peer for channel
+/// A [`query_channel_range`] message is used to query a peer for channel
 /// UTXOs in a range of blocks. The recipient of a query makes a best
-/// effort to reply to the query using one or more reply_channel_range
+/// effort to reply to the query using one or more [`ReplyChannelRange`]
 /// messages.
-#[derive(Clone, Debug, PartialEq)]
+///
+/// [`query_channel_range`]: https://github.com/lightning/bolts/blob/master/07-routing-gossip.md#the-query_channel_range-and-reply_channel_range-messages
+#[derive(Clone, Debug, PartialEq, Eq)]
 pub struct QueryChannelRange {
        /// The genesis hash of the blockchain being queried
        pub chain_hash: BlockHash,
@@ -598,14 +791,18 @@ pub struct QueryChannelRange {
        pub number_of_blocks: u32,
 }
 
-/// A reply_channel_range message is a reply to a query_channel_range
-/// message. Multiple reply_channel_range messages can be sent in reply
-/// to a single query_channel_range message. The query recipient makes a
+/// A [`reply_channel_range`] message is a reply to a [`QueryChannelRange`]
+/// message.
+///
+/// Multiple `reply_channel_range` messages can be sent in reply
+/// to a single [`QueryChannelRange`] message. The query recipient makes a
 /// best effort to respond based on their local network view which may
-/// not be a perfect view of the network. The short_channel_ids in the
-/// reply are encoded. We only support encoding_type=0 uncompressed
-/// serialization and do not support encoding_type=1 zlib serialization.
-#[derive(Clone, Debug, PartialEq)]
+/// not be a perfect view of the network. The `short_channel_id`s in the
+/// reply are encoded. We only support `encoding_type=0` uncompressed
+/// serialization and do not support `encoding_type=1` zlib serialization.
+///
+/// [`reply_channel_range`]: https://github.com/lightning/bolts/blob/master/07-routing-gossip.md#the-query_channel_range-and-reply_channel_range-messages
+#[derive(Clone, Debug, PartialEq, Eq)]
 pub struct ReplyChannelRange {
        /// The genesis hash of the blockchain being queried
        pub chain_hash: BlockHash,
@@ -615,19 +812,22 @@ pub struct ReplyChannelRange {
        pub number_of_blocks: u32,
        /// True when this is the final reply for a query
        pub sync_complete: bool,
-       /// The short_channel_ids in the channel range
+       /// The `short_channel_id`s in the channel range
        pub short_channel_ids: Vec<u64>,
 }
 
-/// A query_short_channel_ids message is used to query a peer for
-/// routing gossip messages related to one or more short_channel_ids.
+/// A [`query_short_channel_ids`] message is used to query a peer for
+/// routing gossip messages related to one or more `short_channel_id`s.
+///
 /// The query recipient will reply with the latest, if available,
-/// channel_announcement, channel_update and node_announcement messages
-/// it maintains for the requested short_channel_ids followed by a
-/// reply_short_channel_ids_end message. The short_channel_ids sent in
-/// this query are encoded. We only support encoding_type=0 uncompressed
-/// serialization and do not support encoding_type=1 zlib serialization.
-#[derive(Clone, Debug, PartialEq)]
+/// [`ChannelAnnouncement`], [`ChannelUpdate`] and [`NodeAnnouncement`] messages
+/// it maintains for the requested `short_channel_id`s followed by a
+/// [`ReplyShortChannelIdsEnd`] message. The `short_channel_id`s sent in
+/// this query are encoded. We only support `encoding_type=0` uncompressed
+/// serialization and do not support `encoding_type=1` zlib serialization.
+///
+/// [`query_short_channel_ids`]: https://github.com/lightning/bolts/blob/master/07-routing-gossip.md#the-query_short_channel_idsreply_short_channel_ids_end-messages
+#[derive(Clone, Debug, PartialEq, Eq)]
 pub struct QueryShortChannelIds {
        /// The genesis hash of the blockchain being queried
        pub chain_hash: BlockHash,
@@ -635,23 +835,27 @@ pub struct QueryShortChannelIds {
        pub short_channel_ids: Vec<u64>,
 }
 
-/// A reply_short_channel_ids_end message is sent as a reply to a
-/// query_short_channel_ids message. The query recipient makes a best
+/// A [`reply_short_channel_ids_end`] message is sent as a reply to a
+/// message. The query recipient makes a best
 /// effort to respond based on their local network view which may not be
 /// a perfect view of the network.
-#[derive(Clone, Debug, PartialEq)]
+///
+/// [`reply_short_channel_ids_end`]: https://github.com/lightning/bolts/blob/master/07-routing-gossip.md#the-query_short_channel_idsreply_short_channel_ids_end-messages
+#[derive(Clone, Debug, PartialEq, Eq)]
 pub struct ReplyShortChannelIdsEnd {
        /// The genesis hash of the blockchain that was queried
        pub chain_hash: BlockHash,
        /// Indicates if the query recipient maintains up-to-date channel
-       /// information for the chain_hash
+       /// information for the `chain_hash`
        pub full_information: bool,
 }
 
-/// A gossip_timestamp_filter message is used by a node to request
+/// A [`gossip_timestamp_filter`] message is used by a node to request
 /// gossip relay for messages in the requested time range when the
-/// gossip_queries feature has been negotiated.
-#[derive(Clone, Debug, PartialEq)]
+/// `gossip_queries` feature has been negotiated.
+///
+/// [`gossip_timestamp_filter`]: https://github.com/lightning/bolts/blob/master/07-routing-gossip.md#the-gossip_timestamp_filter-message
+#[derive(Clone, Debug, PartialEq, Eq)]
 pub struct GossipTimestampFilter {
        /// The genesis hash of the blockchain for channel and node information
        pub chain_hash: BlockHash,
@@ -662,12 +866,14 @@ pub struct GossipTimestampFilter {
 }
 
 /// Encoding type for data compression of collections in gossip queries.
-/// We do not support encoding_type=1 zlib serialization defined in BOLT #7.
+///
+/// We do not support `encoding_type=1` zlib serialization [defined in BOLT
+/// #7](https://github.com/lightning/bolts/blob/master/07-routing-gossip.md#query-messages).
 enum EncodingType {
        Uncompressed = 0x00,
 }
 
-/// Used to put an error message in a LightningError
+/// Used to put an error message in a [`LightningError`].
 #[derive(Clone, Debug)]
 pub enum ErrorAction {
        /// The peer took some action which made us think they were useless. Disconnect them.
@@ -676,11 +882,28 @@ pub enum ErrorAction {
                msg: Option<ErrorMessage>
        },
        /// The peer did something harmless that we weren't able to process, just log and ignore
+       // New code should *not* use this. New code must use IgnoreAndLog, below!
        IgnoreError,
+       /// The peer did something harmless that we weren't able to meaningfully process.
+       /// If the error is logged, log it at the given level.
+       IgnoreAndLog(logger::Level),
+       /// The peer provided us with a gossip message which we'd already seen. In most cases this
+       /// should be ignored, but it may result in the message being forwarded if it is a duplicate of
+       /// our own channel announcements.
+       IgnoreDuplicateGossip,
        /// The peer did something incorrect. Tell them.
        SendErrorMessage {
                /// The message to send.
-               msg: ErrorMessage
+               msg: ErrorMessage,
+       },
+       /// The peer did something incorrect. Tell them without closing any channels.
+       SendWarningMessage {
+               /// The message to send.
+               msg: WarningMessage,
+               /// The peer may have done something harmless that we weren't able to meaningfully process,
+               /// though we should still tell them about it.
+               /// If this event is logged, log it at the given level.
+               log_level: logger::Level,
        },
 }
 
@@ -693,59 +916,31 @@ pub struct LightningError {
        pub action: ErrorAction,
 }
 
-/// Struct used to return values from revoke_and_ack messages, containing a bunch of commitment
+/// Struct used to return values from [`RevokeAndACK`] messages, containing a bunch of commitment
 /// transaction updates if they were pending.
-#[derive(Clone, Debug, PartialEq)]
+#[derive(Clone, Debug, PartialEq, Eq)]
 pub struct CommitmentUpdate {
-       /// update_add_htlc messages which should be sent
+       /// `update_add_htlc` messages which should be sent
        pub update_add_htlcs: Vec<UpdateAddHTLC>,
-       /// update_fulfill_htlc messages which should be sent
+       /// `update_fulfill_htlc` messages which should be sent
        pub update_fulfill_htlcs: Vec<UpdateFulfillHTLC>,
-       /// update_fail_htlc messages which should be sent
+       /// `update_fail_htlc` messages which should be sent
        pub update_fail_htlcs: Vec<UpdateFailHTLC>,
-       /// update_fail_malformed_htlc messages which should be sent
+       /// `update_fail_malformed_htlc` messages which should be sent
        pub update_fail_malformed_htlcs: Vec<UpdateFailMalformedHTLC>,
-       /// An update_fee message which should be sent
+       /// An `update_fee` message which should be sent
        pub update_fee: Option<UpdateFee>,
-       /// Finally, the commitment_signed message which should be sent
+       /// A `commitment_signed` message which should be sent
        pub commitment_signed: CommitmentSigned,
 }
 
-/// The information we received from a peer along the route of a payment we originated. This is
-/// returned by ChannelMessageHandler::handle_update_fail_htlc to be passed into
-/// RoutingMessageHandler::handle_htlc_fail_channel_update to update our network map.
-#[derive(Clone, Debug, PartialEq)]
-pub enum HTLCFailChannelUpdate {
-       /// We received an error which included a full ChannelUpdate message.
-       ChannelUpdateMessage {
-               /// The unwrapped message we received
-               msg: ChannelUpdate,
-       },
-       /// We received an error which indicated only that a channel has been closed
-       ChannelClosed {
-               /// The short_channel_id which has now closed.
-               short_channel_id: u64,
-               /// when this true, this channel should be permanently removed from the
-               /// consideration. Otherwise, this channel can be restored as new channel_update is received
-               is_permanent: bool,
-       },
-       /// We received an error which indicated only that a node has failed
-       NodeFailure {
-               /// The node_id that has failed.
-               node_id: PublicKey,
-               /// when this true, node should be permanently removed from the
-               /// consideration. Otherwise, the channels connected to this node can be
-               /// restored as new channel_update is received
-               is_permanent: bool,
-       }
-}
-
 /// Messages could have optional fields to use with extended features
-/// As we wish to serialize these differently from Option<T>s (Options get a tag byte, but
-/// OptionalFeild simply gets Present if there are enough bytes to read into it), we have a
+/// As we wish to serialize these differently from `Option<T>`s (`Options` get a tag byte, but
+/// [`OptionalField`] simply gets `Present` if there are enough bytes to read into it), we have a
 /// separate enum type for them.
-/// (C-not exported) due to a free generic in T
-#[derive(Clone, Debug, PartialEq)]
+///
+/// This is not exported to bindings users due to a free generic in `T`
+#[derive(Clone, Debug, PartialEq, Eq)]
 pub enum OptionalField<T> {
        /// Optional field is included in message
        Present(T),
@@ -755,100 +950,116 @@ pub enum OptionalField<T> {
 
 /// A trait to describe an object which can receive channel messages.
 ///
-/// Messages MAY be called in parallel when they originate from different their_node_ids, however
-/// they MUST NOT be called in parallel when the two calls have the same their_node_id.
+/// Messages MAY be called in parallel when they originate from different `their_node_ids`, however
+/// they MUST NOT be called in parallel when the two calls have the same `their_node_id`.
 pub trait ChannelMessageHandler : MessageSendEventsProvider {
-       //Channel init:
-       /// Handle an incoming open_channel message from the given peer.
-       fn handle_open_channel(&self, their_node_id: &PublicKey, their_features: InitFeatures, msg: &OpenChannel);
-       /// Handle an incoming accept_channel message from the given peer.
-       fn handle_accept_channel(&self, their_node_id: &PublicKey, their_features: InitFeatures, msg: &AcceptChannel);
-       /// Handle an incoming funding_created message from the given peer.
+       // Channel init:
+       /// Handle an incoming `open_channel` message from the given peer.
+       fn handle_open_channel(&self, their_node_id: &PublicKey, msg: &OpenChannel);
+       /// Handle an incoming `accept_channel` message from the given peer.
+       fn handle_accept_channel(&self, their_node_id: &PublicKey, msg: &AcceptChannel);
+       /// Handle an incoming `funding_created` message from the given peer.
        fn handle_funding_created(&self, their_node_id: &PublicKey, msg: &FundingCreated);
-       /// Handle an incoming funding_signed message from the given peer.
+       /// Handle an incoming `funding_signed` message from the given peer.
        fn handle_funding_signed(&self, their_node_id: &PublicKey, msg: &FundingSigned);
-       /// Handle an incoming funding_locked message from the given peer.
-       fn handle_funding_locked(&self, their_node_id: &PublicKey, msg: &FundingLocked);
+       /// Handle an incoming `channel_ready` message from the given peer.
+       fn handle_channel_ready(&self, their_node_id: &PublicKey, msg: &ChannelReady);
 
        // Channl close:
-       /// Handle an incoming shutdown message from the given peer.
-       fn handle_shutdown(&self, their_node_id: &PublicKey, their_features: &InitFeatures, msg: &Shutdown);
-       /// Handle an incoming closing_signed message from the given peer.
+       /// Handle an incoming `shutdown` message from the given peer.
+       fn handle_shutdown(&self, their_node_id: &PublicKey, msg: &Shutdown);
+       /// Handle an incoming `closing_signed` message from the given peer.
        fn handle_closing_signed(&self, their_node_id: &PublicKey, msg: &ClosingSigned);
 
        // HTLC handling:
-       /// Handle an incoming update_add_htlc message from the given peer.
+       /// Handle an incoming `update_add_htlc` message from the given peer.
        fn handle_update_add_htlc(&self, their_node_id: &PublicKey, msg: &UpdateAddHTLC);
-       /// Handle an incoming update_fulfill_htlc message from the given peer.
+       /// Handle an incoming `update_fulfill_htlc` message from the given peer.
        fn handle_update_fulfill_htlc(&self, their_node_id: &PublicKey, msg: &UpdateFulfillHTLC);
-       /// Handle an incoming update_fail_htlc message from the given peer.
+       /// Handle an incoming `update_fail_htlc` message from the given peer.
        fn handle_update_fail_htlc(&self, their_node_id: &PublicKey, msg: &UpdateFailHTLC);
-       /// Handle an incoming update_fail_malformed_htlc message from the given peer.
+       /// Handle an incoming `update_fail_malformed_htlc` message from the given peer.
        fn handle_update_fail_malformed_htlc(&self, their_node_id: &PublicKey, msg: &UpdateFailMalformedHTLC);
-       /// Handle an incoming commitment_signed message from the given peer.
+       /// Handle an incoming `commitment_signed` message from the given peer.
        fn handle_commitment_signed(&self, their_node_id: &PublicKey, msg: &CommitmentSigned);
-       /// Handle an incoming revoke_and_ack message from the given peer.
+       /// Handle an incoming `revoke_and_ack` message from the given peer.
        fn handle_revoke_and_ack(&self, their_node_id: &PublicKey, msg: &RevokeAndACK);
 
-       /// Handle an incoming update_fee message from the given peer.
+       /// Handle an incoming `update_fee` message from the given peer.
        fn handle_update_fee(&self, their_node_id: &PublicKey, msg: &UpdateFee);
 
        // Channel-to-announce:
-       /// Handle an incoming announcement_signatures message from the given peer.
+       /// Handle an incoming `announcement_signatures` message from the given peer.
        fn handle_announcement_signatures(&self, their_node_id: &PublicKey, msg: &AnnouncementSignatures);
 
        // Connection loss/reestablish:
-       /// Indicates a connection to the peer failed/an existing connection was lost. If no connection
-       /// is believed to be possible in the future (eg they're sending us messages we don't
-       /// understand or indicate they require unknown feature bits), no_connection_possible is set
-       /// and any outstanding channels should be failed.
-       fn peer_disconnected(&self, their_node_id: &PublicKey, no_connection_possible: bool);
-
-       /// Handle a peer reconnecting, possibly generating channel_reestablish message(s).
-       fn peer_connected(&self, their_node_id: &PublicKey, msg: &Init);
-       /// Handle an incoming channel_reestablish message from the given peer.
+       /// Indicates a connection to the peer failed/an existing connection was lost.
+       fn peer_disconnected(&self, their_node_id: &PublicKey);
+
+       /// Handle a peer reconnecting, possibly generating `channel_reestablish` message(s).
+       ///
+       /// May return an `Err(())` if the features the peer supports are not sufficient to communicate
+       /// with us. Implementors should be somewhat conservative about doing so, however, as other
+       /// message handlers may still wish to communicate with this peer.
+       fn peer_connected(&self, their_node_id: &PublicKey, msg: &Init, inbound: bool) -> Result<(), ()>;
+       /// Handle an incoming `channel_reestablish` message from the given peer.
        fn handle_channel_reestablish(&self, their_node_id: &PublicKey, msg: &ChannelReestablish);
 
-       /// Handle an incoming channel update from the given peer.
+       /// Handle an incoming `channel_update` message from the given peer.
        fn handle_channel_update(&self, their_node_id: &PublicKey, msg: &ChannelUpdate);
 
        // Error:
-       /// Handle an incoming error message from the given peer.
+       /// Handle an incoming `error` message from the given peer.
        fn handle_error(&self, their_node_id: &PublicKey, msg: &ErrorMessage);
+
+       // Handler information:
+       /// Gets the node feature flags which this handler itself supports. All available handlers are
+       /// queried similarly and their feature flags are OR'd together to form the [`NodeFeatures`]
+       /// which are broadcasted in our [`NodeAnnouncement`] message.
+       fn provided_node_features(&self) -> NodeFeatures;
+
+       /// Gets the init feature flags which should be sent to the given peer. All available handlers
+       /// are queried similarly and their feature flags are OR'd together to form the [`InitFeatures`]
+       /// which are sent in our [`Init`] message.
+       ///
+       /// Note that this method is called before [`Self::peer_connected`].
+       fn provided_init_features(&self, their_node_id: &PublicKey) -> InitFeatures;
 }
 
 /// A trait to describe an object which can receive routing messages.
 ///
 /// # Implementor DoS Warnings
 ///
-/// For `gossip_queries` messages there are potential DoS vectors when handling
-/// inbound queries. Implementors using an on-disk network graph should be aware of
+/// For messages enabled with the `gossip_queries` feature there are potential DoS vectors when
+/// handling inbound queries. Implementors using an on-disk network graph should be aware of
 /// repeated disk I/O for queries accessing different parts of the network graph.
 pub trait RoutingMessageHandler : MessageSendEventsProvider {
-       /// Handle an incoming node_announcement message, returning true if it should be forwarded on,
-       /// false or returning an Err otherwise.
+       /// Handle an incoming `node_announcement` message, returning `true` if it should be forwarded on,
+       /// `false` or returning an `Err` otherwise.
        fn handle_node_announcement(&self, msg: &NodeAnnouncement) -> Result<bool, LightningError>;
-       /// Handle a channel_announcement message, returning true if it should be forwarded on, false
-       /// or returning an Err otherwise.
+       /// Handle a `channel_announcement` message, returning `true` if it should be forwarded on, `false`
+       /// or returning an `Err` otherwise.
        fn handle_channel_announcement(&self, msg: &ChannelAnnouncement) -> Result<bool, LightningError>;
-       /// Handle an incoming channel_update message, returning true if it should be forwarded on,
-       /// false or returning an Err otherwise.
+       /// Handle an incoming `channel_update` message, returning true if it should be forwarded on,
+       /// `false` or returning an `Err` otherwise.
        fn handle_channel_update(&self, msg: &ChannelUpdate) -> Result<bool, LightningError>;
-       /// Handle some updates to the route graph that we learned due to an outbound failed payment.
-       fn handle_htlc_fail_channel_update(&self, update: &HTLCFailChannelUpdate);
-       /// Gets a subset of the channel announcements and updates required to dump our routing table
-       /// to a remote node, starting at the short_channel_id indicated by starting_point and
-       /// including the batch_amount entries immediately higher in numerical value than starting_point.
-       fn get_next_channel_announcements(&self, starting_point: u64, batch_amount: u8) -> Vec<(ChannelAnnouncement, Option<ChannelUpdate>, Option<ChannelUpdate>)>;
-       /// Gets a subset of the node announcements required to dump our routing table to a remote node,
-       /// starting at the node *after* the provided publickey and including batch_amount entries
-       /// immediately higher (as defined by <PublicKey as Ord>::cmp) than starting_point.
-       /// If None is provided for starting_point, we start at the first node.
-       fn get_next_node_announcements(&self, starting_point: Option<&PublicKey>, batch_amount: u8) -> Vec<NodeAnnouncement>;
+       /// Gets channel announcements and updates required to dump our routing table to a remote node,
+       /// starting at the `short_channel_id` indicated by `starting_point` and including announcements
+       /// for a single channel.
+       fn get_next_channel_announcement(&self, starting_point: u64) -> Option<(ChannelAnnouncement, Option<ChannelUpdate>, Option<ChannelUpdate>)>;
+       /// Gets a node announcement required to dump our routing table to a remote node, starting at
+       /// the node *after* the provided pubkey and including up to one announcement immediately
+       /// higher (as defined by `<PublicKey as Ord>::cmp`) than `starting_point`.
+       /// If `None` is provided for `starting_point`, we start at the first node.
+       fn get_next_node_announcement(&self, starting_point: Option<&NodeId>) -> Option<NodeAnnouncement>;
        /// Called when a connection is established with a peer. This can be used to
        /// perform routing table synchronization using a strategy defined by the
        /// implementor.
-       fn sync_routing_table(&self, their_node_id: &PublicKey, init: &Init);
+       ///
+       /// May return an `Err(())` if the features the peer supports are not sufficient to communicate
+       /// with us. Implementors should be somewhat conservative about doing so, however, as other
+       /// message handlers may still wish to communicate with this peer.
+       fn peer_connected(&self, their_node_id: &PublicKey, init: &Init, inbound: bool) -> Result<(), ()>;
        /// Handles the reply of a query we initiated to learn about channels
        /// for a given range of blocks. We can expect to receive one or more
        /// replies to a single query.
@@ -858,16 +1069,65 @@ pub trait RoutingMessageHandler : MessageSendEventsProvider {
        /// a node has completed its best effort to send us the pertaining routing
        /// gossip messages.
        fn handle_reply_short_channel_ids_end(&self, their_node_id: &PublicKey, msg: ReplyShortChannelIdsEnd) -> Result<(), LightningError>;
-       /// Handles when a peer asks us to send a list of short_channel_ids
+       /// Handles when a peer asks us to send a list of `short_channel_id`s
        /// for the requested range of blocks.
        fn handle_query_channel_range(&self, their_node_id: &PublicKey, msg: QueryChannelRange) -> Result<(), LightningError>;
        /// Handles when a peer asks us to send routing gossip messages for a
-       /// list of short_channel_ids.
+       /// list of `short_channel_id`s.
        fn handle_query_short_channel_ids(&self, their_node_id: &PublicKey, msg: QueryShortChannelIds) -> Result<(), LightningError>;
+
+       // Handler queueing status:
+       /// Indicates that there are a large number of [`ChannelAnnouncement`] (or other) messages
+       /// pending some async action. While there is no guarantee of the rate of future messages, the
+       /// caller should seek to reduce the rate of new gossip messages handled, especially
+       /// [`ChannelAnnouncement`]s.
+       fn processing_queue_high(&self) -> bool;
+
+       // Handler information:
+       /// Gets the node feature flags which this handler itself supports. All available handlers are
+       /// queried similarly and their feature flags are OR'd together to form the [`NodeFeatures`]
+       /// which are broadcasted in our [`NodeAnnouncement`] message.
+       fn provided_node_features(&self) -> NodeFeatures;
+       /// Gets the init feature flags which should be sent to the given peer. All available handlers
+       /// are queried similarly and their feature flags are OR'd together to form the [`InitFeatures`]
+       /// which are sent in our [`Init`] message.
+       ///
+       /// Note that this method is called before [`Self::peer_connected`].
+       fn provided_init_features(&self, their_node_id: &PublicKey) -> InitFeatures;
+}
+
+/// A trait to describe an object that can receive onion messages.
+pub trait OnionMessageHandler : OnionMessageProvider {
+       /// Handle an incoming `onion_message` message from the given peer.
+       fn handle_onion_message(&self, peer_node_id: &PublicKey, msg: &OnionMessage);
+       /// Called when a connection is established with a peer. Can be used to track which peers
+       /// advertise onion message support and are online.
+       ///
+       /// May return an `Err(())` if the features the peer supports are not sufficient to communicate
+       /// with us. Implementors should be somewhat conservative about doing so, however, as other
+       /// message handlers may still wish to communicate with this peer.
+       fn peer_connected(&self, their_node_id: &PublicKey, init: &Init, inbound: bool) -> Result<(), ()>;
+       /// Indicates a connection to the peer failed/an existing connection was lost. Allows handlers to
+       /// drop and refuse to forward onion messages to this peer.
+       fn peer_disconnected(&self, their_node_id: &PublicKey);
+
+       // Handler information:
+       /// Gets the node feature flags which this handler itself supports. All available handlers are
+       /// queried similarly and their feature flags are OR'd together to form the [`NodeFeatures`]
+       /// which are broadcasted in our [`NodeAnnouncement`] message.
+       fn provided_node_features(&self) -> NodeFeatures;
+
+       /// Gets the init feature flags which should be sent to the given peer. All available handlers
+       /// are queried similarly and their feature flags are OR'd together to form the [`InitFeatures`]
+       /// which are sent in our [`Init`] message.
+       ///
+       /// Note that this method is called before [`Self::peer_connected`].
+       fn provided_init_features(&self, their_node_id: &PublicKey) -> InitFeatures;
 }
 
 mod fuzzy_internal_msgs {
-       use ln::PaymentSecret;
+       use crate::prelude::*;
+       use crate::ln::{PaymentPreimage, PaymentSecret};
 
        // These types aren't intended to be pub, but are exposed for direct fuzzing (as we deserialize
        // them from untrusted input):
@@ -880,14 +1140,12 @@ mod fuzzy_internal_msgs {
        }
 
        pub(crate) enum OnionHopDataFormat {
-               Legacy { // aka Realm-0
-                       short_channel_id: u64,
-               },
                NonFinalNode {
                        short_channel_id: u64,
                },
                FinalNode {
                        payment_data: Option<FinalOnionHopData>,
+                       keysend_preimage: Option<PaymentPreimage>,
                },
        }
 
@@ -897,7 +1155,6 @@ mod fuzzy_internal_msgs {
                /// Message serialization may panic if this value is more than 21 million Bitcoin.
                pub(crate) amt_to_forward: u64,
                pub(crate) outgoing_cltv_value: u32,
-               // 12 bytes of 0-padding for Legacy format
        }
 
        pub struct DecodedOnionErrorPacket {
@@ -906,22 +1163,37 @@ mod fuzzy_internal_msgs {
                pub(crate) pad: Vec<u8>,
        }
 }
-#[cfg(feature = "fuzztarget")]
+#[cfg(fuzzing)]
 pub use self::fuzzy_internal_msgs::*;
-#[cfg(not(feature = "fuzztarget"))]
+#[cfg(not(fuzzing))]
 pub(crate) use self::fuzzy_internal_msgs::*;
 
 #[derive(Clone)]
 pub(crate) struct OnionPacket {
        pub(crate) version: u8,
-       /// In order to ensure we always return an error on Onion decode in compliance with BOLT 4, we
-       /// have to deserialize OnionPackets contained in UpdateAddHTLCs even if the ephemeral public
-       /// key (here) is bogus, so we hold a Result instead of a PublicKey as we'd like.
+       /// In order to ensure we always return an error on onion decode in compliance with [BOLT
+       /// #4](https://github.com/lightning/bolts/blob/master/04-onion-routing.md), we have to
+       /// deserialize `OnionPacket`s contained in [`UpdateAddHTLC`] messages even if the ephemeral
+       /// public key (here) is bogus, so we hold a [`Result`] instead of a [`PublicKey`] as we'd
+       /// like.
        pub(crate) public_key: Result<PublicKey, secp256k1::Error>,
        pub(crate) hop_data: [u8; 20*65],
        pub(crate) hmac: [u8; 32],
 }
 
+impl onion_utils::Packet for OnionPacket {
+       type Data = onion_utils::FixedSizeOnionPacket;
+       fn new(pubkey: PublicKey, hop_data: onion_utils::FixedSizeOnionPacket, hmac: [u8; 32]) -> Self {
+               Self {
+                       version: 0,
+                       public_key: Ok(pubkey),
+                       hop_data: hop_data.0,
+                       hmac,
+               }
+       }
+}
+
+impl Eq for OnionPacket { }
 impl PartialEq for OnionPacket {
        fn eq(&self, other: &OnionPacket) -> bool {
                for (i, j) in self.hop_data.iter().zip(other.hop_data.iter()) {
@@ -939,7 +1211,7 @@ impl fmt::Debug for OnionPacket {
        }
 }
 
-#[derive(Clone, Debug, PartialEq)]
+#[derive(Clone, Debug, PartialEq, Eq)]
 pub(crate) struct OnionErrorPacket {
        // This really should be a constant size slice, but the spec lets these things be up to 128KB?
        // (TODO) We limit it in decode to much lower...
@@ -954,15 +1226,15 @@ impl fmt::Display for DecodeError {
                        DecodeError::InvalidValue => f.write_str("Nonsense bytes didn't map to the type they were interpreted as"),
                        DecodeError::ShortRead => f.write_str("Packet extended beyond the provided bytes"),
                        DecodeError::BadLengthDescriptor => f.write_str("A length descriptor in the packet didn't describe the later data correctly"),
-                       DecodeError::Io(ref e) => e.fmt(f),
+                       DecodeError::Io(ref e) => fmt::Debug::fmt(e, f),
                        DecodeError::UnsupportedCompression => f.write_str("We don't support receiving messages with zlib-compressed fields"),
                }
        }
 }
 
-impl From<::std::io::Error> for DecodeError {
-       fn from(e: ::std::io::Error) -> Self {
-               if e.kind() == ::std::io::ErrorKind::UnexpectedEof {
+impl From<io::Error> for DecodeError {
+       fn from(e: io::Error) -> Self {
+               if e.kind() == io::ErrorKind::UnexpectedEof {
                        DecodeError::ShortRead
                } else {
                        DecodeError::Io(e.kind())
@@ -971,7 +1243,7 @@ impl From<::std::io::Error> for DecodeError {
 }
 
 impl Writeable for OptionalField<Script> {
-       fn write<W: Writer>(&self, w: &mut W) -> Result<(), ::std::io::Error> {
+       fn write<W: Writer>(&self, w: &mut W) -> Result<(), io::Error> {
                match *self {
                        OptionalField::Present(ref script) => {
                                // Note that Writeable for script includes the 16-bit length tag for us
@@ -998,7 +1270,7 @@ impl Readable for OptionalField<Script> {
 }
 
 impl Writeable for OptionalField<u64> {
-       fn write<W: Writer>(&self, w: &mut W) -> Result<(), ::std::io::Error> {
+       fn write<W: Writer>(&self, w: &mut W) -> Result<(), io::Error> {
                match *self {
                        OptionalField::Present(ref value) => {
                                value.write(w)?;
@@ -1017,10 +1289,7 @@ impl Readable for OptionalField<u64> {
 }
 
 
-impl_writeable_len_match!(AcceptChannel, {
-               {AcceptChannel{ shutdown_scriptpubkey: OptionalField::Present(ref script), .. }, 270 + 2 + script.len()},
-               {_, 270}
-       }, {
+impl_writeable_msg!(AcceptChannel, {
        temporary_channel_id,
        dust_limit_satoshis,
        max_htlc_value_in_flight_msat,
@@ -1036,18 +1305,19 @@ impl_writeable_len_match!(AcceptChannel, {
        htlc_basepoint,
        first_per_commitment_point,
        shutdown_scriptpubkey
+}, {
+       (1, channel_type, option),
 });
 
-impl_writeable!(AnnouncementSignatures, 32+8+64*2, {
+impl_writeable_msg!(AnnouncementSignatures, {
        channel_id,
        short_channel_id,
        node_signature,
        bitcoin_signature
-});
+}, {});
 
 impl Writeable for ChannelReestablish {
-       fn write<W: Writer>(&self, w: &mut W) -> Result<(), ::std::io::Error> {
-               w.size_hint(if let OptionalField::Present(..) = self.data_loss_protect { 32+2*8+33+32 } else { 32+2*8 });
+       fn write<W: Writer>(&self, w: &mut W) -> Result<(), io::Error> {
                self.channel_id.write(w)?;
                self.next_local_commitment_number.write(w)?;
                self.next_remote_commitment_number.write(w)?;
@@ -1083,51 +1353,57 @@ impl Readable for ChannelReestablish{
        }
 }
 
-impl_writeable!(ClosingSigned, 32+8+64, {
-       channel_id,
-       fee_satoshis,
-       signature
+impl_writeable_msg!(ClosingSigned,
+       { channel_id, fee_satoshis, signature },
+       { (1, fee_range, option) }
+);
+
+impl_writeable!(ClosingSignedFeeRange, {
+       min_fee_satoshis,
+       max_fee_satoshis
 });
 
-impl_writeable_len_match!(CommitmentSigned, {
-               { CommitmentSigned { ref htlc_signatures, .. }, 32+64+2+htlc_signatures.len()*64 }
-       }, {
+impl_writeable_msg!(CommitmentSigned, {
        channel_id,
        signature,
        htlc_signatures
-});
+}, {});
 
-impl_writeable_len_match!(DecodedOnionErrorPacket, {
-               { DecodedOnionErrorPacket { ref failuremsg, ref pad, .. }, 32 + 4 + failuremsg.len() + pad.len() }
-       }, {
+impl_writeable!(DecodedOnionErrorPacket, {
        hmac,
        failuremsg,
        pad
 });
 
-impl_writeable!(FundingCreated, 32+32+2+64, {
+impl_writeable_msg!(FundingCreated, {
        temporary_channel_id,
        funding_txid,
        funding_output_index,
        signature
-});
+}, {});
 
-impl_writeable!(FundingSigned, 32+64, {
+impl_writeable_msg!(FundingSigned, {
        channel_id,
        signature
-});
+}, {});
 
-impl_writeable!(FundingLocked, 32+33, {
+impl_writeable_msg!(ChannelReady, {
        channel_id,
-       next_per_commitment_point
+       next_per_commitment_point,
+}, {
+       (1, short_channel_id_alias, option),
 });
 
 impl Writeable for Init {
-       fn write<W: Writer>(&self, w: &mut W) -> Result<(), ::std::io::Error> {
+       fn write<W: Writer>(&self, w: &mut W) -> Result<(), io::Error> {
                // global_features gets the bottom 13 bits of our features, and local_features gets all of
                // our relevant feature bits. This keeps us compatible with old nodes.
                self.features.write_up_to_13(w)?;
-               self.features.write(w)
+               self.features.write(w)?;
+               encode_tlv_stream!(w, {
+                       (3, self.remote_network_address, option)
+               });
+               Ok(())
        }
 }
 
@@ -1135,16 +1411,18 @@ impl Readable for Init {
        fn read<R: Read>(r: &mut R) -> Result<Self, DecodeError> {
                let global_features: InitFeatures = Readable::read(r)?;
                let features: InitFeatures = Readable::read(r)?;
+               let mut remote_network_address: Option<NetAddress> = None;
+               decode_tlv_stream!(r, {
+                       (3, remote_network_address, option)
+               });
                Ok(Init {
                        features: features.or(global_features),
+                       remote_network_address,
                })
        }
 }
 
-impl_writeable_len_match!(OpenChannel, {
-               { OpenChannel { shutdown_scriptpubkey: OptionalField::Present(ref script), .. }, 319 + 2 + script.len() },
-               { _, 319 }
-       }, {
+impl_writeable_msg!(OpenChannel, {
        chain_hash,
        temporary_channel_id,
        funding_satoshis,
@@ -1164,56 +1442,57 @@ impl_writeable_len_match!(OpenChannel, {
        first_per_commitment_point,
        channel_flags,
        shutdown_scriptpubkey
+}, {
+       (1, channel_type, option),
 });
 
-impl_writeable!(RevokeAndACK, 32+32+33, {
+impl_writeable_msg!(RevokeAndACK, {
        channel_id,
        per_commitment_secret,
        next_per_commitment_point
-});
+}, {});
 
-impl_writeable_len_match!(Shutdown, {
-               { Shutdown { ref scriptpubkey, .. }, 32 + 2 + scriptpubkey.len() }
-       }, {
+impl_writeable_msg!(Shutdown, {
        channel_id,
        scriptpubkey
-});
+}, {});
 
-impl_writeable_len_match!(UpdateFailHTLC, {
-               { UpdateFailHTLC { ref reason, .. }, 32 + 10 + reason.data.len() }
-       }, {
+impl_writeable_msg!(UpdateFailHTLC, {
        channel_id,
        htlc_id,
        reason
-});
+}, {});
 
-impl_writeable!(UpdateFailMalformedHTLC, 32+8+32+2, {
+impl_writeable_msg!(UpdateFailMalformedHTLC, {
        channel_id,
        htlc_id,
        sha256_of_onion,
        failure_code
-});
+}, {});
 
-impl_writeable!(UpdateFee, 32+4, {
+impl_writeable_msg!(UpdateFee, {
        channel_id,
        feerate_per_kw
-});
+}, {});
 
-impl_writeable!(UpdateFulfillHTLC, 32+8+32, {
+impl_writeable_msg!(UpdateFulfillHTLC, {
        channel_id,
        htlc_id,
        payment_preimage
-});
+}, {});
 
-impl_writeable_len_match!(OnionErrorPacket, {
-               { OnionErrorPacket { ref data, .. }, 2 + data.len() }
-       }, {
+// Note that this is written as a part of ChannelManager objects, and thus cannot change its
+// serialization format in a way which assumes we know the total serialized length/message end
+// position.
+impl_writeable!(OnionErrorPacket, {
        data
 });
 
+// Note that this is written as a part of ChannelManager objects, and thus cannot change its
+// serialization format in a way which assumes we know the total serialized length/message end
+// position.
 impl Writeable for OnionPacket {
-       fn write<W: Writer>(&self, w: &mut W) -> Result<(), ::std::io::Error> {
-               w.size_hint(1 + 33 + 20*65 + 32);
+       fn write<W: Writer>(&self, w: &mut W) -> Result<(), io::Error> {
                self.version.write(w)?;
                match self.public_key {
                        Ok(pubkey) => pubkey.write(w)?,
@@ -1240,62 +1519,69 @@ impl Readable for OnionPacket {
        }
 }
 
-impl_writeable!(UpdateAddHTLC, 32+8+8+32+4+1366, {
+impl_writeable_msg!(UpdateAddHTLC, {
        channel_id,
        htlc_id,
        amount_msat,
        payment_hash,
        cltv_expiry,
        onion_routing_packet
-});
+}, {});
+
+impl Readable for OnionMessage {
+       fn read<R: Read>(r: &mut R) -> Result<Self, DecodeError> {
+               let blinding_point: PublicKey = Readable::read(r)?;
+               let len: u16 = Readable::read(r)?;
+               let mut packet_reader = FixedLengthReader::new(r, len as u64);
+               let onion_routing_packet: onion_message::Packet = <onion_message::Packet as LengthReadable>::read(&mut packet_reader)?;
+               Ok(Self {
+                       blinding_point,
+                       onion_routing_packet,
+               })
+       }
+}
+
+impl Writeable for OnionMessage {
+       fn write<W: Writer>(&self, w: &mut W) -> Result<(), io::Error> {
+               self.blinding_point.write(w)?;
+               let onion_packet_len = self.onion_routing_packet.serialized_length();
+               (onion_packet_len as u16).write(w)?;
+               self.onion_routing_packet.write(w)?;
+               Ok(())
+       }
+}
 
 impl Writeable for FinalOnionHopData {
-       fn write<W: Writer>(&self, w: &mut W) -> Result<(), ::std::io::Error> {
-               w.size_hint(32 + 8 - (self.total_msat.leading_zeros()/8) as usize);
+       fn write<W: Writer>(&self, w: &mut W) -> Result<(), io::Error> {
                self.payment_secret.0.write(w)?;
-               HighZeroBytesDroppedVarInt(self.total_msat).write(w)
+               HighZeroBytesDroppedBigSize(self.total_msat).write(w)
        }
 }
 
 impl Readable for FinalOnionHopData {
        fn read<R: Read>(r: &mut R) -> Result<Self, DecodeError> {
                let secret: [u8; 32] = Readable::read(r)?;
-               let amt: HighZeroBytesDroppedVarInt<u64> = Readable::read(r)?;
+               let amt: HighZeroBytesDroppedBigSize<u64> = Readable::read(r)?;
                Ok(Self { payment_secret: PaymentSecret(secret), total_msat: amt.0 })
        }
 }
 
 impl Writeable for OnionHopData {
-       fn write<W: Writer>(&self, w: &mut W) -> Result<(), ::std::io::Error> {
-               w.size_hint(33);
-               // Note that this should never be reachable if Rust-Lightning generated the message, as we
-               // check values are sane long before we get here, though its possible in the future
-               // user-generated messages may hit this.
-               if self.amt_to_forward > MAX_VALUE_MSAT { panic!("We should never be sending infinite/overflow onion payments"); }
+       fn write<W: Writer>(&self, w: &mut W) -> Result<(), io::Error> {
                match self.format {
-                       OnionHopDataFormat::Legacy { short_channel_id } => {
-                               0u8.write(w)?;
-                               short_channel_id.write(w)?;
-                               self.amt_to_forward.write(w)?;
-                               self.outgoing_cltv_value.write(w)?;
-                               w.write_all(&[0;12])?;
-                       },
                        OnionHopDataFormat::NonFinalNode { short_channel_id } => {
-                               encode_varint_length_prefixed_tlv!(w, {
-                                       (2, HighZeroBytesDroppedVarInt(self.amt_to_forward)),
-                                       (4, HighZeroBytesDroppedVarInt(self.outgoing_cltv_value)),
-                                       (6, short_channel_id)
-                               }, { });
+                               _encode_varint_length_prefixed_tlv!(w, {
+                                       (2, HighZeroBytesDroppedBigSize(self.amt_to_forward), required),
+                                       (4, HighZeroBytesDroppedBigSize(self.outgoing_cltv_value), required),
+                                       (6, short_channel_id, required)
+                               });
                        },
-                       OnionHopDataFormat::FinalNode { ref payment_data } => {
-                               if let Some(final_data) = payment_data {
-                                       if final_data.total_msat > MAX_VALUE_MSAT { panic!("We should never be sending infinite/overflow onion payments"); }
-                               }
-                               encode_varint_length_prefixed_tlv!(w, {
-                                       (2, HighZeroBytesDroppedVarInt(self.amt_to_forward)),
-                                       (4, HighZeroBytesDroppedVarInt(self.outgoing_cltv_value))
-                               }, {
-                                       (8, payment_data)
+                       OnionHopDataFormat::FinalNode { ref payment_data, ref keysend_preimage } => {
+                               _encode_varint_length_prefixed_tlv!(w, {
+                                       (2, HighZeroBytesDroppedBigSize(self.amt_to_forward), required),
+                                       (4, HighZeroBytesDroppedBigSize(self.outgoing_cltv_value), required),
+                                       (8, payment_data, option),
+                                       (5482373484, keysend_preimage, option)
                                });
                        },
                }
@@ -1304,68 +1590,59 @@ impl Writeable for OnionHopData {
 }
 
 impl Readable for OnionHopData {
-       fn read<R: Read>(mut r: &mut R) -> Result<Self, DecodeError> {
-               use bitcoin::consensus::encode::{Decodable, Error, VarInt};
-               let v: VarInt = Decodable::consensus_decode(&mut r)
-                       .map_err(|e| match e {
-                               Error::Io(ioe) => DecodeError::from(ioe),
-                               _ => DecodeError::InvalidValue
-                       })?;
-               const LEGACY_ONION_HOP_FLAG: u64 = 0;
-               let (format, amt, cltv_value) = if v.0 != LEGACY_ONION_HOP_FLAG {
-                       let mut rd = FixedLengthReader::new(r, v.0);
-                       let mut amt = HighZeroBytesDroppedVarInt(0u64);
-                       let mut cltv_value = HighZeroBytesDroppedVarInt(0u32);
-                       let mut short_id: Option<u64> = None;
-                       let mut payment_data: Option<FinalOnionHopData> = None;
-                       decode_tlv!(&mut rd, {
-                               (2, amt),
-                               (4, cltv_value)
-                       }, {
-                               (6, short_id),
-                               (8, payment_data)
-                       });
-                       rd.eat_remaining().map_err(|_| DecodeError::ShortRead)?;
-                       let format = if let Some(short_channel_id) = short_id {
-                               if payment_data.is_some() { return Err(DecodeError::InvalidValue); }
-                               OnionHopDataFormat::NonFinalNode {
-                                       short_channel_id,
-                               }
-                       } else {
-                               if let &Some(ref data) = &payment_data {
-                                       if data.total_msat > MAX_VALUE_MSAT {
-                                               return Err(DecodeError::InvalidValue);
-                                       }
-                               }
-                               OnionHopDataFormat::FinalNode {
-                                       payment_data
-                               }
-                       };
-                       (format, amt.0, cltv_value.0)
+       fn read<R: Read>(r: &mut R) -> Result<Self, DecodeError> {
+               let mut amt = HighZeroBytesDroppedBigSize(0u64);
+               let mut cltv_value = HighZeroBytesDroppedBigSize(0u32);
+               let mut short_id: Option<u64> = None;
+               let mut payment_data: Option<FinalOnionHopData> = None;
+               let mut keysend_preimage: Option<PaymentPreimage> = None;
+               read_tlv_fields!(r, {
+                       (2, amt, required),
+                       (4, cltv_value, required),
+                       (6, short_id, option),
+                       (8, payment_data, option),
+                       // See https://github.com/lightning/blips/blob/master/blip-0003.md
+                       (5482373484, keysend_preimage, option)
+               });
+
+               let format = if let Some(short_channel_id) = short_id {
+                       if payment_data.is_some() { return Err(DecodeError::InvalidValue); }
+                       OnionHopDataFormat::NonFinalNode {
+                               short_channel_id,
+                       }
                } else {
-                       let format = OnionHopDataFormat::Legacy {
-                               short_channel_id: Readable::read(r)?,
-                       };
-                       let amt: u64 = Readable::read(r)?;
-                       let cltv_value: u32 = Readable::read(r)?;
-                       r.read_exact(&mut [0; 12])?;
-                       (format, amt, cltv_value)
+                       if let &Some(ref data) = &payment_data {
+                               if data.total_msat > MAX_VALUE_MSAT {
+                                       return Err(DecodeError::InvalidValue);
+                               }
+                       }
+                       OnionHopDataFormat::FinalNode {
+                               payment_data,
+                               keysend_preimage,
+                       }
                };
 
-               if amt > MAX_VALUE_MSAT {
+               if amt.0 > MAX_VALUE_MSAT {
                        return Err(DecodeError::InvalidValue);
                }
                Ok(OnionHopData {
                        format,
-                       amt_to_forward: amt,
-                       outgoing_cltv_value: cltv_value,
+                       amt_to_forward: amt.0,
+                       outgoing_cltv_value: cltv_value.0,
                })
        }
 }
 
+// ReadableArgs because we need onion_utils::decode_next_hop to accommodate payment packets and
+// onion message packets.
+impl ReadableArgs<()> for OnionHopData {
+       fn read<R: Read>(r: &mut R, _arg: ()) -> Result<Self, DecodeError> {
+               <Self as Readable>::read(r)
+       }
+}
+
 impl Writeable for Ping {
-       fn write<W: Writer>(&self, w: &mut W) -> Result<(), ::std::io::Error> {
-               w.size_hint(self.byteslen as usize + 4);
+       fn write<W: Writer>(&self, w: &mut W) -> Result<(), io::Error> {
                self.ponglen.write(w)?;
                vec![0u8; self.byteslen as usize].write(w)?; // size-unchecked write
                Ok(())
@@ -1386,8 +1663,7 @@ impl Readable for Ping {
 }
 
 impl Writeable for Pong {
-       fn write<W: Writer>(&self, w: &mut W) -> Result<(), ::std::io::Error> {
-               w.size_hint(self.byteslen as usize + 2);
+       fn write<W: Writer>(&self, w: &mut W) -> Result<(), io::Error> {
                vec![0u8; self.byteslen as usize].write(w)?; // size-unchecked write
                Ok(())
        }
@@ -1406,8 +1682,7 @@ impl Readable for Pong {
 }
 
 impl Writeable for UnsignedChannelAnnouncement {
-       fn write<W: Writer>(&self, w: &mut W) -> Result<(), ::std::io::Error> {
-               w.size_hint(2 + 32 + 8 + 4*33 + self.features.byte_count() + self.excess_data.len());
+       fn write<W: Writer>(&self, w: &mut W) -> Result<(), io::Error> {
                self.features.write(w)?;
                self.chain_hash.write(w)?;
                self.short_channel_id.write(w)?;
@@ -1430,19 +1705,12 @@ impl Readable for UnsignedChannelAnnouncement {
                        node_id_2: Readable::read(r)?,
                        bitcoin_key_1: Readable::read(r)?,
                        bitcoin_key_2: Readable::read(r)?,
-                       excess_data: {
-                               let mut excess_data = vec![];
-                               r.read_to_end(&mut excess_data)?;
-                               excess_data
-                       },
+                       excess_data: read_to_end(r)?,
                })
        }
 }
 
-impl_writeable_len_match!(ChannelAnnouncement, {
-               { ChannelAnnouncement { contents: UnsignedChannelAnnouncement {ref features, ref excess_data, ..}, .. },
-                       2 + 32 + 8 + 4*33 + features.byte_count() + excess_data.len() + 4*64 }
-       }, {
+impl_writeable!(ChannelAnnouncement, {
        node_signature_1,
        node_signature_2,
        bitcoin_signature_1,
@@ -1451,18 +1719,13 @@ impl_writeable_len_match!(ChannelAnnouncement, {
 });
 
 impl Writeable for UnsignedChannelUpdate {
-       fn write<W: Writer>(&self, w: &mut W) -> Result<(), ::std::io::Error> {
-               let mut size = 64 + self.excess_data.len();
-               let mut message_flags: u8 = 0;
-               if let OptionalField::Present(_) = self.htlc_maximum_msat {
-                       size += 8;
-                       message_flags = 1;
-               }
-               w.size_hint(size);
+       fn write<W: Writer>(&self, w: &mut W) -> Result<(), io::Error> {
+               // `message_flags` used to indicate presence of `htlc_maximum_msat`, but was deprecated in the spec.
+               const MESSAGE_FLAGS: u8 = 1;
                self.chain_hash.write(w)?;
                self.short_channel_id.write(w)?;
                self.timestamp.write(w)?;
-               let all_flags = self.flags as u16 | ((message_flags as u16) << 8);
+               let all_flags = self.flags as u16 | ((MESSAGE_FLAGS as u16) << 8);
                all_flags.write(w)?;
                self.cltv_expiry_delta.write(w)?;
                self.htlc_minimum_msat.write(w)?;
@@ -1476,42 +1739,32 @@ impl Writeable for UnsignedChannelUpdate {
 
 impl Readable for UnsignedChannelUpdate {
        fn read<R: Read>(r: &mut R) -> Result<Self, DecodeError> {
-               let has_htlc_maximum_msat;
                Ok(Self {
                        chain_hash: Readable::read(r)?,
                        short_channel_id: Readable::read(r)?,
                        timestamp: Readable::read(r)?,
                        flags: {
                                let flags: u16 = Readable::read(r)?;
-                               let message_flags = flags >> 8;
-                               has_htlc_maximum_msat = (message_flags as i32 & 1) == 1;
+                               // Note: we ignore the `message_flags` for now, since it was deprecated by the spec.
                                flags as u8
                        },
                        cltv_expiry_delta: Readable::read(r)?,
                        htlc_minimum_msat: Readable::read(r)?,
                        fee_base_msat: Readable::read(r)?,
                        fee_proportional_millionths: Readable::read(r)?,
-                       htlc_maximum_msat: if has_htlc_maximum_msat { Readable::read(r)? } else { OptionalField::Absent },
-                       excess_data: {
-                               let mut excess_data = vec![];
-                               r.read_to_end(&mut excess_data)?;
-                               excess_data
-                       },
+                       htlc_maximum_msat: Readable::read(r)?,
+                       excess_data: read_to_end(r)?,
                })
        }
 }
 
-impl_writeable_len_match!(ChannelUpdate, {
-               { ChannelUpdate { contents: UnsignedChannelUpdate {ref excess_data, ref htlc_maximum_msat, ..}, .. },
-                       64 + 64 + excess_data.len() + if let OptionalField::Present(_) = htlc_maximum_msat { 8 } else { 0 } }
-       }, {
+impl_writeable!(ChannelUpdate, {
        signature,
        contents
 });
 
 impl Writeable for ErrorMessage {
-       fn write<W: Writer>(&self, w: &mut W) -> Result<(), ::std::io::Error> {
-               w.size_hint(32 + 2 + self.data.len());
+       fn write<W: Writer>(&self, w: &mut W) -> Result<(), io::Error> {
                self.channel_id.write(w)?;
                (self.data.len() as u16).write(w)?;
                w.write_all(self.data.as_bytes())?;
@@ -1524,11 +1777,38 @@ impl Readable for ErrorMessage {
                Ok(Self {
                        channel_id: Readable::read(r)?,
                        data: {
-                               let mut sz: usize = <u16 as Readable>::read(r)? as usize;
-                               let mut data = vec![];
-                               let data_len = r.read_to_end(&mut data)?;
-                               sz = cmp::min(data_len, sz);
-                               match String::from_utf8(data[..sz as usize].to_vec()) {
+                               let sz: usize = <u16 as Readable>::read(r)? as usize;
+                               let mut data = Vec::with_capacity(sz);
+                               data.resize(sz, 0);
+                               r.read_exact(&mut data)?;
+                               match String::from_utf8(data) {
+                                       Ok(s) => s,
+                                       Err(_) => return Err(DecodeError::InvalidValue),
+                               }
+                       }
+               })
+       }
+}
+
+impl Writeable for WarningMessage {
+       fn write<W: Writer>(&self, w: &mut W) -> Result<(), io::Error> {
+               self.channel_id.write(w)?;
+               (self.data.len() as u16).write(w)?;
+               w.write_all(self.data.as_bytes())?;
+               Ok(())
+       }
+}
+
+impl Readable for WarningMessage {
+       fn read<R: Read>(r: &mut R) -> Result<Self, DecodeError> {
+               Ok(Self {
+                       channel_id: Readable::read(r)?,
+                       data: {
+                               let sz: usize = <u16 as Readable>::read(r)? as usize;
+                               let mut data = Vec::with_capacity(sz);
+                               data.resize(sz, 0);
+                               r.read_exact(&mut data)?;
+                               match String::from_utf8(data) {
                                        Ok(s) => s,
                                        Err(_) => return Err(DecodeError::InvalidValue),
                                }
@@ -1538,8 +1818,7 @@ impl Readable for ErrorMessage {
 }
 
 impl Writeable for UnsignedNodeAnnouncement {
-       fn write<W: Writer>(&self, w: &mut W) -> Result<(), ::std::io::Error> {
-               w.size_hint(76 + self.features.byte_count() + self.addresses.len()*38 + self.excess_address_data.len() + self.excess_data.len());
+       fn write<W: Writer>(&self, w: &mut W) -> Result<(), io::Error> {
                self.features.write(w)?;
                self.timestamp.write(w)?;
                self.node_id.write(w)?;
@@ -1564,7 +1843,7 @@ impl Readable for UnsignedNodeAnnouncement {
        fn read<R: Read>(r: &mut R) -> Result<Self, DecodeError> {
                let features: NodeFeatures = Readable::read(r)?;
                let timestamp: u32 = Readable::read(r)?;
-               let node_id: PublicKey = Readable::read(r)?;
+               let node_id: NodeId = Readable::read(r)?;
                let mut rgb = [0; 3];
                r.read_exact(&mut rgb)?;
                let alias: [u8; 32] = Readable::read(r)?;
@@ -1608,7 +1887,7 @@ impl Readable for UnsignedNodeAnnouncement {
                        }
                        Vec::new()
                };
-               r.read_to_end(&mut excess_data)?;
+               excess_data.extend(read_to_end(r)?.iter());
                Ok(UnsignedNodeAnnouncement {
                        features,
                        timestamp,
@@ -1622,10 +1901,7 @@ impl Readable for UnsignedNodeAnnouncement {
        }
 }
 
-impl_writeable_len_match!(NodeAnnouncement, <=, {
-               { NodeAnnouncement { contents: UnsignedNodeAnnouncement { ref features, ref addresses, ref excess_address_data, ref excess_data, ..}, .. },
-                       64 + 76 + features.byte_count() + addresses.len()*(NetAddress::MAX_LEN as usize + 1) + excess_address_data.len() + excess_data.len() }
-       }, {
+impl_writeable!(NodeAnnouncement, {
        signature,
        contents
 });
@@ -1665,11 +1941,10 @@ impl Readable for QueryShortChannelIds {
 }
 
 impl Writeable for QueryShortChannelIds {
-       fn write<W: Writer>(&self, w: &mut W) -> Result<(), ::std::io::Error> {
+       fn write<W: Writer>(&self, w: &mut W) -> Result<(), io::Error> {
                // Calculated from 1-byte encoding_type plus 8-bytes per short_channel_id
                let encoding_len: u16 = 1 + self.short_channel_ids.len() as u16 * 8;
 
-               w.size_hint(32 + 2 + encoding_len as usize);
                self.chain_hash.write(w)?;
                encoding_len.write(w)?;
 
@@ -1684,31 +1959,15 @@ impl Writeable for QueryShortChannelIds {
        }
 }
 
-impl Readable for ReplyShortChannelIdsEnd {
-       fn read<R: Read>(r: &mut R) -> Result<Self, DecodeError> {
-               let chain_hash: BlockHash = Readable::read(r)?;
-               let full_information: bool = Readable::read(r)?;
-               Ok(ReplyShortChannelIdsEnd {
-                       chain_hash,
-                       full_information,
-               })
-       }
-}
-
-impl Writeable for ReplyShortChannelIdsEnd {
-       fn write<W: Writer>(&self, w: &mut W) -> Result<(), ::std::io::Error> {
-               w.size_hint(32 + 1);
-               self.chain_hash.write(w)?;
-               self.full_information.write(w)?;
-               Ok(())
-       }
-}
+impl_writeable_msg!(ReplyShortChannelIdsEnd, {
+       chain_hash,
+       full_information,
+}, {});
 
 impl QueryChannelRange {
-       /**
-        * Calculates the overflow safe ending block height for the query.
-        * Overflow returns `0xffffffff`, otherwise returns `first_blocknum + number_of_blocks`
-        */
+       /// Calculates the overflow safe ending block height for the query.
+       ///
+       /// Overflow returns `0xffffffff`, otherwise returns `first_blocknum + number_of_blocks`.
        pub fn end_blocknum(&self) -> u32 {
                match self.first_blocknum.checked_add(self.number_of_blocks) {
                        Some(block) => block,
@@ -1717,28 +1976,11 @@ impl QueryChannelRange {
        }
 }
 
-impl Readable for QueryChannelRange {
-       fn read<R: Read>(r: &mut R) -> Result<Self, DecodeError> {
-               let chain_hash: BlockHash = Readable::read(r)?;
-               let first_blocknum: u32 = Readable::read(r)?;
-               let number_of_blocks: u32 = Readable::read(r)?;
-               Ok(QueryChannelRange {
-                       chain_hash,
-                       first_blocknum,
-                       number_of_blocks
-               })
-       }
-}
-
-impl Writeable for QueryChannelRange {
-       fn write<W: Writer>(&self, w: &mut W) -> Result<(), ::std::io::Error> {
-               w.size_hint(32 + 4 + 4);
-               self.chain_hash.write(w)?;
-               self.first_blocknum.write(w)?;
-               self.number_of_blocks.write(w)?;
-               Ok(())
-       }
-}
+impl_writeable_msg!(QueryChannelRange, {
+       chain_hash,
+       first_blocknum,
+       number_of_blocks
+}, {});
 
 impl Readable for ReplyChannelRange {
        fn read<R: Read>(r: &mut R) -> Result<Self, DecodeError> {
@@ -1781,9 +2023,8 @@ impl Readable for ReplyChannelRange {
 }
 
 impl Writeable for ReplyChannelRange {
-       fn write<W: Writer>(&self, w: &mut W) -> Result<(), ::std::io::Error> {
+       fn write<W: Writer>(&self, w: &mut W) -> Result<(), io::Error> {
                let encoding_len: u16 = 1 + self.short_channel_ids.len() as u16 * 8;
-               w.size_hint(32 + 4 + 4 + 1 + 2 + encoding_len as usize);
                self.chain_hash.write(w)?;
                self.first_blocknum.write(w)?;
                self.number_of_blocks.write(w)?;
@@ -1799,37 +2040,21 @@ impl Writeable for ReplyChannelRange {
        }
 }
 
-impl Readable for GossipTimestampFilter {
-       fn read<R: Read>(r: &mut R) -> Result<Self, DecodeError> {
-               let chain_hash: BlockHash = Readable::read(r)?;
-               let first_timestamp: u32 = Readable::read(r)?;
-               let timestamp_range: u32 = Readable::read(r)?;
-               Ok(GossipTimestampFilter {
-                       chain_hash,
-                       first_timestamp,
-                       timestamp_range,
-               })
-       }
-}
-
-impl Writeable for GossipTimestampFilter {
-       fn write<W: Writer>(&self, w: &mut W) -> Result<(), ::std::io::Error> {
-               w.size_hint(32 + 4 + 4);
-               self.chain_hash.write(w)?;
-               self.first_timestamp.write(w)?;
-               self.timestamp_range.write(w)?;
-               Ok(())
-       }
-}
-
+impl_writeable_msg!(GossipTimestampFilter, {
+       chain_hash,
+       first_timestamp,
+       timestamp_range,
+}, {});
 
 #[cfg(test)]
 mod tests {
        use hex;
-       use ln::{PaymentPreimage, PaymentHash, PaymentSecret};
-       use ln::msgs;
-       use ln::msgs::{ChannelFeatures, FinalOnionHopData, InitFeatures, NodeFeatures, OptionalField, OnionErrorPacket, OnionHopDataFormat};
-       use util::ser::{Writeable, Readable};
+       use crate::ln::{PaymentPreimage, PaymentHash, PaymentSecret};
+       use crate::ln::features::{ChannelFeatures, ChannelTypeFeatures, InitFeatures, NodeFeatures};
+       use crate::ln::msgs;
+       use crate::ln::msgs::{FinalOnionHopData, OptionalField, OnionErrorPacket, OnionHopDataFormat};
+       use crate::routing::gossip::NodeId;
+       use crate::util::ser::{Writeable, Readable, Hostname};
 
        use bitcoin::hashes::hex::FromHex;
        use bitcoin::util::address::Address;
@@ -1838,10 +2063,12 @@ mod tests {
        use bitcoin::blockdata::opcodes;
        use bitcoin::hash_types::{Txid, BlockHash};
 
-       use bitcoin::secp256k1::key::{PublicKey,SecretKey};
+       use bitcoin::secp256k1::{PublicKey,SecretKey};
        use bitcoin::secp256k1::{Secp256k1, Message};
 
-       use std::io::Cursor;
+       use crate::io::{self, Cursor};
+       use crate::prelude::*;
+       use core::convert::TryFrom;
 
        #[test]
        fn encoding_channel_reestablish_no_secret() {
@@ -1894,7 +2121,7 @@ mod tests {
                ($privkey: expr, $ctx: expr, $string: expr) => {
                        {
                                let sighash = Message::from_slice(&$string.into_bytes()[..]).unwrap();
-                               $ctx.sign(&sighash, &$privkey)
+                               $ctx.sign_ecdsa(&sighash, &$privkey)
                        }
                }
        }
@@ -1926,7 +2153,7 @@ mod tests {
                let sig_2 = get_sig_on!(privkey_2, secp_ctx, String::from("01010101010101010101010101010101"));
                let sig_3 = get_sig_on!(privkey_3, secp_ctx, String::from("01010101010101010101010101010101"));
                let sig_4 = get_sig_on!(privkey_4, secp_ctx, String::from("01010101010101010101010101010101"));
-               let mut features = ChannelFeatures::known();
+               let mut features = ChannelFeatures::empty();
                if unknown_features_bits {
                        features = ChannelFeatures::from_le_bytes(vec![0xFF, 0xFF]);
                }
@@ -1934,10 +2161,10 @@ mod tests {
                        features,
                        chain_hash: BlockHash::from_hex("6fe28c0ab6f1b372c1a6a246ae63f74f931e8365e15a089c68d6190000000000").unwrap(),
                        short_channel_id: 2316138423780173,
-                       node_id_1: pubkey_1,
-                       node_id_2: pubkey_2,
-                       bitcoin_key_1: pubkey_3,
-                       bitcoin_key_2: pubkey_4,
+                       node_id_1: NodeId::from_pubkey(&pubkey_1),
+                       node_id_2: NodeId::from_pubkey(&pubkey_2),
+                       bitcoin_key_1: NodeId::from_pubkey(&pubkey_3),
+                       bitcoin_key_2: NodeId::from_pubkey(&pubkey_4),
                        excess_data: if excess_data { vec![10, 0, 0, 20, 0, 0, 30, 0, 0, 40] } else { Vec::new() },
                };
                let channel_announcement = msgs::ChannelAnnouncement {
@@ -1970,7 +2197,7 @@ mod tests {
                do_encoding_channel_announcement(true, true);
        }
 
-       fn do_encoding_node_announcement(unknown_features_bits: bool, ipv4: bool, ipv6: bool, onionv2: bool, onionv3: bool, excess_address_data: bool, excess_data: bool) {
+       fn do_encoding_node_announcement(unknown_features_bits: bool, ipv4: bool, ipv6: bool, onionv2: bool, onionv3: bool, hostname: bool, excess_address_data: bool, excess_data: bool) {
                let secp_ctx = Secp256k1::new();
                let (privkey_1, pubkey_1) = get_keys_from!("0101010101010101010101010101010101010101010101010101010101010101", secp_ctx);
                let sig_1 = get_sig_on!(privkey_1, secp_ctx, String::from("01010101010101010101010101010101"));
@@ -1994,10 +2221,9 @@ mod tests {
                        });
                }
                if onionv2 {
-                       addresses.push(msgs::NetAddress::OnionV2 {
-                               addr: [255, 254, 253, 252, 251, 250, 249, 248, 247, 246],
-                               port: 9735
-                       });
+                       addresses.push(msgs::NetAddress::OnionV2(
+                               [255, 254, 253, 252, 251, 250, 249, 248, 247, 246, 38, 7]
+                       ));
                }
                if onionv3 {
                        addresses.push(msgs::NetAddress::OnionV3 {
@@ -2007,6 +2233,12 @@ mod tests {
                                port: 9735
                        });
                }
+               if hostname {
+                       addresses.push(msgs::NetAddress::Hostname {
+                               hostname: Hostname::try_from(String::from("host")).unwrap(),
+                               port: 9735,
+                       });
+               }
                let mut addr_len = 0;
                for addr in &addresses {
                        addr_len += addr.len() + 1;
@@ -2014,7 +2246,7 @@ mod tests {
                let unsigned_node_announcement = msgs::UnsignedNodeAnnouncement {
                        features,
                        timestamp: 20190119,
-                       node_id: pubkey_1,
+                       node_id: NodeId::from_pubkey(&pubkey_1),
                        rgb: [32; 3],
                        alias: [16;32],
                        addresses,
@@ -2047,6 +2279,9 @@ mod tests {
                if onionv3 {
                        target_value.append(&mut hex::decode("04fffefdfcfbfaf9f8f7f6f5f4f3f2f1f0efeeedecebeae9e8e7e6e5e4e3e2e1e00020102607").unwrap());
                }
+               if hostname {
+                       target_value.append(&mut hex::decode("0504686f73742607").unwrap());
+               }
                if excess_address_data {
                        target_value.append(&mut hex::decode("216c280b5395a2546e7e4b2663e04f811622f15a4f92e83aa2e92ba2a573c139142c54ae63072a1ec1ee7dc0c04bde5c847806172aa05c92c22ae8e308d1d269").unwrap());
                }
@@ -2058,18 +2293,19 @@ mod tests {
 
        #[test]
        fn encoding_node_announcement() {
-               do_encoding_node_announcement(true, true, true, true, true, true, true);
-               do_encoding_node_announcement(false, false, false, false, false, false, false);
-               do_encoding_node_announcement(false, true, false, false, false, false, false);
-               do_encoding_node_announcement(false, false, true, false, false, false, false);
-               do_encoding_node_announcement(false, false, false, true, false, false, false);
-               do_encoding_node_announcement(false, false, false, false, true, false, false);
-               do_encoding_node_announcement(false, false, false, false, false, true, false);
-               do_encoding_node_announcement(false, true, false, true, false, true, false);
-               do_encoding_node_announcement(false, false, true, false, true, false, false);
-       }
-
-       fn do_encoding_channel_update(direction: bool, disable: bool, htlc_maximum_msat: bool, excess_data: bool) {
+               do_encoding_node_announcement(true, true, true, true, true, true, true, true);
+               do_encoding_node_announcement(false, false, false, false, false, false, false, false);
+               do_encoding_node_announcement(false, true, false, false, false, false, false, false);
+               do_encoding_node_announcement(false, false, true, false, false, false, false, false);
+               do_encoding_node_announcement(false, false, false, true, false, false, false, false);
+               do_encoding_node_announcement(false, false, false, false, true, false, false, false);
+               do_encoding_node_announcement(false, false, false, false, false, true, false, false);
+               do_encoding_node_announcement(false, false, false, false, false, false, true, false);
+               do_encoding_node_announcement(false, true, false, true, false, false, true, false);
+               do_encoding_node_announcement(false, false, true, false, true, false, false, false);
+       }
+
+       fn do_encoding_channel_update(direction: bool, disable: bool, excess_data: bool) {
                let secp_ctx = Secp256k1::new();
                let (privkey_1, _) = get_keys_from!("0101010101010101010101010101010101010101010101010101010101010101", secp_ctx);
                let sig_1 = get_sig_on!(privkey_1, secp_ctx, String::from("01010101010101010101010101010101"));
@@ -2080,7 +2316,7 @@ mod tests {
                        flags: if direction { 1 } else { 0 } | if disable { 1 << 1 } else { 0 },
                        cltv_expiry_delta: 144,
                        htlc_minimum_msat: 1000000,
-                       htlc_maximum_msat: if htlc_maximum_msat { OptionalField::Present(131355275467161) } else { OptionalField::Absent },
+                       htlc_maximum_msat: 131355275467161,
                        fee_base_msat: 10000,
                        fee_proportional_millionths: 20,
                        excess_data: if excess_data { vec![0, 0, 0, 0, 59, 154, 202, 0] } else { Vec::new() }
@@ -2093,11 +2329,7 @@ mod tests {
                let mut target_value = hex::decode("d977cb9b53d93a6ff64bb5f1e158b4094b66e798fb12911168a3ccdf80a83096340a6a95da0ae8d9f776528eecdbb747eb6b545495a4319ed5378e35b21e073a").unwrap();
                target_value.append(&mut hex::decode("000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f").unwrap());
                target_value.append(&mut hex::decode("00083a840000034d013413a7").unwrap());
-               if htlc_maximum_msat {
-                       target_value.append(&mut hex::decode("01").unwrap());
-               } else {
-                       target_value.append(&mut hex::decode("00").unwrap());
-               }
+               target_value.append(&mut hex::decode("01").unwrap());
                target_value.append(&mut hex::decode("00").unwrap());
                if direction {
                        let flag = target_value.last_mut().unwrap();
@@ -2108,9 +2340,7 @@ mod tests {
                        *flag = *flag | 1 << 1;
                }
                target_value.append(&mut hex::decode("009000000000000f42400000271000000014").unwrap());
-               if htlc_maximum_msat {
-                       target_value.append(&mut hex::decode("0000777788889999").unwrap());
-               }
+               target_value.append(&mut hex::decode("0000777788889999").unwrap());
                if excess_data {
                        target_value.append(&mut hex::decode("000000003b9aca00").unwrap());
                }
@@ -2119,19 +2349,17 @@ mod tests {
 
        #[test]
        fn encoding_channel_update() {
-               do_encoding_channel_update(false, false, false, false);
-               do_encoding_channel_update(false, false, false, true);
-               do_encoding_channel_update(true, false, false, false);
-               do_encoding_channel_update(true, false, false, true);
-               do_encoding_channel_update(false, true, false, false);
-               do_encoding_channel_update(false, true, false, true);
-               do_encoding_channel_update(false, false, true, false);
-               do_encoding_channel_update(false, false, true, true);
-               do_encoding_channel_update(true, true, true, false);
-               do_encoding_channel_update(true, true, true, true);
-       }
-
-       fn do_encoding_open_channel(random_bit: bool, shutdown: bool) {
+               do_encoding_channel_update(false, false, false);
+               do_encoding_channel_update(false, false, true);
+               do_encoding_channel_update(true, false, false);
+               do_encoding_channel_update(true, false, true);
+               do_encoding_channel_update(false, true, false);
+               do_encoding_channel_update(false, true, true);
+               do_encoding_channel_update(true, true, false);
+               do_encoding_channel_update(true, true, true);
+       }
+
+       fn do_encoding_open_channel(random_bit: bool, shutdown: bool, incl_chan_type: bool) {
                let secp_ctx = Secp256k1::new();
                let (_, pubkey_1) = get_keys_from!("0101010101010101010101010101010101010101010101010101010101010101", secp_ctx);
                let (_, pubkey_2) = get_keys_from!("0202020202020202020202020202020202020202020202020202020202020202", secp_ctx);
@@ -2158,7 +2386,8 @@ mod tests {
                        htlc_basepoint: pubkey_5,
                        first_per_commitment_point: pubkey_6,
                        channel_flags: if random_bit { 1 << 5 } else { 0 },
-                       shutdown_scriptpubkey: if shutdown { OptionalField::Present(Address::p2pkh(&::bitcoin::PublicKey{compressed: true, key: pubkey_1}, Network::Testnet).script_pubkey()) } else { OptionalField::Absent }
+                       shutdown_scriptpubkey: if shutdown { OptionalField::Present(Address::p2pkh(&::bitcoin::PublicKey{compressed: true, inner: pubkey_1}, Network::Testnet).script_pubkey()) } else { OptionalField::Absent },
+                       channel_type: if incl_chan_type { Some(ChannelTypeFeatures::empty()) } else { None },
                };
                let encoded_value = open_channel.encode();
                let mut target_value = Vec::new();
@@ -2172,15 +2401,22 @@ mod tests {
                if shutdown {
                        target_value.append(&mut hex::decode("001976a91479b000887626b294a914501a4cd226b58b23598388ac").unwrap());
                }
+               if incl_chan_type {
+                       target_value.append(&mut hex::decode("0100").unwrap());
+               }
                assert_eq!(encoded_value, target_value);
        }
 
        #[test]
        fn encoding_open_channel() {
-               do_encoding_open_channel(false, false);
-               do_encoding_open_channel(true, false);
-               do_encoding_open_channel(false, true);
-               do_encoding_open_channel(true, true);
+               do_encoding_open_channel(false, false, false);
+               do_encoding_open_channel(false, false, true);
+               do_encoding_open_channel(false, true, false);
+               do_encoding_open_channel(false, true, true);
+               do_encoding_open_channel(true, false, false);
+               do_encoding_open_channel(true, false, true);
+               do_encoding_open_channel(true, true, false);
+               do_encoding_open_channel(true, true, true);
        }
 
        fn do_encoding_accept_channel(shutdown: bool) {
@@ -2206,7 +2442,8 @@ mod tests {
                        delayed_payment_basepoint: pubkey_4,
                        htlc_basepoint: pubkey_5,
                        first_per_commitment_point: pubkey_6,
-                       shutdown_scriptpubkey: if shutdown { OptionalField::Present(Address::p2pkh(&::bitcoin::PublicKey{compressed: true, key: pubkey_1}, Network::Testnet).script_pubkey()) } else { OptionalField::Absent }
+                       shutdown_scriptpubkey: if shutdown { OptionalField::Present(Address::p2pkh(&::bitcoin::PublicKey{compressed: true, inner: pubkey_1}, Network::Testnet).script_pubkey()) } else { OptionalField::Absent },
+                       channel_type: None,
                };
                let encoded_value = accept_channel.encode();
                let mut target_value = hex::decode("020202020202020202020202020202020202020202020202020202020202020212345678901234562334032891223698321446687011447600083a840000034d000c89d4c0bcc0bc031b84c5567b126440995d3ed5aaba0565d71e1834604819ff9c17f5e9d5dd078f024d4b6cd1361032ca9bd2aeb9d900aa4d45d9ead80ac9423374c451a7254d076602531fe6068134503d2723133227c867ac8fa6c83c537e9a44c3c5bdbdcb1fe33703462779ad4aad39514614751a71085f2f10e1c7a593e4e030efb5b8721ce55b0b0362c0a046dacce86ddd0343c6d3c7c79c2208ba0d9c9cf24a6d046d21d21f90f703f006a18d5653c4edf5391ff23a61f03ff83d237e880ee61187fa9f379a028e0a").unwrap();
@@ -2253,14 +2490,15 @@ mod tests {
        }
 
        #[test]
-       fn encoding_funding_locked() {
+       fn encoding_channel_ready() {
                let secp_ctx = Secp256k1::new();
                let (_, pubkey_1,) = get_keys_from!("0101010101010101010101010101010101010101010101010101010101010101", secp_ctx);
-               let funding_locked = msgs::FundingLocked {
+               let channel_ready = msgs::ChannelReady {
                        channel_id: [2; 32],
                        next_per_commitment_point: pubkey_1,
+                       short_channel_id_alias: None,
                };
-               let encoded_value = funding_locked.encode();
+               let encoded_value = channel_ready.encode();
                let target_value = hex::decode("0202020202020202020202020202020202020202020202020202020202020202031b84c5567b126440995d3ed5aaba0565d71e1834604819ff9c17f5e9d5dd078f").unwrap();
                assert_eq!(encoded_value, target_value);
        }
@@ -2272,9 +2510,9 @@ mod tests {
                let shutdown = msgs::Shutdown {
                        channel_id: [2; 32],
                        scriptpubkey:
-                                    if script_type == 1 { Address::p2pkh(&::bitcoin::PublicKey{compressed: true, key: pubkey_1}, Network::Testnet).script_pubkey() }
-                               else if script_type == 2 { Address::p2sh(&script, Network::Testnet).script_pubkey() }
-                               else if script_type == 3 { Address::p2wpkh(&::bitcoin::PublicKey{compressed: true, key: pubkey_1}, Network::Testnet).unwrap().script_pubkey() }
+                                    if script_type == 1 { Address::p2pkh(&::bitcoin::PublicKey{compressed: true, inner: pubkey_1}, Network::Testnet).script_pubkey() }
+                               else if script_type == 2 { Address::p2sh(&script, Network::Testnet).unwrap().script_pubkey() }
+                               else if script_type == 3 { Address::p2wpkh(&::bitcoin::PublicKey{compressed: true, inner: pubkey_1}, Network::Testnet).unwrap().script_pubkey() }
                                else                     { Address::p2wsh(&script, Network::Testnet).script_pubkey() },
                };
                let encoded_value = shutdown.encode();
@@ -2308,10 +2546,27 @@ mod tests {
                        channel_id: [2; 32],
                        fee_satoshis: 2316138423780173,
                        signature: sig_1,
+                       fee_range: None,
                };
                let encoded_value = closing_signed.encode();
                let target_value = hex::decode("020202020202020202020202020202020202020202020202020202020202020200083a840000034dd977cb9b53d93a6ff64bb5f1e158b4094b66e798fb12911168a3ccdf80a83096340a6a95da0ae8d9f776528eecdbb747eb6b545495a4319ed5378e35b21e073a").unwrap();
                assert_eq!(encoded_value, target_value);
+               assert_eq!(msgs::ClosingSigned::read(&mut Cursor::new(&target_value)).unwrap(), closing_signed);
+
+               let closing_signed_with_range = msgs::ClosingSigned {
+                       channel_id: [2; 32],
+                       fee_satoshis: 2316138423780173,
+                       signature: sig_1,
+                       fee_range: Some(msgs::ClosingSignedFeeRange {
+                               min_fee_satoshis: 0xdeadbeef,
+                               max_fee_satoshis: 0x1badcafe01234567,
+                       }),
+               };
+               let encoded_value_with_range = closing_signed_with_range.encode();
+               let target_value_with_range = hex::decode("020202020202020202020202020202020202020202020202020202020202020200083a840000034dd977cb9b53d93a6ff64bb5f1e158b4094b66e798fb12911168a3ccdf80a83096340a6a95da0ae8d9f776528eecdbb747eb6b545495a4319ed5378e35b21e073a011000000000deadbeef1badcafe01234567").unwrap();
+               assert_eq!(encoded_value_with_range, target_value_with_range);
+               assert_eq!(msgs::ClosingSigned::read(&mut Cursor::new(&target_value_with_range)).unwrap(),
+                       closing_signed_with_range);
        }
 
        #[test]
@@ -2437,13 +2692,27 @@ mod tests {
        fn encoding_init() {
                assert_eq!(msgs::Init {
                        features: InitFeatures::from_le_bytes(vec![0xFF, 0xFF, 0xFF]),
+                       remote_network_address: None,
                }.encode(), hex::decode("00023fff0003ffffff").unwrap());
                assert_eq!(msgs::Init {
                        features: InitFeatures::from_le_bytes(vec![0xFF]),
+                       remote_network_address: None,
                }.encode(), hex::decode("0001ff0001ff").unwrap());
                assert_eq!(msgs::Init {
                        features: InitFeatures::from_le_bytes(vec![]),
+                       remote_network_address: None,
                }.encode(), hex::decode("00000000").unwrap());
+
+               let init_msg = msgs::Init { features: InitFeatures::from_le_bytes(vec![]),
+                       remote_network_address: Some(msgs::NetAddress::IPv4 {
+                               addr: [127, 0, 0, 1],
+                               port: 1000,
+                       }),
+               };
+               let encoded_value = init_msg.encode();
+               let target_value = hex::decode("000000000307017f00000103e8").unwrap();
+               assert_eq!(encoded_value, target_value);
+               assert_eq!(msgs::Init::read(&mut Cursor::new(&target_value)).unwrap(), init_msg);
        }
 
        #[test]
@@ -2457,6 +2726,17 @@ mod tests {
                assert_eq!(encoded_value, target_value);
        }
 
+       #[test]
+       fn encoding_warning() {
+               let error = msgs::WarningMessage {
+                       channel_id: [2; 32],
+                       data: String::from("rust-lightning"),
+               };
+               let encoded_value = error.encode();
+               let target_value = hex::decode("0202020202020202020202020202020202020202020202020202020202020202000e727573742d6c696768746e696e67").unwrap();
+               assert_eq!(encoded_value, target_value);
+       }
+
        #[test]
        fn encoding_ping() {
                let ping = msgs::Ping {
@@ -2478,20 +2758,6 @@ mod tests {
                assert_eq!(encoded_value, target_value);
        }
 
-       #[test]
-       fn encoding_legacy_onion_hop_data() {
-               let msg = msgs::OnionHopData {
-                       format: OnionHopDataFormat::Legacy {
-                               short_channel_id: 0xdeadbeef1bad1dea,
-                       },
-                       amt_to_forward: 0x0badf00d01020304,
-                       outgoing_cltv_value: 0xffffffff,
-               };
-               let encoded_value = msg.encode();
-               let target_value = hex::decode("00deadbeef1bad1dea0badf00d01020304ffffffff000000000000000000000000").unwrap();
-               assert_eq!(encoded_value, target_value);
-       }
-
        #[test]
        fn encoding_nonfinal_onion_hop_data() {
                let mut msg = msgs::OnionHopData {
@@ -2517,6 +2783,7 @@ mod tests {
                let mut msg = msgs::OnionHopData {
                        format: OnionHopDataFormat::FinalNode {
                                payment_data: None,
+                               keysend_preimage: None,
                        },
                        amt_to_forward: 0x0badf00d01020304,
                        outgoing_cltv_value: 0xffffffff,
@@ -2525,7 +2792,7 @@ mod tests {
                let target_value = hex::decode("1002080badf00d010203040404ffffffff").unwrap();
                assert_eq!(encoded_value, target_value);
                msg = Readable::read(&mut Cursor::new(&target_value[..])).unwrap();
-               if let OnionHopDataFormat::FinalNode { payment_data: None } = msg.format { } else { panic!(); }
+               if let OnionHopDataFormat::FinalNode { payment_data: None, .. } = msg.format { } else { panic!(); }
                assert_eq!(msg.amt_to_forward, 0x0badf00d01020304);
                assert_eq!(msg.outgoing_cltv_value, 0xffffffff);
        }
@@ -2539,6 +2806,7 @@ mod tests {
                                        payment_secret: expected_payment_secret,
                                        total_msat: 0x1badca1f
                                }),
+                               keysend_preimage: None,
                        },
                        amt_to_forward: 0x0badf00d01020304,
                        outgoing_cltv_value: 0xffffffff,
@@ -2551,7 +2819,8 @@ mod tests {
                        payment_data: Some(FinalOnionHopData {
                                payment_secret,
                                total_msat: 0x1badca1f
-                       })
+                       }),
+                       keysend_preimage: None,
                } = msg.format {
                        assert_eq!(payment_secret, expected_payment_secret);
                } else { panic!(); }
@@ -2694,4 +2963,40 @@ mod tests {
                assert_eq!(gossip_timestamp_filter.first_timestamp, 1590000000);
                assert_eq!(gossip_timestamp_filter.timestamp_range, 0xffff_ffff);
        }
+
+       #[test]
+       fn decode_onion_hop_data_len_as_bigsize() {
+               // Tests that we can decode an onion payload that is >253 bytes.
+               // Previously, receiving a payload of this size could've caused us to fail to decode a valid
+               // payload, because we were decoding the length (a BigSize, big-endian) as a VarInt
+               // (little-endian).
+
+               // Encode a test onion payload with a big custom TLV such that it's >253 bytes, forcing the
+               // payload length to be encoded over multiple bytes rather than a single u8.
+               let big_payload = encode_big_payload().unwrap();
+               let mut rd = Cursor::new(&big_payload[..]);
+               <msgs::OnionHopData as Readable>::read(&mut rd).unwrap();
+       }
+       // see above test, needs to be a separate method for use of the serialization macros.
+       fn encode_big_payload() -> Result<Vec<u8>, io::Error> {
+               use crate::util::ser::HighZeroBytesDroppedBigSize;
+               let payload = msgs::OnionHopData {
+                       format: OnionHopDataFormat::NonFinalNode {
+                               short_channel_id: 0xdeadbeef1bad1dea,
+                       },
+                       amt_to_forward: 1000,
+                       outgoing_cltv_value: 0xffffffff,
+               };
+               let mut encoded_payload = Vec::new();
+               let test_bytes = vec![42u8; 1000];
+               if let OnionHopDataFormat::NonFinalNode { short_channel_id } = payload.format {
+                       _encode_varint_length_prefixed_tlv!(&mut encoded_payload, {
+                               (1, test_bytes, vec_type),
+                               (2, HighZeroBytesDroppedBigSize(payload.amt_to_forward), required),
+                               (4, HighZeroBytesDroppedBigSize(payload.outgoing_cltv_value), required),
+                               (6, short_channel_id, required)
+                       });
+               }
+               Ok(encoded_payload)
+       }
 }