X-Git-Url: http://git.bitcoin.ninja/index.cgi?a=blobdiff_plain;f=lightning%2Fsrc%2Fln%2Fmsgs.rs;h=b078b6f3b5374836a794f5cd1820a46b0509a426;hb=e94e403dda57e1443fdeda9f1d0b2cb8a9e7fb0f;hp=6c04636a4355c96fb0f5158887f855a93977e7b7;hpb=1b3249a1929929170bba9d2553d7a9c8670193e5;p=rust-lightning

diff --git a/lightning/src/ln/msgs.rs b/lightning/src/ln/msgs.rs
index 6c04636a..e063c314 100644
--- a/lightning/src/ln/msgs.rs
+++ b/lightning/src/ln/msgs.rs
@@ -24,23 +24,25 @@
 //! raw socket events into your non-internet-facing system and then send routing events back to
 //! track the network on the less-secure system.
 
-use bitcoin::secp256k1::key::PublicKey;
-use bitcoin::secp256k1::Signature;
+use bitcoin::secp256k1::PublicKey;
+use bitcoin::secp256k1::ecdsa::Signature;
 use bitcoin::secp256k1;
 use bitcoin::blockdata::script::Script;
 use bitcoin::hash_types::{Txid, BlockHash};
 
 use ln::features::{ChannelFeatures, ChannelTypeFeatures, InitFeatures, NodeFeatures};
+use ln::onion_utils;
+use onion_message;
 
 use prelude::*;
-use core::{cmp, fmt};
+use core::fmt;
 use core::fmt::Debug;
 use io::{self, Read};
 use io_extras::read_to_end;
 
-use util::events::MessageSendEventsProvider;
+use util::events::{MessageSendEventsProvider, OnionMessageProvider};
 use util::logger;
-use util::ser::{Readable, Writeable, Writer, FixedLengthReader, HighZeroBytesDroppedVarInt};
+use util::ser::{BigSize, LengthReadable, Readable, ReadableArgs, Writeable, Writer, FixedLengthReader, HighZeroBytesDroppedBigSize, Hostname};
 
 use ln::{PaymentPreimage, PaymentHash, PaymentSecret};
 
@@ -75,6 +77,11 @@ pub enum DecodeError {
 pub struct Init {
 	/// The relevant features which the sender supports
 	pub features: InitFeatures,
+	/// The receipient's network address. This adds the option to report a remote IP address
+	/// back to a connecting peer using the init message. A node can decide to use that information
+	/// to discover a potential update to its public IPv4 address (NAT) and use
+	/// that for a node_announcement update message containing the new address.
+	pub remote_network_address: Option<NetAddress>,
 }
 
 /// An error message to be sent or received from a peer
@@ -204,6 +211,12 @@ pub struct AcceptChannel {
 	pub first_per_commitment_point: PublicKey,
 	/// Optionally, a request to pre-set the to-sender output's scriptPubkey for when we collaboratively close
 	pub shutdown_scriptpubkey: OptionalField<Script>,
+	/// The channel type that this channel will represent. If none is set, we derive the channel
+	/// type from the intersection of our feature bits with our counterparty's feature bits from
+	/// the Init message.
+	///
+	/// This is required to match the equivalent field in [`OpenChannel::channel_type`].
+	pub channel_type: Option<ChannelTypeFeatures>,
 }
 
 /// A funding_created message to be sent or received from a peer
@@ -228,13 +241,16 @@ pub struct FundingSigned {
 	pub signature: Signature,
 }
 
-/// A funding_locked message to be sent or received from a peer
+/// A channel_ready message to be sent or received from a peer
 #[derive(Clone, Debug, PartialEq)]
-pub struct FundingLocked {
+pub struct ChannelReady {
 	/// The channel ID
 	pub channel_id: [u8; 32],
 	/// The per-commitment point of the second commitment transaction
 	pub next_per_commitment_point: PublicKey,
+	/// If set, provides a short_channel_id alias for this channel. The sender will accept payments
+	/// to be forwarded over this SCID and forward them to this messages' recipient.
+	pub short_channel_id_alias: Option<u64>,
 }
 
 /// A shutdown message to be sent or received from a peer
@@ -290,6 +306,14 @@ pub struct UpdateAddHTLC {
 	pub(crate) onion_routing_packet: OnionPacket,
 }
 
+ /// An onion message to be sent or received from a peer
+#[derive(Clone, Debug, PartialEq)]
+pub struct OnionMessage {
+	/// Used in decrypting the onion packet's payload.
+	pub blinding_point: PublicKey,
+	pub(crate) onion_routing_packet: onion_message::Packet,
+}
+
 /// An update_fulfill_htlc message to be sent or received from a peer
 #[derive(Clone, Debug, PartialEq)]
 pub struct UpdateFulfillHTLC {
@@ -428,6 +452,13 @@ pub enum NetAddress {
 		/// The port on which the node is listening
 		port: u16,
 	},
+	/// A hostname/port on which the peer is listening.
+	Hostname {
+		/// The hostname on which the node is listening.
+		hostname: Hostname,
+		/// The port on which the node is listening.
+		port: u16,
+	},
 }
 impl NetAddress {
 	/// Gets the ID of this address type. Addresses in node_announcement messages should be sorted
@@ -438,6 +469,7 @@ impl NetAddress {
 			&NetAddress::IPv6 {..} => { 2 },
 			&NetAddress::OnionV2(_) => { 3 },
 			&NetAddress::OnionV3 {..} => { 4 },
+			&NetAddress::Hostname {..} => { 5 },
 		}
 	}
 
@@ -448,11 +480,15 @@ impl NetAddress {
 			&NetAddress::IPv6 { .. } => { 18 },
 			&NetAddress::OnionV2(_) => { 12 },
 			&NetAddress::OnionV3 { .. } => { 37 },
+			// Consists of 1-byte hostname length, hostname bytes, and 2-byte port.
+			&NetAddress::Hostname { ref hostname, .. } => { u16::from(hostname.len()) + 3 },
 		}
 	}
 
-	/// The maximum length of any address descriptor, not including the 1-byte type
-	pub(crate) const MAX_LEN: u16 = 37;
+	/// The maximum length of any address descriptor, not including the 1-byte type.
+	/// This maximum length is reached by a hostname address descriptor:
+	/// a hostname with a maximum length of 255, its 1-byte length and a 2-byte port.
+	pub(crate) const MAX_LEN: u16 = 258;
 }
 
 impl Writeable for NetAddress {
@@ -478,7 +514,12 @@ impl Writeable for NetAddress {
 				checksum.write(writer)?;
 				version.write(writer)?;
 				port.write(writer)?;
-			}
+			},
+			&NetAddress::Hostname { ref hostname, ref port } => {
+				5u8.write(writer)?;
+				hostname.write(writer)?;
+				port.write(writer)?;
+			},
 		}
 		Ok(())
 	}
@@ -509,6 +550,12 @@ impl Readable for Result<NetAddress, u8> {
 					port: Readable::read(reader)?,
 				}))
 			},
+			5 => {
+				Ok(Ok(NetAddress::Hostname {
+					hostname: Readable::read(reader)?,
+					port: Readable::read(reader)?,
+				}))
+			},
 			_ => return Ok(Err(byte)),
 		}
 	}
@@ -610,13 +657,16 @@ pub struct UnsignedChannelUpdate {
 	pub cltv_expiry_delta: u16,
 	/// The minimum HTLC size incoming to sender, in milli-satoshi
 	pub htlc_minimum_msat: u64,
-	/// Optionally, the maximum HTLC value incoming to sender, in milli-satoshi
-	pub htlc_maximum_msat: OptionalField<u64>,
+	/// The maximum HTLC value incoming to sender, in milli-satoshi. Used to be optional.
+	pub htlc_maximum_msat: u64,
 	/// The base HTLC fee charged by sender, in milli-satoshi
 	pub fee_base_msat: u32,
 	/// The amount to fee multiplier, in micro-satoshi
 	pub fee_proportional_millionths: u32,
-	pub(crate) excess_data: Vec<u8>,
+	/// Excess data which was signed as a part of the message which we do not (yet) understand how
+	/// to decode. This is stored to ensure forward-compatibility as new fields are added to the
+	/// lightning gossip
+	pub excess_data: Vec<u8>,
 }
 /// A channel_update message to be sent or received from a peer
 #[derive(Clone, Debug, PartialEq)]
@@ -798,8 +848,8 @@ pub trait ChannelMessageHandler : MessageSendEventsProvider {
 	fn handle_funding_created(&self, their_node_id: &PublicKey, msg: &FundingCreated);
 	/// Handle an incoming funding_signed message from the given peer.
 	fn handle_funding_signed(&self, their_node_id: &PublicKey, msg: &FundingSigned);
-	/// Handle an incoming funding_locked message from the given peer.
-	fn handle_funding_locked(&self, their_node_id: &PublicKey, msg: &FundingLocked);
+	/// Handle an incoming channel_ready message from the given peer.
+	fn handle_channel_ready(&self, their_node_id: &PublicKey, msg: &ChannelReady);
 
 	// Channl close:
 	/// Handle an incoming shutdown message from the given peer.
@@ -846,6 +896,19 @@ pub trait ChannelMessageHandler : MessageSendEventsProvider {
 	// Error:
 	/// Handle an incoming error message from the given peer.
 	fn handle_error(&self, their_node_id: &PublicKey, msg: &ErrorMessage);
+
+	// Handler information:
+	/// Gets the node feature flags which this handler itself supports. All available handlers are
+	/// queried similarly and their feature flags are OR'd together to form the [`NodeFeatures`]
+	/// which are broadcasted in our node_announcement message.
+	fn provided_node_features(&self) -> NodeFeatures;
+
+	/// Gets the init feature flags which should be sent to the given peer. All available handlers
+	/// are queried similarly and their feature flags are OR'd together to form the [`InitFeatures`]
+	/// which are sent in our [`Init`] message.
+	///
+	/// Note that this method is called before [`Self::peer_connected`].
+	fn provided_init_features(&self, their_node_id: &PublicKey) -> InitFeatures;
 }
 
 /// A trait to describe an object which can receive routing messages.
@@ -865,19 +928,19 @@ pub trait RoutingMessageHandler : MessageSendEventsProvider {
 	/// Handle an incoming channel_update message, returning true if it should be forwarded on,
 	/// false or returning an Err otherwise.
 	fn handle_channel_update(&self, msg: &ChannelUpdate) -> Result<bool, LightningError>;
-	/// Gets a subset of the channel announcements and updates required to dump our routing table
-	/// to a remote node, starting at the short_channel_id indicated by starting_point and
-	/// including the batch_amount entries immediately higher in numerical value than starting_point.
-	fn get_next_channel_announcements(&self, starting_point: u64, batch_amount: u8) -> Vec<(ChannelAnnouncement, Option<ChannelUpdate>, Option<ChannelUpdate>)>;
-	/// Gets a subset of the node announcements required to dump our routing table to a remote node,
-	/// starting at the node *after* the provided publickey and including batch_amount entries
-	/// immediately higher (as defined by <PublicKey as Ord>::cmp) than starting_point.
+	/// Gets channel announcements and updates required to dump our routing table to a remote node,
+	/// starting at the short_channel_id indicated by starting_point and including announcements
+	/// for a single channel.
+	fn get_next_channel_announcement(&self, starting_point: u64) -> Option<(ChannelAnnouncement, Option<ChannelUpdate>, Option<ChannelUpdate>)>;
+	/// Gets a node announcement required to dump our routing table to a remote node, starting at
+	/// the node *after* the provided pubkey and including up to one announcement immediately
+	/// higher (as defined by <PublicKey as Ord>::cmp) than starting_point.
 	/// If None is provided for starting_point, we start at the first node.
-	fn get_next_node_announcements(&self, starting_point: Option<&PublicKey>, batch_amount: u8) -> Vec<NodeAnnouncement>;
+	fn get_next_node_announcement(&self, starting_point: Option<&PublicKey>) -> Option<NodeAnnouncement>;
 	/// Called when a connection is established with a peer. This can be used to
 	/// perform routing table synchronization using a strategy defined by the
 	/// implementor.
-	fn sync_routing_table(&self, their_node_id: &PublicKey, init: &Init);
+	fn peer_connected(&self, their_node_id: &PublicKey, init: &Init);
 	/// Handles the reply of a query we initiated to learn about channels
 	/// for a given range of blocks. We can expect to receive one or more
 	/// replies to a single query.
@@ -893,6 +956,26 @@ pub trait RoutingMessageHandler : MessageSendEventsProvider {
 	/// Handles when a peer asks us to send routing gossip messages for a
 	/// list of short_channel_ids.
 	fn handle_query_short_channel_ids(&self, their_node_id: &PublicKey, msg: QueryShortChannelIds) -> Result<(), LightningError>;
+
+	// Handler information:
+	/// Gets the init feature flags which should be sent to the given peer. All available handlers
+	/// are queried similarly and their feature flags are OR'd together to form the [`InitFeatures`]
+	/// which are sent in our [`Init`] message.
+	///
+	/// Note that this method is called before [`Self::peer_connected`].
+	fn provided_init_features(&self, their_node_id: &PublicKey) -> InitFeatures;
+}
+
+/// A trait to describe an object that can receive onion messages.
+pub trait OnionMessageHandler : OnionMessageProvider {
+	/// Handle an incoming onion_message message from the given peer.
+	fn handle_onion_message(&self, peer_node_id: &PublicKey, msg: &OnionMessage);
+	/// Called when a connection is established with a peer. Can be used to track which peers
+	/// advertise onion message support and are online.
+	fn peer_connected(&self, their_node_id: &PublicKey, init: &Init);
+	/// Indicates a connection to the peer failed/an existing connection was lost. Allows handlers to
+	/// drop and refuse to forward onion messages to this peer.
+	fn peer_disconnected(&self, their_node_id: &PublicKey, no_connection_possible: bool);
 }
 
 mod fuzzy_internal_msgs {
@@ -937,9 +1020,9 @@ mod fuzzy_internal_msgs {
 		pub(crate) pad: Vec<u8>,
 	}
 }
-#[cfg(feature = "fuzztarget")]
+#[cfg(fuzzing)]
 pub use self::fuzzy_internal_msgs::*;
-#[cfg(not(feature = "fuzztarget"))]
+#[cfg(not(fuzzing))]
 pub(crate) use self::fuzzy_internal_msgs::*;
 
 #[derive(Clone)]
@@ -953,6 +1036,18 @@ pub(crate) struct OnionPacket {
 	pub(crate) hmac: [u8; 32],
 }
 
+impl onion_utils::Packet for OnionPacket {
+	type Data = onion_utils::FixedSizeOnionPacket;
+	fn new(pubkey: PublicKey, hop_data: onion_utils::FixedSizeOnionPacket, hmac: [u8; 32]) -> Self {
+		Self {
+			version: 0,
+			public_key: Ok(pubkey),
+			hop_data: hop_data.0,
+			hmac,
+		}
+	}
+}
+
 impl PartialEq for OnionPacket {
 	fn eq(&self, other: &OnionPacket) -> bool {
 		for (i, j) in self.hop_data.iter().zip(other.hop_data.iter()) {
@@ -985,7 +1080,7 @@ impl fmt::Display for DecodeError {
 			DecodeError::InvalidValue => f.write_str("Nonsense bytes didn't map to the type they were interpreted as"),
 			DecodeError::ShortRead => f.write_str("Packet extended beyond the provided bytes"),
 			DecodeError::BadLengthDescriptor => f.write_str("A length descriptor in the packet didn't describe the later data correctly"),
-			DecodeError::Io(ref e) => e.fmt(f),
+			DecodeError::Io(ref e) => fmt::Debug::fmt(e, f),
 			DecodeError::UnsupportedCompression => f.write_str("We don't support receiving messages with zlib-compressed fields"),
 		}
 	}
@@ -1064,7 +1159,9 @@ impl_writeable_msg!(AcceptChannel, {
 	htlc_basepoint,
 	first_per_commitment_point,
 	shutdown_scriptpubkey
-}, {});
+}, {
+	(1, channel_type, option),
+});
 
 impl_writeable_msg!(AnnouncementSignatures, {
 	channel_id,
@@ -1144,17 +1241,23 @@ impl_writeable_msg!(FundingSigned, {
 	signature
 }, {});
 
-impl_writeable_msg!(FundingLocked, {
+impl_writeable_msg!(ChannelReady, {
 	channel_id,
 	next_per_commitment_point,
-}, {});
+}, {
+	(1, short_channel_id_alias, option),
+});
 
 impl Writeable for Init {
 	fn write<W: Writer>(&self, w: &mut W) -> Result<(), io::Error> {
 		// global_features gets the bottom 13 bits of our features, and local_features gets all of
 		// our relevant feature bits. This keeps us compatible with old nodes.
 		self.features.write_up_to_13(w)?;
-		self.features.write(w)
+		self.features.write(w)?;
+		encode_tlv_stream!(w, {
+			(3, self.remote_network_address, option)
+		});
+		Ok(())
 	}
 }
 
@@ -1162,8 +1265,13 @@ impl Readable for Init {
 	fn read<R: Read>(r: &mut R) -> Result<Self, DecodeError> {
 		let global_features: InitFeatures = Readable::read(r)?;
 		let features: InitFeatures = Readable::read(r)?;
+		let mut remote_network_address: Option<NetAddress> = None;
+		decode_tlv_stream!(r, {
+			(3, remote_network_address, option)
+		});
 		Ok(Init {
 			features: features.or(global_features),
+			remote_network_address,
 		})
 	}
 }
@@ -1274,27 +1382,46 @@ impl_writeable_msg!(UpdateAddHTLC, {
 	onion_routing_packet
 }, {});
 
+impl Readable for OnionMessage {
+	fn read<R: Read>(r: &mut R) -> Result<Self, DecodeError> {
+		let blinding_point: PublicKey = Readable::read(r)?;
+		let len: u16 = Readable::read(r)?;
+		let mut packet_reader = FixedLengthReader::new(r, len as u64);
+		let onion_routing_packet: onion_message::Packet = <onion_message::Packet as LengthReadable>::read(&mut packet_reader)?;
+		Ok(Self {
+			blinding_point,
+			onion_routing_packet,
+		})
+	}
+}
+
+impl Writeable for OnionMessage {
+	fn write<W: Writer>(&self, w: &mut W) -> Result<(), io::Error> {
+		self.blinding_point.write(w)?;
+		let onion_packet_len = self.onion_routing_packet.serialized_length();
+		(onion_packet_len as u16).write(w)?;
+		self.onion_routing_packet.write(w)?;
+		Ok(())
+	}
+}
+
 impl Writeable for FinalOnionHopData {
 	fn write<W: Writer>(&self, w: &mut W) -> Result<(), io::Error> {
 		self.payment_secret.0.write(w)?;
-		HighZeroBytesDroppedVarInt(self.total_msat).write(w)
+		HighZeroBytesDroppedBigSize(self.total_msat).write(w)
 	}
 }
 
 impl Readable for FinalOnionHopData {
 	fn read<R: Read>(r: &mut R) -> Result<Self, DecodeError> {
 		let secret: [u8; 32] = Readable::read(r)?;
-		let amt: HighZeroBytesDroppedVarInt<u64> = Readable::read(r)?;
+		let amt: HighZeroBytesDroppedBigSize<u64> = Readable::read(r)?;
 		Ok(Self { payment_secret: PaymentSecret(secret), total_msat: amt.0 })
 	}
 }
 
 impl Writeable for OnionHopData {
 	fn write<W: Writer>(&self, w: &mut W) -> Result<(), io::Error> {
-		// Note that this should never be reachable if Rust-Lightning generated the message, as we
-		// check values are sane long before we get here, though its possible in the future
-		// user-generated messages may hit this.
-		if self.amt_to_forward > MAX_VALUE_MSAT { panic!("We should never be sending infinite/overflow onion payments"); }
 		match self.format {
 			OnionHopDataFormat::Legacy { short_channel_id } => {
 				0u8.write(w)?;
@@ -1305,18 +1432,15 @@ impl Writeable for OnionHopData {
 			},
 			OnionHopDataFormat::NonFinalNode { short_channel_id } => {
 				encode_varint_length_prefixed_tlv!(w, {
-					(2, HighZeroBytesDroppedVarInt(self.amt_to_forward), required),
-					(4, HighZeroBytesDroppedVarInt(self.outgoing_cltv_value), required),
+					(2, HighZeroBytesDroppedBigSize(self.amt_to_forward), required),
+					(4, HighZeroBytesDroppedBigSize(self.outgoing_cltv_value), required),
 					(6, short_channel_id, required)
 				});
 			},
 			OnionHopDataFormat::FinalNode { ref payment_data, ref keysend_preimage } => {
-				if let Some(final_data) = payment_data {
-					if final_data.total_msat > MAX_VALUE_MSAT { panic!("We should never be sending infinite/overflow onion payments"); }
-				}
 				encode_varint_length_prefixed_tlv!(w, {
-					(2, HighZeroBytesDroppedVarInt(self.amt_to_forward), required),
-					(4, HighZeroBytesDroppedVarInt(self.outgoing_cltv_value), required),
+					(2, HighZeroBytesDroppedBigSize(self.amt_to_forward), required),
+					(4, HighZeroBytesDroppedBigSize(self.outgoing_cltv_value), required),
 					(8, payment_data, option),
 					(5482373484, keysend_preimage, option)
 				});
@@ -1327,27 +1451,22 @@ impl Writeable for OnionHopData {
 }
 
 impl Readable for OnionHopData {
-	fn read<R: Read>(mut r: &mut R) -> Result<Self, DecodeError> {
-		use bitcoin::consensus::encode::{Decodable, Error, VarInt};
-		let v: VarInt = Decodable::consensus_decode(&mut r)
-			.map_err(|e| match e {
-				Error::Io(ioe) => DecodeError::from(ioe),
-				_ => DecodeError::InvalidValue
-			})?;
+	fn read<R: Read>(r: &mut R) -> Result<Self, DecodeError> {
+		let b: BigSize = Readable::read(r)?;
 		const LEGACY_ONION_HOP_FLAG: u64 = 0;
-		let (format, amt, cltv_value) = if v.0 != LEGACY_ONION_HOP_FLAG {
-			let mut rd = FixedLengthReader::new(r, v.0);
-			let mut amt = HighZeroBytesDroppedVarInt(0u64);
-			let mut cltv_value = HighZeroBytesDroppedVarInt(0u32);
+		let (format, amt, cltv_value) = if b.0 != LEGACY_ONION_HOP_FLAG {
+			let mut rd = FixedLengthReader::new(r, b.0);
+			let mut amt = HighZeroBytesDroppedBigSize(0u64);
+			let mut cltv_value = HighZeroBytesDroppedBigSize(0u32);
 			let mut short_id: Option<u64> = None;
 			let mut payment_data: Option<FinalOnionHopData> = None;
 			let mut keysend_preimage: Option<PaymentPreimage> = None;
-			// The TLV type is chosen to be compatible with lnd and c-lightning.
 			decode_tlv_stream!(&mut rd, {
 				(2, amt, required),
 				(4, cltv_value, required),
 				(6, short_id, option),
 				(8, payment_data, option),
+				// See https://github.com/lightning/blips/blob/master/blip-0003.md
 				(5482373484, keysend_preimage, option)
 			});
 			rd.eat_remaining().map_err(|_| DecodeError::ShortRead)?;
@@ -1389,6 +1508,14 @@ impl Readable for OnionHopData {
 	}
 }
 
+// ReadableArgs because we need onion_utils::decode_next_hop to accommodate payment packets and
+// onion message packets.
+impl ReadableArgs<()> for OnionHopData {
+	fn read<R: Read>(r: &mut R, _arg: ()) -> Result<Self, DecodeError> {
+		<Self as Readable>::read(r)
+	}
+}
+
 impl Writeable for Ping {
 	fn write<W: Writer>(&self, w: &mut W) -> Result<(), io::Error> {
 		self.ponglen.write(w)?;
@@ -1468,14 +1595,12 @@ impl_writeable!(ChannelAnnouncement, {
 
 impl Writeable for UnsignedChannelUpdate {
 	fn write<W: Writer>(&self, w: &mut W) -> Result<(), io::Error> {
-		let mut message_flags: u8 = 0;
-		if let OptionalField::Present(_) = self.htlc_maximum_msat {
-			message_flags = 1;
-		}
+		// `message_flags` used to indicate presence of `htlc_maximum_msat`, but was deprecated in the spec.
+		const MESSAGE_FLAGS: u8 = 1;
 		self.chain_hash.write(w)?;
 		self.short_channel_id.write(w)?;
 		self.timestamp.write(w)?;
-		let all_flags = self.flags as u16 | ((message_flags as u16) << 8);
+		let all_flags = self.flags as u16 | ((MESSAGE_FLAGS as u16) << 8);
 		all_flags.write(w)?;
 		self.cltv_expiry_delta.write(w)?;
 		self.htlc_minimum_msat.write(w)?;
@@ -1489,22 +1614,20 @@ impl Writeable for UnsignedChannelUpdate {
 
 impl Readable for UnsignedChannelUpdate {
 	fn read<R: Read>(r: &mut R) -> Result<Self, DecodeError> {
-		let has_htlc_maximum_msat;
 		Ok(Self {
 			chain_hash: Readable::read(r)?,
 			short_channel_id: Readable::read(r)?,
 			timestamp: Readable::read(r)?,
 			flags: {
 				let flags: u16 = Readable::read(r)?;
-				let message_flags = flags >> 8;
-				has_htlc_maximum_msat = (message_flags as i32 & 1) == 1;
+				// Note: we ignore the `message_flags` for now, since it was deprecated by the spec.
 				flags as u8
 			},
 			cltv_expiry_delta: Readable::read(r)?,
 			htlc_minimum_msat: Readable::read(r)?,
 			fee_base_msat: Readable::read(r)?,
 			fee_proportional_millionths: Readable::read(r)?,
-			htlc_maximum_msat: if has_htlc_maximum_msat { Readable::read(r)? } else { OptionalField::Absent },
+			htlc_maximum_msat: Readable::read(r)?,
 			excess_data: read_to_end(r)?,
 		})
 	}
@@ -1529,10 +1652,11 @@ impl Readable for ErrorMessage {
 		Ok(Self {
 			channel_id: Readable::read(r)?,
 			data: {
-				let mut sz: usize = <u16 as Readable>::read(r)? as usize;
-				let data = read_to_end(r)?;
-				sz = cmp::min(data.len(), sz);
-				match String::from_utf8(data[..sz as usize].to_vec()) {
+				let sz: usize = <u16 as Readable>::read(r)? as usize;
+				let mut data = Vec::with_capacity(sz);
+				data.resize(sz, 0);
+				r.read_exact(&mut data)?;
+				match String::from_utf8(data) {
 					Ok(s) => s,
 					Err(_) => return Err(DecodeError::InvalidValue),
 				}
@@ -1555,10 +1679,11 @@ impl Readable for WarningMessage {
 		Ok(Self {
 			channel_id: Readable::read(r)?,
 			data: {
-				let mut sz: usize = <u16 as Readable>::read(r)? as usize;
-				let data = read_to_end(r)?;
-				sz = cmp::min(data.len(), sz);
-				match String::from_utf8(data[..sz as usize].to_vec()) {
+				let sz: usize = <u16 as Readable>::read(r)? as usize;
+				let mut data = Vec::with_capacity(sz);
+				data.resize(sz, 0);
+				r.read_exact(&mut data)?;
+				match String::from_utf8(data) {
 					Ok(s) => s,
 					Err(_) => return Err(DecodeError::InvalidValue),
 				}
@@ -1804,7 +1929,7 @@ mod tests {
 	use ln::features::{ChannelFeatures, ChannelTypeFeatures, InitFeatures, NodeFeatures};
 	use ln::msgs;
 	use ln::msgs::{FinalOnionHopData, OptionalField, OnionErrorPacket, OnionHopDataFormat};
-	use util::ser::{Writeable, Readable};
+	use util::ser::{Writeable, Readable, Hostname};
 
 	use bitcoin::hashes::hex::FromHex;
 	use bitcoin::util::address::Address;
@@ -1813,11 +1938,12 @@ mod tests {
 	use bitcoin::blockdata::opcodes;
 	use bitcoin::hash_types::{Txid, BlockHash};
 
-	use bitcoin::secp256k1::key::{PublicKey,SecretKey};
+	use bitcoin::secp256k1::{PublicKey,SecretKey};
 	use bitcoin::secp256k1::{Secp256k1, Message};
 
-	use io::Cursor;
+	use io::{self, Cursor};
 	use prelude::*;
+	use core::convert::TryFrom;
 
 	#[test]
 	fn encoding_channel_reestablish_no_secret() {
@@ -1870,7 +1996,7 @@ mod tests {
 		($privkey: expr, $ctx: expr, $string: expr) => {
 			{
 				let sighash = Message::from_slice(&$string.into_bytes()[..]).unwrap();
-				$ctx.sign(&sighash, &$privkey)
+				$ctx.sign_ecdsa(&sighash, &$privkey)
 			}
 		}
 	}
@@ -1946,7 +2072,7 @@ mod tests {
 		do_encoding_channel_announcement(true, true);
 	}
 
-	fn do_encoding_node_announcement(unknown_features_bits: bool, ipv4: bool, ipv6: bool, onionv2: bool, onionv3: bool, excess_address_data: bool, excess_data: bool) {
+	fn do_encoding_node_announcement(unknown_features_bits: bool, ipv4: bool, ipv6: bool, onionv2: bool, onionv3: bool, hostname: bool, excess_address_data: bool, excess_data: bool) {
 		let secp_ctx = Secp256k1::new();
 		let (privkey_1, pubkey_1) = get_keys_from!("0101010101010101010101010101010101010101010101010101010101010101", secp_ctx);
 		let sig_1 = get_sig_on!(privkey_1, secp_ctx, String::from("01010101010101010101010101010101"));
@@ -1982,6 +2108,12 @@ mod tests {
 				port: 9735
 			});
 		}
+		if hostname {
+			addresses.push(msgs::NetAddress::Hostname {
+				hostname: Hostname::try_from(String::from("host")).unwrap(),
+				port: 9735,
+			});
+		}
 		let mut addr_len = 0;
 		for addr in &addresses {
 			addr_len += addr.len() + 1;
@@ -2022,6 +2154,9 @@ mod tests {
 		if onionv3 {
 			target_value.append(&mut hex::decode("04fffefdfcfbfaf9f8f7f6f5f4f3f2f1f0efeeedecebeae9e8e7e6e5e4e3e2e1e00020102607").unwrap());
 		}
+		if hostname {
+			target_value.append(&mut hex::decode("0504686f73742607").unwrap());
+		}
 		if excess_address_data {
 			target_value.append(&mut hex::decode("216c280b5395a2546e7e4b2663e04f811622f15a4f92e83aa2e92ba2a573c139142c54ae63072a1ec1ee7dc0c04bde5c847806172aa05c92c22ae8e308d1d269").unwrap());
 		}
@@ -2033,18 +2168,19 @@ mod tests {
 
 	#[test]
 	fn encoding_node_announcement() {
-		do_encoding_node_announcement(true, true, true, true, true, true, true);
-		do_encoding_node_announcement(false, false, false, false, false, false, false);
-		do_encoding_node_announcement(false, true, false, false, false, false, false);
-		do_encoding_node_announcement(false, false, true, false, false, false, false);
-		do_encoding_node_announcement(false, false, false, true, false, false, false);
-		do_encoding_node_announcement(false, false, false, false, true, false, false);
-		do_encoding_node_announcement(false, false, false, false, false, true, false);
-		do_encoding_node_announcement(false, true, false, true, false, true, false);
-		do_encoding_node_announcement(false, false, true, false, true, false, false);
-	}
-
-	fn do_encoding_channel_update(direction: bool, disable: bool, htlc_maximum_msat: bool, excess_data: bool) {
+		do_encoding_node_announcement(true, true, true, true, true, true, true, true);
+		do_encoding_node_announcement(false, false, false, false, false, false, false, false);
+		do_encoding_node_announcement(false, true, false, false, false, false, false, false);
+		do_encoding_node_announcement(false, false, true, false, false, false, false, false);
+		do_encoding_node_announcement(false, false, false, true, false, false, false, false);
+		do_encoding_node_announcement(false, false, false, false, true, false, false, false);
+		do_encoding_node_announcement(false, false, false, false, false, true, false, false);
+		do_encoding_node_announcement(false, false, false, false, false, false, true, false);
+		do_encoding_node_announcement(false, true, false, true, false, false, true, false);
+		do_encoding_node_announcement(false, false, true, false, true, false, false, false);
+	}
+
+	fn do_encoding_channel_update(direction: bool, disable: bool, excess_data: bool) {
 		let secp_ctx = Secp256k1::new();
 		let (privkey_1, _) = get_keys_from!("0101010101010101010101010101010101010101010101010101010101010101", secp_ctx);
 		let sig_1 = get_sig_on!(privkey_1, secp_ctx, String::from("01010101010101010101010101010101"));
@@ -2055,7 +2191,7 @@ mod tests {
 			flags: if direction { 1 } else { 0 } | if disable { 1 << 1 } else { 0 },
 			cltv_expiry_delta: 144,
 			htlc_minimum_msat: 1000000,
-			htlc_maximum_msat: if htlc_maximum_msat { OptionalField::Present(131355275467161) } else { OptionalField::Absent },
+			htlc_maximum_msat: 131355275467161,
 			fee_base_msat: 10000,
 			fee_proportional_millionths: 20,
 			excess_data: if excess_data { vec![0, 0, 0, 0, 59, 154, 202, 0] } else { Vec::new() }
@@ -2068,11 +2204,7 @@ mod tests {
 		let mut target_value = hex::decode("d977cb9b53d93a6ff64bb5f1e158b4094b66e798fb12911168a3ccdf80a83096340a6a95da0ae8d9f776528eecdbb747eb6b545495a4319ed5378e35b21e073a").unwrap();
 		target_value.append(&mut hex::decode("000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f").unwrap());
 		target_value.append(&mut hex::decode("00083a840000034d013413a7").unwrap());
-		if htlc_maximum_msat {
-			target_value.append(&mut hex::decode("01").unwrap());
-		} else {
-			target_value.append(&mut hex::decode("00").unwrap());
-		}
+		target_value.append(&mut hex::decode("01").unwrap());
 		target_value.append(&mut hex::decode("00").unwrap());
 		if direction {
 			let flag = target_value.last_mut().unwrap();
@@ -2083,9 +2215,7 @@ mod tests {
 			*flag = *flag | 1 << 1;
 		}
 		target_value.append(&mut hex::decode("009000000000000f42400000271000000014").unwrap());
-		if htlc_maximum_msat {
-			target_value.append(&mut hex::decode("0000777788889999").unwrap());
-		}
+		target_value.append(&mut hex::decode("0000777788889999").unwrap());
 		if excess_data {
 			target_value.append(&mut hex::decode("000000003b9aca00").unwrap());
 		}
@@ -2094,16 +2224,14 @@ mod tests {
 
 	#[test]
 	fn encoding_channel_update() {
-		do_encoding_channel_update(false, false, false, false);
-		do_encoding_channel_update(false, false, false, true);
-		do_encoding_channel_update(true, false, false, false);
-		do_encoding_channel_update(true, false, false, true);
-		do_encoding_channel_update(false, true, false, false);
-		do_encoding_channel_update(false, true, false, true);
-		do_encoding_channel_update(false, false, true, false);
-		do_encoding_channel_update(false, false, true, true);
-		do_encoding_channel_update(true, true, true, false);
-		do_encoding_channel_update(true, true, true, true);
+		do_encoding_channel_update(false, false, false);
+		do_encoding_channel_update(false, false, true);
+		do_encoding_channel_update(true, false, false);
+		do_encoding_channel_update(true, false, true);
+		do_encoding_channel_update(false, true, false);
+		do_encoding_channel_update(false, true, true);
+		do_encoding_channel_update(true, true, false);
+		do_encoding_channel_update(true, true, true);
 	}
 
 	fn do_encoding_open_channel(random_bit: bool, shutdown: bool, incl_chan_type: bool) {
@@ -2133,7 +2261,7 @@ mod tests {
 			htlc_basepoint: pubkey_5,
 			first_per_commitment_point: pubkey_6,
 			channel_flags: if random_bit { 1 << 5 } else { 0 },
-			shutdown_scriptpubkey: if shutdown { OptionalField::Present(Address::p2pkh(&::bitcoin::PublicKey{compressed: true, key: pubkey_1}, Network::Testnet).script_pubkey()) } else { OptionalField::Absent },
+			shutdown_scriptpubkey: if shutdown { OptionalField::Present(Address::p2pkh(&::bitcoin::PublicKey{compressed: true, inner: pubkey_1}, Network::Testnet).script_pubkey()) } else { OptionalField::Absent },
 			channel_type: if incl_chan_type { Some(ChannelTypeFeatures::empty()) } else { None },
 		};
 		let encoded_value = open_channel.encode();
@@ -2189,7 +2317,8 @@ mod tests {
 			delayed_payment_basepoint: pubkey_4,
 			htlc_basepoint: pubkey_5,
 			first_per_commitment_point: pubkey_6,
-			shutdown_scriptpubkey: if shutdown { OptionalField::Present(Address::p2pkh(&::bitcoin::PublicKey{compressed: true, key: pubkey_1}, Network::Testnet).script_pubkey()) } else { OptionalField::Absent }
+			shutdown_scriptpubkey: if shutdown { OptionalField::Present(Address::p2pkh(&::bitcoin::PublicKey{compressed: true, inner: pubkey_1}, Network::Testnet).script_pubkey()) } else { OptionalField::Absent },
+			channel_type: None,
 		};
 		let encoded_value = accept_channel.encode();
 		let mut target_value = hex::decode("020202020202020202020202020202020202020202020202020202020202020212345678901234562334032891223698321446687011447600083a840000034d000c89d4c0bcc0bc031b84c5567b126440995d3ed5aaba0565d71e1834604819ff9c17f5e9d5dd078f024d4b6cd1361032ca9bd2aeb9d900aa4d45d9ead80ac9423374c451a7254d076602531fe6068134503d2723133227c867ac8fa6c83c537e9a44c3c5bdbdcb1fe33703462779ad4aad39514614751a71085f2f10e1c7a593e4e030efb5b8721ce55b0b0362c0a046dacce86ddd0343c6d3c7c79c2208ba0d9c9cf24a6d046d21d21f90f703f006a18d5653c4edf5391ff23a61f03ff83d237e880ee61187fa9f379a028e0a").unwrap();
@@ -2236,14 +2365,15 @@ mod tests {
 	}
 
 	#[test]
-	fn encoding_funding_locked() {
+	fn encoding_channel_ready() {
 		let secp_ctx = Secp256k1::new();
 		let (_, pubkey_1,) = get_keys_from!("0101010101010101010101010101010101010101010101010101010101010101", secp_ctx);
-		let funding_locked = msgs::FundingLocked {
+		let channel_ready = msgs::ChannelReady {
 			channel_id: [2; 32],
 			next_per_commitment_point: pubkey_1,
+			short_channel_id_alias: None,
 		};
-		let encoded_value = funding_locked.encode();
+		let encoded_value = channel_ready.encode();
 		let target_value = hex::decode("0202020202020202020202020202020202020202020202020202020202020202031b84c5567b126440995d3ed5aaba0565d71e1834604819ff9c17f5e9d5dd078f").unwrap();
 		assert_eq!(encoded_value, target_value);
 	}
@@ -2255,9 +2385,9 @@ mod tests {
 		let shutdown = msgs::Shutdown {
 			channel_id: [2; 32],
 			scriptpubkey:
-				     if script_type == 1 { Address::p2pkh(&::bitcoin::PublicKey{compressed: true, key: pubkey_1}, Network::Testnet).script_pubkey() }
-				else if script_type == 2 { Address::p2sh(&script, Network::Testnet).script_pubkey() }
-				else if script_type == 3 { Address::p2wpkh(&::bitcoin::PublicKey{compressed: true, key: pubkey_1}, Network::Testnet).unwrap().script_pubkey() }
+				     if script_type == 1 { Address::p2pkh(&::bitcoin::PublicKey{compressed: true, inner: pubkey_1}, Network::Testnet).script_pubkey() }
+				else if script_type == 2 { Address::p2sh(&script, Network::Testnet).unwrap().script_pubkey() }
+				else if script_type == 3 { Address::p2wpkh(&::bitcoin::PublicKey{compressed: true, inner: pubkey_1}, Network::Testnet).unwrap().script_pubkey() }
 				else                     { Address::p2wsh(&script, Network::Testnet).script_pubkey() },
 		};
 		let encoded_value = shutdown.encode();
@@ -2437,13 +2567,27 @@ mod tests {
 	fn encoding_init() {
 		assert_eq!(msgs::Init {
 			features: InitFeatures::from_le_bytes(vec![0xFF, 0xFF, 0xFF]),
+			remote_network_address: None,
 		}.encode(), hex::decode("00023fff0003ffffff").unwrap());
 		assert_eq!(msgs::Init {
 			features: InitFeatures::from_le_bytes(vec![0xFF]),
+			remote_network_address: None,
 		}.encode(), hex::decode("0001ff0001ff").unwrap());
 		assert_eq!(msgs::Init {
 			features: InitFeatures::from_le_bytes(vec![]),
+			remote_network_address: None,
 		}.encode(), hex::decode("00000000").unwrap());
+
+		let init_msg = msgs::Init { features: InitFeatures::from_le_bytes(vec![]),
+			remote_network_address: Some(msgs::NetAddress::IPv4 {
+				addr: [127, 0, 0, 1],
+				port: 1000,
+			}),
+		};
+		let encoded_value = init_msg.encode();
+		let target_value = hex::decode("000000000307017f00000103e8").unwrap();
+		assert_eq!(encoded_value, target_value);
+		assert_eq!(msgs::Init::read(&mut Cursor::new(&target_value)).unwrap(), init_msg);
 	}
 
 	#[test]
@@ -2708,4 +2852,40 @@ mod tests {
 		assert_eq!(gossip_timestamp_filter.first_timestamp, 1590000000);
 		assert_eq!(gossip_timestamp_filter.timestamp_range, 0xffff_ffff);
 	}
+
+	#[test]
+	fn decode_onion_hop_data_len_as_bigsize() {
+		// Tests that we can decode an onion payload that is >253 bytes.
+		// Previously, receiving a payload of this size could've caused us to fail to decode a valid
+		// payload, because we were decoding the length (a BigSize, big-endian) as a VarInt
+		// (little-endian).
+
+		// Encode a test onion payload with a big custom TLV such that it's >253 bytes, forcing the
+		// payload length to be encoded over multiple bytes rather than a single u8.
+		let big_payload = encode_big_payload().unwrap();
+		let mut rd = Cursor::new(&big_payload[..]);
+		<msgs::OnionHopData as Readable>::read(&mut rd).unwrap();
+	}
+	// see above test, needs to be a separate method for use of the serialization macros.
+	fn encode_big_payload() -> Result<Vec<u8>, io::Error> {
+		use util::ser::HighZeroBytesDroppedBigSize;
+		let payload = msgs::OnionHopData {
+			format: OnionHopDataFormat::NonFinalNode {
+				short_channel_id: 0xdeadbeef1bad1dea,
+			},
+			amt_to_forward: 1000,
+			outgoing_cltv_value: 0xffffffff,
+		};
+		let mut encoded_payload = Vec::new();
+		let test_bytes = vec![42u8; 1000];
+		if let OnionHopDataFormat::NonFinalNode { short_channel_id } = payload.format {
+			encode_varint_length_prefixed_tlv!(&mut encoded_payload, {
+				(1, test_bytes, vec_type),
+				(2, HighZeroBytesDroppedBigSize(payload.amt_to_forward), required),
+				(4, HighZeroBytesDroppedBigSize(payload.outgoing_cltv_value), required),
+				(6, short_channel_id, required)
+			});
+		}
+		Ok(encoded_payload)
+	}
 }