Merge pull request #167 from TheBlueMatt/2018-09-dup-htlc
authorMatt Corallo <649246+TheBlueMatt@users.noreply.github.com>
Wed, 12 Sep 2018 17:07:13 +0000 (13:07 -0400)
committerGitHub <noreply@github.com>
Wed, 12 Sep 2018 17:07:13 +0000 (13:07 -0400)
 Allow duplicate-payment_hash HTLCs for HTLC forwards

37 files changed:
fuzz/Cargo.toml
fuzz/fuzz_targets/channel_target.rs
fuzz/fuzz_targets/msg_ping_target.rs [deleted file]
fuzz/fuzz_targets/msg_pong_target.rs [deleted file]
fuzz/fuzz_targets/msg_targets/gen_target.sh
fuzz/fuzz_targets/msg_targets/msg_accept_channel_target.rs
fuzz/fuzz_targets/msg_targets/msg_announcement_signatures_target.rs
fuzz/fuzz_targets/msg_targets/msg_channel_announcement_target.rs
fuzz/fuzz_targets/msg_targets/msg_channel_reestablish_target.rs
fuzz/fuzz_targets/msg_targets/msg_channel_update_target.rs
fuzz/fuzz_targets/msg_targets/msg_closing_signed_target.rs
fuzz/fuzz_targets/msg_targets/msg_commitment_signed_target.rs
fuzz/fuzz_targets/msg_targets/msg_decoded_onion_error_packet_target.rs
fuzz/fuzz_targets/msg_targets/msg_error_message_target.rs
fuzz/fuzz_targets/msg_targets/msg_funding_created_target.rs
fuzz/fuzz_targets/msg_targets/msg_funding_locked_target.rs
fuzz/fuzz_targets/msg_targets/msg_funding_signed_target.rs
fuzz/fuzz_targets/msg_targets/msg_init_target.rs
fuzz/fuzz_targets/msg_targets/msg_node_announcement_target.rs
fuzz/fuzz_targets/msg_targets/msg_onion_hop_data_target.rs
fuzz/fuzz_targets/msg_targets/msg_open_channel_target.rs
fuzz/fuzz_targets/msg_targets/msg_ping_target.rs [new file with mode: 0644]
fuzz/fuzz_targets/msg_targets/msg_pong_target.rs [new file with mode: 0644]
fuzz/fuzz_targets/msg_targets/msg_revoke_and_ack_target.rs
fuzz/fuzz_targets/msg_targets/msg_shutdown_target.rs
fuzz/fuzz_targets/msg_targets/msg_target_template.txt
fuzz/fuzz_targets/msg_targets/msg_update_add_htlc_target.rs
fuzz/fuzz_targets/msg_targets/msg_update_fail_htlc_target.rs
fuzz/fuzz_targets/msg_targets/msg_update_fail_malformed_htlc_target.rs
fuzz/fuzz_targets/msg_targets/msg_update_fee_target.rs
fuzz/fuzz_targets/msg_targets/msg_update_fulfill_htlc_target.rs
fuzz/fuzz_targets/msg_targets/utils.rs
fuzz/fuzz_targets/router_target.rs
src/ln/msgs.rs
src/ln/peer_handler.rs
src/util/mod.rs
src/util/ser.rs [new file with mode: 0644]

index 3cd249ea8218d7ef115054c5afe37d77014662ee..01311f206776d29725f7e4433612262d4c2bd639 100644 (file)
@@ -54,11 +54,11 @@ path = "fuzz_targets/chanmon_deser_target.rs"
 # message fuzz targets
 [[bin]]
 name = "msg_ping_target"
-path = "fuzz_targets/msg_ping_target.rs"
+path = "fuzz_targets/msg_targets/msg_ping_target.rs"
 
 [[bin]]
 name = "msg_pong_target"
-path = "fuzz_targets/msg_pong_target.rs"
+path = "fuzz_targets/msg_targets/msg_pong_target.rs"
 
 [[bin]]
 name = "msg_error_message_target"
index f6e88417c72ab0f4bd514fc525900b18cd1174c6..d33ac94192a5efeaecc3829dd26c6ab9d7e94751 100644 (file)
@@ -10,11 +10,12 @@ use bitcoin::network::serialize::{serialize, BitcoinHash};
 use lightning::ln::channel::{Channel, ChannelKeys};
 use lightning::ln::channelmanager::{HTLCFailReason, HTLCSource, PendingHTLCStatus};
 use lightning::ln::msgs;
-use lightning::ln::msgs::{MsgDecodable, ErrorAction};
+use lightning::ln::msgs::{ErrorAction};
 use lightning::chain::chaininterface::{FeeEstimator, ConfirmationTarget};
 use lightning::chain::transaction::OutPoint;
 use lightning::util::reset_rng_state;
 use lightning::util::logger::Logger;
+use lightning::util::ser::{Readable, Reader};
 
 mod utils;
 
@@ -119,8 +120,9 @@ pub fn do_test(data: &[u8]) {
        }
 
        macro_rules! decode_msg {
-               ($MsgType: path, $len: expr) => {
-                       match <($MsgType)>::decode(get_slice!($len)) {
+               ($MsgType: path, $len: expr) => {{
+                       let mut reader = Reader::new(::std::io::Cursor::new(get_slice!($len)));
+                       match <($MsgType)>::read(&mut reader) {
                                Ok(msg) => msg,
                                Err(e) => match e {
                                        msgs::DecodeError::UnknownRealmByte => return,
@@ -131,9 +133,11 @@ pub fn do_test(data: &[u8]) {
                                        msgs::DecodeError::ExtraAddressesPerType => return,
                                        msgs::DecodeError::BadLengthDescriptor => return,
                                        msgs::DecodeError::ShortRead => panic!("We picked the length..."),
+                                       msgs::DecodeError::InvalidValue => panic!("Should not happen with p2p message decoding"),
+                                       msgs::DecodeError::Io(e) => panic!(format!("{}", e)),
                                }
                        }
-               }
+               }}
        }
 
        macro_rules! decode_msg_with_len16 {
@@ -143,19 +147,7 @@ pub fn do_test(data: &[u8]) {
                                        Some(slice) => slice,
                                        None => return,
                                }[$begin_len..$begin_len + 2]);
-                               match <($MsgType)>::decode(get_slice!($begin_len as usize + 2 + (extra_len as usize)*$factor)) {
-                                       Ok(msg) => msg,
-                                       Err(e) => match e {
-                                               msgs::DecodeError::UnknownRealmByte => return,
-                                               msgs::DecodeError::UnknownRequiredFeature => return,
-                                               msgs::DecodeError::BadPublicKey => return,
-                                               msgs::DecodeError::BadSignature => return,
-                                               msgs::DecodeError::BadText => return,
-                                               msgs::DecodeError::ExtraAddressesPerType => return,
-                                               msgs::DecodeError::BadLengthDescriptor => return,
-                                               msgs::DecodeError::ShortRead => panic!("We picked the length..."),
-                                       }
-                               }
+                               decode_msg!($MsgType, $begin_len as usize + 2 + (extra_len as usize)*$factor)
                        }
                }
        }
diff --git a/fuzz/fuzz_targets/msg_ping_target.rs b/fuzz/fuzz_targets/msg_ping_target.rs
deleted file mode 100644 (file)
index c9fb341..0000000
+++ /dev/null
@@ -1,45 +0,0 @@
-// This file is auto-generated by gen_target.sh based on msg_target_template.txt
-// To modify it, modify msg_target_template.txt and run gen_target.sh instead.
-
-extern crate lightning;
-
-use lightning::util::reset_rng_state;
-
-use lightning::ln::msgs::{MsgEncodable, MsgDecodable, Ping};
-
-#[inline]
-pub fn do_test(data: &[u8]) {
-       reset_rng_state();
-       if let Ok(msg) = Ping::decode(data) {
-               let _ = msg.encode();
-       }
-}
-
-#[cfg(feature = "afl")]
-#[macro_use] extern crate afl;
-#[cfg(feature = "afl")]
-fn main() {
-       fuzz!(|data| {
-               do_test(data);
-       });
-}
-
-#[cfg(feature = "honggfuzz")]
-#[macro_use] extern crate honggfuzz;
-#[cfg(feature = "honggfuzz")]
-fn main() {
-       loop {
-               fuzz!(|data| {
-                       do_test(data);
-               });
-       }
-}
-
-extern crate hex;
-#[cfg(test)]
-mod tests {
-       #[test]
-       fn duplicate_crash() {
-               super::do_test(&::hex::decode("00").unwrap());
-       }
-}
diff --git a/fuzz/fuzz_targets/msg_pong_target.rs b/fuzz/fuzz_targets/msg_pong_target.rs
deleted file mode 100644 (file)
index d4572d3..0000000
+++ /dev/null
@@ -1,45 +0,0 @@
-// This file is auto-generated by gen_target.sh based on msg_target_template.txt
-// To modify it, modify msg_target_template.txt and run gen_target.sh instead.
-
-extern crate lightning;
-
-use lightning::util::reset_rng_state;
-
-use lightning::ln::msgs::{MsgEncodable, MsgDecodable, Pong};
-
-#[inline]
-pub fn do_test(data: &[u8]) {
-       reset_rng_state();
-       if let Ok(msg) = Pong::decode(data) {
-               let _ = msg.encode();
-       }
-}
-
-#[cfg(feature = "afl")]
-#[macro_use] extern crate afl;
-#[cfg(feature = "afl")]
-fn main() {
-       fuzz!(|data| {
-               do_test(data);
-       });
-}
-
-#[cfg(feature = "honggfuzz")]
-#[macro_use] extern crate honggfuzz;
-#[cfg(feature = "honggfuzz")]
-fn main() {
-       loop {
-               fuzz!(|data| {
-                       do_test(data);
-               });
-       }
-}
-
-extern crate hex;
-#[cfg(test)]
-mod tests {
-       #[test]
-       fn duplicate_crash() {
-               super::do_test(&::hex::decode("00").unwrap());
-       }
-}
index 32f071e85cd5cfb568e118ba57551a434bab7811..cfd100f3dfe73a5b48a1434b4497304d179893e6 100755 (executable)
@@ -31,3 +31,6 @@ GEN_TEST NodeAnnouncement test_msg_exact ""
 GEN_TEST UpdateAddHTLC test_msg_hole ", 85, 33"
 GEN_TEST ErrorMessage test_msg_hole ", 32, 2"
 GEN_TEST OnionHopData test_msg_hole ", 1+8+8+4, 12"
+
+GEN_TEST Ping test_msg_simple ""
+GEN_TEST Pong test_msg_simple ""
index d7f0a88138009085b93f2324d1b26936eba36f60..ef3ae4cdf4e1f6848c6b2be4b3555970dd09188c 100644 (file)
@@ -6,8 +6,6 @@ extern crate lightning;
 use lightning::ln::msgs;
 use lightning::util::reset_rng_state;
 
-use lightning::ln::msgs::{MsgEncodable, MsgDecodable};
-
 mod utils;
 
 #[inline]
index 33c52b33e9eccbee036f4862db0fc5b418f9d9ef..f32aad1a198b2b6c4a63535de4e5a0b771724562 100644 (file)
@@ -6,8 +6,6 @@ extern crate lightning;
 use lightning::ln::msgs;
 use lightning::util::reset_rng_state;
 
-use lightning::ln::msgs::{MsgEncodable, MsgDecodable};
-
 mod utils;
 
 #[inline]
index 1252cfe79dec0492dece71c40bc5b3ad18b50619..e814c4fae52028365ae0589b486d465a2c272d7a 100644 (file)
@@ -6,8 +6,6 @@ extern crate lightning;
 use lightning::ln::msgs;
 use lightning::util::reset_rng_state;
 
-use lightning::ln::msgs::{MsgEncodable, MsgDecodable};
-
 mod utils;
 
 #[inline]
index 38f9160450f8a3c1c38d00d6fb66494d90425093..8532ac120504c04efb6ebe515eceb26871059400 100644 (file)
@@ -6,8 +6,6 @@ extern crate lightning;
 use lightning::ln::msgs;
 use lightning::util::reset_rng_state;
 
-use lightning::ln::msgs::{MsgEncodable, MsgDecodable};
-
 mod utils;
 
 #[inline]
index c488fb6da978e56dfcc93c66f4f67382c3db6928..8968ee33ea0c1df0a13c0d302ec6c0f1bf394fa1 100644 (file)
@@ -6,8 +6,6 @@ extern crate lightning;
 use lightning::ln::msgs;
 use lightning::util::reset_rng_state;
 
-use lightning::ln::msgs::{MsgEncodable, MsgDecodable};
-
 mod utils;
 
 #[inline]
index 504e1e37633ff867595bdeac632e7266d1350e8f..70d969209403c77f95a43751ea5d0258844fd87f 100644 (file)
@@ -6,8 +6,6 @@ extern crate lightning;
 use lightning::ln::msgs;
 use lightning::util::reset_rng_state;
 
-use lightning::ln::msgs::{MsgEncodable, MsgDecodable};
-
 mod utils;
 
 #[inline]
index 701dd1fb9e2085c10084814505b5f23f1c19514d..c70437b8409e3f9756137b84bc5b7cf24521bb68 100644 (file)
@@ -6,8 +6,6 @@ extern crate lightning;
 use lightning::ln::msgs;
 use lightning::util::reset_rng_state;
 
-use lightning::ln::msgs::{MsgEncodable, MsgDecodable};
-
 mod utils;
 
 #[inline]
index 56a5fb9f676203b8b8b0b45a5cff53105ae04456..173f13d3fdeb6b9c40536fc91b425605eab4c60b 100644 (file)
@@ -6,8 +6,6 @@ extern crate lightning;
 use lightning::ln::msgs;
 use lightning::util::reset_rng_state;
 
-use lightning::ln::msgs::{MsgEncodable, MsgDecodable};
-
 mod utils;
 
 #[inline]
index 97e15c3c4b76299aeaf782abdba43b3d0e7595df..f565e80f9195973c9b46e188a28b1a83bcf719fd 100644 (file)
@@ -6,8 +6,6 @@ extern crate lightning;
 use lightning::ln::msgs;
 use lightning::util::reset_rng_state;
 
-use lightning::ln::msgs::{MsgEncodable, MsgDecodable};
-
 mod utils;
 
 #[inline]
index 7289267c23b126852f57efea3d0493aaff6cf1be..b9c67ae33504b27e7820cb0b17095d73200d2264 100644 (file)
@@ -6,8 +6,6 @@ extern crate lightning;
 use lightning::ln::msgs;
 use lightning::util::reset_rng_state;
 
-use lightning::ln::msgs::{MsgEncodable, MsgDecodable};
-
 mod utils;
 
 #[inline]
index bfafdf48979c6678acbf723cc43e2bf262966365..499be0049dc4e0783056a6d8cd6aacb5cf2e982a 100644 (file)
@@ -6,8 +6,6 @@ extern crate lightning;
 use lightning::ln::msgs;
 use lightning::util::reset_rng_state;
 
-use lightning::ln::msgs::{MsgEncodable, MsgDecodable};
-
 mod utils;
 
 #[inline]
index 6a2b6ac3a7dfa7610f740676eac106c4ae82a320..2f63bd0bd166f1dcdc5045058f6cb7ed968261cf 100644 (file)
@@ -6,8 +6,6 @@ extern crate lightning;
 use lightning::ln::msgs;
 use lightning::util::reset_rng_state;
 
-use lightning::ln::msgs::{MsgEncodable, MsgDecodable};
-
 mod utils;
 
 #[inline]
index 8a5ee76f5b5a34f150d1b4385744fa5b486c0495..2013a540650763ed5bdb08cdfd017c49be6f64d5 100644 (file)
@@ -6,8 +6,6 @@ extern crate lightning;
 use lightning::ln::msgs;
 use lightning::util::reset_rng_state;
 
-use lightning::ln::msgs::{MsgEncodable, MsgDecodable};
-
 mod utils;
 
 #[inline]
index 54b9cb684b5b7ee6dd6e845dcc501078e46038c6..1a0c9e062f24492d81c71387900ac6fbcac3ffec 100644 (file)
@@ -6,8 +6,6 @@ extern crate lightning;
 use lightning::ln::msgs;
 use lightning::util::reset_rng_state;
 
-use lightning::ln::msgs::{MsgEncodable, MsgDecodable};
-
 mod utils;
 
 #[inline]
index 70849c18981e08701cad7b66171f670a79b64cdd..398dd2886bfd74698c0ca6320e517bf694e5cca4 100644 (file)
@@ -6,8 +6,6 @@ extern crate lightning;
 use lightning::ln::msgs;
 use lightning::util::reset_rng_state;
 
-use lightning::ln::msgs::{MsgEncodable, MsgDecodable};
-
 mod utils;
 
 #[inline]
index 737bc2f0c63b40905e8937c199f422dc40ee3188..01082d8b77ac4e4931c9bf2e9e9ddcbb82d9274d 100644 (file)
@@ -6,8 +6,6 @@ extern crate lightning;
 use lightning::ln::msgs;
 use lightning::util::reset_rng_state;
 
-use lightning::ln::msgs::{MsgEncodable, MsgDecodable};
-
 mod utils;
 
 #[inline]
diff --git a/fuzz/fuzz_targets/msg_targets/msg_ping_target.rs b/fuzz/fuzz_targets/msg_targets/msg_ping_target.rs
new file mode 100644 (file)
index 0000000..4a792f6
--- /dev/null
@@ -0,0 +1,44 @@
+// This file is auto-generated by gen_target.sh based on msg_target_template.txt
+// To modify it, modify msg_target_template.txt and run gen_target.sh instead.
+
+extern crate lightning;
+
+use lightning::ln::msgs;
+use lightning::util::reset_rng_state;
+
+mod utils;
+
+#[inline]
+pub fn do_test(data: &[u8]) {
+       reset_rng_state();
+       test_msg_simple!(msgs::Ping, data);
+}
+
+#[cfg(feature = "afl")]
+#[macro_use] extern crate afl;
+#[cfg(feature = "afl")]
+fn main() {
+       fuzz!(|data| {
+               do_test(data);
+       });
+}
+
+#[cfg(feature = "honggfuzz")]
+#[macro_use] extern crate honggfuzz;
+#[cfg(feature = "honggfuzz")]
+fn main() {
+       loop {
+               fuzz!(|data| {
+                       do_test(data);
+               });
+       }
+}
+
+extern crate hex;
+#[cfg(test)]
+mod tests {
+       #[test]
+       fn duplicate_crash() {
+               super::do_test(&::hex::decode("00").unwrap());
+       }
+}
diff --git a/fuzz/fuzz_targets/msg_targets/msg_pong_target.rs b/fuzz/fuzz_targets/msg_targets/msg_pong_target.rs
new file mode 100644 (file)
index 0000000..ae5d22a
--- /dev/null
@@ -0,0 +1,44 @@
+// This file is auto-generated by gen_target.sh based on msg_target_template.txt
+// To modify it, modify msg_target_template.txt and run gen_target.sh instead.
+
+extern crate lightning;
+
+use lightning::ln::msgs;
+use lightning::util::reset_rng_state;
+
+mod utils;
+
+#[inline]
+pub fn do_test(data: &[u8]) {
+       reset_rng_state();
+       test_msg_simple!(msgs::Pong, data);
+}
+
+#[cfg(feature = "afl")]
+#[macro_use] extern crate afl;
+#[cfg(feature = "afl")]
+fn main() {
+       fuzz!(|data| {
+               do_test(data);
+       });
+}
+
+#[cfg(feature = "honggfuzz")]
+#[macro_use] extern crate honggfuzz;
+#[cfg(feature = "honggfuzz")]
+fn main() {
+       loop {
+               fuzz!(|data| {
+                       do_test(data);
+               });
+       }
+}
+
+extern crate hex;
+#[cfg(test)]
+mod tests {
+       #[test]
+       fn duplicate_crash() {
+               super::do_test(&::hex::decode("00").unwrap());
+       }
+}
index 6086d279cb261d7ca3a09af20521952dfa813753..804e4ba4b2f194169bc37126c31dff32859a6bb0 100644 (file)
@@ -6,8 +6,6 @@ extern crate lightning;
 use lightning::ln::msgs;
 use lightning::util::reset_rng_state;
 
-use lightning::ln::msgs::{MsgEncodable, MsgDecodable};
-
 mod utils;
 
 #[inline]
index 29243dab6ae27c48c05f2a556aa9d7b666a7cd51..4cc58f28d594845747bcbb28d462073599620a06 100644 (file)
@@ -6,8 +6,6 @@ extern crate lightning;
 use lightning::ln::msgs;
 use lightning::util::reset_rng_state;
 
-use lightning::ln::msgs::{MsgEncodable, MsgDecodable};
-
 mod utils;
 
 #[inline]
index 6053c041cf72c64462cd0fd5047e14a7d36a70ee..d3bda1aa33595038806eed0ff18ad60a67643556 100644 (file)
@@ -6,8 +6,6 @@ extern crate lightning;
 use lightning::ln::msgs;
 use lightning::util::reset_rng_state;
 
-use lightning::ln::msgs::{MsgEncodable, MsgDecodable};
-
 mod utils;
 
 #[inline]
index 64806f20fa4395dff7fa572a4e71ef7f76944e49..cfccc21a3f472c8b763b0934786c363bff5336e9 100644 (file)
@@ -6,8 +6,6 @@ extern crate lightning;
 use lightning::ln::msgs;
 use lightning::util::reset_rng_state;
 
-use lightning::ln::msgs::{MsgEncodable, MsgDecodable};
-
 mod utils;
 
 #[inline]
index 27eaf123e7f6a263a636171892f5cb8020ae1b39..90827f31c95ff7935eef112d046e820d8fc3b701 100644 (file)
@@ -6,8 +6,6 @@ extern crate lightning;
 use lightning::ln::msgs;
 use lightning::util::reset_rng_state;
 
-use lightning::ln::msgs::{MsgEncodable, MsgDecodable};
-
 mod utils;
 
 #[inline]
index eb6dc94aa1de00944b44deeedafabeeead6ab571..a045dea6038de130b5dd097478e33e34eb41fcbf 100644 (file)
@@ -6,8 +6,6 @@ extern crate lightning;
 use lightning::ln::msgs;
 use lightning::util::reset_rng_state;
 
-use lightning::ln::msgs::{MsgEncodable, MsgDecodable};
-
 mod utils;
 
 #[inline]
index eec1d2624598947c36c589c7e113faa2dd9fca17..6f5ff3e47a98867da17d63f5bee4c24ccbae5aeb 100644 (file)
@@ -6,8 +6,6 @@ extern crate lightning;
 use lightning::ln::msgs;
 use lightning::util::reset_rng_state;
 
-use lightning::ln::msgs::{MsgEncodable, MsgDecodable};
-
 mod utils;
 
 #[inline]
index 3dd87b30ca5387e4acb5ebc0c52611747981017e..c5dd32356570bdeceba462839e8bc8a9ae9533d0 100644 (file)
@@ -6,8 +6,6 @@ extern crate lightning;
 use lightning::ln::msgs;
 use lightning::util::reset_rng_state;
 
-use lightning::ln::msgs::{MsgEncodable, MsgDecodable};
-
 mod utils;
 
 #[inline]
index 64bfd89326b4e8d3aa1e12c2bdd6f0323d929b8e..c1bcd8f361887003d4e2823d2e5445fc4ce16496 100644 (file)
@@ -4,9 +4,29 @@
 macro_rules! test_msg {
        ($MsgType: path, $data: ident) => {
                {
-                       if let Ok(msg) = <$MsgType as MsgDecodable>::decode($data){
-                               let enc = msg.encode();
-                               assert_eq!(&$data[..enc.len()], &enc[..]);
+                       use lightning::util::ser::{Writer, Reader, Writeable, Readable};
+                       let mut r = Reader::new(::std::io::Cursor::new($data));
+                       if let Ok(msg) = <$MsgType as Readable<::std::io::Cursor<&[u8]>>>::read(&mut r) {
+                               let p = r.get_ref().position() as usize;
+                               let mut w = Writer::new(::std::io::Cursor::new(vec![]));
+                               msg.write(&mut w).unwrap();
+
+                               let buf = w.into_inner().into_inner();
+                               assert_eq!(buf.len(), p);
+                               assert_eq!(&r.into_inner().into_inner()[..p], &buf[..p]);
+                       }
+               }
+       }
+}
+
+#[macro_export]
+macro_rules! test_msg_simple {
+       ($MsgType: path, $data: ident) => {
+               {
+                       use lightning::util::ser::{Writer, Reader, Writeable, Readable};
+                       let mut r = Reader::new(::std::io::Cursor::new($data));
+                       if let Ok(msg) = <$MsgType as Readable<::std::io::Cursor<&[u8]>>>::read(&mut r) {
+                               msg.write(&mut Writer::new(::std::io::Cursor::new(vec![]))).unwrap();
                        }
                }
        }
@@ -16,9 +36,14 @@ macro_rules! test_msg {
 macro_rules! test_msg_exact {
        ($MsgType: path, $data: ident) => {
                {
-                       if let Ok(msg) = <$MsgType as MsgDecodable>::decode($data){
-                               let enc = msg.encode();
-                               assert_eq!(&$data[..], &enc[..]);
+                       use lightning::util::ser::{Writer, Reader, Writeable, Readable};
+                       let mut r = Reader::new(::std::io::Cursor::new($data));
+                       if let Ok(msg) = <$MsgType as Readable<::std::io::Cursor<&[u8]>>>::read(&mut r) {
+                               let mut w = Writer::new(::std::io::Cursor::new(vec![]));
+                               msg.write(&mut w).unwrap();
+
+                               let buf = w.into_inner().into_inner();
+                               assert_eq!(&r.into_inner().into_inner()[..], &buf[..]);
                        }
                }
        }
@@ -28,10 +53,17 @@ macro_rules! test_msg_exact {
 macro_rules! test_msg_hole {
        ($MsgType: path, $data: ident, $hole: expr, $hole_len: expr) => {
                {
-                       if let Ok(msg) = <$MsgType as MsgDecodable>::decode($data){
-                               let enc = msg.encode();
-                               assert_eq!(&$data[..$hole], &enc[..$hole]);
-                               assert_eq!(&$data[$hole + $hole_len..enc.len()], &enc[$hole + $hole_len..]);
+                       use lightning::util::ser::{Writer, Reader, Writeable, Readable};
+                       let mut r = Reader::new(::std::io::Cursor::new($data));
+                       if let Ok(msg) = <$MsgType as Readable<::std::io::Cursor<&[u8]>>>::read(&mut r) {
+                               let mut w = Writer::new(::std::io::Cursor::new(vec![]));
+                               msg.write(&mut w).unwrap();
+                               let p = w.get_ref().position() as usize;
+
+                               let buf = w.into_inner().into_inner();
+                               assert_eq!(buf.len(),p);
+                               assert_eq!(&r.get_ref().get_ref()[..$hole], &buf[..$hole]);
+                               assert_eq!(&r.get_ref().get_ref()[$hole+$hole_len..p], &buf[$hole+$hole_len..]);
                        }
                }
        }
index 6476f812fb34dd2243ea1a585e6753eb44b3a52b..fc3e9e9ba9ace20c1de0db547b14fb88360e369b 100644 (file)
@@ -8,10 +8,11 @@ use bitcoin::blockdata::script::{Script, Builder};
 use lightning::chain::chaininterface::{ChainError,ChainWatchInterface, ChainListener};
 use lightning::ln::channelmanager::ChannelDetails;
 use lightning::ln::msgs;
-use lightning::ln::msgs::{MsgDecodable, RoutingMessageHandler};
+use lightning::ln::msgs::{RoutingMessageHandler};
 use lightning::ln::router::{Router, RouteHint};
 use lightning::util::reset_rng_state;
 use lightning::util::logger::Logger;
+use lightning::util::ser::{Reader, Readable};
 
 use secp256k1::key::PublicKey;
 use secp256k1::Secp256k1;
@@ -119,8 +120,9 @@ pub fn do_test(data: &[u8]) {
        }
 
        macro_rules! decode_msg {
-               ($MsgType: path, $len: expr) => {
-                       match <($MsgType)>::decode(get_slice!($len)) {
+               ($MsgType: path, $len: expr) => {{
+                       let mut reader = Reader::new(::std::io::Cursor::new(get_slice!($len)));
+                       match <($MsgType)>::read(&mut reader) {
                                Ok(msg) => msg,
                                Err(e) => match e {
                                        msgs::DecodeError::UnknownRealmByte => return,
@@ -131,9 +133,11 @@ pub fn do_test(data: &[u8]) {
                                        msgs::DecodeError::ExtraAddressesPerType => return,
                                        msgs::DecodeError::BadLengthDescriptor => return,
                                        msgs::DecodeError::ShortRead => panic!("We picked the length..."),
+                                       msgs::DecodeError::InvalidValue => panic!("Should not happen with p2p message decoding"),
+                                       msgs::DecodeError::Io(e) => panic!(format!("{}", e)),
                                }
                        }
-               }
+               }}
        }
 
        macro_rules! decode_msg_with_len16 {
index 1d0ea1c637235c02774fe3e111238c6a25ed15d4..fc942ab5a2022dbe30eb303c3d4279362c81d881 100644 (file)
@@ -10,6 +10,7 @@ use std::{cmp, fmt};
 use std::result::Result;
 
 use util::{byte_utils, internal_traits, events};
+use util::ser::{Readable, Reader, Writeable, Writer};
 
 pub trait MsgEncodable {
        fn encode(&self) -> Vec<u8>;
@@ -43,6 +44,10 @@ pub enum DecodeError {
        /// A length descriptor in the packet didn't describe the later data correctly
        /// (currently only generated in node_announcement)
        BadLengthDescriptor,
+       /// Error from std::io
+       Io(::std::io::Error),
+       /// 1 or 0 is not found for boolean value
+       InvalidValue,
 }
 pub trait MsgDecodable: Sized {
        fn decode(v: &[u8]) -> Result<Self, DecodeError>;
@@ -519,6 +524,8 @@ impl Error for DecodeError {
                        DecodeError::ShortRead => "Packet extended beyond the provided bytes",
                        DecodeError::ExtraAddressesPerType => "More than one address of a single type",
                        DecodeError::BadLengthDescriptor => "A length descriptor in the packet didn't describe the later data correctly",
+                       DecodeError::Io(ref e) => e.description(),
+                       DecodeError::InvalidValue => "0 or 1 is not found for boolean",
                }
        }
 }
@@ -534,6 +541,16 @@ impl fmt::Debug for HandleError {
        }
 }
 
+impl From<::std::io::Error> for DecodeError {
+       fn from(e: ::std::io::Error) -> Self {
+               if e.kind() == ::std::io::ErrorKind::UnexpectedEof {
+                       DecodeError::ShortRead
+               } else {
+                       DecodeError::Io(e)
+               }
+       }
+}
+
 macro_rules! secp_pubkey {
        ( $ctx: expr, $slice: expr ) => {
                match PublicKey::from_slice($ctx, $slice) {
@@ -1682,6 +1699,586 @@ impl MsgDecodable for ErrorMessage {
        }
 }
 
+impl_writeable!(AcceptChannel, {
+       temporary_channel_id,
+       dust_limit_satoshis,
+       max_htlc_value_in_flight_msat,
+       channel_reserve_satoshis,
+       htlc_minimum_msat,
+       minimum_depth,
+       to_self_delay,
+       max_accepted_htlcs,
+       funding_pubkey,
+       revocation_basepoint,
+       payment_basepoint,
+       delayed_payment_basepoint,
+       htlc_basepoint,
+       first_per_commitment_point,
+       shutdown_scriptpubkey
+});
+
+impl_writeable!(AnnouncementSignatures, {
+       channel_id,
+       short_channel_id,
+       node_signature,
+       bitcoin_signature
+});
+
+impl<W: ::std::io::Write> Writeable<W> for ChannelReestablish {
+       fn write(&self, w: &mut Writer<W>) -> Result<(), DecodeError> {
+               self.channel_id.write(w)?;
+               self.next_local_commitment_number.write(w)?;
+               self.next_remote_commitment_number.write(w)?;
+               if let Some(ref data_loss_protect) = self.data_loss_protect {
+                       data_loss_protect.your_last_per_commitment_secret.write(w)?;
+                       data_loss_protect.my_current_per_commitment_point.write(w)?;
+               }
+               Ok(())
+       }
+}
+
+impl<R: ::std::io::Read> Readable<R> for ChannelReestablish{
+       fn read(r: &mut Reader<R>) -> Result<Self, DecodeError> {
+               Ok(Self {
+                       channel_id: Readable::read(r)?,
+                       next_local_commitment_number: Readable::read(r)?,
+                       next_remote_commitment_number: Readable::read(r)?,
+                       data_loss_protect: {
+                               match <[u8; 32] as Readable<R>>::read(r) {
+                                       Ok(your_last_per_commitment_secret) =>
+                                               Some(DataLossProtect {
+                                                       your_last_per_commitment_secret,
+                                                       my_current_per_commitment_point: Readable::read(r)?,
+                                               }),
+                                       Err(DecodeError::ShortRead) => None,
+                                       Err(e) => return Err(e)
+                               }
+                       }
+               })
+       }
+}
+
+impl_writeable!(ClosingSigned, {
+       channel_id,
+       fee_satoshis,
+       signature
+});
+
+impl_writeable!(CommitmentSigned, {
+       channel_id,
+       signature,
+       htlc_signatures
+});
+
+impl_writeable!(DecodedOnionErrorPacket, {
+       hmac,
+       failuremsg,
+       pad
+});
+
+impl_writeable!(FundingCreated, {
+       temporary_channel_id,
+       funding_txid,
+       funding_output_index,
+       signature
+});
+
+impl_writeable!(FundingSigned, {
+       channel_id,
+       signature
+});
+
+impl_writeable!(FundingLocked, {
+       channel_id,
+       next_per_commitment_point
+});
+
+impl_writeable!(GlobalFeatures, {
+       flags
+});
+
+impl_writeable!(LocalFeatures, {
+       flags
+});
+
+impl_writeable!(Init, {
+       global_features,
+       local_features
+});
+
+impl_writeable!(OpenChannel, {
+       chain_hash,
+       temporary_channel_id,
+       funding_satoshis,
+       push_msat,
+       dust_limit_satoshis,
+       max_htlc_value_in_flight_msat,
+       channel_reserve_satoshis,
+       htlc_minimum_msat,
+       feerate_per_kw,
+       to_self_delay,
+       max_accepted_htlcs,
+       funding_pubkey,
+       revocation_basepoint,
+       payment_basepoint,
+       delayed_payment_basepoint,
+       htlc_basepoint,
+       first_per_commitment_point,
+       channel_flags,
+       shutdown_scriptpubkey
+});
+
+impl_writeable!(RevokeAndACK, {
+       channel_id,
+       per_commitment_secret,
+       next_per_commitment_point
+});
+
+impl_writeable!(Shutdown, {
+       channel_id,
+       scriptpubkey
+});
+
+impl_writeable!(UpdateFailHTLC, {
+       channel_id,
+       htlc_id,
+       reason
+});
+
+impl_writeable!(UpdateFailMalformedHTLC, {
+       channel_id,
+       htlc_id,
+       sha256_of_onion,
+       failure_code
+});
+
+impl_writeable!(UpdateFee, {
+       channel_id,
+       feerate_per_kw
+});
+
+impl_writeable!(UpdateFulfillHTLC, {
+       channel_id,
+       htlc_id,
+       payment_preimage
+});
+
+impl_writeable!(OnionErrorPacket, {
+       data
+});
+
+impl<W: ::std::io::Write> Writeable<W> for OnionPacket {
+       fn write(&self, w: &mut Writer<W>) -> Result<(), DecodeError> {
+               self.version.write(w)?;
+               match self.public_key {
+                       Ok(pubkey) => pubkey.write(w)?,
+                       Err(_) => [0u8;33].write(w)?,
+               }
+               w.write_all(&self.hop_data)?;
+               self.hmac.write(w)?;
+               Ok(())
+       }
+}
+
+impl<R: ::std::io::Read> Readable<R> for OnionPacket {
+       fn read(r: &mut Reader<R>) -> Result<Self, DecodeError> {
+               Ok(OnionPacket {
+                       version: Readable::read(r)?,
+                       public_key: {
+                               let mut buf = [0u8;33];
+                               r.read_exact(&mut buf)?;
+                               PublicKey::from_slice(&Secp256k1::without_caps(), &buf)
+                       },
+                       hop_data: Readable::read(r)?,
+                       hmac: Readable::read(r)?,
+               })
+       }
+}
+
+impl_writeable!(UpdateAddHTLC, {
+       channel_id,
+       htlc_id,
+       amount_msat,
+       payment_hash,
+       cltv_expiry,
+       onion_routing_packet
+});
+
+impl<W: ::std::io::Write> Writeable<W> for OnionRealm0HopData {
+       fn write(&self, w: &mut Writer<W>) -> Result<(), DecodeError> {
+               self.short_channel_id.write(w)?;
+               self.amt_to_forward.write(w)?;
+               self.outgoing_cltv_value.write(w)?;
+               w.write_all(&[0;12])?;
+               Ok(())
+       }
+}
+
+impl<R: ::std::io::Read> Readable<R> for OnionRealm0HopData {
+       fn read(r: &mut Reader<R>) -> Result<Self, DecodeError> {
+               Ok(OnionRealm0HopData {
+                       short_channel_id: Readable::read(r)?,
+                       amt_to_forward: Readable::read(r)?,
+                       outgoing_cltv_value: {
+                               let v: u32 = Readable::read(r)?;
+                               r.read_exact(&mut [0; 12])?;
+                               v
+                       }
+               })
+       }
+}
+
+impl<W: ::std::io::Write> Writeable<W> for OnionHopData {
+       fn write(&self, w: &mut Writer<W>) -> Result<(), DecodeError> {
+               self.realm.write(w)?;
+               self.data.write(w)?;
+               self.hmac.write(w)?;
+               Ok(())
+       }
+}
+
+impl<R: ::std::io::Read> Readable<R> for OnionHopData {
+       fn read(r: &mut Reader<R>) -> Result<Self, DecodeError> {
+               Ok(OnionHopData {
+                       realm: {
+                               let r: u8 = Readable::read(r)?;
+                               if r != 0 {
+                                       return Err(DecodeError::UnknownRealmByte);
+                               }
+                               r
+                       },
+                       data: Readable::read(r)?,
+                       hmac: Readable::read(r)?,
+               })
+       }
+}
+
+impl<W: ::std::io::Write> Writeable<W> for Ping {
+       fn write(&self, w: &mut Writer<W>) -> Result<(), DecodeError> {
+               self.ponglen.write(w)?;
+               vec![0u8; self.byteslen as usize].write(w)?; // size-unchecked write
+               Ok(())
+       }
+}
+
+impl<R: ::std::io::Read> Readable<R> for Ping {
+       fn read(r: &mut Reader<R>) -> Result<Self, DecodeError> {
+               Ok(Ping {
+                       ponglen: Readable::read(r)?,
+                       byteslen: {
+                               let byteslen = Readable::read(r)?;
+                               r.read_exact(&mut vec![0u8; byteslen as usize][..])?;
+                               byteslen
+                       }
+               })
+       }
+}
+
+impl<W: ::std::io::Write> Writeable<W> for Pong {
+       fn write(&self, w: &mut Writer<W>) -> Result<(), DecodeError> {
+               vec![0u8; self.byteslen as usize].write(w)?; // size-unchecked write
+               Ok(())
+       }
+}
+
+impl<R: ::std::io::Read> Readable<R> for Pong {
+       fn read(r: &mut Reader<R>) -> Result<Self, DecodeError> {
+               Ok(Pong {
+                       byteslen: {
+                               let byteslen = Readable::read(r)?;
+                               r.read_exact(&mut vec![0u8; byteslen as usize][..])?;
+                               byteslen
+                       }
+               })
+       }
+}
+
+impl<W: ::std::io::Write> Writeable<W> for UnsignedChannelAnnouncement {
+       fn write(&self, w: &mut Writer<W>) -> Result<(), DecodeError> {
+               self.features.write(w)?;
+               self.chain_hash.write(w)?;
+               self.short_channel_id.write(w)?;
+               self.node_id_1.write(w)?;
+               self.node_id_2.write(w)?;
+               self.bitcoin_key_1.write(w)?;
+               self.bitcoin_key_2.write(w)?;
+               w.write_all(&self.excess_data[..])?;
+               Ok(())
+       }
+}
+
+impl<R: ::std::io::Read> Readable<R> for UnsignedChannelAnnouncement {
+       fn read(r: &mut Reader<R>) -> Result<Self, DecodeError> {
+               Ok(Self {
+                       features: {
+                               let f: GlobalFeatures = Readable::read(r)?;
+                               if f.requires_unknown_bits() {
+                                       return Err(DecodeError::UnknownRequiredFeature);
+                               }
+                               f
+                       },
+                       chain_hash: Readable::read(r)?,
+                       short_channel_id: Readable::read(r)?,
+                       node_id_1: Readable::read(r)?,
+                       node_id_2: Readable::read(r)?,
+                       bitcoin_key_1: Readable::read(r)?,
+                       bitcoin_key_2: Readable::read(r)?,
+                       excess_data: {
+                               let mut excess_data = vec![];
+                               r.read_to_end(&mut excess_data)?;
+                               excess_data
+                       },
+               })
+       }
+}
+
+impl_writeable!(ChannelAnnouncement,{
+       node_signature_1,
+       node_signature_2,
+       bitcoin_signature_1,
+       bitcoin_signature_2,
+       contents
+});
+
+impl<W: ::std::io::Write> Writeable<W> for UnsignedChannelUpdate {
+       fn write(&self, w: &mut Writer<W>) -> Result<(), DecodeError> {
+               self.chain_hash.write(w)?;
+               self.short_channel_id.write(w)?;
+               self.timestamp.write(w)?;
+               self.flags.write(w)?;
+               self.cltv_expiry_delta.write(w)?;
+               self.htlc_minimum_msat.write(w)?;
+               self.fee_base_msat.write(w)?;
+               self.fee_proportional_millionths.write(w)?;
+               w.write_all(&self.excess_data[..])?;
+               Ok(())
+       }
+}
+
+impl<R: ::std::io::Read> Readable<R> for UnsignedChannelUpdate {
+       fn read(r: &mut Reader<R>) -> Result<Self, DecodeError> {
+               Ok(Self {
+                       chain_hash: Readable::read(r)?,
+                       short_channel_id: Readable::read(r)?,
+                       timestamp: Readable::read(r)?,
+                       flags: Readable::read(r)?,
+                       cltv_expiry_delta: Readable::read(r)?,
+                       htlc_minimum_msat: Readable::read(r)?,
+                       fee_base_msat: Readable::read(r)?,
+                       fee_proportional_millionths: Readable::read(r)?,
+                       excess_data: {
+                               let mut excess_data = vec![];
+                               r.read_to_end(&mut excess_data)?;
+                               excess_data
+                       },
+               })
+       }
+}
+
+impl_writeable!(ChannelUpdate, {
+       signature,
+       contents
+});
+
+impl<W: ::std::io::Write> Writeable<W> for ErrorMessage {
+       fn write(&self, w: &mut Writer<W>) -> Result<(), DecodeError> {
+               self.channel_id.write(w)?;
+               self.data.as_bytes().to_vec().write(w)?; // write with size prefix
+               Ok(())
+       }
+}
+
+impl<R: ::std::io::Read> Readable<R> for ErrorMessage {
+       fn read(r: &mut Reader<R>) -> Result<Self, DecodeError> {
+               Ok(Self {
+                       channel_id: Readable::read(r)?,
+                       data: {
+                               let mut sz: usize = <u16 as Readable<R>>::read(r)? as usize;
+                               let mut data = vec![];
+                               let data_len = r.read_to_end(&mut data)?;
+                               sz = cmp::min(data_len, sz);
+                               match String::from_utf8(data[..sz as usize].to_vec()) {
+                                       Ok(s) => s,
+                                       Err(_) => return Err(DecodeError::BadText),
+                               }
+                       }
+               })
+       }
+}
+
+impl<W: ::std::io::Write> Writeable<W> for UnsignedNodeAnnouncement {
+       fn write(&self, w: &mut Writer<W>) -> Result<(), DecodeError> {
+               self.features.write(w)?;
+               self.timestamp.write(w)?;
+               self.node_id.write(w)?;
+               w.write_all(&self.rgb)?;
+               self.alias.write(w)?;
+
+               let mut addr_slice = Vec::with_capacity(self.addresses.len() * 18);
+               let mut addrs_to_encode = self.addresses.clone();
+               addrs_to_encode.sort_unstable_by(|a, b| { a.get_id().cmp(&b.get_id()) });
+               addrs_to_encode.dedup_by(|a, b| { a.get_id() == b.get_id() });
+               for addr in addrs_to_encode.iter() {
+                       match addr {
+                               &NetAddress::IPv4{addr, port} => {
+                                       addr_slice.push(1);
+                                       addr_slice.extend_from_slice(&addr);
+                                       addr_slice.extend_from_slice(&byte_utils::be16_to_array(port));
+                               },
+                               &NetAddress::IPv6{addr, port} => {
+                                       addr_slice.push(2);
+                                       addr_slice.extend_from_slice(&addr);
+                                       addr_slice.extend_from_slice(&byte_utils::be16_to_array(port));
+                               },
+                               &NetAddress::OnionV2{addr, port} => {
+                                       addr_slice.push(3);
+                                       addr_slice.extend_from_slice(&addr);
+                                       addr_slice.extend_from_slice(&byte_utils::be16_to_array(port));
+                               },
+                               &NetAddress::OnionV3{ed25519_pubkey, checksum, version, port} => {
+                                       addr_slice.push(4);
+                                       addr_slice.extend_from_slice(&ed25519_pubkey);
+                                       addr_slice.extend_from_slice(&byte_utils::be16_to_array(checksum));
+                                       addr_slice.push(version);
+                                       addr_slice.extend_from_slice(&byte_utils::be16_to_array(port));
+                               },
+                       }
+               }
+               ((addr_slice.len() + self.excess_address_data.len()) as u16).write(w)?;
+               w.write_all(&addr_slice[..])?;
+               w.write_all(&self.excess_address_data[..])?;
+               w.write_all(&self.excess_data[..])?;
+               Ok(())
+       }
+}
+
+impl<R: ::std::io::Read> Readable<R> for UnsignedNodeAnnouncement {
+       fn read(r: &mut Reader<R>) -> Result<Self, DecodeError> {
+               let features: GlobalFeatures = Readable::read(r)?;
+               if features.requires_unknown_bits() {
+                       return Err(DecodeError::UnknownRequiredFeature);
+               }
+               let timestamp: u32 = Readable::read(r)?;
+               let node_id: PublicKey = Readable::read(r)?;
+               let mut rgb = [0; 3];
+               r.read_exact(&mut rgb)?;
+               let alias: [u8; 32] = Readable::read(r)?;
+
+               let addrlen: u16 = Readable::read(r)?;
+               let mut addr_readpos = 0;
+               let mut addresses = Vec::with_capacity(4);
+               let mut f: u8 = 0;
+               let mut excess = 0;
+               loop {
+                       if addrlen <= addr_readpos { break; }
+                       f = Readable::read(r)?;
+                       match f {
+                               1 => {
+                                       if addresses.len() > 0 {
+                                               return Err(DecodeError::ExtraAddressesPerType);
+                                       }
+                                       if addrlen < addr_readpos + 1 + 6 {
+                                               return Err(DecodeError::BadLengthDescriptor);
+                                       }
+                                       addresses.push(NetAddress::IPv4 {
+                                               addr: {
+                                                       let mut addr = [0; 4];
+                                                       r.read_exact(&mut addr)?;
+                                                       addr
+                                               },
+                                               port: Readable::read(r)?,
+                                       });
+                                       addr_readpos += 1 + 6
+                               },
+                               2 => {
+                                       if addresses.len() > 1 || (addresses.len() == 1 && addresses[0].get_id() != 1) {
+                                               return Err(DecodeError::ExtraAddressesPerType);
+                                       }
+                                       if addrlen < addr_readpos + 1 + 18 {
+                                               return Err(DecodeError::BadLengthDescriptor);
+                                       }
+                                       addresses.push(NetAddress::IPv6 {
+                                               addr: {
+                                                       let mut addr = [0; 16];
+                                                       r.read_exact(&mut addr)?;
+                                                       addr
+                                               },
+                                               port: Readable::read(r)?,
+                                       });
+                                       addr_readpos += 1 + 18
+                               },
+                               3 => {
+                                       if addresses.len() > 2 || (addresses.len() > 0 && addresses.last().unwrap().get_id() > 2) {
+                                               return Err(DecodeError::ExtraAddressesPerType);
+                                       }
+                                       if addrlen < addr_readpos + 1 + 12 {
+                                               return Err(DecodeError::BadLengthDescriptor);
+                                       }
+                                       addresses.push(NetAddress::OnionV2 {
+                                               addr: {
+                                                       let mut addr = [0; 10];
+                                                       r.read_exact(&mut addr)?;
+                                                       addr
+                                               },
+                                               port: Readable::read(r)?,
+                                       });
+                                       addr_readpos += 1 + 12
+                               },
+                               4 => {
+                                       if addresses.len() > 3 || (addresses.len() > 0 && addresses.last().unwrap().get_id() > 3) {
+                                               return Err(DecodeError::ExtraAddressesPerType);
+                                       }
+                                       if addrlen < addr_readpos + 1 + 37 {
+                                               return Err(DecodeError::BadLengthDescriptor);
+                                       }
+                                       addresses.push(NetAddress::OnionV3 {
+                                               ed25519_pubkey: Readable::read(r)?,
+                                               checksum: Readable::read(r)?,
+                                               version: Readable::read(r)?,
+                                               port: Readable::read(r)?,
+                                       });
+                                       addr_readpos += 1 + 37
+                               },
+                               _ => { excess = 1; break; }
+                       }
+               }
+
+               let mut excess_data = vec![];
+               let excess_address_data = if addr_readpos < addrlen {
+                       let mut excess_address_data = vec![0; (addrlen - addr_readpos) as usize];
+                       r.read_exact(&mut excess_address_data[excess..])?;
+                       if excess == 1 {
+                               excess_address_data[0] = f;
+                       }
+                       excess_address_data
+               } else {
+                       if excess == 1 {
+                               excess_data.push(f);
+                       }
+                       Vec::new()
+               };
+
+               Ok(UnsignedNodeAnnouncement {
+                       features: features,
+                       timestamp: timestamp,
+                       node_id: node_id,
+                       rgb: rgb,
+                       alias: alias,
+                       addresses: addresses,
+                       excess_address_data: excess_address_data,
+                       excess_data: {
+                               r.read_to_end(&mut excess_data)?;
+                               excess_data
+                       },
+               })
+       }
+}
+
+impl_writeable!(NodeAnnouncement, {
+       signature,
+       contents
+});
+
 #[cfg(test)]
 mod tests {
        use hex;
index 8a4228851b7a7e3981e698ccc6cd05f3cb3e5d3b..4e710117bb805037a80457086a434de70a0bb29b 100644 (file)
@@ -1,7 +1,7 @@
 use secp256k1::key::{SecretKey,PublicKey};
 
 use ln::msgs;
-use ln::msgs::{MsgEncodable,MsgDecodable};
+use util::ser::{Writer, Reader, Writeable, Readable};
 use ln::peer_channel_encryptor::{PeerChannelEncryptor,NextNoiseStep};
 use util::byte_utils;
 use util::events::{EventsProvider,Event};
@@ -113,15 +113,15 @@ pub struct PeerManager<Descriptor: SocketDescriptor> {
 }
 
 macro_rules! encode_msg {
-       ($msg: expr, $msg_code: expr) => {
-               {
-                       let just_msg = $msg.encode();
-                       let mut encoded_msg = Vec::with_capacity(just_msg.len() + 2);
-                       encoded_msg.extend_from_slice(&byte_utils::be16_to_array($msg_code));
-                       encoded_msg.extend_from_slice(&just_msg[..]);
-                       encoded_msg
-               }
-       }
+       ($msg: expr, $msg_code: expr) => {{
+               let mut w = Writer::new(::std::io::Cursor::new(vec![]));
+               0u16.write(&mut w).unwrap();
+               $msg.write(&mut w).unwrap();
+               let mut msg = w.into_inner().into_inner();
+               let len = msg.len();
+               msg[..2].copy_from_slice(&byte_utils::be16_to_array(len as u16 - 2));
+               msg
+       }}
 }
 
 //TODO: Really should do something smarter for this
@@ -363,6 +363,8 @@ impl<Descriptor: SocketDescriptor> PeerManager<Descriptor> {
                                                                                                        continue;
                                                                                                },
                                                                                                msgs::DecodeError::BadLengthDescriptor => return Err(PeerHandleError{ no_connection_possible: false }),
+                                                                                               msgs::DecodeError::Io(_) => return Err(PeerHandleError{ no_connection_possible: false }),
+                                                                                               msgs::DecodeError::InvalidValue => panic!("should not happen with message decoding"),
                                                                                        }
                                                                                }
                                                                        };
@@ -435,10 +437,11 @@ impl<Descriptor: SocketDescriptor> PeerManager<Descriptor> {
                                                                                        // Need an init message as first message
                                                                                        return Err(PeerHandleError{ no_connection_possible: false });
                                                                                }
+                                                                               let mut reader = Reader::new(::std::io::Cursor::new(&msg_data[2..]));
                                                                                match msg_type {
                                                                                        // Connection control:
                                                                                        16 => {
-                                                                                               let msg = try_potential_decodeerror!(msgs::Init::decode(&msg_data[2..]));
+                                                                                               let msg = try_potential_decodeerror!(msgs::Init::read(&mut reader));
                                                                                                if msg.global_features.requires_unknown_bits() {
                                                                                                        return Err(PeerHandleError{ no_connection_possible: true });
                                                                                                }
@@ -464,7 +467,7 @@ impl<Descriptor: SocketDescriptor> PeerManager<Descriptor> {
                                                                                                }
                                                                                        },
                                                                                        17 => {
-                                                                                               let msg = try_potential_decodeerror!(msgs::ErrorMessage::decode(&msg_data[2..]));
+                                                                                               let msg = try_potential_decodeerror!(msgs::ErrorMessage::read(&mut reader));
                                                                                                let mut data_is_printable = true;
                                                                                                for b in msg.data.bytes() {
                                                                                                        if b < 32 || b > 126 {
@@ -485,38 +488,38 @@ impl<Descriptor: SocketDescriptor> PeerManager<Descriptor> {
                                                                                        },
 
                                                                                        18 => {
-                                                                                               let msg = try_potential_decodeerror!(msgs::Ping::decode(&msg_data[2..]));
+                                                                                               let msg = try_potential_decodeerror!(msgs::Ping::read(&mut reader));
                                                                                                if msg.ponglen < 65532 {
                                                                                                        let resp = msgs::Pong { byteslen: msg.ponglen };
                                                                                                        encode_and_send_msg!(resp, 19);
                                                                                                }
                                                                                        },
                                                                                        19 => {
-                                                                                               try_potential_decodeerror!(msgs::Pong::decode(&msg_data[2..]));
+                                                                                               try_potential_decodeerror!(msgs::Pong::read(&mut reader));
                                                                                        },
 
                                                                                        // Channel control:
                                                                                        32 => {
-                                                                                               let msg = try_potential_decodeerror!(msgs::OpenChannel::decode(&msg_data[2..]));
+                                                                                               let msg = try_potential_decodeerror!(msgs::OpenChannel::read(&mut reader));
                                                                                                let resp = try_potential_handleerror!(self.message_handler.chan_handler.handle_open_channel(&peer.their_node_id.unwrap(), &msg));
                                                                                                encode_and_send_msg!(resp, 33);
                                                                                        },
                                                                                        33 => {
-                                                                                               let msg = try_potential_decodeerror!(msgs::AcceptChannel::decode(&msg_data[2..]));
+                                                                                               let msg = try_potential_decodeerror!(msgs::AcceptChannel::read(&mut reader));
                                                                                                try_potential_handleerror!(self.message_handler.chan_handler.handle_accept_channel(&peer.their_node_id.unwrap(), &msg));
                                                                                        },
 
                                                                                        34 => {
-                                                                                               let msg = try_potential_decodeerror!(msgs::FundingCreated::decode(&msg_data[2..]));
+                                                                                               let msg = try_potential_decodeerror!(msgs::FundingCreated::read(&mut reader));
                                                                                                let resp = try_potential_handleerror!(self.message_handler.chan_handler.handle_funding_created(&peer.their_node_id.unwrap(), &msg));
                                                                                                encode_and_send_msg!(resp, 35);
                                                                                        },
                                                                                        35 => {
-                                                                                               let msg = try_potential_decodeerror!(msgs::FundingSigned::decode(&msg_data[2..]));
+                                                                                               let msg = try_potential_decodeerror!(msgs::FundingSigned::read(&mut reader));
                                                                                                try_potential_handleerror!(self.message_handler.chan_handler.handle_funding_signed(&peer.their_node_id.unwrap(), &msg));
                                                                                        },
                                                                                        36 => {
-                                                                                               let msg = try_potential_decodeerror!(msgs::FundingLocked::decode(&msg_data[2..]));
+                                                                                               let msg = try_potential_decodeerror!(msgs::FundingLocked::read(&mut reader));
                                                                                                let resp_option = try_potential_handleerror!(self.message_handler.chan_handler.handle_funding_locked(&peer.their_node_id.unwrap(), &msg));
                                                                                                match resp_option {
                                                                                                        Some(resp) => encode_and_send_msg!(resp, 259),
@@ -525,7 +528,7 @@ impl<Descriptor: SocketDescriptor> PeerManager<Descriptor> {
                                                                                        },
 
                                                                                        38 => {
-                                                                                               let msg = try_potential_decodeerror!(msgs::Shutdown::decode(&msg_data[2..]));
+                                                                                               let msg = try_potential_decodeerror!(msgs::Shutdown::read(&mut reader));
                                                                                                let resp_options = try_potential_handleerror!(self.message_handler.chan_handler.handle_shutdown(&peer.their_node_id.unwrap(), &msg));
                                                                                                if let Some(resp) = resp_options.0 {
                                                                                                        encode_and_send_msg!(resp, 38);
@@ -535,7 +538,7 @@ impl<Descriptor: SocketDescriptor> PeerManager<Descriptor> {
                                                                                                }
                                                                                        },
                                                                                        39 => {
-                                                                                               let msg = try_potential_decodeerror!(msgs::ClosingSigned::decode(&msg_data[2..]));
+                                                                                               let msg = try_potential_decodeerror!(msgs::ClosingSigned::read(&mut reader));
                                                                                                let resp_option = try_potential_handleerror!(self.message_handler.chan_handler.handle_closing_signed(&peer.their_node_id.unwrap(), &msg));
                                                                                                if let Some(resp) = resp_option {
                                                                                                        encode_and_send_msg!(resp, 39);
@@ -543,27 +546,27 @@ impl<Descriptor: SocketDescriptor> PeerManager<Descriptor> {
                                                                                        },
 
                                                                                        128 => {
-                                                                                               let msg = try_potential_decodeerror!(msgs::UpdateAddHTLC::decode(&msg_data[2..]));
+                                                                                               let msg = try_potential_decodeerror!(msgs::UpdateAddHTLC::read(&mut reader));
                                                                                                try_potential_handleerror!(self.message_handler.chan_handler.handle_update_add_htlc(&peer.their_node_id.unwrap(), &msg));
                                                                                        },
                                                                                        130 => {
-                                                                                               let msg = try_potential_decodeerror!(msgs::UpdateFulfillHTLC::decode(&msg_data[2..]));
+                                                                                               let msg = try_potential_decodeerror!(msgs::UpdateFulfillHTLC::read(&mut reader));
                                                                                                try_potential_handleerror!(self.message_handler.chan_handler.handle_update_fulfill_htlc(&peer.their_node_id.unwrap(), &msg));
                                                                                        },
                                                                                        131 => {
-                                                                                               let msg = try_potential_decodeerror!(msgs::UpdateFailHTLC::decode(&msg_data[2..]));
+                                                                                               let msg = try_potential_decodeerror!(msgs::UpdateFailHTLC::read(&mut reader));
                                                                                                let chan_update = try_potential_handleerror!(self.message_handler.chan_handler.handle_update_fail_htlc(&peer.their_node_id.unwrap(), &msg));
                                                                                                if let Some(update) = chan_update {
                                                                                                        self.message_handler.route_handler.handle_htlc_fail_channel_update(&update);
                                                                                                }
                                                                                        },
                                                                                        135 => {
-                                                                                               let msg = try_potential_decodeerror!(msgs::UpdateFailMalformedHTLC::decode(&msg_data[2..]));
+                                                                                               let msg = try_potential_decodeerror!(msgs::UpdateFailMalformedHTLC::read(&mut reader));
                                                                                                try_potential_handleerror!(self.message_handler.chan_handler.handle_update_fail_malformed_htlc(&peer.their_node_id.unwrap(), &msg));
                                                                                        },
 
                                                                                        132 => {
-                                                                                               let msg = try_potential_decodeerror!(msgs::CommitmentSigned::decode(&msg_data[2..]));
+                                                                                               let msg = try_potential_decodeerror!(msgs::CommitmentSigned::read(&mut reader));
                                                                                                let resps = try_potential_handleerror!(self.message_handler.chan_handler.handle_commitment_signed(&peer.their_node_id.unwrap(), &msg));
                                                                                                encode_and_send_msg!(resps.0, 133);
                                                                                                if let Some(resp) = resps.1 {
@@ -571,7 +574,7 @@ impl<Descriptor: SocketDescriptor> PeerManager<Descriptor> {
                                                                                                }
                                                                                        },
                                                                                        133 => {
-                                                                                               let msg = try_potential_decodeerror!(msgs::RevokeAndACK::decode(&msg_data[2..]));
+                                                                                               let msg = try_potential_decodeerror!(msgs::RevokeAndACK::read(&mut reader));
                                                                                                let resp_option = try_potential_handleerror!(self.message_handler.chan_handler.handle_revoke_and_ack(&peer.their_node_id.unwrap(), &msg));
                                                                                                match resp_option {
                                                                                                        Some(resps) => {
@@ -590,18 +593,18 @@ impl<Descriptor: SocketDescriptor> PeerManager<Descriptor> {
                                                                                                }
                                                                                        },
                                                                                        134 => {
-                                                                                               let msg = try_potential_decodeerror!(msgs::UpdateFee::decode(&msg_data[2..]));
+                                                                                               let msg = try_potential_decodeerror!(msgs::UpdateFee::read(&mut reader));
                                                                                                try_potential_handleerror!(self.message_handler.chan_handler.handle_update_fee(&peer.their_node_id.unwrap(), &msg));
                                                                                        },
                                                                                        136 => { }, // TODO: channel_reestablish
 
                                                                                        // Routing control:
                                                                                        259 => {
-                                                                                               let msg = try_potential_decodeerror!(msgs::AnnouncementSignatures::decode(&msg_data[2..]));
+                                                                                               let msg = try_potential_decodeerror!(msgs::AnnouncementSignatures::read(&mut reader));
                                                                                                try_potential_handleerror!(self.message_handler.chan_handler.handle_announcement_signatures(&peer.their_node_id.unwrap(), &msg));
                                                                                        },
                                                                                        256 => {
-                                                                                               let msg = try_potential_decodeerror!(msgs::ChannelAnnouncement::decode(&msg_data[2..]));
+                                                                                               let msg = try_potential_decodeerror!(msgs::ChannelAnnouncement::read(&mut reader));
                                                                                                let should_forward = try_potential_handleerror!(self.message_handler.route_handler.handle_channel_announcement(&msg));
 
                                                                                                if should_forward {
@@ -609,7 +612,7 @@ impl<Descriptor: SocketDescriptor> PeerManager<Descriptor> {
                                                                                                }
                                                                                        },
                                                                                        257 => {
-                                                                                               let msg = try_potential_decodeerror!(msgs::NodeAnnouncement::decode(&msg_data[2..]));
+                                                                                               let msg = try_potential_decodeerror!(msgs::NodeAnnouncement::read(&mut reader));
                                                                                                let should_forward = try_potential_handleerror!(self.message_handler.route_handler.handle_node_announcement(&msg));
 
                                                                                                if should_forward {
@@ -617,7 +620,7 @@ impl<Descriptor: SocketDescriptor> PeerManager<Descriptor> {
                                                                                                }
                                                                                        },
                                                                                        258 => {
-                                                                                               let msg = try_potential_decodeerror!(msgs::ChannelUpdate::decode(&msg_data[2..]));
+                                                                                               let msg = try_potential_decodeerror!(msgs::ChannelUpdate::read(&mut reader));
                                                                                                let should_forward = try_potential_handleerror!(self.message_handler.route_handler.handle_channel_update(&msg));
 
                                                                                                if should_forward {
index 571a4dbed1010049c2cd7590df55397a221af39c..308825855bd934f46e314477bb432cdda2fb7aa1 100644 (file)
@@ -21,4 +21,11 @@ pub(crate) mod test_utils;
 #[macro_use]
 pub(crate) mod macro_logger;
 
+#[cfg(feature = "fuzztarget")]
+#[macro_use]
+pub mod ser;
+#[cfg(not(feature = "fuzztarget"))]
+#[macro_use]
+pub(crate) mod ser;
+
 pub mod logger;
diff --git a/src/util/ser.rs b/src/util/ser.rs
new file mode 100644 (file)
index 0000000..0a247d4
--- /dev/null
@@ -0,0 +1,327 @@
+use std::result::Result;
+use std::io::{Read, Write};
+use std::collections::HashMap;
+use std::hash::Hash;
+use std::mem;
+
+use secp256k1::{Secp256k1, Signature};
+use secp256k1::key::PublicKey;
+use bitcoin::util::hash::Sha256dHash;
+use bitcoin::blockdata::script::Script;
+use std::marker::Sized;
+use ln::msgs::DecodeError;
+
+use util::byte_utils::{be64_to_array, be32_to_array, be16_to_array, slice_to_be16, slice_to_be32, slice_to_be64};
+
+const MAX_BUF_SIZE: usize = 16 * 1024;
+
+pub struct Writer<W> { writer: W }
+pub struct Reader<R> { reader: R }
+
+pub trait Writeable<W: Write> {
+       fn write(&self, writer: &mut Writer<W>) -> Result<(), DecodeError>;
+}
+
+pub trait Readable<R>
+       where Self: Sized,
+             R: Read
+{
+       fn read(reader: &mut Reader<R>) -> Result<Self, DecodeError>;
+}
+
+impl<W: Write> Writer<W> {
+       pub fn new(writer: W) -> Writer<W> {
+               return Writer { writer }
+       }
+       pub fn into_inner(self) -> W { self.writer }
+       pub fn get_ref(&self) -> &W { &self.writer }
+       fn write_u64(&mut self, v: u64) -> Result<(), DecodeError> {
+               Ok(self.writer.write_all(&be64_to_array(v))?)
+       }
+       fn write_u32(&mut self, v: u32) -> Result<(), DecodeError> {
+               Ok(self.writer.write_all(&be32_to_array(v))?)
+       }
+       fn write_u16(&mut self, v: u16) -> Result<(), DecodeError> {
+               Ok(self.writer.write_all(&be16_to_array(v))?)
+       }
+       fn write_u8(&mut self, v: u8) -> Result<(), DecodeError> {
+               Ok(self.writer.write_all(&[v])?)
+       }
+       fn write_bool(&mut self, v: bool) -> Result<(), DecodeError> {
+               Ok(self.writer.write_all(&[if v {1} else {0}])?)
+       }
+       pub fn write_all(&mut self, v: &[u8]) -> Result<(), DecodeError> {
+               Ok(self.writer.write_all(v)?)
+       }
+}
+
+impl<R: Read> Reader<R> {
+       pub fn new(reader: R) -> Reader<R> {
+               return Reader { reader }
+       }
+       pub fn into_inner(self) -> R { self.reader }
+       pub fn get_ref(&self) -> &R { &self.reader }
+
+       fn read_u64(&mut self) -> Result<u64, DecodeError> {
+               let mut buf = [0; 8];
+               self.reader.read_exact(&mut buf)?;
+               Ok(slice_to_be64(&buf))
+       }
+
+       fn read_u32(&mut self) -> Result<u32, DecodeError> {
+               let mut buf = [0; 4];
+               self.reader.read_exact(&mut buf)?;
+               Ok(slice_to_be32(&buf))
+       }
+
+       fn read_u16(&mut self) -> Result<u16, DecodeError> {
+               let mut buf = [0; 2];
+               self.reader.read_exact(&mut buf)?;
+               Ok(slice_to_be16(&buf))
+       }
+
+       fn read_u8(&mut self) -> Result<u8, DecodeError> {
+               let mut buf = [0; 1];
+               self.reader.read_exact(&mut buf)?;
+               Ok(buf[0])
+       }
+       fn read_bool(&mut self) -> Result<bool, DecodeError> {
+               let mut buf = [0; 1];
+               self.reader.read_exact(&mut buf)?;
+               if buf[0] != 0 && buf[0] != 1 {
+                       return Err(DecodeError::InvalidValue);
+               }
+               Ok(buf[0] == 1)
+       }
+       pub fn read_exact(&mut self, buf: &mut [u8]) -> Result<(), DecodeError> {
+               Ok(self.reader.read_exact(buf)?)
+       }
+       pub fn read_to_end(&mut self, buf: &mut Vec<u8>) -> Result<usize, DecodeError> {
+               Ok(self.reader.read_to_end(buf)?)
+       }
+}
+
+macro_rules! impl_writeable_primitive {
+       ($val_type:ty, $meth_write:ident, $meth_read:ident) => {
+               impl<W:Write> Writeable<W> for $val_type {
+                       #[inline]
+                       fn write(&self, writer: &mut Writer<W>) -> Result<(), DecodeError> {
+                               writer.$meth_write(*self)
+                       }
+               }
+               impl<R:Read> Readable<R> for $val_type {
+                       #[inline]
+                       fn read(reader: &mut Reader<R>) -> Result<$val_type, DecodeError> {
+                               reader.$meth_read()
+                       }
+               }
+       }
+}
+
+impl_writeable_primitive!(u64, write_u64, read_u64);
+impl_writeable_primitive!(u32, write_u32, read_u32);
+impl_writeable_primitive!(u16, write_u16, read_u16);
+impl_writeable_primitive!(u8, write_u8, read_u8);
+impl_writeable_primitive!(bool, write_bool, read_bool);
+
+// u8 arrays
+macro_rules! impl_array {
+       ( $size:expr ) => (
+               impl<W> Writeable<W> for [u8; $size]
+                       where W: Write
+               {
+                       #[inline]
+                       fn write(&self, w: &mut Writer<W>) -> Result<(), DecodeError> {
+                               w.write_all(self)?;
+                               Ok(())
+                       }
+               }
+
+               impl<R> Readable<R> for [u8; $size]
+                       where R: Read
+               {
+                       #[inline]
+                       fn read(r: &mut Reader<R>) -> Result<Self, DecodeError> {
+                               let mut buf = [0u8; $size];
+                               r.read_exact(&mut buf)?;
+                               Ok(buf)
+                       }
+               }
+       );
+}
+
+//TODO: performance issue with [u8; size] with impl_array!()
+impl_array!(32); // for channel id & hmac
+impl_array!(33); // for PublicKey
+impl_array!(64); // for Signature
+impl_array!(1300); // for OnionPacket.hop_data
+
+// HashMap
+impl<W, K, V> Writeable<W> for HashMap<K, V>
+       where W: Write,
+             K: Writeable<W> + Eq + Hash,
+             V: Writeable<W>
+{
+       #[inline]
+       fn write(&self, w: &mut Writer<W>) -> Result<(), DecodeError> {
+       (self.len() as u16).write(w)?;
+               for (key, value) in self.iter() {
+                       key.write(w)?;
+                       value.write(w)?;
+               }
+               Ok(())
+       }
+}
+
+impl<R, K, V> Readable<R> for HashMap<K, V>
+       where R: Read,
+             K: Readable<R> + Eq + Hash,
+             V: Readable<R>
+{
+       #[inline]
+       fn read(r: &mut Reader<R>) -> Result<Self, DecodeError> {
+               let len: u16 = Readable::read(r)?;
+               let mut ret = HashMap::with_capacity(len as usize);
+               for _ in 0..len {
+                               ret.insert(K::read(r)?, V::read(r)?);
+               }
+               Ok(ret)
+       }
+}
+
+// Vectors
+impl<W: Write, T: Writeable<W>> Writeable<W> for Vec<T> {
+       #[inline]
+       fn write(&self, w: &mut Writer<W>) -> Result<(), DecodeError> {
+               let byte_size = (self.len() as usize)
+                               .checked_mul(mem::size_of::<T>())
+                               .ok_or(DecodeError::BadLengthDescriptor)?;
+               if byte_size > MAX_BUF_SIZE {
+                               return Err(DecodeError::BadLengthDescriptor);
+               }
+               (self.len() as u16).write(w)?;
+               // performance with Vec<u8>
+               for e in self.iter() {
+                       e.write(w)?;
+               }
+               Ok(())
+       }
+}
+
+impl<R: Read, T: Readable<R>> Readable<R> for Vec<T> {
+       #[inline]
+       fn read(r: &mut Reader<R>) -> Result<Self, DecodeError> {
+                       let len: u16 = Readable::read(r)?;
+                       let byte_size = (len as usize)
+                                       .checked_mul(mem::size_of::<T>())
+                                       .ok_or(DecodeError::BadLengthDescriptor)?;
+                       if byte_size > MAX_BUF_SIZE {
+                                       return Err(DecodeError::BadLengthDescriptor);
+                       }
+                       let mut ret = Vec::with_capacity(len as usize);
+                       for _ in 0..len { ret.push(T::read(r)?); }
+                       Ok(ret)
+       }
+}
+
+impl<W: Write> Writeable<W> for Script {
+       fn write(&self, w: &mut Writer<W>) -> Result<(), DecodeError> {
+               self.to_bytes().to_vec().write(w)
+       }
+}
+
+impl<R: Read> Readable<R> for Script {
+       fn read(r: &mut Reader<R>) -> Result<Self, DecodeError> {
+               let len = <u16 as Readable<R>>::read(r)? as usize;
+               let mut buf = vec![0; len];
+               r.read_exact(&mut buf)?;
+               Ok(Script::from(buf))
+       }
+}
+
+impl<W: Write> Writeable<W> for Option<Script> {
+       fn write(&self, w: &mut Writer<W>) -> Result<(), DecodeError> {
+               if let &Some(ref script) = self {
+                       script.write(w)?;
+               }
+               Ok(())
+       }
+}
+
+impl<R: Read> Readable<R> for Option<Script> {
+       fn read(r: &mut Reader<R>) -> Result<Self, DecodeError> {
+               match <u16 as Readable<R>>::read(r) {
+                       Ok(len) => {
+                               let mut buf = vec![0; len as usize];
+                               r.read_exact(&mut buf)?;
+                               Ok(Some(Script::from(buf)))
+                       },
+                       Err(DecodeError::ShortRead) => Ok(None),
+                       Err(e) => Err(e)
+               }
+       }
+}
+
+impl<W: Write> Writeable<W> for PublicKey {
+       fn write(&self, w: &mut Writer<W>) -> Result<(), DecodeError> {
+               self.serialize().write(w)
+       }
+}
+
+impl<R: Read> Readable<R> for PublicKey {
+       fn read(r: &mut Reader<R>) -> Result<Self, DecodeError> {
+               let buf: [u8; 33] = Readable::read(r)?;
+               match PublicKey::from_slice(&Secp256k1::without_caps(), &buf) {
+                       Ok(key) => Ok(key),
+                       Err(_) => return Err(DecodeError::BadPublicKey),
+               }
+       }
+}
+
+impl<W: Write> Writeable<W> for Sha256dHash {
+       fn write(&self, w: &mut Writer<W>) -> Result<(), DecodeError> {
+               self.as_bytes().write(w)
+       }
+}
+
+impl<R: Read> Readable<R> for Sha256dHash {
+       fn read(r: &mut Reader<R>) -> Result<Self, DecodeError> {
+               let buf: [u8; 32] = Readable::read(r)?;
+               Ok(From::from(&buf[..]))
+       }
+}
+
+impl<W: Write> Writeable<W> for Signature {
+       fn write(&self, w: &mut Writer<W>) -> Result<(), DecodeError> {
+               self.serialize_compact(&Secp256k1::without_caps()).write(w)
+       }
+}
+
+impl<R: Read> Readable<R> for Signature {
+       fn read(r: &mut Reader<R>) -> Result<Self, DecodeError> {
+               let buf: [u8; 64] = Readable::read(r)?;
+               match Signature::from_compact(&Secp256k1::without_caps(), &buf) {
+                       Ok(sig) => Ok(sig),
+                       Err(_) => return Err(DecodeError::BadSignature),
+               }
+       }
+}
+
+macro_rules! impl_writeable {
+       ($st:ident, {$($field:ident),*}) => {
+               impl<W: ::std::io::Write> Writeable<W> for $st {
+                       fn write(&self, w: &mut Writer<W>) -> Result<(), DecodeError> {
+                               $( self.$field.write(w)?; )*
+                               Ok(())
+                       }
+               }
+
+               impl<R: ::std::io::Read> Readable<R> for $st {
+                       fn read(r: &mut Reader<R>) -> Result<Self, DecodeError> {
+                               Ok(Self {
+                                       $($field: Readable::read(r)?),*
+                               })
+                       }
+               }
+       }
+}