From c8b51228d6c7012659aa4b4a37591dc27222937c Mon Sep 17 00:00:00 2001 From: Matt Corallo Date: Fri, 24 Apr 2020 14:22:26 -0400 Subject: [PATCH] Run with mutagen on travis. Sadly our test coverage isn't very good and I had to hunt for functions to mutate where we fail tests on every mutation mutagen creates, but committing the framework is a start. --- .github/workflows/build.yml | 15 +++++++++++++-- .travis.yml | 11 +++++++++-- lightning/Cargo.toml | 6 +++++- lightning/src/lib.rs | 6 ++++++ lightning/src/ln/channel.rs | 4 ++++ 5 files changed, 37 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index cd9b9f63..50a3b7c0 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -7,7 +7,7 @@ jobs: strategy: matrix: toolchain: [ stable, - beta, + nightly, # 1.22.0 is MSRV for rust-lightning in general: 1.22.0, # 1.34.2 is Debian stable @@ -17,8 +17,9 @@ jobs: include: - toolchain: stable build-net-tokio: true - - toolchain: beta + - toolchain: nightly build-net-tokio: true + run-mutagen: true - toolchain: 1.39.0 build-net-tokio: true coverage: true @@ -44,6 +45,16 @@ jobs: - name: Test on Rust ${{ matrix.toolchain }} if: "! matrix.build-net-tokio" run: RUSTFLAGS="-C link-dead-code" cargo test --verbose --color always -p lightning + - name: Run mutagen tests on ${{ matrix.toolchain }} + if: matrix.run-mutagen + # Run mutagen on nightly with TheBlueMatt's fork which exits with non-0 status + # if any mutations resulted in anything except test failures to prevent regressions. + run: | + rm -rf mutagen && git clone https://github.com/TheBlueMatt/mutagen + cargo install --force --path mutagen/mutagen-runner + cd lightning + ~/.cargo/bin/cargo-mutagen --features mutation_testing + # Generate code cov information on Rust 1.39.0 - name: Install deps for kcov if: matrix.coverage run: | diff --git a/.travis.yml b/.travis.yml index 0c36806c..dcb3a11b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,7 @@ language: rust rust: - stable - - beta + - nightly # 1.22.0 is MSRV for rust-lightning in general: - 1.22.0 # 1.34.2 is Debian stable @@ -15,7 +15,7 @@ before_install: - sudo apt-get install -y binutils-dev libunwind8-dev libcurl4-openssl-dev libelf-dev libdw-dev cmake gcc binutils-dev libiberty-dev script: - # Support lightning-net-tokio only on Rust stable, beta, and 1.39.0 + # Support lightning-net-tokio only on Rust stable, nightly, and 1.39.0 - if [ "$(rustup show | grep default | grep '1.39.0')" != "" ]; then export BUILD_NET_TOKIO=1; fi - if [ "$(rustup show | grep default | grep '1\.')" == "" ]; then export BUILD_NET_TOKIO=1; fi # Build the appropriate workspace(s) @@ -27,6 +27,13 @@ script: - if [ "$BUILD_NET_TOKIO" != "1" ]; then RUSTFLAGS="-C link-dead-code" cargo test --verbose -p lightning; fi # Run lightning workspace fuzz tests on Rust stable - if [ "$(rustup show | grep default | grep stable)" != "" ]; then cd fuzz && cargo test --verbose && ./ci-fuzz.sh; fi + # Run mutagen on nightly with TheBlueMatt's fork which exits with non-0 status + # if any mutations resulted in anything except test failures to prevent regressions. + - if [ "$(rustup show | grep default | grep nightly)" != "" ]; then + rm -rf mutagen && git clone https://github.com/TheBlueMatt/mutagen && + cargo install --force --path mutagen/mutagen-runner && + cd lightning && + ~/.cargo/bin/cargo-mutagen --features mutation_testing; fi # Generate code cov information on Rust 1.39.0 - if [ "$(rustup show | grep default | grep 1.39.0)" != "" ]; then wget https://github.com/SimonKagstrom/kcov/archive/master.tar.gz && diff --git a/lightning/Cargo.toml b/lightning/Cargo.toml index f89b5481..3be6b7d3 100644 --- a/lightning/Cargo.toml +++ b/lightning/Cargo.toml @@ -11,7 +11,6 @@ Still missing tons of error-handling. See GitHub issues for suggested projects i """ [features] -fuzztarget = ["bitcoin/fuzztarget"] # Unlog messages superior at targeted level. max_level_off = [] max_level_error = [] @@ -19,8 +18,13 @@ max_level_warn = [] max_level_info = [] max_level_debug = [] +# Testing only features, don't enable these unless you want to run rust-lightning tests! +fuzztarget = ["bitcoin/fuzztarget"] +mutation_testing = ["mutagen"] + [dependencies] bitcoin = "0.23" +mutagen = { git = "https://github.com/TheBlueMatt/mutagen", optional = true } [dev-dependencies.bitcoin] version = "0.23" diff --git a/lightning/src/lib.rs b/lightning/src/lib.rs index 23645ab1..0cea7ad5 100644 --- a/lightning/src/lib.rs +++ b/lightning/src/lib.rs @@ -21,6 +21,7 @@ extern crate bitcoin; #[cfg(test)] extern crate rand; #[cfg(test)] extern crate hex; +#[cfg(all(test, feature = "mutation_testing"))] extern crate mutagen; #[macro_use] pub mod util; @@ -28,3 +29,8 @@ pub mod chain; pub mod ln; pub mod routing; +#[cfg(all( + any(feature = "mutation_testing", feature = "fuzztarget"), + not(any(test, debug_assertions)) + ))] +const ERR: () = "You should never be building with feature = mutation_testing or feature = fuzztarget! They are used to compile with broken code for testing only!"; diff --git a/lightning/src/ln/channel.rs b/lightning/src/ln/channel.rs index c9eebf41..262342d4 100644 --- a/lightning/src/ln/channel.rs +++ b/lightning/src/ln/channel.rs @@ -35,6 +35,9 @@ use std::default::Default; use std::{cmp,mem,fmt}; use std::ops::Deref; +#[cfg(all(test, feature = "mutation_testing"))] +use mutagen::mutate; + #[cfg(test)] pub struct ChannelValueStat { pub value_to_self_msat: u64, @@ -2513,6 +2516,7 @@ impl Channel { Ok(()) } + #[cfg_attr(all(test, feature = "mutation_testing"), mutate)] fn get_last_revoke_and_ack(&self) -> msgs::RevokeAndACK { let next_per_commitment_point = PublicKey::from_secret_key(&self.secp_ctx, &self.build_local_commitment_secret(self.cur_local_commitment_transaction_number)); let per_commitment_secret = chan_utils::build_commitment_secret(self.local_keys.commitment_seed(), self.cur_local_commitment_transaction_number + 2); -- 2.30.2