Manually download `bitcoind`/`electrs` for CI tests

Previously, we used the auto-download feature of the
`electrsd`/`bitcoind` crates. While convenient, they unnecessarily
introduced a lot of dependecies (`zip`, `zstd`, `time`, etc.) to our
test environment which needed pinning for the MSRV of 1.63.

Here, we introduce a new `no_download` config flag to the
`lightning-transaction-sync` crate allowing us to disable this
auto-download feature in CI, where we now opt to download the
corresponding binaries manually. We keep the default-auto-download as a
convenience feature for running tests locally though.

diff --git a/ci/ci-tests.sh b/ci/ci-tests.sh
index 8dfb432666dbb5ffc92a61294026439a9c6f785f..11934a8307a6f0723aeeb7a5094f3853a0c0a981 100755 (executable)
--- a/ci/ci-tests.sh
+++ b/ci/ci-tests.sh
@@ -11,6 +11,50 @@ function PIN_RELEASE_DEPS {
        return 0 # Don't fail the script if our rustc is higher than the last check
 }
 
+# The tests of `lightning-transaction-sync` require `electrs` and `bitcoind`
+# binaries. Here, we download the binaries, validate them, and export their
+# location via `ELECTRS_EXE`/`BITCOIND_EXE` which will be used by the
+# `electrsd`/`bitcoind` crates in our tests.
+function DOWNLOAD_ELECTRS_AND_BITCOIND {
+       ELECTRS_DL_ENDPOINT="https://github.com/RCasatta/electrsd/releases/download/electrs_releases"
+       ELECTRS_VERSION="esplora_a33e97e1a1fc63fa9c20a116bb92579bbf43b254"
+       BITCOIND_DL_ENDPOINT="https://bitcoincore.org/bin/"
+       BITCOIND_VERSION="25.1"
+       if [[ "$HOST_PLATFORM" == *linux* ]]; then
+               ELECTRS_DL_FILE_NAME=electrs_linux_"$ELECTRS_VERSION".zip
+               ELECTRS_DL_HASH="865e26a96e8df77df01d96f2f569dcf9622fc87a8d99a9b8fe30861a4db9ddf1"
+               BITCOIND_DL_FILE_NAME=bitcoin-"$BITCOIND_VERSION"-x86_64-linux-gnu.tar.gz
+               BITCOIND_DL_HASH="a978c407b497a727f0444156e397b50491ce862d1f906fef9b521415b3611c8b"
+       elif [[ "$HOST_PLATFORM" == *darwin* ]]; then
+               ELECTRS_DL_FILE_NAME=electrs_macos_"$ELECTRS_VERSION".zip
+               ELECTRS_DL_HASH="2d5ff149e8a2482d3658e9b386830dfc40c8fbd7c175ca7cbac58240a9505bcd"
+               BITCOIND_DL_FILE_NAME=bitcoin-"$BITCOIND_VERSION"-x86_64-apple-darwin.tar.gz
+               BITCOIND_DL_HASH="1acfde0ec3128381b83e3e5f54d1c7907871d324549129592144dd12a821eff1"
+       else
+               echo -e "\n\nUnsupported platform. Exiting.."
+               exit 1
+       fi
+
+       DL_TMP_DIR=$(mktemp -d)
+       trap 'rm -rf -- "$DL_TMP_DIR"' EXIT
+
+       pushd "$DL_TMP_DIR"
+       ELECTRS_DL_URL="$ELECTRS_DL_ENDPOINT"/"$ELECTRS_DL_FILE_NAME"
+       curl -L -o "$ELECTRS_DL_FILE_NAME" "$ELECTRS_DL_URL"
+       echo "$ELECTRS_DL_HASH  $ELECTRS_DL_FILE_NAME"|shasum -a 256 -c
+       unzip "$ELECTRS_DL_FILE_NAME"
+       export ELECTRS_EXE="$DL_TMP_DIR"/electrs
+       chmod +x "$ELECTRS_EXE"
+
+       BITCOIND_DL_URL="$BITCOIND_DL_ENDPOINT"/bitcoin-core-"$BITCOIND_VERSION"/"$BITCOIND_DL_FILE_NAME"
+       curl -L -o "$BITCOIND_DL_FILE_NAME" "$BITCOIND_DL_URL"
+       echo "$BITCOIND_DL_HASH  $BITCOIND_DL_FILE_NAME"|shasum -a 256 -c
+       tar xzf "$BITCOIND_DL_FILE_NAME"
+       export BITCOIND_EXE="$DL_TMP_DIR"/bitcoin-"$BITCOIND_VERSION"/bin/bitcoind
+       chmod +x "$BITCOIND_EXE"
+       popd
+}
+
 PIN_RELEASE_DEPS # pin the release dependencies in our main workspace
 
 # Starting with version 1.10.0, the `regex` crate has an MSRV of rustc 1.65.0.
@@ -41,23 +85,21 @@ if [[ "$HOST_PLATFORM" != *windows* ]]; then
        echo -e "\n\nBuilding and testing Transaction Sync Clients with features"
        pushd lightning-transaction-sync
 
-       # zstd-sys 2.0.9+zstd.1.5.5 requires rustc 1.64.0
-       [ "$RUSTC_MINOR_VERSION" -lt 64 ] && cargo update -p zstd-sys --precise "2.0.8+zstd.1.5.5" --verbose
        # reqwest 0.11.21 had a regression that broke its 1.63.0 MSRV
        [ "$RUSTC_MINOR_VERSION" -lt 65 ] && cargo update -p reqwest --precise "0.11.20" --verbose
-       # jobserver 0.1.27 requires rustc 1.66.0
-       [ "$RUSTC_MINOR_VERSION" -lt 66 ] && cargo update -p jobserver --precise "0.1.26" --verbose
        # Starting with version 1.10.0, the `regex` crate has an MSRV of rustc 1.65.0.
        [ "$RUSTC_MINOR_VERSION" -lt 65 ] && cargo update -p regex --precise "1.9.6" --verbose
 
-       cargo test --verbose --color always --features esplora-blocking
-       cargo check --verbose --color always --features esplora-blocking
-       cargo test --verbose --color always --features esplora-async
-       cargo check --verbose --color always --features esplora-async
-       cargo test --verbose --color always --features esplora-async-https
-       cargo check --verbose --color always --features esplora-async-https
-       cargo test --verbose --color always --features electrum
-       cargo check --verbose --color always --features electrum
+       DOWNLOAD_ELECTRS_AND_BITCOIND
+
+       RUSTFLAGS="--cfg no_download" cargo test --verbose --color always --features esplora-blocking
+       RUSTFLAGS="--cfg no_download" cargo check --verbose --color always --features esplora-blocking
+       RUSTFLAGS="--cfg no_download" cargo test --verbose --color always --features esplora-async
+       RUSTFLAGS="--cfg no_download" cargo check --verbose --color always --features esplora-async
+       RUSTFLAGS="--cfg no_download" cargo test --verbose --color always --features esplora-async-https
+       RUSTFLAGS="--cfg no_download" cargo check --verbose --color always --features esplora-async-https
+       RUSTFLAGS="--cfg no_download" cargo test --verbose --color always --features electrum
+       RUSTFLAGS="--cfg no_download" cargo check --verbose --color always --features electrum
 
        popd
 fi

diff --git a/lightning-transaction-sync/Cargo.toml b/lightning-transaction-sync/Cargo.toml
index ca46953a209c63be1c24ba3e2b0fa7c5e9cbdd94..d712229b0a820b841579f883594dcfed7a5f44ac 100644 (file)
--- a/lightning-transaction-sync/Cargo.toml
+++ b/lightning-transaction-sync/Cargo.toml
@@ -31,5 +31,10 @@ electrum-client = { version = "0.18.0", optional = true }
 
 [dev-dependencies]
 lightning = { version = "0.0.118", path = "../lightning", default-features = false, features = ["std", "_test_utils"] }
-electrsd = { version = "0.26.0", features = ["legacy", "esplora_a33e97e1", "bitcoind_25_0"] }
 tokio = { version = "1.14.0", features = ["full"] }
+
+[target.'cfg(not(no_download))'.dev-dependencies]
+electrsd = { version = "0.26.0", default-features = false, features = ["legacy", "esplora_a33e97e1", "bitcoind_25_0"] }
+
+[target.'cfg(no_download)'.dev-dependencies]
+electrsd = { version = "0.26.0", default-features = false, features = ["legacy"] }