From adf0d2f5069e794c773de70454442c67f41abff4 Mon Sep 17 00:00:00 2001 From: Matt Corallo Date: Tue, 19 Oct 2021 18:28:38 +0000 Subject: [PATCH] Use gitweb's snapshots to fetch binaries for comparison, not git --- .github/workflows/build.yml | 93 ++++++++++++++++++++++++------------- 1 file changed, 62 insertions(+), 31 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 885f0b83..3ec1440d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -17,7 +17,7 @@ jobs: run: | apt-get update apt-get -y dist-upgrade - apt-get -y install cargo libstd-rust-dev-wasm32 valgrind lld git g++ clang openjdk-11-jdk maven faketime zip unzip llvm + apt-get -y install cargo libstd-rust-dev-wasm32 valgrind lld git g++ clang openjdk-11-jdk maven faketime zip unzip llvm curl - name: Checkout source code uses: actions/checkout@v2 with: @@ -60,7 +60,16 @@ jobs: - name: Checkout latest MacOS binaries run: | export LDK_GARBAGECOLLECTED_GIT_OVERRIDE="$(git describe --tag HEAD)" - git clone --depth 1 https://git.bitcoin.ninja/ldk-java-bins + echo "Fetching deterministic binaries for LDK-GC ${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}" + # Gitweb only allows snapshots of folders by providing the object hash, which we have to extract: + SNAPSHOT_LINK="$(curl "https://git.bitcoin.ninja/index.cgi?p=ldk-java-bins;a=tree;f=${LDK_GARBAGECOLLECTED_GIT_OVERRIDE};hb=refs/heads/main" | grep snapshot | grep -o 'href="[a-zA-Z0-9/?\.=;\-]*"' | sed 's/href="//' | tr -d '"' | grep snapshot)" + curl -o bins-snapshot.tgz "https://git.bitcoin.ninja${SNAPSHOT_LINK}" + mkdir -p ldk-java-bins/"${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}" + cd ldk-java-bins/"${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}" + tar xvvf ../../bins-snapshot.tgz + mv ldk-java-bins-*/* ./ + rm -r ldk-java-bins-* + cd ../.. mkdir -p src/main/resources/ cp "ldk-java-bins/${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}/liblightningjni_MacOSX-"* src/main/resources/ - name: Build Java/TS Release Bindings @@ -76,24 +85,23 @@ jobs: - name: Check latest library and jars are in bins repo run: | export LDK_GARBAGECOLLECTED_GIT_OVERRIDE="$(git describe --tag HEAD)" - rm "ldk-java-bins/${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}/"*.jar - rm "ldk-java-bins/${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}/"liblightningjni_Linux*.nativelib - cp src/main/resources/liblightningjni_Linux-amd64.nativelib "ldk-java-bins/${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}/" - cp ldk-java-sources.jar "ldk-java-bins/${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}/" - cp ldk-java.jar "ldk-java-bins/${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}/" - cp ldk-java-classes.jar "ldk-java-bins/${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}/" - cp ldk-java-javadoc.jar "ldk-java-bins/${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}/" - cd ldk-java-bins + mkdir ldk-java-bins/new/ + cp "ldk-java-bins/${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}/"liblightningjni_Mac*.nativelib ldk-java-bins/new/ + cp "ldk-java-bins/${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}/"LDK-release.aar ldk-java-bins/new/ # ldk-java-leaktracking is only for debug purposes and we don't bother with determinism - git checkout "${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}/ldk-java-leaktracking.jar" - if ! git diff --exit-code; then - mv "${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}/"* ./ - git checkout "${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}/" + cp "ldk-java-bins/${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}/"ldk-java-leaktracking.jar ldk-java-bins/new/ + cp src/main/resources/liblightningjni_Linux-amd64.nativelib ldk-java-bins/new/ + cp ldk-java-sources.jar ldk-java-bins/new/ + cp ldk-java.jar ldk-java-bins/new/ + cp ldk-java-classes.jar ldk-java-bins/new/ + cp ldk-java-javadoc.jar ldk-java-bins/new/ + cd ldk-java-bins + if ! diff -r "${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}/" new/; then apt-get -y install diffoscope - diffoscope ldk-java-sources.jar "${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}/ldk-java-sources.jar" || echo - diffoscope ldk-java-javadoc.jar "${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}/ldk-java-javadoc.jar" || echo - diffoscope ldk-java-classes.jar "${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}/ldk-java-classes.jar" || echo - diffoscope ldk-java.jar "${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}/ldk-java.jar" || echo + diffoscope new/ldk-java-sources.jar "${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}/ldk-java-sources.jar" || echo + diffoscope new/ldk-java-javadoc.jar "${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}/ldk-java-javadoc.jar" || echo + diffoscope new/ldk-java-classes.jar "${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}/ldk-java-classes.jar" || echo + diffoscope new/ldk-java.jar "${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}/ldk-java.jar" || echo exit 1 fi - name: Run Java Tests against built release jar @@ -155,7 +163,29 @@ jobs: cd ldk-c-bindings git checkout 0.0.102 - name: Checkout Android AAR binaries and artifacts - run: git clone --depth 1 https://git.bitcoin.ninja/ldk-java-bins + run: | + # Gitweb only allows snapshots of folders by providing the object hash, which we have to extract: + export LDK_GARBAGECOLLECTED_GIT_OVERRIDE="$(git describe --tag HEAD)" + echo "Fetching deterministic binaries for LDK-GC ${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}" + RELEASE_PAGE="https://git.bitcoin.ninja/index.cgi?p=ldk-java-bins;a=tree;f=${LDK_GARBAGECOLLECTED_GIT_OVERRIDE};hb=refs/heads/main" + SNAPSHOT_LINK="$(curl "$RELEASE_PAGE" | grep snapshot | grep -o 'href="[a-zA-Z0-9/?\.=;\-]*"' | sed 's/href="//' | tr -d '"' | grep snapshot)" + curl -o bins-snapshot.tgz "https://git.bitcoin.ninja${SNAPSHOT_LINK}" + + ANDROID_PAGE="https://git.bitcoin.ninja/index.cgi?p=ldk-java-bins;a=tree;f=android-artifacts;hb=refs/heads/main" + SNAPSHOT_LINK="$(curl "$ANDROID_PAGE" | grep snapshot | grep -o 'href="[a-zA-Z0-9/?\.=;\-]*"' | sed 's/href="//' | tr -d '"' | grep snapshot)" + curl -o android-snapshot.tgz "https://git.bitcoin.ninja${SNAPSHOT_LINK}" + + mkdir -p ldk-java-bins/"${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}" + cd ldk-java-bins/"${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}" + tar xvvf ../../bins-snapshot.tgz + mv ldk-java-bins-*/* ./ + rm -r ldk-java-bins-* + + mkdir -p ../android-artifacts + cd ../android-artifacts + tar xvvf ../../android-snapshot.tgz + mv ldk-java-bins-*/* ./ + rm -r ldk-java-bins-* - name: Build Android aar run: | . $HOME/.cargo/env @@ -167,14 +197,9 @@ jobs: - name: Check latest library and jars are in bins repo run: | export LDK_GARBAGECOLLECTED_GIT_OVERRIDE="$(git describe --tag HEAD)" - rm "ldk-java-bins/${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}/LDK-release.aar" - cp LDK-release.aar "ldk-java-bins/${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}/" - cd ldk-java-bins - if ! git diff --exit-code; then - mv "${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}/LDK-release.aar" ./ - git checkout "${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}/LDK-release.aar" + if ! diff LDK-release.aar "ldk-java-bins/${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}/"LDK-release.aar; then apt-get -y install diffoscope - diffoscope LDK-release.aar "${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}/LDK-release.aar" + diffoscope LDK-release.aar "ldk-java-bins/${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}/LDK-release.aar" exit 1 fi @@ -238,7 +263,15 @@ jobs: - name: Checkout latest Linux binaries run: | export LDK_GARBAGECOLLECTED_GIT_OVERRIDE="$(git describe --tag HEAD)" - git clone --depth 1 https://git.bitcoin.ninja/ldk-java-bins + echo "Fetching deterministic binaries for LDK-GC ${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}" + # Gitweb only allows snapshots of folders by providing the object hash, which we have to extract: + SNAPSHOT_LINK="$(wget -O /dev/stdout "https://git.bitcoin.ninja/index.cgi?p=ldk-java-bins;a=tree;f=${LDK_GARBAGECOLLECTED_GIT_OVERRIDE};hb=refs/heads/main" | grep snapshot | grep -o 'href="[a-zA-Z0-9/?\.=;\-]*"' | sed 's/href="//' | tr -d '"' | grep snapshot)" + wget -O bins-snapshot.tgz "https://git.bitcoin.ninja${SNAPSHOT_LINK}" + mkdir -p ldk-java-bins/"${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}" + cd ldk-java-bins/"${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}" + tar xvvf ../../bins-snapshot.tgz + mv ldk-java-bins-*/* ./ + cd ../.. mkdir -p src/main/resources/ cp "ldk-java-bins/${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}/liblightningjni_Linux-"* src/main/resources/ - name: Build Java/TS Release Bindings @@ -280,11 +313,9 @@ jobs: run: | if [ "${{ matrix.platform }}" = "macos-11" ]; then export LDK_GARBAGECOLLECTED_GIT_OVERRIDE="$(git describe --tag HEAD)" - rm "ldk-java-bins/${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}/"liblightningjni_MacOSX-{x86_64,aarch64}.nativelib - cp src/main/resources/liblightningjni_MacOSX-*.nativelib "ldk-java-bins/${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}/" - cd ldk-java-bins # Sadly, OSX binaries are not currently deterministic, more debugging is needed. - #git diff --exit-code + diff "ldk-java-bins/${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}/"liblightningjni_MacOSX-x86_64.nativelib src/main/resources/liblightningjni_MacOSX-x86_64.nativelib || echo + diff "ldk-java-bins/${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}/"liblightningjni_MacOSX-aarch64.nativelib src/main/resources/liblightningjni_MacOSX-aarch64.nativelib || echo fi - name: Run Java Tests against release bins run: | -- 2.39.5