X-Git-Url: http://git.bitcoin.ninja/index.cgi?a=blobdiff_plain;f=.github%2Fworkflows%2Fbuild.yml;h=14a8595cd7ca9123d033b8178a7f062a3f39a75c;hb=a766f4f8f0e421a0510d10aa39259f633551c14a;hp=90d6b194ba469dce623afecb43cd477b25459afa;hpb=a62565e3f563839ad4cdc008f031d2342e40406e;p=ldk-java

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 90d6b194..14a8595c 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -17,7 +17,7 @@ jobs:
         run: |
           apt-get update
           apt-get -y dist-upgrade
-          apt-get -y install cargo libstd-rust-dev-wasm32 valgrind lld git g++ clang openjdk-11-jdk maven faketime zip unzip llvm
+          apt-get -y install cargo libstd-rust-dev-wasm32 valgrind lld git g++ clang openjdk-11-jdk maven faketime zip unzip llvm curl
       - name: Checkout source code
         uses: actions/checkout@v2
         with:
@@ -35,7 +35,7 @@ jobs:
           cd ..
           git clone https://github.com/lightningdevkit/ldk-c-bindings
           cd ldk-c-bindings
-          git checkout 0.0.102
+          git checkout 0.0.103
       - name: Rebuild C bindings without STD for WASM
         run: |
           cd ldk-c-bindings
@@ -60,7 +60,16 @@ jobs:
       - name: Checkout latest MacOS binaries
         run: |
           export LDK_GARBAGECOLLECTED_GIT_OVERRIDE="$(git describe --tag HEAD)"
-          git clone --depth 1 https://git.bitcoin.ninja/ldk-java-bins
+          echo "Fetching deterministic binaries for LDK-GC ${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}"
+          # Gitweb only allows snapshots of folders by providing the object hash, which we have to extract:
+          SNAPSHOT_LINK="$(curl "https://git.bitcoin.ninja/index.cgi?p=ldk-java-bins;a=tree;f=${LDK_GARBAGECOLLECTED_GIT_OVERRIDE};hb=refs/heads/main" | grep snapshot | grep -o 'href="[a-zA-Z0-9/?\.=;\-]*"' | sed 's/href="//' | tr -d '"' | grep snapshot)"
+          curl -o bins-snapshot.tgz "https://git.bitcoin.ninja${SNAPSHOT_LINK}"
+          mkdir -p ldk-java-bins/"${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}"
+          cd ldk-java-bins/"${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}"
+          tar xvvf ../../bins-snapshot.tgz
+          mv ldk-java-bins-*/* ./
+          rm -r ldk-java-bins-*
+          cd ../..
           mkdir -p src/main/resources/
           cp "ldk-java-bins/${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}/liblightningjni_MacOSX-"* src/main/resources/
       - name: Build Java/TS Release Bindings
@@ -76,24 +85,23 @@ jobs:
       - name: Check latest library and jars are in bins repo
         run: |
           export LDK_GARBAGECOLLECTED_GIT_OVERRIDE="$(git describe --tag HEAD)"
-          rm "ldk-java-bins/${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}/"*.jar
-          rm "ldk-java-bins/${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}/"liblightningjni_Linux*.nativelib
-          cp src/main/resources/liblightningjni_Linux-amd64.nativelib "ldk-java-bins/${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}/"
-          cp ldk-java-sources.jar "ldk-java-bins/${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}/"
-          cp ldk-java.jar "ldk-java-bins/${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}/"
-          cp ldk-java-classes.jar "ldk-java-bins/${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}/"
-          cp ldk-java-javadoc.jar "ldk-java-bins/${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}/"
-          cd ldk-java-bins
+          mkdir ldk-java-bins/new/
+          cp "ldk-java-bins/${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}/"liblightningjni_Mac*.nativelib ldk-java-bins/new/
+          cp "ldk-java-bins/${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}/"LDK-release.aar ldk-java-bins/new/
           # ldk-java-leaktracking is only for debug purposes and we don't bother with determinism
-          git checkout "${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}/ldk-java-leaktracking.jar"
-          if ! git diff --exit-code; then
-            mv "${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}/"* ./
-            git checkout "${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}/"
+          cp "ldk-java-bins/${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}/"ldk-java-leaktracking.jar ldk-java-bins/new/
+          cp src/main/resources/liblightningjni_Linux-amd64.nativelib ldk-java-bins/new/
+          cp ldk-java-sources.jar ldk-java-bins/new/
+          cp ldk-java.jar ldk-java-bins/new/
+          cp ldk-java-classes.jar ldk-java-bins/new/
+          cp ldk-java-javadoc.jar ldk-java-bins/new/
+          cd ldk-java-bins
+          if ! diff -r "${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}/" new/; then
             apt-get -y install diffoscope
-            diffoscope ldk-java-sources.jar "${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}/ldk-java-sources.jar" || echo
-            diffoscope ldk-java-javadoc.jar "${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}/ldk-java-javadoc.jar" || echo
-            diffoscope ldk-java-classes.jar "${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}/ldk-java-classes.jar" || echo
-            diffoscope ldk-java.jar "${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}/ldk-java.jar" || echo
+            diffoscope new/ldk-java-sources.jar "${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}/ldk-java-sources.jar" || echo
+            diffoscope new/ldk-java-javadoc.jar "${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}/ldk-java-javadoc.jar" || echo
+            diffoscope new/ldk-java-classes.jar "${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}/ldk-java-classes.jar" || echo
+            diffoscope new/ldk-java.jar "${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}/ldk-java.jar" || echo
             exit 1
           fi
       - name: Run Java Tests against built release jar
@@ -153,9 +161,31 @@ jobs:
           cd ..
           git clone https://github.com/lightningdevkit/ldk-c-bindings
           cd ldk-c-bindings
-          git checkout 0.0.102
+          git checkout 0.0.103
       - name: Checkout Android AAR binaries and artifacts
-        run: git clone --depth 1 https://git.bitcoin.ninja/ldk-java-bins
+        run: |
+          # Gitweb only allows snapshots of folders by providing the object hash, which we have to extract:
+          export LDK_GARBAGECOLLECTED_GIT_OVERRIDE="$(git describe --tag HEAD)"
+          echo "Fetching deterministic binaries for LDK-GC ${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}"
+          RELEASE_PAGE="https://git.bitcoin.ninja/index.cgi?p=ldk-java-bins;a=tree;f=${LDK_GARBAGECOLLECTED_GIT_OVERRIDE};hb=refs/heads/main"
+          SNAPSHOT_LINK="$(curl "$RELEASE_PAGE" | grep snapshot | grep -o 'href="[a-zA-Z0-9/?\.=;\-]*"' | sed 's/href="//' | tr -d '"' | grep snapshot)"
+          curl -o bins-snapshot.tgz "https://git.bitcoin.ninja${SNAPSHOT_LINK}"
+
+          ANDROID_PAGE="https://git.bitcoin.ninja/index.cgi?p=ldk-java-bins;a=tree;f=android-artifacts;hb=refs/heads/main"
+          SNAPSHOT_LINK="$(curl "$ANDROID_PAGE" | grep snapshot | grep -o 'href="[a-zA-Z0-9/?\.=;\-]*"' | sed 's/href="//' | tr -d '"' | grep snapshot)"
+          curl -o android-snapshot.tgz "https://git.bitcoin.ninja${SNAPSHOT_LINK}"
+
+          mkdir -p ldk-java-bins/"${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}"
+          cd ldk-java-bins/"${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}"
+          tar xvvf ../../bins-snapshot.tgz
+          mv ldk-java-bins-*/* ./
+          rm -r ldk-java-bins-*
+
+          mkdir -p ../android-artifacts
+          cd ../android-artifacts
+          tar xvvf ../../android-snapshot.tgz
+          mv ldk-java-bins-*/* ./
+          rm -r ldk-java-bins-*
       - name: Build Android aar
         run: |
           . $HOME/.cargo/env
@@ -167,14 +197,9 @@ jobs:
       - name: Check latest library and jars are in bins repo
         run: |
           export LDK_GARBAGECOLLECTED_GIT_OVERRIDE="$(git describe --tag HEAD)"
-          rm "ldk-java-bins/${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}/LDK-release.aar"
-          cp LDK-release.aar "ldk-java-bins/${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}/"
-          cd ldk-java-bins
-          if ! git diff --exit-code; then
-            mv "${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}/LDK-release.aar" ./
-            git checkout "${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}/LDK-release.aar"
+          if ! diff LDK-release.aar "ldk-java-bins/${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}/"LDK-release.aar; then
             apt-get -y install diffoscope
-            diffoscope LDK-release.aar "${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}/LDK-release.aar"
+            diffoscope LDK-release.aar "ldk-java-bins/${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}/LDK-release.aar"
             exit 1
           fi
 
@@ -193,14 +218,14 @@ jobs:
         run: rustup target install aarch64-apple-darwin
       - name: Fetch upstream LLVM/clang snapshot
         run: |
-          wget -O clang+llvm-12.0.0-x86_64-apple-darwin.tar.xz https://github.com/llvm/llvm-project/releases/download/llvmorg-12.0.0/clang+llvm-12.0.0-x86_64-apple-darwin.tar.xz
-          if [ "$(shasum -a 256 clang+llvm-12.0.0-x86_64-apple-darwin.tar.xz | awk '{ print $1 }')" != "7bc2259bf75c003f644882460fc8e844ddb23b27236fe43a2787870a4cd8ab50" ]; then
+          wget -O clang+llvm-13.0.0-x86_64-apple-darwin.tar.xz https://github.com/llvm/llvm-project/releases/download/llvmorg-13.0.0/clang+llvm-13.0.0-x86_64-apple-darwin.tar.xz
+          if [ "$(shasum -a 256 clang+llvm-13.0.0-x86_64-apple-darwin.tar.xz | awk '{ print $1 }')" != "d051234eca1db1f5e4bc08c64937c879c7098900f7a0370f3ceb7544816a8b09" ]; then
             echo "Bad hash"
             exit 1
           fi
       - name: Unpack upstream LLVM+clang and use it by default
         run: |
-          tar xvvf clang+llvm-12.0.0-x86_64-apple-darwin.tar.xz
+          tar xvvf clang+llvm-13.0.0-x86_64-apple-darwin.tar.xz
       - name: Checkout source code
         uses: actions/checkout@v2
         with:
@@ -218,11 +243,11 @@ jobs:
           cd ..
           git clone https://github.com/lightningdevkit/ldk-c-bindings
           cd ldk-c-bindings
-          git checkout 0.0.102
+          git checkout 0.0.103
       - name: Rebuild C bindings with upstream clang, and check the sample app builds + links
         run: |
           cd ldk-c-bindings
-          export PATH=`pwd`/clang+llvm-12.0.0-x86_64-apple-darwin/bin:$PATH
+          export PATH=`pwd`/clang+llvm-13.0.0-x86_64-apple-darwin/bin:$PATH
           CC=clang ./genbindings.sh ../rust-lightning true
           cd ..
       - name: Fetch OpenJDK 16
@@ -238,7 +263,15 @@ jobs:
       - name: Checkout latest Linux binaries
         run: |
           export LDK_GARBAGECOLLECTED_GIT_OVERRIDE="$(git describe --tag HEAD)"
-          git clone --depth 1 https://git.bitcoin.ninja/ldk-java-bins
+          echo "Fetching deterministic binaries for LDK-GC ${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}"
+          # Gitweb only allows snapshots of folders by providing the object hash, which we have to extract:
+          SNAPSHOT_LINK="$(wget -O /dev/stdout "https://git.bitcoin.ninja/index.cgi?p=ldk-java-bins;a=tree;f=${LDK_GARBAGECOLLECTED_GIT_OVERRIDE};hb=refs/heads/main" | grep snapshot | grep -o 'href="[a-zA-Z0-9/?\.=;\-]*"' | sed 's/href="//' | tr -d '"' | grep snapshot)"
+          wget -O bins-snapshot.tgz "https://git.bitcoin.ninja${SNAPSHOT_LINK}"
+          mkdir -p ldk-java-bins/"${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}"
+          cd ldk-java-bins/"${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}"
+          tar xvvf ../../bins-snapshot.tgz
+          mv ldk-java-bins-*/* ./
+          cd ../..
           mkdir -p src/main/resources/
           cp "ldk-java-bins/${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}/liblightningjni_Linux-"* src/main/resources/
       - name: Build Java/TS Release Bindings
@@ -247,7 +280,7 @@ jobs:
           # genbindings.sh always fails as there is no wasm32-wasi library
           # available, so instead we check that the expected JNI library
           # is created.
-          export PATH=`pwd`/clang+llvm-12.0.0-x86_64-apple-darwin/bin:$PATH
+          export PATH=`pwd`/clang+llvm-13.0.0-x86_64-apple-darwin/bin:$PATH
           ./genbindings.sh ./ldk-c-bindings/ "-I$JAVA_HOME/include/ -I$JAVA_HOME/include/darwin -isysroot$(xcrun --show-sdk-path)" false false || echo
           cat src/main/resources/liblightningjni_MacOSX-x86_64.nativelib > /dev/null
           if [ "${{ matrix.platform }}" = "macos-11" ]; then
@@ -257,34 +290,35 @@ jobs:
             ./genbindings.sh ./ldk-c-bindings/ "-I$JAVA_HOME/include/ -I$JAVA_HOME/include/darwin -isysroot$(xcrun --show-sdk-path)" false false || echo
             cat src/main/resources/liblightningjni_MacOSX-aarch64.nativelib > /dev/null
           fi
-      - name: Fetch Maven 3.8.3
+      - name: Fetch Maven 3.8.4
         run: |
-          wget -O apache-maven-3.8.3-bin.tar.gz https://apache.osuosl.org/maven/maven-3/3.8.3/binaries/apache-maven-3.8.3-bin.tar.gz
-          if [ "$(shasum -a 256 apache-maven-3.8.3-bin.tar.gz | awk '{ print $1 }')" != "0f1597d11085b8fe93d84652a18c6deea71ece9fabba45a02cf6600c7758fd5b" ]; then
+          # We don't bother using the upstream mirrors as they remove prior
+          # releases aggressively, causing spurious CI failures when we don't
+          # care about the version used.
+          wget -O apache-maven-3.8.4-bin.tar.gz https://bitcoin.ninja/apache-maven-3.8.4-bin.tar.gz
+          if [ "$(shasum -a 256 apache-maven-3.8.4-bin.tar.gz | awk '{ print $1 }')" != "2cdc9c519427bb20fdc25bef5a9063b790e4abd930e7b14b4e9f4863d6f9f13c" ]; then
             echo "Bad hash"
             exit 1
           fi
-          tar xvvf apache-maven-3.8.3-bin.tar.gz
-          export PATH=apache-maven-3.8.3/bin:$PATH
+          tar xvvf apache-maven-3.8.4-bin.tar.gz
+          export PATH=apache-maven-3.8.4/bin:$PATH
       - name: Run Java Tests against built jar
         run: |
-          mvn -DskipTests=true package
+          mvn -q -B -DskipTests=true package
           export LDK_GARBAGECOLLECTED_GIT_OVERRIDE="$(git describe --tag HEAD)"
           JAR_VERSION=${LDK_GARBAGECOLLECTED_GIT_OVERRIDE:1:100}
           mvn install:install-file -Dfile=target/ldk-java-${JAR_VERSION}.jar -DgroupId=org.lightningdevkit -DartifactId=ldk-java -Dversion=1.0-SNAPSHOT -Dpackaging=jar
           cd javatester
-          mvn package
+          mvn -q -B package
           java -ea -jar target/ldk-java-tests-1.0-SNAPSHOT-jar-with-dependencies.jar
           cd ..
       - name: Check latest release libs are in git
         run: |
           if [ "${{ matrix.platform }}" = "macos-11" ]; then
             export LDK_GARBAGECOLLECTED_GIT_OVERRIDE="$(git describe --tag HEAD)"
-            rm "ldk-java-bins/${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}/"liblightningjni_MacOSX-{x86_64,aarch64}.nativelib
-            cp src/main/resources/liblightningjni_MacOSX-*.nativelib "ldk-java-bins/${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}/"
-            cd ldk-java-bins
             # Sadly, OSX binaries are not currently deterministic, more debugging is needed.
-            #git diff --exit-code
+            diff "ldk-java-bins/${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}/"liblightningjni_MacOSX-x86_64.nativelib src/main/resources/liblightningjni_MacOSX-x86_64.nativelib || echo
+            diff "ldk-java-bins/${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}/"liblightningjni_MacOSX-aarch64.nativelib src/main/resources/liblightningjni_MacOSX-aarch64.nativelib || echo
           fi
       - name: Run Java Tests against release bins
         run: |
@@ -292,11 +326,11 @@ jobs:
             export LDK_GARBAGECOLLECTED_GIT_OVERRIDE="$(git describe --tag HEAD)"
             cp "ldk-java-bins/${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}/"liblightningjni_MacOSX-{x86_64,aarch64}.nativelib src/main/resources/
             mvn clean
-            mvn -DskipTests=true package
+            mvn -q -B -DskipTests=true package
             JAR_VERSION=${LDK_GARBAGECOLLECTED_GIT_OVERRIDE:1:100}
             mvn install:install-file -Dfile=target/ldk-java-${JAR_VERSION}.jar -DgroupId=org.lightningdevkit -DartifactId=ldk-java -Dversion=1.0-SNAPSHOT -Dpackaging=jar
             cd javatester
-            mvn package
+            mvn -q -B package
             java -ea -jar target/ldk-java-tests-1.0-SNAPSHOT-jar-with-dependencies.jar
             cd ..
           fi