Merge pull request #22 from TheBlueMatt/main
authorMatt Corallo <649246+TheBlueMatt@users.noreply.github.com>
Wed, 16 Jun 2021 19:26:37 +0000 (19:26 +0000)
committerGitHub <noreply@github.com>
Wed, 16 Jun 2021 19:26:37 +0000 (19:26 +0000)
Support building on OSX

18 files changed:
.github/workflows/build.yml
README.md
genbindings.sh
java_strings.py
javatester/pom.xml [new file with mode: 0644]
javatester/src/main/java/org/ldk/HumanObjectPeerTest.java [new symlink]
liblightningjni.so [deleted symlink]
liblightningjni_debug.so [deleted file]
liblightningjni_release.so [deleted file]
src/main/java/org/ldk/batteries/ChannelManagerConstructor.java
src/main/java/org/ldk/impl/bindings.java
src/main/java/org/ldk/structs/CommonBase.java
src/main/jni/bindings.c
src/main/jni/bindings.c.body
src/main/resources/liblightningjni_Linux-amd64.nativelib [new file with mode: 0755]
src/main/resources/liblightningjni_MacOSX-aarch64.nativelib [new file with mode: 0755]
src/main/resources/liblightningjni_MacOSX-x86_64.nativelib [new file with mode: 0755]
src/test/java/org/ldk/HumanObjectPeerTest.java

index 3145675ac66eaacd43cdbd5a6bdb1d3c38532220..c470471e0b98109aaf804d867c062cdcd1fc8a1c 100644 (file)
@@ -51,13 +51,9 @@ jobs:
         run: ./genbindings.sh ./ldk-c-bindings/ "-I/usr/lib/jvm/java-11-openjdk-amd64/include/ -I/usr/lib/jvm/java-11-openjdk-amd64/include/linux/" true false
       - name: Run Java Tests against Debug Bindings
         run: |
-          rm liblightningjni.so
-          ln -s liblightningjni_debug.so liblightningjni.so
-          export LD_LIBRARY_PATH=.
-          export LD_PRELOAD=/usr/lib/llvm-11/lib/clang/11.0.1/lib/linux/libclang_rt.asan-x86_64.so
+          mv liblightningjni_debug_Linux-amd64.so liblightningjni.so
           export ASAN_OPTIONS=detect_leaks=0
-          mvn test
-          git checkout liblightningjni.so
+          LD_PRELOAD=/usr/lib/llvm-11/lib/clang/11.0.1/lib/linux/libclang_rt.asan-x86_64.so LD_LIBRARY_PATH=. mvn test
       - name: Build Java/TS Release Bindings
         run: |
           # We assume the top commit is just a bindings update commit, so we
@@ -73,8 +69,113 @@ jobs:
           fi
           echo "Using $LDK_GARBAGECOLLECTED_GIT_OVERRIDE as git version"
           ./genbindings.sh ./ldk-c-bindings/ "-I/usr/lib/jvm/java-11-openjdk-amd64/include/ -I/usr/lib/jvm/java-11-openjdk-amd64/include/linux/" false false
-      - name: Check latest headers are in git
+      - name: Check latest headers and release lib are in git
         run: |
-          # For some reason the debug library is not deterministic, this may be fixed in a future rustc
-          git checkout liblightningjni_debug.so
           git diff --exit-code
+      - name: Run Java Tests against built release jar
+        run: |
+          mvn -DskipTests=true package
+          mvn install:install-file -Dfile=target/ldk-java-1.0-SNAPSHOT.jar -DgroupId=org.ldk -DartifactId=ldk-java -Dversion=1.0-SNAPSHOT -Dpackaging=jar
+          cd javatester
+          mvn package
+          java -ea -jar target/ldk-java-tests-1.0-SNAPSHOT-jar-with-dependencies.jar
+          cd ..
+
+  osx:
+    strategy:
+      matrix:
+        include:
+          - platform: macos-10.15
+          # MacOS 11 is currently in private preview, we've applied for access
+          # - platform: macos-11
+    runs-on: ${{ matrix.platform }}
+    env:
+      TOOLCHAIN: stable
+    steps:
+      - name: Install other Rust platforms
+        run: rustup target install aarch64-apple-darwin
+      - name: Fetch upstream LLVM/clang snapshot
+        run: |
+          wget -O clang+llvm-12.0.0-x86_64-apple-darwin.tar.xz https://github.com/llvm/llvm-project/releases/download/llvmorg-12.0.0/clang+llvm-12.0.0-x86_64-apple-darwin.tar.xz
+          if [ "$(shasum -a 256 clang+llvm-12.0.0-x86_64-apple-darwin.tar.xz | awk '{ print $1 }')" != "7bc2259bf75c003f644882460fc8e844ddb23b27236fe43a2787870a4cd8ab50" ]; then
+            echo "Bad hash"
+            exit 1
+          fi
+      - name: Unpack upstream LLVM+clang and use it by default
+        run: |
+          tar xvvf clang+llvm-12.0.0-x86_64-apple-darwin.tar.xz
+      - name: Checkout source code
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+      - name: Install cbindgen
+        run: cargo install --force cbindgen
+      - name: Checkout Rust-Lightning and LDK-C-Bindings git
+        run: |
+          git config --global user.email "ldk-ci@example.com"
+          git config --global user.name "LDK CI"
+          git clone https://github.com/rust-bitcoin/rust-lightning
+          cd rust-lightning
+          git remote add matt https://git.bitcoin.ninja/rust-lightning
+          git fetch matt
+          git merge matt/2021-03-java-bindings-base
+          cd ..
+          git clone https://github.com/lightningdevkit/ldk-c-bindings
+      - name: Rebuild C bindings with upstream clang, and check the sample app builds + links
+        run: |
+          cd ldk-c-bindings
+          export PATH=`pwd`/clang+llvm-12.0.0-x86_64-apple-darwin/bin:$PATH
+          CC=clang ./genbindings.sh ../rust-lightning true
+          cd ..
+      - name: Fetch OpenJDK 16
+        run: |
+          wget -O openjdk-16.0.1_osx-x64_bin.tar.gz https://download.java.net/java/GA/jdk16.0.1/7147401fd7354114ac51ef3e1328291f/9/GPL/openjdk-16.0.1_osx-x64_bin.tar.gz
+          if [ "$(shasum -a 256 openjdk-16.0.1_osx-x64_bin.tar.gz | awk '{ print $1 }')" != "6098f839954439d4916444757c542c1b8778a32461706812d41cc8bbefce7f2f" ]; then
+            echo "Bad hash"
+            exit 1
+          fi
+          tar xvvf openjdk-16.0.1_osx-x64_bin.tar.gz
+          export JAVA_HOME=`pwd`/jdk-16.0.1.jdk/Contents/Home
+          export PATH=$JAVA_HOME/bin:$PATH
+      - name: Build Java/TS Release Bindings
+        run: |
+          # We assume the top commit is just a bindings update commit, so we
+          # check out the previous commit to use as the commit we git describe.
+          # If the top commit is a merge commit, we need to get the last merge
+          # head and assume the latest bindings are built against its parent.
+          COMMIT_PARENTS=$(git show -s --pretty=format:%P HEAD)
+          if [ "${#COMMIT_PARENTS}" = 40 ]; then
+            export LDK_GARBAGECOLLECTED_GIT_OVERRIDE="$(git describe --tag HEAD^1)"
+          else
+            MERGE_HEAD=$(git show --pretty=format:%P HEAD | ( for last in $(cat /dev/stdin); do true; done; echo $last ))
+            export LDK_GARBAGECOLLECTED_GIT_OVERRIDE="$(git describe --tag $MERGE_HEAD^1)"
+          fi
+          echo "Using $LDK_GARBAGECOLLECTED_GIT_OVERRIDE as git version"
+          # genbindings.sh always fails as there is no wasm32-wasi library
+          # available, so instead we delete the expected JNI library and check
+          # that it was created.
+          rm src/main/resources/liblightningjni_MacOSX-x86_64.nativelib
+          ./genbindings.sh ./ldk-c-bindings/ "-I$JAVA_HOME/include/ -I$JAVA_HOME/include/darwin -isysroot$(xcrun --show-sdk-path)" false false || echo
+          cat src/main/resources/liblightningjni_MacOSX-x86_64.nativelib > /dev/null
+      - name: Fetch Maven 3.8.1
+        run: |
+          wget -O apache-maven-3.8.1-bin.tar.gz https://apache.osuosl.org/maven/maven-3/3.8.1/binaries/apache-maven-3.8.1-bin.tar.gz
+          if [ "$(shasum -a 256 apache-maven-3.8.1-bin.tar.gz | awk '{ print $1 }')" != "b98a1905eb554d07427b2e5509ff09bd53e2f1dd7a0afa38384968b113abef02" ]; then
+            echo "Bad hash"
+            exit 1
+          fi
+          tar xvvf apache-maven-3.8.1-bin.tar.gz
+          export PATH=apache-maven-3.8.1/bin:$PATH
+      - name: Run Java Tests against built jar
+        run: |
+          mvn -DskipTests=true package
+          mvn install:install-file -Dfile=target/ldk-java-1.0-SNAPSHOT.jar -DgroupId=org.ldk -DartifactId=ldk-java -Dversion=1.0-SNAPSHOT -Dpackaging=jar
+          cd javatester
+          mvn package
+          java -ea -jar target/ldk-java-tests-1.0-SNAPSHOT-jar-with-dependencies.jar
+          cd ..
+      - name: Check latest headers and release lib are in git
+        run: |
+          if [ "${{ matrix.platform }}" = "macos-11" ]; then
+            git diff --exit-code
+          fi
index daa27633f5644ee63f0d4cebfed9828153d07b2d..8fe3e29e6d1705803d84e36aa9bb72303a693ccc 100644 (file)
--- a/README.md
+++ b/README.md
@@ -25,6 +25,17 @@ bindings should be built with workarounds required for Android. JNI CFLAGS on de
 When running a program linking against the library in debug mode, LD_PRELOAD should likely include
 the relevant `libclang_rt.asan-platform.so` path.
 
+Note that building with address sanitizer is only supported on MacOS with upstream clang (ie where
+the LLVM version in use matches the LLVM version against which rustc was built), not with Apple clang.
+Builds with Apple clang will work fine, but largely only be useful in a release context.
+To build on Mac with address sanitizer, you will need to run `ldk-c-bindings`' `genbindings.sh`
+script with upstream clang in your PATH and likely replace your $JAVA_HOME/bin/java with a simple
+wrapper which calls java after an export like:
+`export DYLD_INSERT_LIBRARIES=/path/to/upstream/llvm/lib/clang/12.0.0/lib/darwin/libclang_rt.asan_osx_dynamic.dylib`
+
+To build for Apple M1 (ie aarch64-apple-darwin), you probably want something like
+`CC="clang --target=aarch64-apple-darwin" LDK_TARGET=aarch64-apple-darwin LDK_TARGET_CPU=generic ./genbindings.sh ...`
+
 Status
 ======
 
index c10ff2412a49e9e34f2681037d4ef8761b79002a..6a4109c8f447b0a909e3ce0e2562e058231db700 100755 (executable)
@@ -10,15 +10,30 @@ usage() {
 [ "$3" != "true" -a "$3" != "false" ] && usage
 [ "$4" != "true" -a "$4" != "false" ] && usage
 
+set -x
+
 if [ "$CC" != "" ]; then
        COMMON_COMPILE="$CC -std=c11 -Wall -Wextra -Wno-unused-parameter -Wno-ignored-qualifiers -Wno-unused-function -Wno-nullability-completeness -Wno-pointer-sign -Wdate-time -ffile-prefix-map=$(pwd)="
 else
+       CC=clang
        COMMON_COMPILE="clang -std=c11 -Wall -Wextra -Wno-unused-parameter -Wno-ignored-qualifiers -Wno-unused-function -Wno-nullability-completeness -Wno-pointer-sign -Wdate-time -ffile-prefix-map=$(pwd)="
 fi
 
-if [ "$LDK_TARGET" != "" ]; then
-       LDK_TARGET_SUFFIX="_$LDK_TARGET"
+TARGET_STRING="$LDK_TARGET"
+if [ "$TARGET_STRING" = "" ]; then
+       # We assume clang-style $CC --version here, but worst-case we just get an empty suffix
+       TARGET_STRING="$($CC --version | grep Target | awk '{ print $2 }')"
 fi
+case "$TARGET_STRING" in
+       "x86_64-pc-linux"*)
+               LDK_TARGET_SUFFIX="_Linux-amd64" ;;
+       "x86_64-apple-darwin"*)
+               LDK_TARGET_SUFFIX="_MacOSX-x86_64" ;;
+       "aarch64-apple-darwin"*)
+               LDK_TARGET_SUFFIX="_MacOSX-aarch64" ;;
+       *)
+               LDK_TARGET_SUFFIX=""
+esac
 if [ "$LDK_TARGET_CPU" = "" ]; then
        LDK_TARGET_CPU="sandybridge"
 fi
@@ -52,12 +67,23 @@ cat src/main/jni/bindings.c.body >> src/main/jni/bindings.c
 javac -h src/main/jni src/main/java/org/ldk/enums/*.java src/main/java/org/ldk/impl/bindings.java
 rm src/main/java/org/ldk/enums/*.class src/main/java/org/ldk/impl/bindings*.class
 
+IS_MAC=false
+[ "$($CC --version | grep apple-darwin)" != "" ] && IS_MAC=true
+
 echo "Building Java bindings..."
-COMPILE="$COMMON_COMPILE -mcpu=$LDK_TARGET_CPU -Isrc/main/jni -pthread -ldl -Wl,--no-undefined -shared -fPIC"
+COMPILE="$COMMON_COMPILE -mcpu=$LDK_TARGET_CPU -Isrc/main/jni -pthread -ldl -shared -fPIC"
+[ "$IS_MAC" = "false" ] && COMPILE="$COMPILE -Wl,--no-undefined"
+[ "$IS_MAC" = "true" ] && COMPILE="$COMPILE -mmacosx-version-min=10.9"
 if [ "$3" = "true" ]; then
-       $COMPILE -o liblightningjni_debug$LDK_TARGET_SUFFIX.so -g -fsanitize=address -shared-libasan -Wl,-wrap,calloc -Wl,-wrap,realloc -Wl,-wrap,reallocarray -Wl,-wrap,malloc -Wl,-wrap,free -rdynamic -I"$1"/lightning-c-bindings/include/ $2 src/main/jni/bindings.c "$1"/lightning-c-bindings/target/$LDK_TARGET/debug/libldk.a -lm
+       [ "$IS_MAC" = "false" ] && COMPILE="$COMPILE -Wl,-wrap,calloc -Wl,-wrap,realloc -Wl,-wrap,reallocarray -Wl,-wrap,malloc -Wl,-wrap,free"
+       $COMPILE -o liblightningjni_debug$LDK_TARGET_SUFFIX.so -g -fsanitize=address -shared-libasan -rdynamic -I"$1"/lightning-c-bindings/include/ $2 src/main/jni/bindings.c "$1"/lightning-c-bindings/target/$LDK_TARGET/debug/libldk.a -lm
 else
-       $COMPILE -o liblightningjni_release$LDK_TARGET_SUFFIX.so -Wl,--version-script=libcode.version -flto -fuse-ld=lld -O3 -I"$1"/lightning-c-bindings/include/ $2 src/main/jni/bindings.c "$1"/lightning-c-bindings/target/$LDK_TARGET/release/libldk.a
+       [ "$IS_MAC" = "false" ] && COMPILE="$COMPILE -Wl,--version-script=libcode.version -fuse-ld=lld"
+       $COMPILE -o liblightningjni_release$LDK_TARGET_SUFFIX.so -flto -O3 -I"$1"/lightning-c-bindings/include/ $2 src/main/jni/bindings.c "$1"/lightning-c-bindings/target/$LDK_TARGET/release/libldk.a
+       if [ "$LDK_TARGET_SUFFIX" != "" ]; then
+               # Copy to JNI native directory for inclusion in JARs
+               cp liblightningjni_release$LDK_TARGET_SUFFIX.so src/main/resources/liblightningjni$LDK_TARGET_SUFFIX.nativelib
+       fi
 fi
 
 echo "Creating TS bindings..."
index 3d5a2a0150cd1c5680d6a51495e27183a93e5ba9..7f82567ef408adbb16ab25c58bb0ac857d026cc7 100644 (file)
@@ -1,5 +1,6 @@
 from bindingstypes import *
 from enum import Enum
+import sys
 
 class Target(Enum):
     JAVA = 1,
@@ -23,6 +24,12 @@ class Consts:
 
         self.bindings_header = """package org.ldk.impl;
 import org.ldk.enums.*;
+import java.io.File;
+import java.io.InputStream;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardCopyOption;
 
 public class bindings {
        public static class VecOrSliceDef {
@@ -34,7 +41,24 @@ public class bindings {
                }
        }
        static {
-               System.loadLibrary(\"lightningjni\");
+               try {
+                       // Try to load natively first, this works on Android and in testing.
+                       System.loadLibrary(\"lightningjni\");
+               } catch (UnsatisfiedLinkError _ignored) {
+                       // Otherwise try to load from the library jar.
+                       File tmpdir = new File(System.getProperty("java.io.tmpdir"), "ldk-java-nativelib");
+                       tmpdir.mkdir(); // If it fails to create, assume it was there already
+                       tmpdir.deleteOnExit();
+                       String libname = "liblightningjni_" + System.getProperty("os.name").replaceAll(" ", "") +
+                               "-" + System.getProperty("os.arch").replaceAll(" ", "") + ".nativelib";
+                       try (InputStream is = bindings.class.getResourceAsStream("/" + libname)) {
+                               Path libpath = new File(tmpdir.toPath().toString(), "liblightningjni.so").toPath();
+                               Files.copy(is, libpath, StandardCopyOption.REPLACE_EXISTING);
+                               Runtime.getRuntime().load(libpath.toString());
+                       } catch (IOException e) {
+                               throw new IllegalArgumentException(e);
+                       }
+               }
                init(java.lang.Enum.class, VecOrSliceDef.class);
                init_class_cache();
                if (!get_lib_version_string().equals(get_ldk_java_bindings_version()))
@@ -83,11 +107,13 @@ class CommonBase {
        long ptr;
        LinkedList<Object> ptrs_to = new LinkedList();
        protected CommonBase(long ptr) { this.ptr = ptr; }
-       public long _test_only_get_ptr() { return this.ptr; }
 }
 """
 
-        self.c_file_pfx = """#include \"org_ldk_impl_bindings.h\"
+        self.c_file_pfx = """#include <jni.h>
+// On OSX jlong (ie long long) is not equivalent to int64_t, so we override here
+#define int64_t jlong
+#include \"org_ldk_impl_bindings.h\"
 #include <lightning.h>
 #include <string.h>
 #include <stdatomic.h>
@@ -124,13 +150,15 @@ void __attribute__((constructor)) spawn_stderr_redirection() {
         else:
             self.c_file_pfx = self.c_file_pfx + "#define DEBUG_PRINT(...) fprintf(stderr, __VA_ARGS__)\n"
 
-        if not DEBUG:
+        if not DEBUG or sys.platform == "darwin":
             self.c_file_pfx = self.c_file_pfx + """#define MALLOC(a, _) malloc(a)
 #define FREE(p) if ((uint64_t)(p) > 1024) { free(p); }
-#define DO_ASSERT(a) (void)(a)
+"""
+        if not DEBUG:
+            self.c_file_pfx += """#define DO_ASSERT(a) (void)(a)
 #define CHECK(a)
 """
-        else:
+        if DEBUG:
             self.c_file_pfx = self.c_file_pfx + """#include <assert.h>
 // Always run a, then assert it is true:
 #define DO_ASSERT(a) do { bool _assert_val = (a); assert(_assert_val); } while(0)
@@ -144,7 +172,10 @@ void __attribute__((constructor)) debug_log_version() {
                DEBUG_PRINT("LDK C Bindings version did not match the header we built against\\n");
        DEBUG_PRINT("Loaded LDK-Java Bindings with LDK %s and LDK-C-Bindings %s\\n", check_get_ldk_version(), check_get_ldk_bindings_version());
 }
+"""
 
+            if sys.platform != "darwin":
+                self.c_file_pfx += """
 // Running a leak check across all the allocations and frees of the JDK is a mess,
 // so instead we implement our own naive leak checker here, relying on the -wrap
 // linker option to wrap malloc/calloc/realloc/free, tracking everyhing allocated
@@ -152,8 +183,8 @@ void __attribute__((constructor)) debug_log_version() {
 #include <threads.h>
 """
 
-            if self.target == Target.ANDROID:
-                self.c_file_pfx = self.c_file_pfx + """
+                if self.target == Target.ANDROID:
+                    self.c_file_pfx = self.c_file_pfx + """
 #include <unwind.h>
 #include <dlfcn.h>
 
@@ -194,9 +225,9 @@ void backtrace_symbols_fd(void ** buffer, int count, int _fd) {
        }
 }
 """
-            else:
-                self.c_file_pfx = self.c_file_pfx + "#include <execinfo.h>\n"
-            self.c_file_pfx = self.c_file_pfx + """
+                else:
+                    self.c_file_pfx = self.c_file_pfx + "#include <execinfo.h>\n"
+                self.c_file_pfx = self.c_file_pfx + """
 #include <unistd.h>
 static mtx_t allocation_mtx;
 
diff --git a/javatester/pom.xml b/javatester/pom.xml
new file mode 100644 (file)
index 0000000..02137e3
--- /dev/null
@@ -0,0 +1,68 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <groupId>org.ldk</groupId>
+    <artifactId>ldk-java-tests</artifactId>
+    <version>1.0-SNAPSHOT</version>
+       <dependencies>
+        <dependency>
+            <groupId>org.junit.jupiter</groupId>
+            <artifactId>junit-jupiter-api</artifactId>
+            <version>RELEASE</version>
+            <scope>compile</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.bitcoinj</groupId>
+            <artifactId>bitcoinj-core</artifactId>
+            <version>0.15.10</version>
+            <scope>compile</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.jetbrains</groupId>
+            <artifactId>annotations</artifactId>
+            <version>RELEASE</version>
+            <scope>compile</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.ldk</groupId>
+            <artifactId>ldk-java</artifactId>
+            <version>1.0-SNAPSHOT</version>
+            <scope>compile</scope>
+        </dependency>
+    </dependencies>
+    <properties>
+        <maven.compiler.source>1.8</maven.compiler.source>
+        <maven.compiler.target>1.8</maven.compiler.target>
+    </properties>
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-assembly-plugin</artifactId>
+                <version>3.2.0</version>
+                <executions>
+                    <execution>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>single</goal>
+                        </goals>
+                        <configuration>
+                            <archive>
+                                <manifest>
+                                    <mainClass>org.ldk.HumanObjectPeerTest</mainClass>
+                                </manifest>
+                            </archive>
+                            <descriptorRefs>
+                                <descriptorRef>jar-with-dependencies</descriptorRef>
+                            </descriptorRefs>
+                        </configuration>
+                    </execution>
+                 </executions>
+             </plugin>
+        </plugins>
+    </build>
+</project>
diff --git a/javatester/src/main/java/org/ldk/HumanObjectPeerTest.java b/javatester/src/main/java/org/ldk/HumanObjectPeerTest.java
new file mode 120000 (symlink)
index 0000000..59d3673
--- /dev/null
@@ -0,0 +1 @@
+../../../../../../src/test/java/org/ldk/HumanObjectPeerTest.java
\ No newline at end of file
diff --git a/liblightningjni.so b/liblightningjni.so
deleted file mode 120000 (symlink)
index 1b6bb88..0000000
+++ /dev/null
@@ -1 +0,0 @@
-liblightningjni_release.so
\ No newline at end of file
diff --git a/liblightningjni_debug.so b/liblightningjni_debug.so
deleted file mode 100755 (executable)
index 8f58fe0..0000000
Binary files a/liblightningjni_debug.so and /dev/null differ
diff --git a/liblightningjni_release.so b/liblightningjni_release.so
deleted file mode 100755 (executable)
index e30297b..0000000
Binary files a/liblightningjni_release.so and /dev/null differ
index d55854084053b629f50e2de311fe8a9c04b1c37b..da5e3fb9c6f3bf9c2ab33200d7b1e80df5bfc5d3 100644 (file)
@@ -88,7 +88,11 @@ public class ChannelManagerConstructor {
             this.peer_manager = PeerManager.of(channel_manager.as_ChannelMessageHandler(), (IgnoringMessageHandler.of()).as_RoutingMessageHandler(), keys_interface.get_node_secret(), random_data, logger);
         }
         NioPeerHandler nio_peer_handler = null;
-        try { nio_peer_handler = new NioPeerHandler(this.peer_manager); } catch (IOException e) { assert false; }
+        try {
+            nio_peer_handler = new NioPeerHandler(this.peer_manager);
+        } catch (IOException e) {
+            throw new IllegalStateException("We should never fail to construct nio objects unless we're on a platform that cannot run LDK.");
+        }
         this.nio_peer_handler = nio_peer_handler;
         if (filter != null) {
             for (ChannelMonitor monitor : monitors) {
@@ -118,7 +122,11 @@ public class ChannelManagerConstructor {
             this.peer_manager = PeerManager.of(channel_manager.as_ChannelMessageHandler(), (IgnoringMessageHandler.of()).as_RoutingMessageHandler(), keys_interface.get_node_secret(), random_data, logger);
         }
         NioPeerHandler nio_peer_handler = null;
-        try { nio_peer_handler = new NioPeerHandler(this.peer_manager); } catch (IOException e) { assert false; }
+        try {
+            nio_peer_handler = new NioPeerHandler(this.peer_manager);
+        } catch (IOException e) {
+            throw new IllegalStateException("We should never fail to construct nio objects unless we're on a platform that cannot run LDK.");
+        }
         this.nio_peer_handler = nio_peer_handler;
     }
 
index 511f6839d654b27c996e47797530406319463a2a..49f921d2deef9cb8a24e88cc4683bb69dd874e28 100644 (file)
@@ -1,5 +1,11 @@
 package org.ldk.impl;
 import org.ldk.enums.*;
+import java.io.File;
+import java.io.InputStream;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardCopyOption;
 
 public class bindings {
        public static class VecOrSliceDef {
@@ -11,7 +17,24 @@ public class bindings {
                }
        }
        static {
-               System.loadLibrary("lightningjni");
+               try {
+                       // Try to load natively first, this works on Android and in testing.
+                       System.loadLibrary("lightningjni");
+               } catch (UnsatisfiedLinkError _ignored) {
+                       // Otherwise try to load from the library jar.
+                       File tmpdir = new File(System.getProperty("java.io.tmpdir"), "ldk-java-nativelib");
+                       tmpdir.mkdir(); // If it fails to create, assume it was there already
+                       tmpdir.deleteOnExit();
+                       String libname = "liblightningjni_" + System.getProperty("os.name").replaceAll(" ", "") +
+                               "-" + System.getProperty("os.arch").replaceAll(" ", "") + ".nativelib";
+                       try (InputStream is = bindings.class.getResourceAsStream("/" + libname)) {
+                               Path libpath = new File(tmpdir.toPath().toString(), "liblightningjni.so").toPath();
+                               Files.copy(is, libpath, StandardCopyOption.REPLACE_EXISTING);
+                               Runtime.getRuntime().load(libpath.toString());
+                       } catch (IOException e) {
+                               throw new IllegalArgumentException(e);
+                       }
+               }
                init(java.lang.Enum.class, VecOrSliceDef.class);
                init_class_cache();
                if (!get_lib_version_string().equals(get_ldk_java_bindings_version()))
@@ -25,7 +48,7 @@ public class bindings {
        static native String get_lib_version_string();
 
        public static String get_ldk_java_bindings_version() {
-               return "v0.0.98.1";
+               return "v0.0.98.2";
        }
        public static native String get_ldk_c_bindings_version();
        public static native String get_ldk_version();
index acff6661da49875fe3379c8c4dab9052827246ab..1794158b79f8e19164e3c2a838a0926c7fd13d62 100644 (file)
@@ -4,5 +4,4 @@ class CommonBase {
        long ptr;
        LinkedList<Object> ptrs_to = new LinkedList();
        protected CommonBase(long ptr) { this.ptr = ptr; }
-       public long _test_only_get_ptr() { return this.ptr; }
 }
index 4d1fdc889cd1581fa311fa447acc85377b1e796f..68b11b177756f43799db67a96cb041b397c18d47 100644 (file)
@@ -1,5 +1,8 @@
 #define LDKCVec_C2Tuple_TxidCVec_C2Tuple_u32TxOutZZZZ LDKCVec_TransactionOutputsZ
 #define CVec_C2Tuple_TxidCVec_C2Tuple_u32TxOutZZZZ_free CVec_TransactionOutputsZ_free
+#include <jni.h>
+// On OSX jlong (ie long long) is not equivalent to int64_t, so we override here
+#define int64_t jlong
 #include "org_ldk_impl_bindings.h"
 #include <lightning.h>
 #include <string.h>
@@ -130,7 +133,7 @@ static inline LDKStr java_to_owned_str(JNIEnv *env, jstring str) {
 }
 
 JNIEXPORT jstring JNICALL Java_org_ldk_impl_bindings_get_1lib_1version_1string(JNIEnv *env, jclass _c) {
-       return str_ref_to_java(env, "v0.0.98.1", strlen("v0.0.98.1"));
+       return str_ref_to_java(env, "v0.0.98.2", strlen("v0.0.98.2"));
 }
 JNIEXPORT jstring JNICALL Java_org_ldk_impl_bindings_get_1ldk_1c_1bindings_1version(JNIEnv *env, jclass _c) {
        return str_ref_to_java(env, check_get_ldk_bindings_version(), strlen(check_get_ldk_bindings_version()));
index 95ae6d52afd2d554cf56da72af27a197751d4ec8..e3fe4d198790c9cda3d0085c41aaad73c4a2c6ab 100644 (file)
@@ -1,3 +1,6 @@
+#include <jni.h>
+// On OSX jlong (ie long long) is not equivalent to int64_t, so we override here
+#define int64_t jlong
 #include "org_ldk_impl_bindings.h"
 #include <lightning.h>
 #include <string.h>
@@ -128,7 +131,7 @@ static inline LDKStr java_to_owned_str(JNIEnv *env, jstring str) {
 }
 
 JNIEXPORT jstring JNICALL Java_org_ldk_impl_bindings_get_1lib_1version_1string(JNIEnv *env, jclass _c) {
-       return str_ref_to_java(env, "v0.0.98.1", strlen("v0.0.98.1"));
+       return str_ref_to_java(env, "v0.0.98.2", strlen("v0.0.98.2"));
 }
 JNIEXPORT jstring JNICALL Java_org_ldk_impl_bindings_get_1ldk_1c_1bindings_1version(JNIEnv *env, jclass _c) {
        return str_ref_to_java(env, check_get_ldk_bindings_version(), strlen(check_get_ldk_bindings_version()));
diff --git a/src/main/resources/liblightningjni_Linux-amd64.nativelib b/src/main/resources/liblightningjni_Linux-amd64.nativelib
new file mode 100755 (executable)
index 0000000..8acc71a
Binary files /dev/null and b/src/main/resources/liblightningjni_Linux-amd64.nativelib differ
diff --git a/src/main/resources/liblightningjni_MacOSX-aarch64.nativelib b/src/main/resources/liblightningjni_MacOSX-aarch64.nativelib
new file mode 100755 (executable)
index 0000000..06f615e
Binary files /dev/null and b/src/main/resources/liblightningjni_MacOSX-aarch64.nativelib differ
diff --git a/src/main/resources/liblightningjni_MacOSX-x86_64.nativelib b/src/main/resources/liblightningjni_MacOSX-x86_64.nativelib
new file mode 100755 (executable)
index 0000000..2c5b8ca
Binary files /dev/null and b/src/main/resources/liblightningjni_MacOSX-x86_64.nativelib differ
index 9d6acedb0d994222c9b1206590ba479e6bc1f3e8..c6a6f475fbf969fbcb174e0e7f479f7de8d08f76 100644 (file)
@@ -7,7 +7,6 @@ import org.ldk.batteries.ChannelManagerConstructor;
 import org.ldk.batteries.NioPeerHandler;
 import org.ldk.enums.Currency;
 import org.ldk.enums.Network;
-import org.ldk.impl.bindings;
 import org.ldk.structs.*;
 import org.ldk.util.TwoTuple;
 
@@ -202,7 +201,10 @@ class HumanObjectPeerTestInstance {
             logger = Logger.new_impl((String arg) -> System.out.println(seed + ": " + arg));
             fee_estimator = FeeEstimator.new_impl((confirmation_target -> 253));
             tx_broadcaster = BroadcasterInterface.new_impl(tx -> {
-                broadcast_set.add(tx);
+                synchronized (broadcast_set) {
+                    broadcast_set.add(tx);
+                    broadcast_set.notifyAll();
+                }
             });
             monitors = new HashMap<>();
             this.seed = seed;
@@ -418,12 +420,16 @@ class HumanObjectPeerTestInstance {
             if (use_chan_manager_constructor) {
                 while (true) {
                     synchronized (this.pending_manager_events) {
-                        if (expected_len != 0 && this.pending_manager_events.size() != expected_len) {
+                        if (expected_len != 0 && this.pending_manager_events.size() == expected_len) {
                             break;
                         }
                     }
-                    try { Thread.sleep(500); } catch (InterruptedException e) { assert false; }
-                    break;
+                    if (expected_len == 0) {
+                        try { Thread.sleep(500); } catch (InterruptedException e) { assert false; }
+                        break;
+                    } else {
+                        Thread.yield();
+                    }
                 }
                 synchronized (this.pending_manager_events) {
                     Event[] res = this.pending_manager_events.toArray(new Event[0]);
@@ -449,7 +455,11 @@ class HumanObjectPeerTestInstance {
             }
         }
 
-        Event[] get_manager_events(int expected_len) {
+        Event[] get_manager_events(int expected_len, Peer peer1, Peer peer2) {
+            assert expected_len != 0;
+            if (!use_nio_peer_handler) {
+                maybe_exchange_peer_messages(peer1, peer2);
+            }
             Event[] res = new Event[0];
             if (use_chan_manager_constructor) {
                 while (res.length < expected_len) {
@@ -464,7 +474,15 @@ class HumanObjectPeerTestInstance {
                 }
             } else {
                 ArrayList<Event> l = new ArrayList<Event>();
-                chan_manager.as_EventsProvider().process_pending_events(EventHandler.new_impl(l::add));
+                while (l.size() < expected_len) {
+                    Thread.yield();
+                    if (use_nio_peer_handler) {
+                        peer1.nio_peer_handler.check_events();
+                        peer2.nio_peer_handler.check_events();
+                    }
+                    chan_manager.as_EventsProvider().process_pending_events(EventHandler.new_impl(l::add));
+                    assert l.size() == expected_len || l.size() == 0; // We don't handle partial results
+                }
                 return l.toArray(new Event[0]);
             }
             assert res.length == expected_len;
@@ -498,12 +516,9 @@ class HumanObjectPeerTestInstance {
                 }
             }
     });
-    void wait_events_processed(Peer peer1, Peer peer2) {
-        if (use_nio_peer_handler) {
-            peer1.nio_peer_handler.check_events();
-            peer2.nio_peer_handler.check_events();
-            try { Thread.sleep(500); } catch (InterruptedException e) { assert false; }
-        } else {
+
+    void maybe_exchange_peer_messages(Peer peer1, Peer peer2) {
+        if (!use_nio_peer_handler) {
             synchronized (runqueue) {
                 ran = false;
             }
@@ -520,6 +535,9 @@ class HumanObjectPeerTestInstance {
                     try { runqueue.wait(); } catch (InterruptedException e) { assert false; }
                 }
             }
+        } else if (!use_chan_manager_constructor) {
+            peer1.nio_peer_handler.check_events();
+            peer2.nio_peer_handler.check_events();
         }
     }
     void do_read_event(PeerManager pm, SocketDescriptor descriptor, byte[] data) {
@@ -540,6 +558,9 @@ class HumanObjectPeerTestInstance {
             try {
                 peer1.nio_peer_handler.connect(peer2.chan_manager.get_our_node_id(), new InetSocketAddress("127.0.0.1", peer2.nio_port), 100);
             } catch (IOException e) { assert false; }
+            while (peer1.peer_manager.get_peer_node_ids().length == 0 || peer2.peer_manager.get_peer_node_ids().length == 0) {
+                Thread.yield();
+            }
         } else {
             DescriptorHolder descriptor1 = new DescriptorHolder();
             DescriptorHolder descriptor1ref = descriptor1;
@@ -573,6 +594,8 @@ class HumanObjectPeerTestInstance {
             Result_NonePeerHandleErrorZ inbound_conn_res = peer2.peer_manager.new_inbound_connection(descriptor2);
             assert inbound_conn_res instanceof Result_NonePeerHandleErrorZ.Result_NonePeerHandleErrorZ_OK;
             do_read_event(peer2.peer_manager, descriptor2, ((Result_CVec_u8ZPeerHandleErrorZ.Result_CVec_u8ZPeerHandleErrorZ_OK) conn_res).res);
+
+            maybe_exchange_peer_messages(peer1, peer2);
         }
     }
 
@@ -581,13 +604,11 @@ class HumanObjectPeerTestInstance {
         Peer peer2 = new Peer((byte) 2);
 
         connect_peers(peer1, peer2);
-        wait_events_processed(peer1, peer2);
 
         Result_NoneAPIErrorZ cc_res = peer1.chan_manager.create_channel(peer2.node_id, 10000, 1000, 42, null);
         assert cc_res instanceof Result_NoneAPIErrorZ.Result_NoneAPIErrorZ_OK;
-        wait_events_processed(peer1, peer2);
 
-        Event[] events = peer1.get_manager_events(1);
+        Event[] events = peer1.get_manager_events(1, peer1, peer2);
         assert events[0] instanceof Event.FundingGenerationReady;
         assert ((Event.FundingGenerationReady) events[0]).channel_value_satoshis == 10000;
         assert ((Event.FundingGenerationReady) events[0]).user_channel_id == 42;
@@ -604,7 +625,13 @@ class HumanObjectPeerTestInstance {
         funding.addOutput(Coin.SATOSHI.multiply(10000), new Script(funding_spk));
         Result_NoneAPIErrorZ funding_res = peer1.chan_manager.funding_transaction_generated(chan_id, funding.bitcoinSerialize());
         assert funding_res instanceof Result_NoneAPIErrorZ.Result_NoneAPIErrorZ_OK;
-        wait_events_processed(peer1, peer2);
+
+        maybe_exchange_peer_messages(peer1, peer2);
+        synchronized (peer1.broadcast_set) {
+            while (peer1.broadcast_set.size() != 1) {
+                peer1.broadcast_set.wait();
+            }
+        }
 
         assert peer1.broadcast_set.size() == 1;
         assert Arrays.equals(peer1.broadcast_set.get(0), funding.bitcoinSerialize());
@@ -619,7 +646,9 @@ class HumanObjectPeerTestInstance {
             peer1.connect_block(b, height, 0);
             peer2.connect_block(b, height, 0);
         }
-        wait_events_processed(peer1, peer2);
+
+        maybe_exchange_peer_messages(peer1, peer2);
+        while (peer1.chan_manager.list_usable_channels().length != 1 || peer2.chan_manager.list_usable_channels().length != 1)
 
         peer1.chan_manager.list_channels();
         ChannelDetails[] peer1_chans = peer1.chan_manager.list_usable_channels();
@@ -664,7 +693,6 @@ class HumanObjectPeerTestInstance {
 
         Result_NonePaymentSendFailureZ payment_res = peer1.chan_manager.send_payment(route, payment_hash, payment_secret);
         assert payment_res instanceof Result_NonePaymentSendFailureZ.Result_NonePaymentSendFailureZ_OK;
-        wait_events_processed(peer1, peer2);
 
         RouteHop[][] hops = new RouteHop[1][1];
         byte[] hop_pubkey = new byte[33];
@@ -675,21 +703,13 @@ class HumanObjectPeerTestInstance {
         payment_res = peer1.chan_manager.send_payment(r2, payment_hash, payment_secret);
         assert payment_res instanceof Result_NonePaymentSendFailureZ.Result_NonePaymentSendFailureZ_Err;
 
-        if (!use_chan_manager_constructor) {
-            peer1.get_monitor_events(0);
-            peer2.get_monitor_events(0);
-        } else {
-            // The events are combined across manager + monitors but peer1 still has no events
-        }
-
         if (reload_peers) {
-            if (use_nio_peer_handler) {
-                peer1.nio_peer_handler.interrupt();
-                peer2.nio_peer_handler.interrupt();
-            }
             if (use_chan_manager_constructor) {
                 peer1.constructor.interrupt();
                 peer2.constructor.interrupt();
+            } else if (use_nio_peer_handler) {
+                peer1.nio_peer_handler.interrupt();
+                peer2.nio_peer_handler.interrupt();
             }
             WeakReference<Peer> op1 = new WeakReference<Peer>(peer1);
             peer1 = new Peer(peer1);
@@ -713,7 +733,7 @@ class HumanObjectPeerTestInstance {
             this.best_blockhash = best_blockhash;
         }
     }
-    void do_test_message_handler_b(TestState state) {
+    void do_test_message_handler_b(TestState state) throws InterruptedException {
         GcCheck obj = new GcCheck();
         if (state.ref_block != null) {
             // Ensure the original peers get freed before we move on. Note that we have to be in a different function
@@ -724,36 +744,52 @@ class HumanObjectPeerTestInstance {
             }
             connect_peers(state.peer1, state.peer2);
         }
-        wait_events_processed(state.peer1, state.peer2);
 
-        Event[] events = state.peer2.get_manager_events(1);
+        Event[] events = state.peer2.get_manager_events(1, state.peer1, state.peer2);
         assert events[0] instanceof Event.PendingHTLCsForwardable;
         state.peer2.chan_manager.process_pending_htlc_forwards();
 
-        events = state.peer2.get_manager_events(1);
+        events = state.peer2.get_manager_events(1, state.peer1, state.peer2);
         assert events[0] instanceof Event.PaymentReceived;
         byte[] payment_preimage = ((Event.PaymentReceived)events[0]).payment_preimage;
         assert !Arrays.equals(payment_preimage, new byte[32]);
         state.peer2.chan_manager.claim_funds(payment_preimage);
-        wait_events_processed(state.peer1, state.peer2);
 
-        events = state.peer1.get_manager_events(1);
+        events = state.peer1.get_manager_events(1, state.peer1, state.peer2);
         assert events[0] instanceof Event.PaymentSent;
         assert Arrays.equals(((Event.PaymentSent) events[0]).payment_preimage, payment_preimage);
-        wait_events_processed(state.peer1, state.peer2);
+
+        if (use_nio_peer_handler) {
+            // We receive PaymentSent immediately upon receipt of the payment preimage, but we expect to not have an
+            // HTLC transaction to broadcast below, which requires a bit more time to fully complete the
+            // commitment-transaction-update dance between both peers.
+            Thread.sleep(100);
+        }
 
         ChannelDetails[] peer1_chans = state.peer1.chan_manager.list_channels();
 
         if (nice_close) {
             Result_NoneAPIErrorZ close_res = state.peer1.chan_manager.close_channel(peer1_chans[0].get_channel_id());
             assert close_res instanceof Result_NoneAPIErrorZ.Result_NoneAPIErrorZ_OK;
-            wait_events_processed(state.peer1, state.peer2);
+            maybe_exchange_peer_messages(state.peer1, state.peer2);
+            synchronized (state.peer1.broadcast_set) {
+                while (state.peer1.broadcast_set.size() != 1) state.peer1.broadcast_set.wait();
+            }
+            synchronized (state.peer2.broadcast_set) {
+                while (state.peer2.broadcast_set.size() != 1) state.peer2.broadcast_set.wait();
+            }
 
             assert state.peer1.broadcast_set.size() == 1;
             assert state.peer2.broadcast_set.size() == 1;
         } else {
             state.peer1.chan_manager.force_close_all_channels();
-            wait_events_processed(state.peer1, state.peer2);
+            maybe_exchange_peer_messages(state.peer1, state.peer2);
+            synchronized (state.peer1.broadcast_set) {
+                while (state.peer1.broadcast_set.size() != 1) state.peer1.broadcast_set.wait();
+            }
+            synchronized (state.peer2.broadcast_set) {
+                while (state.peer2.broadcast_set.size() != 1) state.peer2.broadcast_set.wait();
+            }
 
             assert state.peer1.broadcast_set.size() == 1;
             assert state.peer2.broadcast_set.size() == 1;
@@ -794,9 +830,8 @@ class HumanObjectPeerTestInstance {
 
         if (use_nio_peer_handler) {
             state.peer1.peer_manager.disconnect_by_node_id(state.peer2.chan_manager.get_our_node_id(), false);
-            wait_events_processed(state.peer1, state.peer2);
-            assert state.peer1.peer_manager.get_peer_node_ids().length == 0;
-            assert state.peer2.peer_manager.get_peer_node_ids().length == 0;
+            while (state.peer1.peer_manager.get_peer_node_ids().length != 0) Thread.yield();
+            while (state.peer2.peer_manager.get_peer_node_ids().length != 0) Thread.yield();
             state.peer1.nio_peer_handler.interrupt();
             state.peer2.nio_peer_handler.interrupt();
         }
@@ -862,4 +897,15 @@ public class HumanObjectPeerTest {
             do_test(nice_close, use_km_wrapper, use_manual_watch, reload_peers, break_cross_refs, nio_peer_handler, use_chan_manager_constructor);
         }
     }
+
+    // This is used in the test jar to test the built jar is runnable
+    public static void main(String[] args) {
+        try {
+            new HumanObjectPeerTest().test_message_handler();
+        } catch (Exception e) {
+            System.err.println("Caught exception:");
+            System.err.println(e);
+            System.exit(1);
+        }
+    }
 }