From: Matt Corallo <649246+TheBlueMatt@users.noreply.github.com> Date: Wed, 16 Jun 2021 19:26:37 +0000 (+0000) Subject: Merge pull request #22 from TheBlueMatt/main X-Git-Tag: v0.0.99.0~4 X-Git-Url: http://git.bitcoin.ninja/index.cgi?p=ldk-java;a=commitdiff_plain;h=0061ee2bb89ec994aeefd1e540558ce975067c7b;hp=3d9f4531ff7468b53e86997b1474ccfca018e75d Merge pull request #22 from TheBlueMatt/main Support building on OSX --- diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 3145675a..c470471e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -51,13 +51,9 @@ jobs: run: ./genbindings.sh ./ldk-c-bindings/ "-I/usr/lib/jvm/java-11-openjdk-amd64/include/ -I/usr/lib/jvm/java-11-openjdk-amd64/include/linux/" true false - name: Run Java Tests against Debug Bindings run: | - rm liblightningjni.so - ln -s liblightningjni_debug.so liblightningjni.so - export LD_LIBRARY_PATH=. - export LD_PRELOAD=/usr/lib/llvm-11/lib/clang/11.0.1/lib/linux/libclang_rt.asan-x86_64.so + mv liblightningjni_debug_Linux-amd64.so liblightningjni.so export ASAN_OPTIONS=detect_leaks=0 - mvn test - git checkout liblightningjni.so + LD_PRELOAD=/usr/lib/llvm-11/lib/clang/11.0.1/lib/linux/libclang_rt.asan-x86_64.so LD_LIBRARY_PATH=. mvn test - name: Build Java/TS Release Bindings run: | # We assume the top commit is just a bindings update commit, so we @@ -73,8 +69,113 @@ jobs: fi echo "Using $LDK_GARBAGECOLLECTED_GIT_OVERRIDE as git version" ./genbindings.sh ./ldk-c-bindings/ "-I/usr/lib/jvm/java-11-openjdk-amd64/include/ -I/usr/lib/jvm/java-11-openjdk-amd64/include/linux/" false false - - name: Check latest headers are in git + - name: Check latest headers and release lib are in git run: | - # For some reason the debug library is not deterministic, this may be fixed in a future rustc - git checkout liblightningjni_debug.so git diff --exit-code + - name: Run Java Tests against built release jar + run: | + mvn -DskipTests=true package + mvn install:install-file -Dfile=target/ldk-java-1.0-SNAPSHOT.jar -DgroupId=org.ldk -DartifactId=ldk-java -Dversion=1.0-SNAPSHOT -Dpackaging=jar + cd javatester + mvn package + java -ea -jar target/ldk-java-tests-1.0-SNAPSHOT-jar-with-dependencies.jar + cd .. + + osx: + strategy: + matrix: + include: + - platform: macos-10.15 + # MacOS 11 is currently in private preview, we've applied for access + # - platform: macos-11 + runs-on: ${{ matrix.platform }} + env: + TOOLCHAIN: stable + steps: + - name: Install other Rust platforms + run: rustup target install aarch64-apple-darwin + - name: Fetch upstream LLVM/clang snapshot + run: | + wget -O clang+llvm-12.0.0-x86_64-apple-darwin.tar.xz https://github.com/llvm/llvm-project/releases/download/llvmorg-12.0.0/clang+llvm-12.0.0-x86_64-apple-darwin.tar.xz + if [ "$(shasum -a 256 clang+llvm-12.0.0-x86_64-apple-darwin.tar.xz | awk '{ print $1 }')" != "7bc2259bf75c003f644882460fc8e844ddb23b27236fe43a2787870a4cd8ab50" ]; then + echo "Bad hash" + exit 1 + fi + - name: Unpack upstream LLVM+clang and use it by default + run: | + tar xvvf clang+llvm-12.0.0-x86_64-apple-darwin.tar.xz + - name: Checkout source code + uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Install cbindgen + run: cargo install --force cbindgen + - name: Checkout Rust-Lightning and LDK-C-Bindings git + run: | + git config --global user.email "ldk-ci@example.com" + git config --global user.name "LDK CI" + git clone https://github.com/rust-bitcoin/rust-lightning + cd rust-lightning + git remote add matt https://git.bitcoin.ninja/rust-lightning + git fetch matt + git merge matt/2021-03-java-bindings-base + cd .. + git clone https://github.com/lightningdevkit/ldk-c-bindings + - name: Rebuild C bindings with upstream clang, and check the sample app builds + links + run: | + cd ldk-c-bindings + export PATH=`pwd`/clang+llvm-12.0.0-x86_64-apple-darwin/bin:$PATH + CC=clang ./genbindings.sh ../rust-lightning true + cd .. + - name: Fetch OpenJDK 16 + run: | + wget -O openjdk-16.0.1_osx-x64_bin.tar.gz https://download.java.net/java/GA/jdk16.0.1/7147401fd7354114ac51ef3e1328291f/9/GPL/openjdk-16.0.1_osx-x64_bin.tar.gz + if [ "$(shasum -a 256 openjdk-16.0.1_osx-x64_bin.tar.gz | awk '{ print $1 }')" != "6098f839954439d4916444757c542c1b8778a32461706812d41cc8bbefce7f2f" ]; then + echo "Bad hash" + exit 1 + fi + tar xvvf openjdk-16.0.1_osx-x64_bin.tar.gz + export JAVA_HOME=`pwd`/jdk-16.0.1.jdk/Contents/Home + export PATH=$JAVA_HOME/bin:$PATH + - name: Build Java/TS Release Bindings + run: | + # We assume the top commit is just a bindings update commit, so we + # check out the previous commit to use as the commit we git describe. + # If the top commit is a merge commit, we need to get the last merge + # head and assume the latest bindings are built against its parent. + COMMIT_PARENTS=$(git show -s --pretty=format:%P HEAD) + if [ "${#COMMIT_PARENTS}" = 40 ]; then + export LDK_GARBAGECOLLECTED_GIT_OVERRIDE="$(git describe --tag HEAD^1)" + else + MERGE_HEAD=$(git show --pretty=format:%P HEAD | ( for last in $(cat /dev/stdin); do true; done; echo $last )) + export LDK_GARBAGECOLLECTED_GIT_OVERRIDE="$(git describe --tag $MERGE_HEAD^1)" + fi + echo "Using $LDK_GARBAGECOLLECTED_GIT_OVERRIDE as git version" + # genbindings.sh always fails as there is no wasm32-wasi library + # available, so instead we delete the expected JNI library and check + # that it was created. + rm src/main/resources/liblightningjni_MacOSX-x86_64.nativelib + ./genbindings.sh ./ldk-c-bindings/ "-I$JAVA_HOME/include/ -I$JAVA_HOME/include/darwin -isysroot$(xcrun --show-sdk-path)" false false || echo + cat src/main/resources/liblightningjni_MacOSX-x86_64.nativelib > /dev/null + - name: Fetch Maven 3.8.1 + run: | + wget -O apache-maven-3.8.1-bin.tar.gz https://apache.osuosl.org/maven/maven-3/3.8.1/binaries/apache-maven-3.8.1-bin.tar.gz + if [ "$(shasum -a 256 apache-maven-3.8.1-bin.tar.gz | awk '{ print $1 }')" != "b98a1905eb554d07427b2e5509ff09bd53e2f1dd7a0afa38384968b113abef02" ]; then + echo "Bad hash" + exit 1 + fi + tar xvvf apache-maven-3.8.1-bin.tar.gz + export PATH=apache-maven-3.8.1/bin:$PATH + - name: Run Java Tests against built jar + run: | + mvn -DskipTests=true package + mvn install:install-file -Dfile=target/ldk-java-1.0-SNAPSHOT.jar -DgroupId=org.ldk -DartifactId=ldk-java -Dversion=1.0-SNAPSHOT -Dpackaging=jar + cd javatester + mvn package + java -ea -jar target/ldk-java-tests-1.0-SNAPSHOT-jar-with-dependencies.jar + cd .. + - name: Check latest headers and release lib are in git + run: | + if [ "${{ matrix.platform }}" = "macos-11" ]; then + git diff --exit-code + fi diff --git a/README.md b/README.md index daa27633..8fe3e29e 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,17 @@ bindings should be built with workarounds required for Android. JNI CFLAGS on de When running a program linking against the library in debug mode, LD_PRELOAD should likely include the relevant `libclang_rt.asan-platform.so` path. +Note that building with address sanitizer is only supported on MacOS with upstream clang (ie where +the LLVM version in use matches the LLVM version against which rustc was built), not with Apple clang. +Builds with Apple clang will work fine, but largely only be useful in a release context. +To build on Mac with address sanitizer, you will need to run `ldk-c-bindings`' `genbindings.sh` +script with upstream clang in your PATH and likely replace your $JAVA_HOME/bin/java with a simple +wrapper which calls java after an export like: +`export DYLD_INSERT_LIBRARIES=/path/to/upstream/llvm/lib/clang/12.0.0/lib/darwin/libclang_rt.asan_osx_dynamic.dylib` + +To build for Apple M1 (ie aarch64-apple-darwin), you probably want something like +`CC="clang --target=aarch64-apple-darwin" LDK_TARGET=aarch64-apple-darwin LDK_TARGET_CPU=generic ./genbindings.sh ...` + Status ====== diff --git a/genbindings.sh b/genbindings.sh index c10ff241..6a4109c8 100755 --- a/genbindings.sh +++ b/genbindings.sh @@ -10,15 +10,30 @@ usage() { [ "$3" != "true" -a "$3" != "false" ] && usage [ "$4" != "true" -a "$4" != "false" ] && usage +set -x + if [ "$CC" != "" ]; then COMMON_COMPILE="$CC -std=c11 -Wall -Wextra -Wno-unused-parameter -Wno-ignored-qualifiers -Wno-unused-function -Wno-nullability-completeness -Wno-pointer-sign -Wdate-time -ffile-prefix-map=$(pwd)=" else + CC=clang COMMON_COMPILE="clang -std=c11 -Wall -Wextra -Wno-unused-parameter -Wno-ignored-qualifiers -Wno-unused-function -Wno-nullability-completeness -Wno-pointer-sign -Wdate-time -ffile-prefix-map=$(pwd)=" fi -if [ "$LDK_TARGET" != "" ]; then - LDK_TARGET_SUFFIX="_$LDK_TARGET" +TARGET_STRING="$LDK_TARGET" +if [ "$TARGET_STRING" = "" ]; then + # We assume clang-style $CC --version here, but worst-case we just get an empty suffix + TARGET_STRING="$($CC --version | grep Target | awk '{ print $2 }')" fi +case "$TARGET_STRING" in + "x86_64-pc-linux"*) + LDK_TARGET_SUFFIX="_Linux-amd64" ;; + "x86_64-apple-darwin"*) + LDK_TARGET_SUFFIX="_MacOSX-x86_64" ;; + "aarch64-apple-darwin"*) + LDK_TARGET_SUFFIX="_MacOSX-aarch64" ;; + *) + LDK_TARGET_SUFFIX="" +esac if [ "$LDK_TARGET_CPU" = "" ]; then LDK_TARGET_CPU="sandybridge" fi @@ -52,12 +67,23 @@ cat src/main/jni/bindings.c.body >> src/main/jni/bindings.c javac -h src/main/jni src/main/java/org/ldk/enums/*.java src/main/java/org/ldk/impl/bindings.java rm src/main/java/org/ldk/enums/*.class src/main/java/org/ldk/impl/bindings*.class +IS_MAC=false +[ "$($CC --version | grep apple-darwin)" != "" ] && IS_MAC=true + echo "Building Java bindings..." -COMPILE="$COMMON_COMPILE -mcpu=$LDK_TARGET_CPU -Isrc/main/jni -pthread -ldl -Wl,--no-undefined -shared -fPIC" +COMPILE="$COMMON_COMPILE -mcpu=$LDK_TARGET_CPU -Isrc/main/jni -pthread -ldl -shared -fPIC" +[ "$IS_MAC" = "false" ] && COMPILE="$COMPILE -Wl,--no-undefined" +[ "$IS_MAC" = "true" ] && COMPILE="$COMPILE -mmacosx-version-min=10.9" if [ "$3" = "true" ]; then - $COMPILE -o liblightningjni_debug$LDK_TARGET_SUFFIX.so -g -fsanitize=address -shared-libasan -Wl,-wrap,calloc -Wl,-wrap,realloc -Wl,-wrap,reallocarray -Wl,-wrap,malloc -Wl,-wrap,free -rdynamic -I"$1"/lightning-c-bindings/include/ $2 src/main/jni/bindings.c "$1"/lightning-c-bindings/target/$LDK_TARGET/debug/libldk.a -lm + [ "$IS_MAC" = "false" ] && COMPILE="$COMPILE -Wl,-wrap,calloc -Wl,-wrap,realloc -Wl,-wrap,reallocarray -Wl,-wrap,malloc -Wl,-wrap,free" + $COMPILE -o liblightningjni_debug$LDK_TARGET_SUFFIX.so -g -fsanitize=address -shared-libasan -rdynamic -I"$1"/lightning-c-bindings/include/ $2 src/main/jni/bindings.c "$1"/lightning-c-bindings/target/$LDK_TARGET/debug/libldk.a -lm else - $COMPILE -o liblightningjni_release$LDK_TARGET_SUFFIX.so -Wl,--version-script=libcode.version -flto -fuse-ld=lld -O3 -I"$1"/lightning-c-bindings/include/ $2 src/main/jni/bindings.c "$1"/lightning-c-bindings/target/$LDK_TARGET/release/libldk.a + [ "$IS_MAC" = "false" ] && COMPILE="$COMPILE -Wl,--version-script=libcode.version -fuse-ld=lld" + $COMPILE -o liblightningjni_release$LDK_TARGET_SUFFIX.so -flto -O3 -I"$1"/lightning-c-bindings/include/ $2 src/main/jni/bindings.c "$1"/lightning-c-bindings/target/$LDK_TARGET/release/libldk.a + if [ "$LDK_TARGET_SUFFIX" != "" ]; then + # Copy to JNI native directory for inclusion in JARs + cp liblightningjni_release$LDK_TARGET_SUFFIX.so src/main/resources/liblightningjni$LDK_TARGET_SUFFIX.nativelib + fi fi echo "Creating TS bindings..." diff --git a/java_strings.py b/java_strings.py index 3d5a2a01..7f82567e 100644 --- a/java_strings.py +++ b/java_strings.py @@ -1,5 +1,6 @@ from bindingstypes import * from enum import Enum +import sys class Target(Enum): JAVA = 1, @@ -23,6 +24,12 @@ class Consts: self.bindings_header = """package org.ldk.impl; import org.ldk.enums.*; +import java.io.File; +import java.io.InputStream; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; public class bindings { public static class VecOrSliceDef { @@ -34,7 +41,24 @@ public class bindings { } } static { - System.loadLibrary(\"lightningjni\"); + try { + // Try to load natively first, this works on Android and in testing. + System.loadLibrary(\"lightningjni\"); + } catch (UnsatisfiedLinkError _ignored) { + // Otherwise try to load from the library jar. + File tmpdir = new File(System.getProperty("java.io.tmpdir"), "ldk-java-nativelib"); + tmpdir.mkdir(); // If it fails to create, assume it was there already + tmpdir.deleteOnExit(); + String libname = "liblightningjni_" + System.getProperty("os.name").replaceAll(" ", "") + + "-" + System.getProperty("os.arch").replaceAll(" ", "") + ".nativelib"; + try (InputStream is = bindings.class.getResourceAsStream("/" + libname)) { + Path libpath = new File(tmpdir.toPath().toString(), "liblightningjni.so").toPath(); + Files.copy(is, libpath, StandardCopyOption.REPLACE_EXISTING); + Runtime.getRuntime().load(libpath.toString()); + } catch (IOException e) { + throw new IllegalArgumentException(e); + } + } init(java.lang.Enum.class, VecOrSliceDef.class); init_class_cache(); if (!get_lib_version_string().equals(get_ldk_java_bindings_version())) @@ -83,11 +107,13 @@ class CommonBase { long ptr; LinkedList ptrs_to = new LinkedList(); protected CommonBase(long ptr) { this.ptr = ptr; } - public long _test_only_get_ptr() { return this.ptr; } } """ - self.c_file_pfx = """#include \"org_ldk_impl_bindings.h\" + self.c_file_pfx = """#include +// On OSX jlong (ie long long) is not equivalent to int64_t, so we override here +#define int64_t jlong +#include \"org_ldk_impl_bindings.h\" #include #include #include @@ -124,13 +150,15 @@ void __attribute__((constructor)) spawn_stderr_redirection() { else: self.c_file_pfx = self.c_file_pfx + "#define DEBUG_PRINT(...) fprintf(stderr, __VA_ARGS__)\n" - if not DEBUG: + if not DEBUG or sys.platform == "darwin": self.c_file_pfx = self.c_file_pfx + """#define MALLOC(a, _) malloc(a) #define FREE(p) if ((uint64_t)(p) > 1024) { free(p); } -#define DO_ASSERT(a) (void)(a) +""" + if not DEBUG: + self.c_file_pfx += """#define DO_ASSERT(a) (void)(a) #define CHECK(a) """ - else: + if DEBUG: self.c_file_pfx = self.c_file_pfx + """#include // Always run a, then assert it is true: #define DO_ASSERT(a) do { bool _assert_val = (a); assert(_assert_val); } while(0) @@ -144,7 +172,10 @@ void __attribute__((constructor)) debug_log_version() { DEBUG_PRINT("LDK C Bindings version did not match the header we built against\\n"); DEBUG_PRINT("Loaded LDK-Java Bindings with LDK %s and LDK-C-Bindings %s\\n", check_get_ldk_version(), check_get_ldk_bindings_version()); } +""" + if sys.platform != "darwin": + self.c_file_pfx += """ // Running a leak check across all the allocations and frees of the JDK is a mess, // so instead we implement our own naive leak checker here, relying on the -wrap // linker option to wrap malloc/calloc/realloc/free, tracking everyhing allocated @@ -152,8 +183,8 @@ void __attribute__((constructor)) debug_log_version() { #include """ - if self.target == Target.ANDROID: - self.c_file_pfx = self.c_file_pfx + """ + if self.target == Target.ANDROID: + self.c_file_pfx = self.c_file_pfx + """ #include #include @@ -194,9 +225,9 @@ void backtrace_symbols_fd(void ** buffer, int count, int _fd) { } } """ - else: - self.c_file_pfx = self.c_file_pfx + "#include \n" - self.c_file_pfx = self.c_file_pfx + """ + else: + self.c_file_pfx = self.c_file_pfx + "#include \n" + self.c_file_pfx = self.c_file_pfx + """ #include static mtx_t allocation_mtx; diff --git a/javatester/pom.xml b/javatester/pom.xml new file mode 100644 index 00000000..02137e3f --- /dev/null +++ b/javatester/pom.xml @@ -0,0 +1,68 @@ + + + 4.0.0 + + org.ldk + ldk-java-tests + 1.0-SNAPSHOT + + + org.junit.jupiter + junit-jupiter-api + RELEASE + compile + + + org.bitcoinj + bitcoinj-core + 0.15.10 + compile + + + org.jetbrains + annotations + RELEASE + compile + + + org.ldk + ldk-java + 1.0-SNAPSHOT + compile + + + + 1.8 + 1.8 + + + + + + org.apache.maven.plugins + maven-assembly-plugin + 3.2.0 + + + package + + single + + + + + org.ldk.HumanObjectPeerTest + + + + jar-with-dependencies + + + + + + + + diff --git a/javatester/src/main/java/org/ldk/HumanObjectPeerTest.java b/javatester/src/main/java/org/ldk/HumanObjectPeerTest.java new file mode 120000 index 00000000..59d36734 --- /dev/null +++ b/javatester/src/main/java/org/ldk/HumanObjectPeerTest.java @@ -0,0 +1 @@ +../../../../../../src/test/java/org/ldk/HumanObjectPeerTest.java \ No newline at end of file diff --git a/liblightningjni.so b/liblightningjni.so deleted file mode 120000 index 1b6bb881..00000000 --- a/liblightningjni.so +++ /dev/null @@ -1 +0,0 @@ -liblightningjni_release.so \ No newline at end of file diff --git a/liblightningjni_debug.so b/liblightningjni_debug.so deleted file mode 100755 index 8f58fe0e..00000000 Binary files a/liblightningjni_debug.so and /dev/null differ diff --git a/liblightningjni_release.so b/liblightningjni_release.so deleted file mode 100755 index e30297b5..00000000 Binary files a/liblightningjni_release.so and /dev/null differ diff --git a/src/main/java/org/ldk/batteries/ChannelManagerConstructor.java b/src/main/java/org/ldk/batteries/ChannelManagerConstructor.java index d5585408..da5e3fb9 100644 --- a/src/main/java/org/ldk/batteries/ChannelManagerConstructor.java +++ b/src/main/java/org/ldk/batteries/ChannelManagerConstructor.java @@ -88,7 +88,11 @@ public class ChannelManagerConstructor { this.peer_manager = PeerManager.of(channel_manager.as_ChannelMessageHandler(), (IgnoringMessageHandler.of()).as_RoutingMessageHandler(), keys_interface.get_node_secret(), random_data, logger); } NioPeerHandler nio_peer_handler = null; - try { nio_peer_handler = new NioPeerHandler(this.peer_manager); } catch (IOException e) { assert false; } + try { + nio_peer_handler = new NioPeerHandler(this.peer_manager); + } catch (IOException e) { + throw new IllegalStateException("We should never fail to construct nio objects unless we're on a platform that cannot run LDK."); + } this.nio_peer_handler = nio_peer_handler; if (filter != null) { for (ChannelMonitor monitor : monitors) { @@ -118,7 +122,11 @@ public class ChannelManagerConstructor { this.peer_manager = PeerManager.of(channel_manager.as_ChannelMessageHandler(), (IgnoringMessageHandler.of()).as_RoutingMessageHandler(), keys_interface.get_node_secret(), random_data, logger); } NioPeerHandler nio_peer_handler = null; - try { nio_peer_handler = new NioPeerHandler(this.peer_manager); } catch (IOException e) { assert false; } + try { + nio_peer_handler = new NioPeerHandler(this.peer_manager); + } catch (IOException e) { + throw new IllegalStateException("We should never fail to construct nio objects unless we're on a platform that cannot run LDK."); + } this.nio_peer_handler = nio_peer_handler; } diff --git a/src/main/java/org/ldk/impl/bindings.java b/src/main/java/org/ldk/impl/bindings.java index 511f6839..49f921d2 100644 --- a/src/main/java/org/ldk/impl/bindings.java +++ b/src/main/java/org/ldk/impl/bindings.java @@ -1,5 +1,11 @@ package org.ldk.impl; import org.ldk.enums.*; +import java.io.File; +import java.io.InputStream; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; public class bindings { public static class VecOrSliceDef { @@ -11,7 +17,24 @@ public class bindings { } } static { - System.loadLibrary("lightningjni"); + try { + // Try to load natively first, this works on Android and in testing. + System.loadLibrary("lightningjni"); + } catch (UnsatisfiedLinkError _ignored) { + // Otherwise try to load from the library jar. + File tmpdir = new File(System.getProperty("java.io.tmpdir"), "ldk-java-nativelib"); + tmpdir.mkdir(); // If it fails to create, assume it was there already + tmpdir.deleteOnExit(); + String libname = "liblightningjni_" + System.getProperty("os.name").replaceAll(" ", "") + + "-" + System.getProperty("os.arch").replaceAll(" ", "") + ".nativelib"; + try (InputStream is = bindings.class.getResourceAsStream("/" + libname)) { + Path libpath = new File(tmpdir.toPath().toString(), "liblightningjni.so").toPath(); + Files.copy(is, libpath, StandardCopyOption.REPLACE_EXISTING); + Runtime.getRuntime().load(libpath.toString()); + } catch (IOException e) { + throw new IllegalArgumentException(e); + } + } init(java.lang.Enum.class, VecOrSliceDef.class); init_class_cache(); if (!get_lib_version_string().equals(get_ldk_java_bindings_version())) @@ -25,7 +48,7 @@ public class bindings { static native String get_lib_version_string(); public static String get_ldk_java_bindings_version() { - return "v0.0.98.1"; + return "v0.0.98.2"; } public static native String get_ldk_c_bindings_version(); public static native String get_ldk_version(); diff --git a/src/main/java/org/ldk/structs/CommonBase.java b/src/main/java/org/ldk/structs/CommonBase.java index acff6661..1794158b 100644 --- a/src/main/java/org/ldk/structs/CommonBase.java +++ b/src/main/java/org/ldk/structs/CommonBase.java @@ -4,5 +4,4 @@ class CommonBase { long ptr; LinkedList ptrs_to = new LinkedList(); protected CommonBase(long ptr) { this.ptr = ptr; } - public long _test_only_get_ptr() { return this.ptr; } } diff --git a/src/main/jni/bindings.c b/src/main/jni/bindings.c index 4d1fdc88..68b11b17 100644 --- a/src/main/jni/bindings.c +++ b/src/main/jni/bindings.c @@ -1,5 +1,8 @@ #define LDKCVec_C2Tuple_TxidCVec_C2Tuple_u32TxOutZZZZ LDKCVec_TransactionOutputsZ #define CVec_C2Tuple_TxidCVec_C2Tuple_u32TxOutZZZZ_free CVec_TransactionOutputsZ_free +#include +// On OSX jlong (ie long long) is not equivalent to int64_t, so we override here +#define int64_t jlong #include "org_ldk_impl_bindings.h" #include #include @@ -130,7 +133,7 @@ static inline LDKStr java_to_owned_str(JNIEnv *env, jstring str) { } JNIEXPORT jstring JNICALL Java_org_ldk_impl_bindings_get_1lib_1version_1string(JNIEnv *env, jclass _c) { - return str_ref_to_java(env, "v0.0.98.1", strlen("v0.0.98.1")); + return str_ref_to_java(env, "v0.0.98.2", strlen("v0.0.98.2")); } JNIEXPORT jstring JNICALL Java_org_ldk_impl_bindings_get_1ldk_1c_1bindings_1version(JNIEnv *env, jclass _c) { return str_ref_to_java(env, check_get_ldk_bindings_version(), strlen(check_get_ldk_bindings_version())); diff --git a/src/main/jni/bindings.c.body b/src/main/jni/bindings.c.body index 95ae6d52..e3fe4d19 100644 --- a/src/main/jni/bindings.c.body +++ b/src/main/jni/bindings.c.body @@ -1,3 +1,6 @@ +#include +// On OSX jlong (ie long long) is not equivalent to int64_t, so we override here +#define int64_t jlong #include "org_ldk_impl_bindings.h" #include #include @@ -128,7 +131,7 @@ static inline LDKStr java_to_owned_str(JNIEnv *env, jstring str) { } JNIEXPORT jstring JNICALL Java_org_ldk_impl_bindings_get_1lib_1version_1string(JNIEnv *env, jclass _c) { - return str_ref_to_java(env, "v0.0.98.1", strlen("v0.0.98.1")); + return str_ref_to_java(env, "v0.0.98.2", strlen("v0.0.98.2")); } JNIEXPORT jstring JNICALL Java_org_ldk_impl_bindings_get_1ldk_1c_1bindings_1version(JNIEnv *env, jclass _c) { return str_ref_to_java(env, check_get_ldk_bindings_version(), strlen(check_get_ldk_bindings_version())); diff --git a/src/main/resources/liblightningjni_Linux-amd64.nativelib b/src/main/resources/liblightningjni_Linux-amd64.nativelib new file mode 100755 index 00000000..8acc71a4 Binary files /dev/null and b/src/main/resources/liblightningjni_Linux-amd64.nativelib differ diff --git a/src/main/resources/liblightningjni_MacOSX-aarch64.nativelib b/src/main/resources/liblightningjni_MacOSX-aarch64.nativelib new file mode 100755 index 00000000..06f615e8 Binary files /dev/null and b/src/main/resources/liblightningjni_MacOSX-aarch64.nativelib differ diff --git a/src/main/resources/liblightningjni_MacOSX-x86_64.nativelib b/src/main/resources/liblightningjni_MacOSX-x86_64.nativelib new file mode 100755 index 00000000..2c5b8ca7 Binary files /dev/null and b/src/main/resources/liblightningjni_MacOSX-x86_64.nativelib differ diff --git a/src/test/java/org/ldk/HumanObjectPeerTest.java b/src/test/java/org/ldk/HumanObjectPeerTest.java index 9d6acedb..c6a6f475 100644 --- a/src/test/java/org/ldk/HumanObjectPeerTest.java +++ b/src/test/java/org/ldk/HumanObjectPeerTest.java @@ -7,7 +7,6 @@ import org.ldk.batteries.ChannelManagerConstructor; import org.ldk.batteries.NioPeerHandler; import org.ldk.enums.Currency; import org.ldk.enums.Network; -import org.ldk.impl.bindings; import org.ldk.structs.*; import org.ldk.util.TwoTuple; @@ -202,7 +201,10 @@ class HumanObjectPeerTestInstance { logger = Logger.new_impl((String arg) -> System.out.println(seed + ": " + arg)); fee_estimator = FeeEstimator.new_impl((confirmation_target -> 253)); tx_broadcaster = BroadcasterInterface.new_impl(tx -> { - broadcast_set.add(tx); + synchronized (broadcast_set) { + broadcast_set.add(tx); + broadcast_set.notifyAll(); + } }); monitors = new HashMap<>(); this.seed = seed; @@ -418,12 +420,16 @@ class HumanObjectPeerTestInstance { if (use_chan_manager_constructor) { while (true) { synchronized (this.pending_manager_events) { - if (expected_len != 0 && this.pending_manager_events.size() != expected_len) { + if (expected_len != 0 && this.pending_manager_events.size() == expected_len) { break; } } - try { Thread.sleep(500); } catch (InterruptedException e) { assert false; } - break; + if (expected_len == 0) { + try { Thread.sleep(500); } catch (InterruptedException e) { assert false; } + break; + } else { + Thread.yield(); + } } synchronized (this.pending_manager_events) { Event[] res = this.pending_manager_events.toArray(new Event[0]); @@ -449,7 +455,11 @@ class HumanObjectPeerTestInstance { } } - Event[] get_manager_events(int expected_len) { + Event[] get_manager_events(int expected_len, Peer peer1, Peer peer2) { + assert expected_len != 0; + if (!use_nio_peer_handler) { + maybe_exchange_peer_messages(peer1, peer2); + } Event[] res = new Event[0]; if (use_chan_manager_constructor) { while (res.length < expected_len) { @@ -464,7 +474,15 @@ class HumanObjectPeerTestInstance { } } else { ArrayList l = new ArrayList(); - chan_manager.as_EventsProvider().process_pending_events(EventHandler.new_impl(l::add)); + while (l.size() < expected_len) { + Thread.yield(); + if (use_nio_peer_handler) { + peer1.nio_peer_handler.check_events(); + peer2.nio_peer_handler.check_events(); + } + chan_manager.as_EventsProvider().process_pending_events(EventHandler.new_impl(l::add)); + assert l.size() == expected_len || l.size() == 0; // We don't handle partial results + } return l.toArray(new Event[0]); } assert res.length == expected_len; @@ -498,12 +516,9 @@ class HumanObjectPeerTestInstance { } } }); - void wait_events_processed(Peer peer1, Peer peer2) { - if (use_nio_peer_handler) { - peer1.nio_peer_handler.check_events(); - peer2.nio_peer_handler.check_events(); - try { Thread.sleep(500); } catch (InterruptedException e) { assert false; } - } else { + + void maybe_exchange_peer_messages(Peer peer1, Peer peer2) { + if (!use_nio_peer_handler) { synchronized (runqueue) { ran = false; } @@ -520,6 +535,9 @@ class HumanObjectPeerTestInstance { try { runqueue.wait(); } catch (InterruptedException e) { assert false; } } } + } else if (!use_chan_manager_constructor) { + peer1.nio_peer_handler.check_events(); + peer2.nio_peer_handler.check_events(); } } void do_read_event(PeerManager pm, SocketDescriptor descriptor, byte[] data) { @@ -540,6 +558,9 @@ class HumanObjectPeerTestInstance { try { peer1.nio_peer_handler.connect(peer2.chan_manager.get_our_node_id(), new InetSocketAddress("127.0.0.1", peer2.nio_port), 100); } catch (IOException e) { assert false; } + while (peer1.peer_manager.get_peer_node_ids().length == 0 || peer2.peer_manager.get_peer_node_ids().length == 0) { + Thread.yield(); + } } else { DescriptorHolder descriptor1 = new DescriptorHolder(); DescriptorHolder descriptor1ref = descriptor1; @@ -573,6 +594,8 @@ class HumanObjectPeerTestInstance { Result_NonePeerHandleErrorZ inbound_conn_res = peer2.peer_manager.new_inbound_connection(descriptor2); assert inbound_conn_res instanceof Result_NonePeerHandleErrorZ.Result_NonePeerHandleErrorZ_OK; do_read_event(peer2.peer_manager, descriptor2, ((Result_CVec_u8ZPeerHandleErrorZ.Result_CVec_u8ZPeerHandleErrorZ_OK) conn_res).res); + + maybe_exchange_peer_messages(peer1, peer2); } } @@ -581,13 +604,11 @@ class HumanObjectPeerTestInstance { Peer peer2 = new Peer((byte) 2); connect_peers(peer1, peer2); - wait_events_processed(peer1, peer2); Result_NoneAPIErrorZ cc_res = peer1.chan_manager.create_channel(peer2.node_id, 10000, 1000, 42, null); assert cc_res instanceof Result_NoneAPIErrorZ.Result_NoneAPIErrorZ_OK; - wait_events_processed(peer1, peer2); - Event[] events = peer1.get_manager_events(1); + Event[] events = peer1.get_manager_events(1, peer1, peer2); assert events[0] instanceof Event.FundingGenerationReady; assert ((Event.FundingGenerationReady) events[0]).channel_value_satoshis == 10000; assert ((Event.FundingGenerationReady) events[0]).user_channel_id == 42; @@ -604,7 +625,13 @@ class HumanObjectPeerTestInstance { funding.addOutput(Coin.SATOSHI.multiply(10000), new Script(funding_spk)); Result_NoneAPIErrorZ funding_res = peer1.chan_manager.funding_transaction_generated(chan_id, funding.bitcoinSerialize()); assert funding_res instanceof Result_NoneAPIErrorZ.Result_NoneAPIErrorZ_OK; - wait_events_processed(peer1, peer2); + + maybe_exchange_peer_messages(peer1, peer2); + synchronized (peer1.broadcast_set) { + while (peer1.broadcast_set.size() != 1) { + peer1.broadcast_set.wait(); + } + } assert peer1.broadcast_set.size() == 1; assert Arrays.equals(peer1.broadcast_set.get(0), funding.bitcoinSerialize()); @@ -619,7 +646,9 @@ class HumanObjectPeerTestInstance { peer1.connect_block(b, height, 0); peer2.connect_block(b, height, 0); } - wait_events_processed(peer1, peer2); + + maybe_exchange_peer_messages(peer1, peer2); + while (peer1.chan_manager.list_usable_channels().length != 1 || peer2.chan_manager.list_usable_channels().length != 1) peer1.chan_manager.list_channels(); ChannelDetails[] peer1_chans = peer1.chan_manager.list_usable_channels(); @@ -664,7 +693,6 @@ class HumanObjectPeerTestInstance { Result_NonePaymentSendFailureZ payment_res = peer1.chan_manager.send_payment(route, payment_hash, payment_secret); assert payment_res instanceof Result_NonePaymentSendFailureZ.Result_NonePaymentSendFailureZ_OK; - wait_events_processed(peer1, peer2); RouteHop[][] hops = new RouteHop[1][1]; byte[] hop_pubkey = new byte[33]; @@ -675,21 +703,13 @@ class HumanObjectPeerTestInstance { payment_res = peer1.chan_manager.send_payment(r2, payment_hash, payment_secret); assert payment_res instanceof Result_NonePaymentSendFailureZ.Result_NonePaymentSendFailureZ_Err; - if (!use_chan_manager_constructor) { - peer1.get_monitor_events(0); - peer2.get_monitor_events(0); - } else { - // The events are combined across manager + monitors but peer1 still has no events - } - if (reload_peers) { - if (use_nio_peer_handler) { - peer1.nio_peer_handler.interrupt(); - peer2.nio_peer_handler.interrupt(); - } if (use_chan_manager_constructor) { peer1.constructor.interrupt(); peer2.constructor.interrupt(); + } else if (use_nio_peer_handler) { + peer1.nio_peer_handler.interrupt(); + peer2.nio_peer_handler.interrupt(); } WeakReference op1 = new WeakReference(peer1); peer1 = new Peer(peer1); @@ -713,7 +733,7 @@ class HumanObjectPeerTestInstance { this.best_blockhash = best_blockhash; } } - void do_test_message_handler_b(TestState state) { + void do_test_message_handler_b(TestState state) throws InterruptedException { GcCheck obj = new GcCheck(); if (state.ref_block != null) { // Ensure the original peers get freed before we move on. Note that we have to be in a different function @@ -724,36 +744,52 @@ class HumanObjectPeerTestInstance { } connect_peers(state.peer1, state.peer2); } - wait_events_processed(state.peer1, state.peer2); - Event[] events = state.peer2.get_manager_events(1); + Event[] events = state.peer2.get_manager_events(1, state.peer1, state.peer2); assert events[0] instanceof Event.PendingHTLCsForwardable; state.peer2.chan_manager.process_pending_htlc_forwards(); - events = state.peer2.get_manager_events(1); + events = state.peer2.get_manager_events(1, state.peer1, state.peer2); assert events[0] instanceof Event.PaymentReceived; byte[] payment_preimage = ((Event.PaymentReceived)events[0]).payment_preimage; assert !Arrays.equals(payment_preimage, new byte[32]); state.peer2.chan_manager.claim_funds(payment_preimage); - wait_events_processed(state.peer1, state.peer2); - events = state.peer1.get_manager_events(1); + events = state.peer1.get_manager_events(1, state.peer1, state.peer2); assert events[0] instanceof Event.PaymentSent; assert Arrays.equals(((Event.PaymentSent) events[0]).payment_preimage, payment_preimage); - wait_events_processed(state.peer1, state.peer2); + + if (use_nio_peer_handler) { + // We receive PaymentSent immediately upon receipt of the payment preimage, but we expect to not have an + // HTLC transaction to broadcast below, which requires a bit more time to fully complete the + // commitment-transaction-update dance between both peers. + Thread.sleep(100); + } ChannelDetails[] peer1_chans = state.peer1.chan_manager.list_channels(); if (nice_close) { Result_NoneAPIErrorZ close_res = state.peer1.chan_manager.close_channel(peer1_chans[0].get_channel_id()); assert close_res instanceof Result_NoneAPIErrorZ.Result_NoneAPIErrorZ_OK; - wait_events_processed(state.peer1, state.peer2); + maybe_exchange_peer_messages(state.peer1, state.peer2); + synchronized (state.peer1.broadcast_set) { + while (state.peer1.broadcast_set.size() != 1) state.peer1.broadcast_set.wait(); + } + synchronized (state.peer2.broadcast_set) { + while (state.peer2.broadcast_set.size() != 1) state.peer2.broadcast_set.wait(); + } assert state.peer1.broadcast_set.size() == 1; assert state.peer2.broadcast_set.size() == 1; } else { state.peer1.chan_manager.force_close_all_channels(); - wait_events_processed(state.peer1, state.peer2); + maybe_exchange_peer_messages(state.peer1, state.peer2); + synchronized (state.peer1.broadcast_set) { + while (state.peer1.broadcast_set.size() != 1) state.peer1.broadcast_set.wait(); + } + synchronized (state.peer2.broadcast_set) { + while (state.peer2.broadcast_set.size() != 1) state.peer2.broadcast_set.wait(); + } assert state.peer1.broadcast_set.size() == 1; assert state.peer2.broadcast_set.size() == 1; @@ -794,9 +830,8 @@ class HumanObjectPeerTestInstance { if (use_nio_peer_handler) { state.peer1.peer_manager.disconnect_by_node_id(state.peer2.chan_manager.get_our_node_id(), false); - wait_events_processed(state.peer1, state.peer2); - assert state.peer1.peer_manager.get_peer_node_ids().length == 0; - assert state.peer2.peer_manager.get_peer_node_ids().length == 0; + while (state.peer1.peer_manager.get_peer_node_ids().length != 0) Thread.yield(); + while (state.peer2.peer_manager.get_peer_node_ids().length != 0) Thread.yield(); state.peer1.nio_peer_handler.interrupt(); state.peer2.nio_peer_handler.interrupt(); } @@ -862,4 +897,15 @@ public class HumanObjectPeerTest { do_test(nice_close, use_km_wrapper, use_manual_watch, reload_peers, break_cross_refs, nio_peer_handler, use_chan_manager_constructor); } } + + // This is used in the test jar to test the built jar is runnable + public static void main(String[] args) { + try { + new HumanObjectPeerTest().test_message_handler(); + } catch (Exception e) { + System.err.println("Caught exception:"); + System.err.println(e); + System.exit(1); + } + } }