From 0fee03a8095f5599f8eccd920c1b1af6b3a20260 Mon Sep 17 00:00:00 2001 From: Matt Corallo Date: Wed, 5 Jan 2022 06:32:04 +0000 Subject: [PATCH] Make genbindings.sh only build one of java/wasm, test wasm in CI --- .github/workflows/build.yml | 99 ++++++--- genbindings.sh | 368 ++++++++++++++++---------------- ts/test/browser.mjs | 15 ++ ts/test/index.html | 9 +- ts/test/node.mjs | 4 + ts/test/{test.mjs => tests.mjs} | 8 +- 6 files changed, 293 insertions(+), 210 deletions(-) create mode 100644 ts/test/browser.mjs create mode 100644 ts/test/node.mjs rename ts/test/{test.mjs => tests.mjs} (90%) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f1f72c68..334e106d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -3,21 +3,24 @@ name: Continuous Integration Checks on: [push, pull_request] jobs: - check_bindings: + wasm_bindings: runs-on: ubuntu-latest - # Ubuntu's version of rustc uses its own LLVM instead of being a real native package. - # This leaves us with an incompatible LLVM version when linking. Instead, use a real OS. - container: debian:bullseye - env: - TOOLCHAIN: stable + # Ubuntu's version of clang doesn't support....anything that should work. + # Ubuntu is an utter trash OS and should generally never be used for anything. + # We also require at least rustc 1.51 (ie https://github.com/rust-lang/rust/pull/79998), + # so we use debian testing + container: debian:bookworm strategy: fail-fast: false steps: - - name: Install native Rust toolchain, Valgrind, and build utilitis + - name: Install build utilities and test that clang can build for wasm run: | apt-get update apt-get -y dist-upgrade - apt-get -y install cargo libstd-rust-dev-wasm32 valgrind lld git g++ clang openjdk-11-jdk maven faketime zip unzip llvm curl + apt-get -y install cargo libstd-rust-dev-wasm32 lld git g++ clang curl node-typescript npm python3 + echo "int main() {}" > genbindings_wasm_test_file.c + clang -nostdlib -o /dev/null --target=wasm32-wasi -Wl,--no-entry genbindings_wasm_test_file.c + rm genbindings_wasm_test_file.c - name: Checkout source code uses: actions/checkout@v2 with: @@ -36,21 +39,75 @@ jobs: git clone https://github.com/lightningdevkit/ldk-c-bindings cd ldk-c-bindings git checkout 0.0.104 - - name: Rebuild C bindings without STD for WASM + - name: Rebuild C bindings without STD run: | cd ldk-c-bindings ./genbindings.sh ../rust-lightning false - mv lightning-c-bindings/target/wasm32-wasi ./ + - name: Build and Test TS Bindings for Node + run: | + # We need FinalizationRegistry and top-level await support, which requires node 14.6/8, + # however Debian ships with Node 12 + # Thus, we install the binary nodejs from nodejs.org and test with that. + curl https://nodejs.org/dist/v16.13.1/node-v16.13.1-linux-x64.tar.xz > nodejs.tar.xz + tar xvvf nodejs.tar.xz + export PATH=$(echo node-*/bin):$PATH + ./genbindings.sh ./ldk-c-bindings/ wasm false false + - name: Build and Test TS Bindings for Web + run: | + export PATH=$(echo node-*/bin):$PATH + ./genbindings.sh ./ldk-c-bindings/ wasm false true + export HOME=/root/ # Github actions is apparently broken + npm i -D playwright + # npx playwright install-deps is broken so we do it manually, see https://github.com/microsoft/playwright/issues/11165 + apt-get install -y --no-install-recommends fonts-liberation libenchant-2-2 libicu67 libjpeg62-turbo libasound2 libatk-bridge2.0-0 libatk1.0-0 libatspi2.0-0 libcairo2 libcups2 libdbus-1-3 libdrm2 libegl1 libgbm1 libglib2.0-0 libgtk-3-0 libnspr4 libnss3 libpango-1.0-0 libx11-6 libx11-xcb1 libxcb1 libxcomposite1 libxdamage1 libxext6 libxfixes3 libxrandr2 libxshmfence1 xvfb fonts-noto-color-emoji fonts-unifont libfontconfig libfreetype6 xfonts-scalable fonts-ipafont-gothic fonts-wqy-zenhei fonts-tlwg-loma-otf ffmpeg libcairo-gobject2 libdbus-glib-1-2 libfontconfig1 libgdk-pixbuf2.0-0 libpangocairo-1.0-0 libpangoft2-1.0-0 libxcb-shm0 libxcursor1 libxi6 libxrender1 libxt6 gstreamer1.0-libav gstreamer1.0-plugins-bad gstreamer1.0-plugins-base gstreamer1.0-plugins-good libepoxy0 libevdev2 libgl1 libgles2 libgstreamer-gl1.0-0 libgstreamer1.0-0 libharfbuzz-icu0 libharfbuzz0b libhyphen0 libnotify4 libopenjp2-7 libopus0 libpng16-16 libsecret-1-0 libsoup2.4-1 libwayland-client0 libwayland-egl1 libwayland-server0 libwebp6 libwebpdemux2 libwoff1 libxkbcommon0 libxml2 libxslt1.1 + mkdir -p $HOME/.cache/ms-playwright # `playwright install` is too dumb to create this for us + chmod -R 777 $HOME/ + npx playwright install + export PATH=$(echo node-*/bin):$PATH + python3 -m http.server & + node ts/test/browser.mjs + - name: Check latest TS files are in git + run: | + git diff --exit-code + + java_bindings: + runs-on: ubuntu-latest + # Ubuntu's version of rustc uses its own LLVM instead of being a real native package. + # This leaves us with an incompatible LLVM version when linking. Instead, use a real OS. + container: debian:bullseye + strategy: + fail-fast: false + steps: + - name: Install native Rust toolchain, Valgrind, and build utilitis + run: | + apt-get update + apt-get -y dist-upgrade + apt-get -y install cargo valgrind lld git g++ clang openjdk-11-jdk maven faketime zip unzip llvm curl + - name: Checkout source code + uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Install cbindgen + run: cargo install --force cbindgen + - name: Checkout Rust-Lightning and LDK-C-Bindings git + run: | + git config --global user.email "ldk-ci@example.com" + git config --global user.name "LDK CI" + # Note this is a different endpoint, as we need one non-upstream commit! + git clone https://git.bitcoin.ninja/rust-lightning + cd rust-lightning + git checkout origin/2021-03-java-bindings-base cd .. + git clone https://github.com/lightningdevkit/ldk-c-bindings + cd ldk-c-bindings + git checkout 0.0.104 - name: Rebuild C bindings, and check the sample app builds + links run: | cd ldk-c-bindings # Reset the Cargo.toml file so that git describe doesn't think we're "-dirty" git checkout lightning-c-bindings/Cargo.toml ./genbindings.sh ../rust-lightning true - mv wasm32-wasi lightning-c-bindings/target/ - cd .. - - name: Build Java/TS Debug Bindings + - name: Build Java Debug Bindings run: ./genbindings.sh ./ldk-c-bindings/ "-I/usr/lib/jvm/java-11-openjdk-amd64/include/ -I/usr/lib/jvm/java-11-openjdk-amd64/include/linux/" true false - name: Run Java Tests against Debug Bindings run: | @@ -72,7 +129,7 @@ jobs: cd ../.. mkdir -p src/main/resources/ cp "ldk-java-bins/${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}/liblightningjni_MacOSX-"* src/main/resources/ - - name: Build Java/TS Release Bindings + - name: Build Java Release Bindings run: | export LDK_GARBAGECOLLECTED_GIT_OVERRIDE="$(git describe --tag HEAD)" ./genbindings.sh ./ldk-c-bindings/ "-I/usr/lib/jvm/java-11-openjdk-amd64/include/ -I/usr/lib/jvm/java-11-openjdk-amd64/include/linux/" false false @@ -116,8 +173,6 @@ jobs: runs-on: ubuntu-latest # Frankly, I'm not really sure why debian and ubuntu differ in the results here, they really shouldn't container: debian:bullseye - env: - TOOLCHAIN: stable strategy: fail-fast: false steps: @@ -211,8 +266,6 @@ jobs: - platform: macos-11 fail-fast: false runs-on: ${{ matrix.platform }} - env: - TOOLCHAIN: stable steps: - name: Install other Rust platforms run: rustup target install aarch64-apple-darwin @@ -272,22 +325,18 @@ jobs: cd ../.. mkdir -p src/main/resources/ cp "ldk-java-bins/${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}/liblightningjni_Linux-"* src/main/resources/ - - name: Build Java/TS Release Bindings + - name: Build Java Release Bindings run: | export LDK_GARBAGECOLLECTED_GIT_OVERRIDE="$(git describe --tag HEAD)" export JAVA_HOME=`pwd`/jdk-16.0.1.jdk/Contents/Home export PATH=$JAVA_HOME/bin:$PATH - # genbindings.sh always fails as there is no wasm32-wasi library - # available, so instead we check that the expected JNI library - # is created. export PATH=`pwd`/clang+llvm-13.0.0-x86_64-apple-darwin/bin:$PATH - ./genbindings.sh ./ldk-c-bindings/ "-I$JAVA_HOME/include/ -I$JAVA_HOME/include/darwin -isysroot$(xcrun --show-sdk-path)" false false || echo - cat src/main/resources/liblightningjni_MacOSX-x86_64.nativelib > /dev/null + ./genbindings.sh ./ldk-c-bindings/ "-I$JAVA_HOME/include/ -I$JAVA_HOME/include/darwin -isysroot$(xcrun --show-sdk-path)" false false if [ "${{ matrix.platform }}" = "macos-11" ]; then export CC="clang --target=aarch64-apple-darwin" export LDK_TARGET=aarch64-apple-darwin export LDK_TARGET_CPU=apple-a14 - ./genbindings.sh ./ldk-c-bindings/ "-I$JAVA_HOME/include/ -I$JAVA_HOME/include/darwin -isysroot$(xcrun --show-sdk-path)" false false || echo + ./genbindings.sh ./ldk-c-bindings/ "-I$JAVA_HOME/include/ -I$JAVA_HOME/include/darwin -isysroot$(xcrun --show-sdk-path)" false false cat src/main/resources/liblightningjni_MacOSX-aarch64.nativelib > /dev/null fi - name: Fetch Maven 3.8.4 diff --git a/genbindings.sh b/genbindings.sh index a378a1e1..6c7f6501 100755 --- a/genbindings.sh +++ b/genbindings.sh @@ -1,18 +1,18 @@ #!/bin/bash usage() { - echo "USAGE: path/to/ldk-c-bindings \"JNI_CFLAGS\" debug android web" + echo "USAGE: path/to/ldk-c-bindings [wasm|\"JNI_CFLAGS\"] debug android_web" echo "For JNI_CFLAGS you probably want -I/usr/lib/jvm/java-11-openjdk-amd64/include/ -I/usr/lib/jvm/java-11-openjdk-amd64/include/linux/" + echo "If JNI_CFLAGS is instead set to wasm, we build for wasm/TypeScript instead of Java" echo "debug should either be true, false, or leaks" echo "debug of leaks turns on leak tracking on an optimized release bianry" - echo "android should either be true or false" - echo "web should either be true or false" + echo "android_web should either be true or false and indicates if we build for android (Java) or web (WASM)" exit 1 } [ "$1" = "" ] && usage [ "$3" != "true" -a "$3" != "false" -a "$3" != "leaks" ] && usage [ "$4" != "true" -a "$4" != "false" ] && usage -[ "$5" != "true" -a "$5" != "false" ] && usage +set -e set -x if [ "$CC" != "" ]; then @@ -22,201 +22,211 @@ else COMMON_COMPILE="clang -std=c11 -Wall -Wextra -Wno-unused-parameter -Wno-ignored-qualifiers -Wno-unused-function -Wno-nullability-completeness -Wno-pointer-sign -Wdate-time -ffile-prefix-map=$(pwd)=" fi -TARGET_STRING="$LDK_TARGET" -if [ "$TARGET_STRING" = "" ]; then - # We assume clang-style $CC --version here, but worst-case we just get an empty suffix - TARGET_STRING="$($CC --version | grep Target | awk '{ print $2 }')" -fi -case "$TARGET_STRING" in - "x86_64-pc-linux"*) - LDK_TARGET_SUFFIX="_Linux-amd64" - LDK_JAR_TARGET=true - ;; - "x86_64-apple-darwin"*) - LDK_TARGET_SUFFIX="_MacOSX-x86_64" - LDK_JAR_TARGET=true - ;; - "aarch64-apple-darwin"*) - LDK_TARGET_SUFFIX="_MacOSX-aarch64" - LDK_JAR_TARGET=true - ;; - *) - LDK_TARGET_SUFFIX="_${TARGET_STRING}" -esac -if [ "$LDK_TARGET_CPU" = "" ]; then - LDK_TARGET_CPU="sandybridge" -fi - -set -e - -if [ "$LDK_GARBAGECOLLECTED_GIT_OVERRIDE" = "" ]; then - export LDK_GARBAGECOLLECTED_GIT_OVERRIDE=$(git describe --tag --dirty) -fi -if [ "${LDK_GARBAGECOLLECTED_GIT_OVERRIDE:0:1}" != "v" ]; then - echo "Version tag should start with a v" > /dev/stderr - exit 1 +DEBUG_ARG="$3" +if [ "$3" = "leaks" ]; then + DEBUG_ARG="true" fi cp "$1/lightning-c-bindings/include/lightning.h" ./ if [ "$(rustc --version --verbose | grep "host:")" = "host: x86_64-apple-darwin" ]; then # OSX sed is for some reason not compatible with GNU sed sed -i '' "s/TransactionOutputs/C2Tuple_TxidCVec_C2Tuple_u32TxOutZZZ/g" ./lightning.h - sed -i '' "s/^ .*<\/version>/ ${LDK_GARBAGECOLLECTED_GIT_OVERRIDE:1:100}<\/version>/g" pom.xml else sed -i "s/TransactionOutputs/C2Tuple_TxidCVec_C2Tuple_u32TxOutZZZ/g" ./lightning.h - sed -i "s/^ .*<\/version>/ ${LDK_GARBAGECOLLECTED_GIT_OVERRIDE:1:100}<\/version>/g" pom.xml fi -echo "Creating Java bindings..." -mkdir -p src/main/java/org/ldk/{enums,structs} -rm -f src/main/java/org/ldk/{enums,structs}/*.java -rm -f src/main/jni/*.h -DEBUG_ARG="$3" -if [ "$3" = "leaks" ]; then - DEBUG_ARG="true" -fi -if [ "$4" = "true" ]; then - ./genbindings.py "./lightning.h" src/main/java/org/ldk/impl src/main/java/org/ldk src/main/jni/ $DEBUG_ARG android $4 -else - ./genbindings.py "./lightning.h" src/main/java/org/ldk/impl src/main/java/org/ldk src/main/jni/ $DEBUG_ARG java $4 -fi -rm -f src/main/jni/bindings.c -if [ "$3" = "true" ]; then - echo "#define LDK_DEBUG_BUILD" > src/main/jni/bindings.c -elif [ "$3" = "leaks" ]; then - # For leak checking we use release libldk which doesn't expose - # __unmangle_inner_ptr, but the C code expects to be able to call it. - echo "#define __unmangle_inner_ptr(a) (a)" > src/main/jni/bindings.c -fi -echo "#define LDKCVec_C2Tuple_TxidCVec_C2Tuple_u32TxOutZZZZ LDKCVec_TransactionOutputsZ" >> src/main/jni/bindings.c -echo "#define CVec_C2Tuple_TxidCVec_C2Tuple_u32TxOutZZZZ_free CVec_TransactionOutputsZ_free" >> src/main/jni/bindings.c -cat src/main/jni/bindings.c.body >> src/main/jni/bindings.c -javac -h src/main/jni src/main/java/org/ldk/enums/*.java src/main/java/org/ldk/impl/*.java -rm src/main/java/org/ldk/enums/*.class src/main/java/org/ldk/impl/bindings*.class +if [ "$2" != "wasm" ]; then + TARGET_STRING="$LDK_TARGET" + if [ "$TARGET_STRING" = "" ]; then + # We assume clang-style $CC --version here, but worst-case we just get an empty suffix + TARGET_STRING="$($CC --version | grep Target | awk '{ print $2 }')" + fi + case "$TARGET_STRING" in + "x86_64-pc-linux"*) + LDK_TARGET_SUFFIX="_Linux-amd64" + LDK_JAR_TARGET=true + ;; + "x86_64-apple-darwin"*) + LDK_TARGET_SUFFIX="_MacOSX-x86_64" + LDK_JAR_TARGET=true + ;; + "aarch64-apple-darwin"*) + LDK_TARGET_SUFFIX="_MacOSX-aarch64" + LDK_JAR_TARGET=true + ;; + *) + LDK_TARGET_SUFFIX="_${TARGET_STRING}" + esac + if [ "$LDK_TARGET_CPU" = "" ]; then + LDK_TARGET_CPU="sandybridge" + fi -IS_MAC=false -[ "$($CC --version | grep apple-darwin)" != "" ] && IS_MAC=true -IS_APPLE_CLANG=false -[ "$($CC --version | grep "Apple clang version")" != "" ] && IS_APPLE_CLANG=true + if [ "$LDK_GARBAGECOLLECTED_GIT_OVERRIDE" = "" ]; then + export LDK_GARBAGECOLLECTED_GIT_OVERRIDE=$(git describe --tag --dirty) + fi + if [ "${LDK_GARBAGECOLLECTED_GIT_OVERRIDE:0:1}" != "v" ]; then + echo "Version tag should start with a v" > /dev/stderr + exit 1 + fi -echo "Building Java bindings..." -COMPILE="$COMMON_COMPILE -mcpu=$LDK_TARGET_CPU -Isrc/main/jni -pthread -ldl -shared -fPIC" -[ "$IS_MAC" = "false" ] && COMPILE="$COMPILE -Wl,--no-undefined" -[ "$IS_MAC" = "true" ] && COMPILE="$COMPILE -mmacosx-version-min=10.9" -[ "$IS_MAC" = "true" -a "$IS_APPLE_CLANG" = "false" ] && COMPILE="$COMPILE -fuse-ld=lld" -[ "$IS_MAC" = "true" -a "$IS_APPLE_CLANG" = "false" ] && echo "WARNING: Need at least upstream clang 13!" -[ "$IS_MAC" = "false" -a "$3" != "false" ] && COMPILE="$COMPILE -Wl,-wrap,calloc -Wl,-wrap,realloc -Wl,-wrap,malloc -Wl,-wrap,free" -if [ "$3" = "true" ]; then - $COMPILE -o liblightningjni_debug$LDK_TARGET_SUFFIX.so -g -fsanitize=address -shared-libasan -rdynamic -I"$1"/lightning-c-bindings/include/ $2 src/main/jni/bindings.c "$1"/lightning-c-bindings/target/$LDK_TARGET/debug/libldk.a -lm -else - LDK_LIB="$1"/lightning-c-bindings/target/$LDK_TARGET/release/libldk.a - if [ "$IS_MAC" = "false" -a "$4" = "false" ]; then - COMPILE="$COMPILE -Wl,--version-script=libcode.version -fuse-ld=lld" - # __cxa_thread_atexit_impl is used to more effeciently cleanup per-thread local storage by rust libstd. - # However, it is not available on glibc versions 2.17 or earlier, and rust libstd has a null-check and - # fallback in case it is missing. - # Because it is weak-linked on the rust side, we should be able to simply define it - # explicitly, forcing rust to use the fallback. However, for some reason involving ancient - # dark magic and haunted code segments, overriding the weak symbol only impacts sites which - # *call* the symbol in question, not sites which *compare with* the symbol in question. - # This means that the NULL check in rust's libstd will always think the function is - # callable while the function which is called ends up being NULL (leading to a jmp to the - # zero page and a quick SEGFAULT). - # This issue persists not only with directly providing a symbol, but also ld.lld's -wrap - # and --defsym arguments. - # In smaller programs, it appears to be possible to work around this with -Bsymbolic and - # -nostdlib, however when applied the full-sized JNI library here it no longer works. - # After exhausting nearly every flag documented in lld, the only reliable method appears - # to be editing the LDK binary. Luckily, LLVM's tooling makes this rather easy as we can - # disassemble it into very readable code, edit it, and then reassemble it. - # Note that if we do so we don't have to bother overriding the actual call, LLVM should - # optimize it away, which also provides a good check that there isn't anything actually - # relying on it elsewhere. - [ ! -f "$1"/lightning-c-bindings/target/$LDK_TARGET/release/libldk.a ] && exit 1 - if [ "$(ar t "$1"/lightning-c-bindings/target/$LDK_TARGET/release/libldk.a | grep -v "\.o$" || echo)" != "" ]; then - echo "Archive contained non-object files!" - exit 1 - fi - if [ "$(ar t "$1"/lightning-c-bindings/target/$LDK_TARGET/release/libldk.a | grep ldk.ldk.*-cgu.*.rcgu.o | wc -l)" != "1" ]; then - echo "Archive contained more than one LDK object file" - exit 1 - fi - mkdir -p tmp - rm -f tmp/* - ar x --output=tmp "$1"/lightning-c-bindings/target/$LDK_TARGET/release/libldk.a - pushd tmp - llvm-dis ldk.ldk.*-cgu.*.rcgu.o - sed -i 's/br i1 icmp eq (i8\* @__cxa_thread_atexit_impl, i8\* null)/br i1 icmp eq (i8* null, i8* null)/g' ldk.ldk.*-cgu.*.rcgu.o.ll - llvm-as ldk.ldk.*-cgu.*.rcgu.o.ll -o ./libldk.bc - ar q libldk.a *.o - popd - LDK_LIB="tmp/libldk.bc tmp/libldk.a" + if [ "$(rustc --version --verbose | grep "host:")" = "host: x86_64-apple-darwin" ]; then + # OSX sed is for some reason not compatible with GNU sed + sed -i '' "s/^ .*<\/version>/ ${LDK_GARBAGECOLLECTED_GIT_OVERRIDE:1:100}<\/version>/g" pom.xml + else + sed -i "s/^ .*<\/version>/ ${LDK_GARBAGECOLLECTED_GIT_OVERRIDE:1:100}<\/version>/g" pom.xml fi - $COMPILE -o liblightningjni_release$LDK_TARGET_SUFFIX.so -flto -O3 -I"$1"/lightning-c-bindings/include/ $2 src/main/jni/bindings.c $LDK_LIB - if [ "$IS_MAC" = "false" -a "$4" = "false" ]; then - GLIBC_SYMBS="$(objdump -T liblightningjni_release$LDK_TARGET_SUFFIX.so | grep GLIBC_ | grep -v "GLIBC_2\.2\." | grep -v "GLIBC_2\.3\(\.\| \)" | grep -v "GLIBC_2.\(14\|17\) " || echo)" - if [ "$GLIBC_SYMBS" != "" ]; then - echo "Unexpected glibc version dependency! Some users need glibc 2.17 support, symbols for newer glibcs cannot be included." - echo "$GLIBC_SYMBS" - exit 1 - fi - REALLOC_ARRAY_SYMBS="$(objdump -T liblightningjni_release$LDK_TARGET_SUFFIX.so | grep reallocarray || echo)" - if [ "$REALLOC_ARRAY_SYMBS" != "" ]; then - echo "Unexpected reallocarray dependency!" - exit 1 - fi + + echo "Creating Java bindings..." + mkdir -p src/main/java/org/ldk/{enums,structs} + rm -f src/main/java/org/ldk/{enums,structs}/*.java + rm -f src/main/jni/*.h + if [ "$4" = "true" ]; then + ./genbindings.py "./lightning.h" src/main/java/org/ldk/impl src/main/java/org/ldk src/main/jni/ $DEBUG_ARG android $4 + else + ./genbindings.py "./lightning.h" src/main/java/org/ldk/impl src/main/java/org/ldk src/main/jni/ $DEBUG_ARG java $4 fi - if [ "$LDK_JAR_TARGET" = "true" ]; then - # Copy to JNI native directory for inclusion in JARs - mkdir -p src/main/resources/ - cp liblightningjni_release$LDK_TARGET_SUFFIX.so src/main/resources/liblightningjni$LDK_TARGET_SUFFIX.nativelib + rm -f src/main/jni/bindings.c + if [ "$3" = "true" ]; then + echo "#define LDK_DEBUG_BUILD" > src/main/jni/bindings.c + elif [ "$3" = "leaks" ]; then + # For leak checking we use release libldk which doesn't expose + # __unmangle_inner_ptr, but the C code expects to be able to call it. + echo "#define __unmangle_inner_ptr(a) (a)" > src/main/jni/bindings.c fi -fi + echo "#define LDKCVec_C2Tuple_TxidCVec_C2Tuple_u32TxOutZZZZ LDKCVec_TransactionOutputsZ" >> src/main/jni/bindings.c + echo "#define CVec_C2Tuple_TxidCVec_C2Tuple_u32TxOutZZZZ_free CVec_TransactionOutputsZ_free" >> src/main/jni/bindings.c + cat src/main/jni/bindings.c.body >> src/main/jni/bindings.c + javac -h src/main/jni src/main/java/org/ldk/enums/*.java src/main/java/org/ldk/impl/*.java + rm src/main/java/org/ldk/enums/*.class src/main/java/org/ldk/impl/bindings*.class -echo "Creating TS bindings..." -mkdir -p ts/{enums,structs} -rm -f ts/{enums,structs,}/*.{mjs,mts} -if [ "$5" = "true" ]; then - ./genbindings.py "./lightning.h" ts ts ts $DEBUG_ARG typescript node -else - ./genbindings.py "./lightning.h" ts ts ts $DEBUG_ARG typescript browser -fi -rm -f ts/bindings.c -if [ "$3" = "true" ]; then - echo "#define LDK_DEBUG_BUILD" > ts/bindings.c -elif [ "$3" = "leaks" ]; then - # For leak checking we use release libldk which doesn't expose - # __unmangle_inner_ptr, but the C code expects to be able to call it. - echo "#define __unmangle_inner_ptr(a) (a)" > ts/bindings.c -fi -echo "#define LDKCVec_C2Tuple_TxidCVec_C2Tuple_u32TxOutZZZZ LDKCVec_TransactionOutputsZ" >> ts/bindings.c -echo "#define CVec_C2Tuple_TxidCVec_C2Tuple_u32TxOutZZZZ_free CVec_TransactionOutputsZ_free" >> ts/bindings.c -cat ts/bindings.c.body >> ts/bindings.c + IS_MAC=false + [ "$($CC --version | grep apple-darwin)" != "" ] && IS_MAC=true + IS_APPLE_CLANG=false + [ "$($CC --version | grep "Apple clang version")" != "" ] && IS_APPLE_CLANG=true -echo "Building TS bindings..." -COMPILE="$COMMON_COMPILE -flto -Wl,--no-entry -Wl,--export-dynamic -Wl,-allow-undefined -nostdlib --target=wasm32-wasi" -# We only need malloc and assert/abort, but for now just use WASI for those: -#EXTRA_LINK=/usr/lib/wasm32-wasi/libc.a -EXTRA_LINK= -[ "$3" != "false" ] && COMPILE="$COMPILE -Wl,-wrap,calloc -Wl,-wrap,realloc -Wl,-wrap,reallocarray -Wl,-wrap,malloc -Wl,-wrap,free" -if [ "$3" = "true" ]; then - $COMPILE -o liblightningjs_debug.wasm -g -I"$1"/lightning-c-bindings/include/ ts/bindings.c "$1"/lightning-c-bindings/target/wasm32-wasi/debug/libldk.a $EXTRA_LINK + echo "Building Java bindings..." + COMPILE="$COMMON_COMPILE -mcpu=$LDK_TARGET_CPU -Isrc/main/jni -pthread -ldl -shared -fPIC" + [ "$IS_MAC" = "false" ] && COMPILE="$COMPILE -Wl,--no-undefined" + [ "$IS_MAC" = "true" ] && COMPILE="$COMPILE -mmacosx-version-min=10.9" + [ "$IS_MAC" = "true" -a "$IS_APPLE_CLANG" = "false" ] && COMPILE="$COMPILE -fuse-ld=lld" + [ "$IS_MAC" = "true" -a "$IS_APPLE_CLANG" = "false" ] && echo "WARNING: Need at least upstream clang 13!" + [ "$IS_MAC" = "false" -a "$3" != "false" ] && COMPILE="$COMPILE -Wl,-wrap,calloc -Wl,-wrap,realloc -Wl,-wrap,malloc -Wl,-wrap,free" + if [ "$3" = "true" ]; then + $COMPILE -o liblightningjni_debug$LDK_TARGET_SUFFIX.so -g -fsanitize=address -shared-libasan -rdynamic -I"$1"/lightning-c-bindings/include/ $2 src/main/jni/bindings.c "$1"/lightning-c-bindings/target/$LDK_TARGET/debug/libldk.a -lm + else + LDK_LIB="$1"/lightning-c-bindings/target/$LDK_TARGET/release/libldk.a + if [ "$IS_MAC" = "false" -a "$4" = "false" ]; then + COMPILE="$COMPILE -Wl,--version-script=libcode.version -fuse-ld=lld" + # __cxa_thread_atexit_impl is used to more effeciently cleanup per-thread local storage by rust libstd. + # However, it is not available on glibc versions 2.17 or earlier, and rust libstd has a null-check and + # fallback in case it is missing. + # Because it is weak-linked on the rust side, we should be able to simply define it + # explicitly, forcing rust to use the fallback. However, for some reason involving ancient + # dark magic and haunted code segments, overriding the weak symbol only impacts sites which + # *call* the symbol in question, not sites which *compare with* the symbol in question. + # This means that the NULL check in rust's libstd will always think the function is + # callable while the function which is called ends up being NULL (leading to a jmp to the + # zero page and a quick SEGFAULT). + # This issue persists not only with directly providing a symbol, but also ld.lld's -wrap + # and --defsym arguments. + # In smaller programs, it appears to be possible to work around this with -Bsymbolic and + # -nostdlib, however when applied the full-sized JNI library here it no longer works. + # After exhausting nearly every flag documented in lld, the only reliable method appears + # to be editing the LDK binary. Luckily, LLVM's tooling makes this rather easy as we can + # disassemble it into very readable code, edit it, and then reassemble it. + # Note that if we do so we don't have to bother overriding the actual call, LLVM should + # optimize it away, which also provides a good check that there isn't anything actually + # relying on it elsewhere. + [ ! -f "$1"/lightning-c-bindings/target/$LDK_TARGET/release/libldk.a ] && exit 1 + if [ "$(ar t "$1"/lightning-c-bindings/target/$LDK_TARGET/release/libldk.a | grep -v "\.o$" || echo)" != "" ]; then + echo "Archive contained non-object files!" + exit 1 + fi + if [ "$(ar t "$1"/lightning-c-bindings/target/$LDK_TARGET/release/libldk.a | grep ldk.ldk.*-cgu.*.rcgu.o | wc -l)" != "1" ]; then + echo "Archive contained more than one LDK object file" + exit 1 + fi + mkdir -p tmp + rm -f tmp/* + ar x --output=tmp "$1"/lightning-c-bindings/target/$LDK_TARGET/release/libldk.a + pushd tmp + llvm-dis ldk.ldk.*-cgu.*.rcgu.o + sed -i 's/br i1 icmp eq (i8\* @__cxa_thread_atexit_impl, i8\* null)/br i1 icmp eq (i8* null, i8* null)/g' ldk.ldk.*-cgu.*.rcgu.o.ll + llvm-as ldk.ldk.*-cgu.*.rcgu.o.ll -o ./libldk.bc + ar q libldk.a *.o + popd + LDK_LIB="tmp/libldk.bc tmp/libldk.a" + fi + $COMPILE -o liblightningjni_release$LDK_TARGET_SUFFIX.so -flto -O3 -I"$1"/lightning-c-bindings/include/ $2 src/main/jni/bindings.c $LDK_LIB + if [ "$IS_MAC" = "false" -a "$4" = "false" ]; then + GLIBC_SYMBS="$(objdump -T liblightningjni_release$LDK_TARGET_SUFFIX.so | grep GLIBC_ | grep -v "GLIBC_2\.2\." | grep -v "GLIBC_2\.3\(\.\| \)" | grep -v "GLIBC_2.\(14\|17\) " || echo)" + if [ "$GLIBC_SYMBS" != "" ]; then + echo "Unexpected glibc version dependency! Some users need glibc 2.17 support, symbols for newer glibcs cannot be included." + echo "$GLIBC_SYMBS" + exit 1 + fi + REALLOC_ARRAY_SYMBS="$(objdump -T liblightningjni_release$LDK_TARGET_SUFFIX.so | grep reallocarray || echo)" + if [ "$REALLOC_ARRAY_SYMBS" != "" ]; then + echo "Unexpected reallocarray dependency!" + exit 1 + fi + fi + if [ "$LDK_JAR_TARGET" = "true" ]; then + # Copy to JNI native directory for inclusion in JARs + mkdir -p src/main/resources/ + cp liblightningjni_release$LDK_TARGET_SUFFIX.so src/main/resources/liblightningjni$LDK_TARGET_SUFFIX.nativelib + fi + fi else - $COMPILE -o liblightningjs_release.wasm -s -Os -I"$1"/lightning-c-bindings/include/ ts/bindings.c "$1"/lightning-c-bindings/target/wasm32-wasi/release/libldk.a $EXTRA_LINK -fi + echo "Creating TS bindings..." + mkdir -p ts/{enums,structs} + rm -f ts/{enums,structs,}/*.{mjs,mts} + if [ "$4" = "false" ]; then + ./genbindings.py "./lightning.h" ts ts ts $DEBUG_ARG typescript node + else + ./genbindings.py "./lightning.h" ts ts ts $DEBUG_ARG typescript browser + fi + rm -f ts/bindings.c + if [ "$3" = "true" ]; then + echo "#define LDK_DEBUG_BUILD" > ts/bindings.c + elif [ "$3" = "leaks" ]; then + # For leak checking we use release libldk which doesn't expose + # __unmangle_inner_ptr, but the C code expects to be able to call it. + echo "#define __unmangle_inner_ptr(a) (a)" > ts/bindings.c + fi + echo "#define LDKCVec_C2Tuple_TxidCVec_C2Tuple_u32TxOutZZZZ LDKCVec_TransactionOutputsZ" >> ts/bindings.c + echo "#define CVec_C2Tuple_TxidCVec_C2Tuple_u32TxOutZZZZ_free CVec_TransactionOutputsZ_free" >> ts/bindings.c + cat ts/bindings.c.body >> ts/bindings.c -if [ -x "$(which tsc)" ]; then - cd ts -rm -r structs # TODO: Make the human-types compile - if [ "$5" = "false" ]; then - tsc + echo "Building TS bindings..." + COMPILE="$COMMON_COMPILE -flto -Wl,--no-entry -Wl,--export-dynamic -Wl,-allow-undefined -nostdlib --target=wasm32-wasi" + # We only need malloc and assert/abort, but for now just use WASI for those: + #EXTRA_LINK=/usr/lib/wasm32-wasi/libc.a + EXTRA_LINK= + [ "$3" != "false" ] && COMPILE="$COMPILE -Wl,-wrap,calloc -Wl,-wrap,realloc -Wl,-wrap,reallocarray -Wl,-wrap,malloc -Wl,-wrap,free" + if [ "$3" = "true" ]; then + WASM_FILE=liblightningjs_debug.wasm + $COMPILE -o liblightningjs_debug.wasm -g -I"$1"/lightning-c-bindings/include/ ts/bindings.c "$1"/lightning-c-bindings/target/wasm32-wasi/debug/libldk.a $EXTRA_LINK else - tsc --types node --typeRoots . - cd .. - if [ -x "$(which node)" ]; then - NODE_V="$(node --version)" - if [ "${NODE_V:1:2}" -gt 14 ]; then - node ts/test/ + WASM_FILE=liblightningjs_release.wasm + $COMPILE -o liblightningjs_release.wasm -s -Os -I"$1"/lightning-c-bindings/include/ ts/bindings.c "$1"/lightning-c-bindings/target/wasm32-wasi/release/libldk.a $EXTRA_LINK + fi + + if [ -x "$(which tsc)" ]; then + cd ts + rm -r structs # TODO: Make the human-types compile + if [ "$4" = "true" ]; then + tsc + else + tsc --types node --typeRoots . + cd .. + if [ -x "$(which node)" ]; then + NODE_V="$(node --version)" + if [ "${NODE_V:1:2}" -gt 14 ]; then + rm -f liblightningjs.wasm + ln -s $WASM_FILE liblightningjs.wasm + node ts/test/node.mjs + fi fi fi fi diff --git a/ts/test/browser.mjs b/ts/test/browser.mjs new file mode 100644 index 00000000..fecaa3ec --- /dev/null +++ b/ts/test/browser.mjs @@ -0,0 +1,15 @@ +import { chromium, firefox, webkit } from 'playwright'; +import { strict as assert } from 'assert'; + +for (const browserType of [chromium, firefox]) { // We'd like to test webkit, but playwright doesn't support it on Debian (?!) + const browser = await browserType.launch(); + const context = await browser.newContext(); + const page = await context.newPage(); + await page.goto('http://localhost:8000/ts/test/index.html'); + const ret = await page.evaluate(() => { + return test_runner('../../liblightningjs.wasm'); + }); + assert(ret); + + await browser.close(); +} diff --git a/ts/test/index.html b/ts/test/index.html index 3b0ba69f..a1003954 100644 --- a/ts/test/index.html +++ b/ts/test/index.html @@ -1,9 +1,14 @@ - - + diff --git a/ts/test/node.mjs b/ts/test/node.mjs new file mode 100644 index 00000000..dfabe66e --- /dev/null +++ b/ts/test/node.mjs @@ -0,0 +1,4 @@ +import { run_tests } from "./tests.mjs"; +import { strict as assert } from 'assert'; +const res = await run_tests('./liblightningjs.wasm'); +assert(res); diff --git a/ts/test/test.mjs b/ts/test/tests.mjs similarity index 90% rename from ts/test/test.mjs rename to ts/test/tests.mjs index 1bcf0f1a..96e50ec6 100644 --- a/ts/test/test.mjs +++ b/ts/test/tests.mjs @@ -1,6 +1,7 @@ import * as ldk from "../bindings.mjs"; -async function run_tests() { - await ldk.initializeWasm("../../liblightningjs.wasm"); + +export async function run_tests(wasm_path) { + await ldk.initializeWasm(wasm_path); const result = ldk.CResult_boolLightningErrorZ_ok(true); console.assert(ldk.CResult_boolLightningErrorZ_is_ok(result)); console.assert(ldk.CResult_boolLightningErrorZ_get_ok(result)); @@ -19,6 +20,5 @@ async function run_tests() { wasm.TS_CResult_PublicKeyErrorZ_free(pk_res);*/ console.log("pass"); + return true; } - -run_tests(); -- 2.30.2