X-Git-Url: http://git.bitcoin.ninja/index.cgi?p=ldk-c-bindings;a=blobdiff_plain;f=genbindings.sh;h=533ee7c1722f13fe405f53790a45203bab22b471;hp=c5025982dc36a15704da88bbd69580ee1bcc2d49;hb=HEAD;hpb=cd94859a3ef20447674f393d3f169bd9e29023a1 diff --git a/genbindings.sh b/genbindings.sh index c502598..bc2e073 100755 --- a/genbindings.sh +++ b/genbindings.sh @@ -9,6 +9,18 @@ if [ ! -d "$1/lightning" -o "$2" != "true" -a "$2" != "false" ]; then exit 1 fi +SKIP_TESTS_ARGUMENT=$3 +RUN_CPP_TESTS=true + +if [ ! -z "$SKIP_TESTS_ARGUMENT" ]; then + if [ "$SKIP_TESTS_ARGUMENT" != "skip-tests" ]; then + echo "To skip tests, usage must be: $0 path-to-rust-lightning allow-std skip-tests" + exit 1 + else + RUN_CPP_TESTS=false + fi +fi + export LC_ALL=C # On reasonable systems, we can use realpath here, but OSX is a diva with 20-year-old software. @@ -74,7 +86,7 @@ rm genbindings_path_map_test_file.c case "$ENV_TARGET" in "x86_64"*) export RUSTFLAGS="$BASE_RUSTFLAGS -C target-cpu=sandybridge" - export BASE_HOST_CFLAGS="$BASE_HOST_CFLAGS -march=sandybridge -mcpu=sandybridge -mtune=sandybridge" + export BASE_HOST_CFLAGS="$BASE_HOST_CFLAGS -march=sandybridge -mtune=sandybridge" export CFLAGS_$ENV_TARGET="$BASE_HOST_CFLAGS" ;; "aarch64_apple_darwin") @@ -85,7 +97,7 @@ case "$ENV_TARGET" in *) # Assume this isn't targeted at another host and build for the host's CPU. export RUSTFLAGS="$BASE_RUSTFLAGS -C target-cpu=native" - export BASE_HOST_CFLAGS="$BASE_HOST_CFLAGS -mcpu=native" + export BASE_HOST_CFLAGS="$BASE_HOST_CFLAGS -march=native -mtune=native" export CFLAGS_$ENV_TARGET="$BASE_HOST_CFLAGS" ;; esac @@ -156,7 +168,7 @@ function is_gnu_sed(){ function add_crate() { pushd "$LIGHTNING_PATH/$1" - RUSTC_BOOTSTRAP=1 cargo rustc --profile=check --no-default-features $3 -- --cfg=c_bindings -Zunpretty=expanded > /tmp/$1-crate-source.txt + RUSTC_BOOTSTRAP=1 cargo rustc --profile=check -Z avoid-dev-deps --no-default-features $3 -- --cfg=c_bindings -Zunpretty=expanded > /tmp/$1-crate-source.txt popd if [ "$HOST_OSX" = "true" ]; then sed -i".original" "1i\\ @@ -191,12 +203,12 @@ if [ "$2" = "true" ]; then add_crate "lightning-persister" "lightning_persister" add_crate "lightning-background-processor" "lightning_background_processor" --features=std add_crate "lightning-invoice" "lightning_invoice" --features=std - add_crate "lightning-rapid-gossip-sync" "lightning_rapid_gossip_sync" + add_crate "lightning-rapid-gossip-sync" "lightning_rapid_gossip_sync" --features=std CARGO_BUILD_ARGS="--features=std" else add_crate lightning lightning --features=no-std drop_crate "lightning-persister" - add_crate "lightning-background-processor" "lightning_background_processor" + add_crate "lightning-background-processor" "lightning_background_processor" --features=no-std add_crate "lightning-rapid-gossip-sync" "lightning_rapid_gossip_sync" --features=no-std add_crate "lightning-invoice" "lightning_invoice" --features=no-std CARGO_BUILD_ARGS="--features=no-std" @@ -232,7 +244,7 @@ if is_gnu_sed; then sed -i 's/typedef LDKnative.*Import.*LDKnative.*;//g' include/lightning.h # UnsafeCell is `repr(transparent)` so should be ignored here - sed -i 's/LDKUnsafeCell<\(.*\)> /\1 /g' include/lightning.h + sed -i 's/LDKUnsafeCell<\(.*\)> /struct \1 /g' include/lightning.h # stdlib.h doesn't exist in clang's wasm sysroot, and cbindgen # doesn't actually use it anyway, so drop the import. @@ -242,7 +254,7 @@ else sed -i '' 's/typedef LDKnative.*Import.*LDKnative.*;//g' include/lightning.h # UnsafeCell is `repr(transparent)` so should be ignored by cbindgen - sed -i '' 's/LDKUnsafeCell<\(.*\)> /\1 /g' include/lightning.h + sed -i '' 's/LDKUnsafeCell<\(.*\)> /struct \1 /g' include/lightning.h # stdlib.h doesn't exist in clang's wasm sysroot, and cbindgen # doesn't actually use it anyway, so drop the import. @@ -317,41 +329,78 @@ export IFS="$OLD_IFS" set -x mv include/lightningpp_new.hpp include/lightningpp.hpp -# Finally, sanity-check the generated C and C++ bindings with demo apps: -# Naively run the C demo app: -gcc $LOCAL_CFLAGS -Wall -g -pthread demo.c target/debug/libldk.a -ldl -lm -./a.out +if $RUN_CPP_TESTS; then + # Finally, sanity-check the generated C and C++ bindings with demo apps: + # Naively run the C demo app: + gcc $LOCAL_CFLAGS -Wall -g -pthread demo.c target/debug/libldk.a -ldl -lm + ./a.out + + # And run the C++ demo app + if [ "$2" = "true" ]; then + g++ $LOCAL_CFLAGS -std=c++11 -Wall -g -pthread demo.cpp -Ltarget/debug/ -lldk -ldl + LD_LIBRARY_PATH=target/debug/ ./a.out > /dev/null + fi + + # Finally, run the C++ demo app with our native networking library + # in valgrind to test memory model correctness and lack of leaks. + gcc $LOCAL_CFLAGS -fPIC -std=c99 -Wall -g -pthread -I../ldk-net ../ldk-net/ldk_net.c -c -o ldk_net.o + if [ "$2" = "true" ]; then + g++ $LOCAL_CFLAGS -std=c++11 -Wall -g -pthread -DREAL_NET -I../ldk-net ldk_net.o demo.cpp target/debug/libldk.a -ldl -lm + if [ -x "`which valgrind`" -a "$(uname -m)" != "ppc64le" ]; then + valgrind --error-exitcode=4 --memcheck:leak-check=full --show-leak-kinds=all ./a.out + echo + else + echo "WARNING: Please install valgrind for more testing" + ./a.out + fi + fi + + + # Test a statically-linked C++ version, tracking the resulting binary size and runtime + # across debug, LTO, and cross-language LTO builds (using the same compiler each time). + if [ "$2" = "true" ]; then + clang++ $LOCAL_CFLAGS -std=c++11 demo.cpp target/debug/libldk.a -ldl + strip ./a.out + time ./a.out + echo " C++ Bin size and runtime w/o optimization:" + ls -lha a.out + fi -# And run the C++ demo app -if [ "$2" = "true" ]; then - g++ $LOCAL_CFLAGS -std=c++11 -Wall -g -pthread demo.cpp -Ltarget/debug/ -lldk -ldl - LD_LIBRARY_PATH=target/debug/ ./a.out > /dev/null +else + echo "Skipping tests!" fi -# Finally, run the C++ demo app with our native networking library -# in valgrind to test memory model correctness and lack of leaks. -gcc $LOCAL_CFLAGS -fPIC -std=c99 -Wall -g -pthread -I../ldk-net ../ldk-net/ldk_net.c -c -o ldk_net.o -if [ "$2" = "true" ]; then - g++ $LOCAL_CFLAGS -std=c++11 -Wall -g -pthread -DREAL_NET -I../ldk-net ldk_net.o demo.cpp target/debug/libldk.a -ldl -lm - if [ -x "`which valgrind`" -a "$(uname -m)" != "ppc64le" ]; then - valgrind --error-exitcode=4 --memcheck:leak-check=full --show-leak-kinds=all ./a.out - echo +function REALLY_PIN_CC { + # -Zbuild-std fails if we have any dependencies of build-deps, which + # cc added in 1.0.80, thus we pin back to 1.0.79 to avoid that. + cargo update -p cc --precise "1.0.79" --verbose + ( RUSTC_BOOTSTRAP=1 cargo build --features=std -v --release --target x86_64-apple-darwin -Zbuild-std=std,panic_abort > /dev/null 2>&1 ) || echo -n + ( RUSTC_BOOTSTRAP=1 cargo build --features=std -v --release --target aarch64-apple-darwin -Zbuild-std=std,panic_abort > /dev/null 2>&1 ) || echo -n + pushd ../deterministic-build-wrappers/compiler-builtins-dummy + cargo build > /dev/null 2>&1 || echo -n + popd + # Sadly, std also depends on cc, and we can't pin it in that tree + # directly. Instead, we have to delete the file out of the cargo + # registry and build --offline to avoid it using the latest version. + NEW_CC_DEP="$CARGO_HOME" + [ "$NEW_CC_DEP" = "" ] && NEW_CC_DEP="$HOME" + [ -d "$NEW_CC_DEP/.cargo/registry/cache/"github.com-* ] && CARGO_REGISTRY_CACHE="$(echo "$NEW_CC_DEP/.cargo/registry/cache/"github.com-*)" + [ -d "$NEW_CC_DEP/.cargo/registry/cache/"index.crates.io-* ] && CARGO_REGISTRY_CACHE="$(echo "$NEW_CC_DEP/.cargo/registry/cache/"index.crates.io-*)" + if [ -d "$CARGO_REGISTRY_CACHE" ]; then + if [ -f "$CARGO_REGISTRY_CACHE/compiler_builtins-0.1.109.crate" ]; then + mv "$CARGO_REGISTRY_CACHE/compiler_builtins-0.1.109.crate" ./ + fi + if [ -f "$CARGO_REGISTRY_CACHE/cc-1.0.79.crate" ]; then + mv "$CARGO_REGISTRY_CACHE/cc-1.0.79.crate" ./ + fi + rm -f "$CARGO_REGISTRY_CACHE/"cc-*.crate + [ -f ./cc-1.0.79.crate ] && mv ./cc-1.0.79.crate "$CARGO_REGISTRY_CACHE/" + rm -f "$CARGO_REGISTRY_CACHE/"compiler_builtins-0.1.11*.crate + [ -f ./compiler_builtins-0.1.109.crate ] && mv ./compiler_builtins-0.1.109.crate "$CARGO_REGISTRY_CACHE/" else - echo "WARNING: Please install valgrind for more testing" - ./a.out + echo "Couldn't find cargo cache, build-std builds are likely to fail!" fi -fi - - -# Test a statically-linked C++ version, tracking the resulting binary size and runtime -# across debug, LTO, and cross-language LTO builds (using the same compiler each time). -if [ "$2" = "true" ]; then - clang++ $LOCAL_CFLAGS -std=c++11 demo.cpp target/debug/libldk.a -ldl - strip ./a.out - time ./a.out - echo " C++ Bin size and runtime w/o optimization:" - ls -lha a.out -fi +} # Then, check with memory sanitizer, if we're on Linux and have rustc nightly if [ "$HOST_PLATFORM" = "x86_64-unknown-linux-gnu" ]; then @@ -359,7 +408,9 @@ if [ "$HOST_PLATFORM" = "x86_64-unknown-linux-gnu" ]; then LLVM_V=$(rustc +nightly --version --verbose | grep "LLVM version" | awk '{ print substr($3, 0, 2); }') if [ -x "$(which clang-$LLVM_V)" ]; then cargo +nightly clean - cargo +nightly rustc $CARGO_BUILD_ARGS -Zbuild-std=std,panic_abort --target x86_64-unknown-linux-gnu -v -- -Zsanitizer=memory -Zsanitizer-memory-track-origins -Cforce-frame-pointers=yes + + REALLY_PIN_CC + cargo +nightly rustc --offline $CARGO_BUILD_ARGS -Zbuild-std=std,panic_abort --target x86_64-unknown-linux-gnu -v -- -Zsanitizer=memory -Zsanitizer-memory-track-origins -Cforce-frame-pointers=yes mv target/x86_64-unknown-linux-gnu/debug/libldk.* target/debug/ # Sadly, std doesn't seem to compile into something that is memsan-safe as of Aug 2020, @@ -409,7 +460,9 @@ if [ "$HOST_OSX" = "true" ]; then fi fi else - CLANG_LLVM_V=$(clang --version | head -n1 | awk '{ print substr($4, 0, 2); }') + # Output is something like clang version 17.0.3 (Fedora 17.0.3-1.fc39) or Debian clang version 14.0.6 + CLANG_LLVM_V=$(clang --version | head -n1 | awk '{ print substr($3, 0, 2); }') + [ "$CLANG_LLVM_V" = "ve" ] && CLANG_LLVM_V=$(clang --version | head -n1 | awk '{ print substr($4, 0, 2); }') if [ -x "$(which ld.lld)" ]; then LLD_LLVM_V="$(ld.lld --version | awk '{ print $2; }')" if [ "$LLD_LLVM_V" = "LLD" ]; then # eg if the output is "Debian LLD ..." @@ -469,20 +522,23 @@ if [ "$HOST_PLATFORM" = "x86_64-unknown-linux-gnu" ]; then RUSTFLAGS="$RUSTFLAGS --cfg=test_mod_pointers" RUSTC_BOOTSTRAP=1 cargo rustc $CARGO_BUILD_ARGS -v -- -Zsanitizer=address -Cforce-frame-pointers=yes || ( mv Cargo.toml.bk Cargo.toml; exit 1) mv Cargo.toml.bk Cargo.toml - # First the C demo app... - $CLANG $LOCAL_CFLAGS -fsanitize=address -g demo.c target/debug/libldk.a -ldl - ASAN_OPTIONS='detect_leaks=1 detect_invalid_pointer_pairs=1 detect_stack_use_after_return=1' ./a.out + # Sadly, address sanitizer appears to have had some regression on Debian and now fails to + # get past its init stage, so we disable it for now. - if [ "$2" = "true" ]; then - # ...then the C++ demo app - $CLANGPP $LOCAL_CFLAGS -std=c++11 -fsanitize=address -g demo.cpp target/debug/libldk.a -ldl - ASAN_OPTIONS='detect_leaks=1 detect_invalid_pointer_pairs=1 detect_stack_use_after_return=1' ./a.out >/dev/null - - # ...then the C++ demo app with the ldk_net network implementation - $CLANG $LOCAL_CFLAGS -fPIC -fsanitize=address -g -I../ldk-net ../ldk-net/ldk_net.c -c -o ldk_net.o - $CLANGPP $LOCAL_CFLAGS -std=c++11 -fsanitize=address -g -DREAL_NET -I../ldk-net ldk_net.o demo.cpp target/debug/libldk.a -ldl - ASAN_OPTIONS='detect_leaks=1 detect_invalid_pointer_pairs=1 detect_stack_use_after_return=1' ./a.out >/dev/null - fi + # First the C demo app... + #$CLANG $LOCAL_CFLAGS -fsanitize=address -g demo.c target/debug/libldk.a -ldl + #ASAN_OPTIONS='detect_leaks=1 detect_invalid_pointer_pairs=1 detect_stack_use_after_return=1' ./a.out + + #if [ "$2" = "true" ]; then + # # ...then the C++ demo app + # $CLANGPP $LOCAL_CFLAGS -std=c++11 -fsanitize=address -g demo.cpp target/debug/libldk.a -ldl + # ASAN_OPTIONS='detect_leaks=1 detect_invalid_pointer_pairs=1 detect_stack_use_after_return=1' ./a.out >/dev/null + + # # ...then the C++ demo app with the ldk_net network implementation + # $CLANG $LOCAL_CFLAGS -fPIC -fsanitize=address -g -I../ldk-net ../ldk-net/ldk_net.c -c -o ldk_net.o + # $CLANGPP $LOCAL_CFLAGS -std=c++11 -fsanitize=address -g -DREAL_NET -I../ldk-net ldk_net.o demo.cpp target/debug/libldk.a -ldl + # ASAN_OPTIONS='detect_leaks=1 detect_invalid_pointer_pairs=1 detect_stack_use_after_return=1' ./a.out >/dev/null + #fi else echo "WARNING: Please install clang-$RUSTC_LLVM_V and clang++-$RUSTC_LLVM_V to build with address sanitizer" fi @@ -530,6 +586,7 @@ fi EXTRA_TARGETS=( $LDK_C_BINDINGS_EXTRA_TARGETS ) EXTRA_CCS=( $LDK_C_BINDINGS_EXTRA_TARGET_CCS ) +EXTRA_LINK_LTO=( $LDK_C_BINDINGS_EXTRA_TARGET_LINK_LTO ) if [ ${#EXTRA_TARGETS[@]} != ${#EXTRA_CCS[@]} ]; then echo "LDK_C_BINDINGS_EXTRA_TARGETS and LDK_C_BINDINGS_EXTRA_TARGET_CCS didn't have the same number of elements!" @@ -540,7 +597,15 @@ for IDX in ${!EXTRA_TARGETS[@]}; do EXTRA_ENV_TARGET=$(echo "${EXTRA_TARGETS[$IDX]}" | sed 's/-/_/g') export CFLAGS_$EXTRA_ENV_TARGET="$BASE_CFLAGS" export CC_$EXTRA_ENV_TARGET=${EXTRA_CCS[$IDX]} - RUSTFLAGS="$BASE_RUSTFLAGS -C embed-bitcode=yes -C lto -C linker=${EXTRA_CCS[$IDX]}" CARGO_PROFILE_RELEASE_LTO=true cargo build $CARGO_BUILD_ARGS -v --release --target "${EXTRA_TARGETS[$IDX]}" + EXTRA_RUSTFLAGS="" + case "$EXTRA_ENV_TARGET" in + "x86_64"*) + export CFLAGS_$EXTRA_ENV_TARGET="$BASE_CFLAGS -march=sandybridge -mtune=sandybridge" + EXTRA_RUSTFLAGS="-C target-cpu=sandybridge" + ;; + esac + [ "${EXTRA_LINK_LTO[$IDX]}" != "" ] && EXTRA_RUSTFLAGS="-C linker-plugin-lto" + RUSTFLAGS="$BASE_RUSTFLAGS -C embed-bitcode=yes -C lto -C linker=${EXTRA_CCS[$IDX]} $EXTRA_RUSTFLAGS" CARGO_PROFILE_RELEASE_LTO=true cargo build $CARGO_BUILD_ARGS -v --release --target "${EXTRA_TARGETS[$IDX]}" done if [ "$CLANGPP" != "" -a "$LLD" != "" ]; then @@ -552,12 +617,13 @@ if [ "$CLANGPP" != "" -a "$LLD" != "" ]; then LINK_ARG_FLAGS="-C link-arg=-fuse-ld=$LLD" export LDK_CLANG_PATH=$(which $CLANG) if [ "$MACOS_SDK" != "" ]; then + REALLY_PIN_CC export CLANG="$(pwd)/../deterministic-build-wrappers/clang-lto-link-osx" for ARG in $CFLAGS_aarch64_apple_darwin; do MANUAL_LINK_CFLAGS="$MANUAL_LINK_CFLAGS -C link-arg=$ARG" done export CFLAGS_aarch64_apple_darwin="$CFLAGS_aarch64_apple_darwin -O3 -fPIC -fembed-bitcode" - RUSTC_BOOTSTRAP=1 RUSTFLAGS="$BASE_RUSTFLAGS -C target-cpu=apple-a14 -C embed-bitcode=yes -C linker-plugin-lto -C lto -C linker=$CLANG $MANUAL_LINK_CFLAGS $LINK_ARG_FLAGS -C link-arg=-mcpu=apple-a14" CARGO_PROFILE_RELEASE_LTO=true cargo build $CARGO_BUILD_ARGS -v --release --target aarch64-apple-darwin -Zbuild-std=std,panic_abort + RUSTC_BOOTSTRAP=1 RUSTFLAGS="$BASE_RUSTFLAGS -C target-cpu=apple-a14 -C embed-bitcode=yes -C linker-plugin-lto -C lto -C linker=$CLANG $MANUAL_LINK_CFLAGS $LINK_ARG_FLAGS -C link-arg=-mcpu=apple-a14" CARGO_PROFILE_RELEASE_LTO=true cargo build $CARGO_BUILD_ARGS --offline -v --release --target aarch64-apple-darwin -Zbuild-std=std,panic_abort if [ "$HOST_OSX" != "true" ]; then # If we're not on OSX but can build OSX binaries, build the x86_64 OSX release now MANUAL_LINK_CFLAGS="" @@ -565,7 +631,7 @@ if [ "$CLANGPP" != "" -a "$LLD" != "" ]; then MANUAL_LINK_CFLAGS="$MANUAL_LINK_CFLAGS -C link-arg=$ARG" done export CFLAGS_x86_64_apple_darwin="$CFLAGS_x86_64_apple_darwin -O3 -fPIC -fembed-bitcode" - RUSTC_BOOTSTRAP=1 RUSTFLAGS="$BASE_RUSTFLAGS -C target-cpu=sandybridge -C embed-bitcode=yes -C linker-plugin-lto -C lto -C linker=$CLANG $MANUAL_LINK_CFLAGS $LINK_ARG_FLAGS -C link-arg=-mcpu=sandybridge -C link-arg=-mtune=sandybridge" CARGO_PROFILE_RELEASE_LTO=true cargo build $CARGO_BUILD_ARGS -v --release --target x86_64-apple-darwin -Zbuild-std=std,panic_abort + RUSTC_BOOTSTRAP=1 RUSTFLAGS="$BASE_RUSTFLAGS -C target-cpu=sandybridge -C embed-bitcode=yes -C linker-plugin-lto -C lto -C linker=$CLANG $MANUAL_LINK_CFLAGS $LINK_ARG_FLAGS -C link-arg=-march=sandybridge -C link-arg=-mtune=sandybridge" CARGO_PROFILE_RELEASE_LTO=true cargo build $CARGO_BUILD_ARGS --offline -v --release --target x86_64-apple-darwin -Zbuild-std=std,panic_abort fi fi # If we're on an M1 don't bother building X86 binaries @@ -574,7 +640,7 @@ if [ "$CLANGPP" != "" -a "$LLD" != "" ]; then export CFLAGS_$ENV_TARGET="$BASE_HOST_CFLAGS -O3 -fPIC -fembed-bitcode" # Rust doesn't recognize CFLAGS changes, so we need to clean build artifacts cargo clean --release - CARGO_PROFILE_RELEASE_LTO=true RUSTFLAGS="$RUSTFLAGS -C embed-bitcode=yes -C linker-plugin-lto -C lto -C linker=$CLANG $LINK_ARG_FLAGS -C link-arg=-march=sandybridge -C link-arg=-mcpu=sandybridge -C link-arg=-mtune=sandybridge" cargo build $CARGO_BUILD_ARGS -v --release + CARGO_PROFILE_RELEASE_LTO=true RUSTFLAGS="$RUSTFLAGS -C embed-bitcode=yes -C linker-plugin-lto -C lto -C linker=$CLANG $LINK_ARG_FLAGS -C link-arg=-march=sandybridge -C link-arg=-mtune=sandybridge" cargo build $CARGO_BUILD_ARGS -v --release if [ "$2" = "true" ]; then $CLANGPP $LOCAL_CFLAGS -flto -fuse-ld=$LLD -O2 -c demo.cpp -o demo.o