From adabc2f552aed94b4ca6eeb82e51bb595ea653f2 Mon Sep 17 00:00:00 2001 From: Martin Kinkelin Date: Sat, 18 Jan 2025 23:45:14 +0100 Subject: [PATCH 1/8] GHA: Integrate Alpine musl job in main workflow Incl. prebuilt package etc. --- .github/actions/3-build-native/action.yml | 4 + .github/actions/4c-test-dmd/action.yml | 15 +++- .github/actions/6-integration-test/action.yml | 10 ++- .github/actions/7-package/action.yml | 17 ++-- .github/workflows/alpine_musl.yml | 84 ------------------- .github/workflows/main.yml | 41 ++++++++- 6 files changed, 73 insertions(+), 98 deletions(-) delete mode 100644 .github/workflows/alpine_musl.yml diff --git a/.github/actions/3-build-native/action.yml b/.github/actions/3-build-native/action.yml index 3683f67fd8c..14bbd8f812f 100644 --- a/.github/actions/3-build-native/action.yml +++ b/.github/actions/3-build-native/action.yml @@ -31,3 +31,7 @@ runs: - run: ../build/bin/ldc2 --version shell: bash + + - run: ldd ../build/bin/ldc2 + if: runner.os == 'Linux' + shell: bash diff --git a/.github/actions/4c-test-dmd/action.yml b/.github/actions/4c-test-dmd/action.yml index 22dbea1f942..cd7368dc858 100644 --- a/.github/actions/4c-test-dmd/action.yml +++ b/.github/actions/4c-test-dmd/action.yml @@ -9,7 +9,20 @@ runs: - name: 'Posix: Run DMD testsuite' if: runner.os != 'Windows' shell: bash - run: cd ../build && ctest -V -R "dmd-testsuite" + run: | + set -eux + repoDir=$PWD + cd ../build + if type -P apk &>/dev/null; then + # Alpine: run full dmd-testsuite-debug + ctest -V -R 'dmd-testsuite' -E '^dmd-testsuite$' + # these two tests require extra flags "-link-defaultlib-debug -frame-pointer=all": https://github.com/ldc-developers/ldc/issues/4694 + # => remove before running optimized dmd-testsuite separately + rm $repoDir/tests/dmd/runnable/{test17559.d,test19086.d} + ctest -V -R '^dmd-testsuite$' + else + ctest -V -R "dmd-testsuite" + fi - name: 'Windows: Run DMD testsuite' if: runner.os == 'Windows' diff --git a/.github/actions/6-integration-test/action.yml b/.github/actions/6-integration-test/action.yml index 59c018c2d8c..79729b4223a 100644 --- a/.github/actions/6-integration-test/action.yml +++ b/.github/actions/6-integration-test/action.yml @@ -15,13 +15,16 @@ runs: run: | set -eux cd .. + if type -P apk &>/dev/null; then + exit 0 # Alpine: no shared libs + fi if [[ '${{ runner.os }}' == Windows ]]; then # add druntime/Phobos DLL dir to PATH export PATH="$PWD/installed/bin:$PATH" fi installed/bin/ldc2 hello.d -link-defaultlib-shared ./hello - if [[ '${{ runner.os }}-${{ inputs.arch }}' == Linux-x86_64 ]]; then + if [[ -d installed/lib32 ]]; then installed/bin/ldc2 hello.d -m32 -link-defaultlib-shared ./hello fi @@ -34,7 +37,7 @@ runs: for mode in thin full; do installed/bin/ldc2 hello.d -of=hello_$mode -flto=$mode -defaultlib=phobos2-ldc-lto,druntime-ldc-lto ./hello_$mode - if [[ '${{ runner.os }}-${{ inputs.arch }}' == Linux-x86_64 ]]; then + if [[ -d installed/lib32 ]]; then installed/bin/ldc2 hello.d -m32 -of=hello_$mode-32 -flto=$mode -defaultlib=phobos2-ldc-lto,druntime-ldc-lto ./hello_$mode-32 fi @@ -45,6 +48,9 @@ runs: run: | set -eux cd .. + if type -P apk &>/dev/null; then + exit 0 # FIXME: Alpine: LLVM too old for dynamic-compile support + fi if [[ '${{ runner.os }}' == Windows ]]; then # add ldc-jit.dll dir to PATH export PATH="$PWD/installed/bin:$PATH" diff --git a/.github/actions/7-package/action.yml b/.github/actions/7-package/action.yml index e33a7df1412..e6c91b46211 100644 --- a/.github/actions/7-package/action.yml +++ b/.github/actions/7-package/action.yml @@ -73,10 +73,12 @@ runs: "$(dirname "$DMD")/ldc2" -run list_payload.d . fi - # use host compiler's dub, which is guaranteed to be native - DFLAGS="-O -linkonce-templates ${DFLAGS:-}" dub build -v \ - --build-mode=allAtOnce --combined $archFlag \ - --compiler="$(dirname "$DMD")/ldc2" + # prefer host compiler's dub, which is guaranteed to be native + PATH="$PATH:$PWD/../installed/bin" \ + DFLAGS="-O -linkonce-templates ${DFLAGS:-}" \ + dub build -v \ + --build-mode=allAtOnce --combined $archFlag \ + --compiler="$(dirname "$DMD")/ldc2" cp bin/reggae ../installed/bin/ if [[ '${{ inputs.cross_target_triple }}' == '' ]]; then @@ -120,10 +122,11 @@ runs: chmod -R go=rX $artifactName if [[ '${{ runner.os }}' == macOS ]]; then sudo chown -R root:wheel $artifactName - tar -cf - $artifactName | 7za a artifacts/$artifactName.tar.xz -si -txz -mx9 - else - tar -cf - --owner=0 --group=0 $artifactName | 7za a artifacts/$artifactName.tar.xz -si -txz -mx9 + # note: already running as root in Alpine container, *and* potential busybox complication + elif [[ $os != alpine ]]; then + sudo chown -R 0:0 $artifactName fi + tar -cf - $artifactName | 7za a artifacts/$artifactName.tar.xz -si -txz -mx9 fi # export ARTIFACT_{ID,NAME} diff --git a/.github/workflows/alpine_musl.yml b/.github/workflows/alpine_musl.yml deleted file mode 100644 index 978d07893e4..00000000000 --- a/.github/workflows/alpine_musl.yml +++ /dev/null @@ -1,84 +0,0 @@ -name: Alpine Linux, musl libc, vanilla LLVM -on: - - pull_request - - push - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - test: - name: Alpine Linux (musl libc) - runs-on: ubuntu-latest - container: alpine:3.20 - steps: - - name: Install prerequisites - run: apk add ldc git g++ cmake ninja llvm-dev llvm-static compiler-rt libxml2-static zstd-static zlib-static bash grep diffutils make - - - uses: actions/checkout@v4 - with: - submodules: true - fetch-depth: 50 - - - name: Build LDC bootstrap - run: | - ninja --version - set -eux - mkdir bootstrap - cd bootstrap - cmake -G Ninja .. \ - -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_INSTALL_PREFIX="/opt/ldc2" \ - -DCMAKE_EXE_LINKER_FLAGS=-static-libstdc++ \ - -DD_COMPILER_FLAGS=-link-defaultlib-shared=false \ - -DBUILD_SHARED_LIBS=OFF - ninja obj/ldc2.o all - bin/ldc2 --version - cd .. - - # TODO: Add '-DLLVM_IS_SHARED=OFF' when static linking is fully supported - # TSan and XRay do not work. - - name: Build LDC & LDC D unittests & defaultlib unittest runners - run: | - set -eux - cmake -G Ninja . \ - -DD_COMPILER=./bootstrap/bin/ldmd2 \ - -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_INSTALL_PREFIX="/opt/ldc2" \ - -DCMAKE_EXE_LINKER_FLAGS=-static-libstdc++ \ - -DD_COMPILER_FLAGS=-link-defaultlib-shared=false \ - -DBUILD_SHARED_LIBS=OFF \ - -DTEST_COMPILER_RT_LIBRARIES="profile;lsan;asan;msan;fuzzer" - ninja obj/ldc2.o all ldc2-unittest all-test-runners - bin/ldc2 --version - ldd bin/ldc2 - - - name: Run LDC D unittests - if: success() || failure() - run: ctest --output-on-failure -R "ldc2-unittest" - - - name: Run LIT testsuite - if: success() || failure() - run: | - set -eux - ctest -V -R "lit-tests" - - - name: Run DMD testsuite - if: success() || failure() - run: | - # These two tests require extra flags "-link-defaultlib-debug -frame-pointer=all", https://github.com/ldc-developers/ldc/issues/4694. - # Run them separately with these flags, and then remove them before running the full testsuite. - bin/ldc2 -g -L-export-dynamic -link-defaultlib-debug -frame-pointer=all -run tests/dmd/runnable/test17559.d - bin/ldc2 -O -g -L-export-dynamic -link-defaultlib-debug -frame-pointer=all -run tests/dmd/runnable/test17559.d - bin/ldc2 -g -L-export-dynamic -link-defaultlib-debug -frame-pointer=all -run tests/dmd/runnable/test19086.d - bin/ldc2 -O -g -L-export-dynamic -link-defaultlib-debug -frame-pointer=all -run tests/dmd/runnable/test19086.d - rm tests/dmd/runnable/test17559.d - rm tests/dmd/runnable/test19086.d - ctest -V -R "dmd-testsuite" - - - name: Run defaultlib unittests & druntime integration tests - if: success() || failure() - run: | - set -eux - ctest -j$(nproc) --output-on-failure -E "dmd-testsuite|lit-tests|ldc2-unittest" diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index c0f202de9e2..3fa42187a67 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -53,6 +53,23 @@ jobs: -DEXTRA_CXXFLAGS=-flto=full with_pgo: true + - job_name: Alpine musl x86_64 + os: ubuntu-latest + container_image: alpine:3.20 + arch: x86_64 + bootstrap_cmake_flags: -DBUILD_LTO_LIBS=ON + # TODO: Add '-DLLVM_IS_SHARED=OFF' when static linking is fully supported + # TSan and XRay do not work. + extra_cmake_flags: >- + -DBUILD_LTO_LIBS=ON + -DBUILD_SHARED_LIBS=OFF + -DTEST_COMPILER_RT_LIBRARIES="profile;lsan;asan;msan;fuzzer" + -DCMAKE_EXE_LINKER_FLAGS=-static-libstdc++ + -DJITRT_EXTRA_LDFLAGS=-static-libstdc++ + -DD_COMPILER_FLAGS="-O -flto=full" + # FIXME: -defaultlib=phobos2-ldc-lto,druntime-ldc-lto + with_pgo: false + - job_name: macOS x86_64 os: macos-13 arch: x86_64 @@ -108,18 +125,33 @@ jobs: env: MACOSX_DEPLOYMENT_TARGET: ${{ matrix.arch == 'arm64' && '11.0' || '10.12' }} steps: - - name: 'Container: Install git and sudo' + - name: 'Set up Linux container' if: matrix.container_image - shell: bash run: | set -eux - apt-get -q update - DEBIAN_FRONTEND=noninteractive apt-get -yq install git-core sudo + if type -P apt-get &>/dev/null; then + # Ubuntu: pre-install git and sudo + apt-get -q update + DEBIAN_FRONTEND=noninteractive apt-get -yq install git-core sudo + else + # set up Alpine container + apk add \ + git cmake ninja-is-really-ninja g++ ldc llvm-dev llvm-static compiler-rt \ + libxml2-static zstd-static zlib-static \ + bash grep diffutils make curl 7zip perl + # create missing 7za symlink + ln -s 7z /usr/bin/7za + # create ../llvm symlink to distro LLVM (no prebuilt LDC-LLVM for musl) + ln -s /usr/lib/llvm17 $(dirname $(pwd))/llvm + ../llvm/bin/llvm-config --version + fi - uses: actions/checkout@v4 with: submodules: true fetch-depth: 50 - name: Install prerequisites + if: | + !startsWith(matrix.container_image, 'alpine') uses: ./.github/actions/1-setup with: llvm_version: ${{ matrix.llvm_version || env.LLVM_VERSION }} @@ -194,6 +226,7 @@ jobs: uses: ./.github/actions/7-package with: arch: ${{ matrix.arch }} + os: ${{ startsWith(matrix.container_image, 'alpine') && 'alpine' || '' }} # Cross-compilation jobs for non-native targets. From 76531bde024af4756395273e7999294cb7ac0926 Mon Sep 17 00:00:00 2001 From: Johan Engelen Date: Wed, 8 May 2024 17:20:41 +0200 Subject: [PATCH 2/8] Enable user-configuration of LLVM static/shared linking --- cmake/Modules/FindLLVM.cmake | 32 +++++++++++++++++++++++--------- 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/cmake/Modules/FindLLVM.cmake b/cmake/Modules/FindLLVM.cmake index 3e15fa9a81a..03b868f3c3c 100644 --- a/cmake/Modules/FindLLVM.cmake +++ b/cmake/Modules/FindLLVM.cmake @@ -80,9 +80,16 @@ else() if(LLVM_FIND_QUIETLY) set(_quiet_arg ERROR_QUIET) endif() + if(DEFINED LLVM_IS_SHARED) + if(LLVM_IS_SHARED) + set(_sharedstatic "--link-shared") + else() + set(_sharedstatic "--link-static") + endif() + endif() set(result_code) execute_process( - COMMAND ${LLVM_CONFIG} --${flag} + COMMAND ${LLVM_CONFIG} ${_sharedstatic} --${flag} RESULT_VARIABLE result_code OUTPUT_VARIABLE LLVM_${var} OUTPUT_STRIP_TRAILING_WHITESPACE @@ -100,9 +107,14 @@ else() if(LLVM_FIND_QUIETLY) set(_quiet_arg ERROR_QUIET) endif() + if (LLVM_IS_SHARED) + set(_sharedstatic "--link-shared") + else() + set(_sharedstatic "--link-static") + endif() set(result_code) execute_process( - COMMAND ${LLVM_CONFIG} --${flag} ${components} + COMMAND ${LLVM_CONFIG} ${_sharedstatic} --${flag} ${components} RESULT_VARIABLE result_code OUTPUT_VARIABLE tmplibs OUTPUT_STRIP_TRAILING_WHITESPACE @@ -116,6 +128,15 @@ else() endif() endmacro() + if (NOT DEFINED LLVM_IS_SHARED) + llvm_set(SHARED_MODE shared-mode) + if(LLVM_SHARED_MODE STREQUAL "shared") + set(LLVM_IS_SHARED ON) + else() + set(LLVM_IS_SHARED OFF) + endif() + endif() + llvm_set(VERSION_STRING version) llvm_set(CXXFLAGS cxxflags) llvm_set(INCLUDE_DIRS includedir true) @@ -127,13 +148,6 @@ else() string(REGEX REPLACE "([0-9]+).*" "\\1" LLVM_VERSION_MAJOR "${LLVM_VERSION_STRING}" ) string(REGEX REPLACE "[0-9]+\\.([0-9]+).*[A-Za-z]*" "\\1" LLVM_VERSION_MINOR "${LLVM_VERSION_STRING}" ) - llvm_set(SHARED_MODE shared-mode) - if(LLVM_SHARED_MODE STREQUAL "shared") - set(LLVM_IS_SHARED ON) - else() - set(LLVM_IS_SHARED OFF) - endif() - llvm_set(LDFLAGS ldflags) llvm_set(SYSTEM_LIBS system-libs) string(REPLACE "\n" " " LLVM_LDFLAGS "${LLVM_LDFLAGS} ${LLVM_SYSTEM_LIBS}") From 4f51525746ed8d18e52e4ab401b333a530702be4 Mon Sep 17 00:00:00 2001 From: Martin Kinkelin Date: Sun, 19 Jan 2025 15:13:04 +0100 Subject: [PATCH 3/8] GHA Alpine: Link LLVM statically --- .github/workflows/main.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 3fa42187a67..69d27872707 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -58,11 +58,14 @@ jobs: container_image: alpine:3.20 arch: x86_64 bootstrap_cmake_flags: -DBUILD_LTO_LIBS=ON - # TODO: Add '-DLLVM_IS_SHARED=OFF' when static linking is fully supported # TSan and XRay do not work. extra_cmake_flags: >- -DBUILD_LTO_LIBS=ON -DBUILD_SHARED_LIBS=OFF + -DLLVM_IS_SHARED=OFF + -DLDC_INSTALL_LTOPLUGIN=OFF + -DCOMPILER_RT_LIBDIR_OS=linux + -DLDC_INSTALL_LLVM_RUNTIME_LIBS_ARCH=x86_64 -DTEST_COMPILER_RT_LIBRARIES="profile;lsan;asan;msan;fuzzer" -DCMAKE_EXE_LINKER_FLAGS=-static-libstdc++ -DJITRT_EXTRA_LDFLAGS=-static-libstdc++ From 8165790b9821dc2833b959c7ffdf461958f1c517 Mon Sep 17 00:00:00 2001 From: Martin Kinkelin Date: Sun, 19 Jan 2025 15:56:28 +0100 Subject: [PATCH 4/8] GHA Alpine: Link remaining lib{zstd,z,c} statically too --- .github/actions/3-build-native/action.yml | 4 ---- .github/actions/6-integration-test/action.yml | 2 +- .github/actions/7-package/action.yml | 18 ++++++++++++++++++ .github/workflows/main.yml | 5 +++-- runtime/CMakeLists.txt | 3 --- 5 files changed, 22 insertions(+), 10 deletions(-) diff --git a/.github/actions/3-build-native/action.yml b/.github/actions/3-build-native/action.yml index 14bbd8f812f..3683f67fd8c 100644 --- a/.github/actions/3-build-native/action.yml +++ b/.github/actions/3-build-native/action.yml @@ -31,7 +31,3 @@ runs: - run: ../build/bin/ldc2 --version shell: bash - - - run: ldd ../build/bin/ldc2 - if: runner.os == 'Linux' - shell: bash diff --git a/.github/actions/6-integration-test/action.yml b/.github/actions/6-integration-test/action.yml index 79729b4223a..b6c4cb57d28 100644 --- a/.github/actions/6-integration-test/action.yml +++ b/.github/actions/6-integration-test/action.yml @@ -49,7 +49,7 @@ runs: set -eux cd .. if type -P apk &>/dev/null; then - exit 0 # FIXME: Alpine: LLVM too old for dynamic-compile support + exit 0 # Alpine: disabled dynamic-compile support fi if [[ '${{ runner.os }}' == Windows ]]; then # add ldc-jit.dll dir to PATH diff --git a/.github/actions/7-package/action.yml b/.github/actions/7-package/action.yml index e6c91b46211..569e5b69227 100644 --- a/.github/actions/7-package/action.yml +++ b/.github/actions/7-package/action.yml @@ -24,6 +24,13 @@ runs: fi echo "DMD=$DMD" >> $GITHUB_ENV + - name: 'Alpine: Set DFLAGS environment variable (-Xcc=-static)' # for fully statically linked dub etc. + if: inputs.os == 'alpine' + shell: bash + run: | + set -eux + echo "DFLAGS=-Xcc=-static${DFLAGS:+ $DFLAGS}" >> $GITHUB_ENV + - name: Build & copy dub shell: bash run: | @@ -85,6 +92,17 @@ runs: ../installed/bin/reggae --version -b ninja fi + - name: 'Linux: List executable dependencies' + if: runner.os == 'Linux' + shell: bash + run: | + set -euxo pipefail + cd ../installed/bin + for i in *; do + ls -lh $i + readelf -d $i | grep NEEDED || true + done + - name: Pack installation dir shell: bash run: | diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 69d27872707..ff292b2bea0 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -63,12 +63,13 @@ jobs: -DBUILD_LTO_LIBS=ON -DBUILD_SHARED_LIBS=OFF -DLLVM_IS_SHARED=OFF + -DLDC_ENABLE_PLUGINS=OFF + -DLDC_DYNAMIC_COMPILE=OFF -DLDC_INSTALL_LTOPLUGIN=OFF -DCOMPILER_RT_LIBDIR_OS=linux -DLDC_INSTALL_LLVM_RUNTIME_LIBS_ARCH=x86_64 -DTEST_COMPILER_RT_LIBRARIES="profile;lsan;asan;msan;fuzzer" - -DCMAKE_EXE_LINKER_FLAGS=-static-libstdc++ - -DJITRT_EXTRA_LDFLAGS=-static-libstdc++ + -DCMAKE_EXE_LINKER_FLAGS=-static -DD_COMPILER_FLAGS="-O -flto=full" # FIXME: -defaultlib=phobos2-ldc-lto,druntime-ldc-lto with_pgo: false diff --git a/runtime/CMakeLists.txt b/runtime/CMakeLists.txt index 2d2d7e81309..ce4b6007b08 100644 --- a/runtime/CMakeLists.txt +++ b/runtime/CMakeLists.txt @@ -136,9 +136,6 @@ if("${C_SYSTEM_LIBS}" STREQUAL "AUTO") set(C_SYSTEM_LIBS m c) elseif("${TARGET_SYSTEM}" MATCHES "Linux") set(C_SYSTEM_LIBS m pthread rt dl) - if("${TARGET_SYSTEM}" MATCHES "musl") - list(APPEND C_SYSTEM_LIBS "unwind") - endif() elseif("${TARGET_SYSTEM}" MATCHES "FreeBSD") set(C_SYSTEM_LIBS m pthread execinfo z) else() From 094ab42627535e355db746a68423fc382330efda Mon Sep 17 00:00:00 2001 From: Martin Kinkelin Date: Sun, 19 Jan 2025 17:36:44 +0100 Subject: [PATCH 5/8] [overcome problem of fully static executables not working with std.net.curl] Incl. the Phobos test runner, and dub. --- .github/actions/7-package/action.yml | 12 +++++------- .github/workflows/main.yml | 3 ++- cmake/Modules/BuildDExecutable.cmake | 2 +- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/.github/actions/7-package/action.yml b/.github/actions/7-package/action.yml index 569e5b69227..6e31581d00d 100644 --- a/.github/actions/7-package/action.yml +++ b/.github/actions/7-package/action.yml @@ -24,13 +24,6 @@ runs: fi echo "DMD=$DMD" >> $GITHUB_ENV - - name: 'Alpine: Set DFLAGS environment variable (-Xcc=-static)' # for fully statically linked dub etc. - if: inputs.os == 'alpine' - shell: bash - run: | - set -eux - echo "DFLAGS=-Xcc=-static${DFLAGS:+ $DFLAGS}" >> $GITHUB_ENV - - name: Build & copy dub shell: bash run: | @@ -56,6 +49,11 @@ runs: cd dlang-tools git checkout "$(cat ../ldc/packaging/dlang-tools_version)" + # Alpine: build these tools as fully static executables + if [[ '${{ inputs.os }}' == 'alpine' ]]; then + export DFLAGS="-Xcc=-static${DFLAGS:+ $DFLAGS}" + fi + mkdir bin $DMD -w -de -dip1000 rdmd.d -of=bin/rdmd $DMD -w -de -dip1000 ddemangle.d -of=bin/ddemangle diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index ff292b2bea0..111093204c9 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -69,7 +69,8 @@ jobs: -DCOMPILER_RT_LIBDIR_OS=linux -DLDC_INSTALL_LLVM_RUNTIME_LIBS_ARCH=x86_64 -DTEST_COMPILER_RT_LIBRARIES="profile;lsan;asan;msan;fuzzer" - -DCMAKE_EXE_LINKER_FLAGS=-static + -DCMAKE_EXE_LINKER_FLAGS=-static-libstdc++ + -DD_LINKER_ARGS=-static -DD_COMPILER_FLAGS="-O -flto=full" # FIXME: -defaultlib=phobos2-ldc-lto,druntime-ldc-lto with_pgo: false diff --git a/cmake/Modules/BuildDExecutable.cmake b/cmake/Modules/BuildDExecutable.cmake index d5ee8cddf83..fe71780b8cb 100644 --- a/cmake/Modules/BuildDExecutable.cmake +++ b/cmake/Modules/BuildDExecutable.cmake @@ -95,7 +95,7 @@ function(build_d_executable target_name output_exe d_src_files compiler_args lin list(APPEND dep_libs "-L$") endforeach() - set(full_linker_args ${CMAKE_EXE_LINKER_FLAGS} ${linker_args}) + set(full_linker_args ${CMAKE_EXE_LINKER_FLAGS} ${linker_args} ${D_LINKER_ARGS}) translate_linker_args(full_linker_args translated_linker_args) # We need to link against the C++ runtime library. From 768759b06e0ba9b8f4a89665bf3e6cbc3d28cd1f Mon Sep 17 00:00:00 2001 From: Martin Kinkelin Date: Sun, 19 Jan 2025 19:43:01 +0100 Subject: [PATCH 6/8] GHA Alpine: include shared druntime+Phobos too --- .github/actions/6-integration-test/action.yml | 3 --- .github/workflows/main.yml | 1 - runtime/druntime/test/shared/Makefile | 10 ++++---- runtime/druntime/test/shared/src/finalize.d | 18 ++++++++++++--- runtime/druntime/test/shared/src/load_13414.d | 23 ++++++++----------- 5 files changed, 28 insertions(+), 27 deletions(-) diff --git a/.github/actions/6-integration-test/action.yml b/.github/actions/6-integration-test/action.yml index b6c4cb57d28..4ab0bd226fe 100644 --- a/.github/actions/6-integration-test/action.yml +++ b/.github/actions/6-integration-test/action.yml @@ -15,9 +15,6 @@ runs: run: | set -eux cd .. - if type -P apk &>/dev/null; then - exit 0 # Alpine: no shared libs - fi if [[ '${{ runner.os }}' == Windows ]]; then # add druntime/Phobos DLL dir to PATH export PATH="$PWD/installed/bin:$PATH" diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 111093204c9..69e440879e2 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -61,7 +61,6 @@ jobs: # TSan and XRay do not work. extra_cmake_flags: >- -DBUILD_LTO_LIBS=ON - -DBUILD_SHARED_LIBS=OFF -DLLVM_IS_SHARED=OFF -DLDC_ENABLE_PLUGINS=OFF -DLDC_DYNAMIC_COMPILE=OFF diff --git a/runtime/druntime/test/shared/Makefile b/runtime/druntime/test/shared/Makefile index 43f977b690a..e25dc184379 100644 --- a/runtime/druntime/test/shared/Makefile +++ b/runtime/druntime/test/shared/Makefile @@ -4,7 +4,7 @@ LINK_SHARED:=$(SHARED) include ../common.mak # affected by LINK_SHARED! ifneq (,$(LINK_SHARED)) - # LDC: disable 3 tests on Mac, 1 on Windows + # LDC: disable 2 tests on Mac, 1 on Windows ifeq (,$(findstring ldmd2,$(DMD))) # TODO: enable tests on Windows ifeq (windows,$(OS)) @@ -14,20 +14,18 @@ ifneq (,$(LINK_SHARED)) link_linkdep load_linkdep link_loaddep load_loaddep load_13414 endif else - TESTS:=link load linkD linkDR loadDR dynamiccast \ + TESTS:=link load linkD linkDR loadDR finalize dynamiccast \ link_linkdep link_loaddep load_loaddep load_13414 ifneq ($(OS),osx) # * `host` loads two modules with the same name, which is currently disallowed # by the (potentially overly eager) module collision detection on OS X. - # * `finalize` fails starting with macOS 10.13, as .dylibs with TLS can't be - # unloaded anymore (https://github.com/ldc-developers/ldc/issues/3002). # * FIXME: `load_linkdep` # it might fail because of unimplemented `getDependencies()` in rt.sections_elf_shared ifeq (windows,$(OS)) # LDC FIXME: disable `load_linkdep` on Windows - needs `getDependencies()` - TESTS+=host finalize + TESTS+=host else - TESTS+=host finalize load_linkdep + TESTS+=host load_linkdep endif endif endif diff --git a/runtime/druntime/test/shared/src/finalize.d b/runtime/druntime/test/shared/src/finalize.d index d61cd490a43..9c5dfd5b786 100644 --- a/runtime/druntime/test/shared/src/finalize.d +++ b/runtime/druntime/test/shared/src/finalize.d @@ -45,7 +45,7 @@ void main(string[] args) auto nf1 = new NoFinalize; auto nf2 = new NoFinalizeBig; - shared size_t finalizeCounter; + shared static size_t finalizeCounter; SetFinalizeCounter setFinalizeCounter; loadSym(h, setFinalizeCounter, "setFinalizeCounter"); setFinalizeCounter(&finalizeCounter); @@ -58,8 +58,20 @@ void main(string[] args) auto r = Runtime.unloadLibrary(h); if (!r) assert(0); - if (finalizeCounter != 4) - assert(0); + version (darwin) + { + // Fails starting with macOS 10.13, as .dylibs with TLS can't be + // unloaded anymore (https://github.com/ldc-developers/ldc/issues/3002). + } + else version (CRuntime_Musl) + { + // On Musl, dlclose is a no-op + } + else + { + if (finalizeCounter != 4) + assert(0); + } if (nf1._finalizeCounter) assert(0); if (nf2._finalizeCounter) diff --git a/runtime/druntime/test/shared/src/load_13414.d b/runtime/druntime/test/shared/src/load_13414.d index cf782f8b081..435f11c264b 100644 --- a/runtime/druntime/test/shared/src/load_13414.d +++ b/runtime/druntime/test/shared/src/load_13414.d @@ -22,25 +22,20 @@ void runTest(string name) *pLibSharedStaticDtorHook = &sharedStaticDtorHook; const unloaded = Runtime.unloadLibrary(h); - version (CRuntime_Musl) + assert(unloaded); + assert(tlsDtor == 1); + version (LDC_darwin) { - // On Musl, unloadLibrary is a no-op because dlclose is a no-op - assert(!unloaded); - assert(tlsDtor == 0); + // Since 10.13: https://github.com/ldc-developers/ldc/issues/3002 assert(dtor == 0); } - else + else version (CRuntime_Musl) { - assert(unloaded); - assert(tlsDtor == 1); - version (LDC_darwin) - { - // Since 10.13: https://github.com/ldc-developers/ldc/issues/3002 - assert(dtor == 0); - } - else - assert(dtor == 1); + // On Musl, dlclose is a no-op + assert(dtor == 0); } + else + assert(dtor == 1); } void main(string[] args) From 9adc6b6a8ceb0f185daab90d9a07ad57c1eff8d8 Mon Sep 17 00:00:00 2001 From: Martin Kinkelin Date: Sun, 19 Jan 2025 22:54:52 +0100 Subject: [PATCH 7/8] [add changelog entry] --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 87a6313c541..d9626ea60ac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ #### Big news - LLVM for prebuilt packages bumped to v19.1.7. (#4822) +- New prebuilt package for Alpine Linux x86_64 with musl libc. It's currently generated on Alpine v3.20, using its default LLVM 17. Most bundled executables are fully static and can be run on ~all distros. (#4826) - Revived dynamic-compile (JIT) functionality (formerly unsupported since LLVM 12), supporting LLVM 18+ now. (#4774) - ldc2.conf: `%%ldcversion%%` placeholder added, allowing to refer to version-specific directories. From f89fc195f6a6de1d0041f285287742e06ae22b45 Mon Sep 17 00:00:00 2001 From: Martin Kinkelin Date: Mon, 20 Jan 2025 01:57:39 +0100 Subject: [PATCH 8/8] CMake: Add LDC_FULLY_STATIC option --- .github/workflows/main.yml | 2 +- CMakeLists.txt | 14 ++++++++++---- tools/CMakeLists.txt | 10 +++++----- 3 files changed, 16 insertions(+), 10 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 69e440879e2..7abdcaa0500 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -69,7 +69,7 @@ jobs: -DLDC_INSTALL_LLVM_RUNTIME_LIBS_ARCH=x86_64 -DTEST_COMPILER_RT_LIBRARIES="profile;lsan;asan;msan;fuzzer" -DCMAKE_EXE_LINKER_FLAGS=-static-libstdc++ - -DD_LINKER_ARGS=-static + -DLDC_FULLY_STATIC=ON -DD_COMPILER_FLAGS="-O -flto=full" # FIXME: -defaultlib=phobos2-ldc-lto,druntime-ldc-lto with_pgo: false diff --git a/CMakeLists.txt b/CMakeLists.txt index 12c0ea8412b..dc4c83dd69e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -315,6 +315,12 @@ else() separate_arguments(LLVM_LDFLAGS UNIX_COMMAND "${LLVM_LDFLAGS}") endif() +option(LDC_FULLY_STATIC "Posix: Link most executables (compiler and tools) as fully-static binaries" OFF) +set(FULLY_STATIC_LDFLAG) +if(UNIX AND LDC_FULLY_STATIC) + set(FULLY_STATIC_LDFLAG "-static") +endif() + # Suppress superfluous randlib warnings about "*.a" having no symbols on MacOSX. if (APPLE) set(CMAKE_C_ARCHIVE_CREATE " Scr ") @@ -708,7 +714,7 @@ build_d_executable( "${LDC_EXE_FULL}" "${LDC_D_SOURCE_FILES}" "${DFLAGS_BUILD_TYPE} ${DFLAGS_LDC}" - "${ALTERNATIVE_MALLOC_O};${LDC_LINKERFLAG_LIST}" + "${ALTERNATIVE_MALLOC_O};${LDC_LINKERFLAG_LIST};${FULLY_STATIC_LDFLAG}" "${FE_RES}" "${LDC_LIB}" ${COMPILE_D_MODULES_SEPARATELY} @@ -743,7 +749,7 @@ build_d_executable( "${LDMD_EXE_FULL}" "${LDMD_D_SOURCE_FILES}" "${DFLAGS_BUILD_TYPE}" - "${LDC_LINKERFLAG_LIST}" + "${LDC_LINKERFLAG_LIST};${FULLY_STATIC_LDFLAG}" "" "LDMD_CXX_LIB" ${COMPILE_D_MODULES_SEPARATELY} @@ -927,7 +933,7 @@ build_d_executable( "${LDC_UNITTEST_EXE_FULL}" "${LDC_D_SOURCE_FILES}" "-g -unittest ${DFLAGS_LDC}" - "${LDC_LINKERFLAG_LIST}" + "${LDC_LINKERFLAG_LIST};${FULLY_STATIC_LDFLAG}" "" "${LDC_LIB}" ${COMPILE_D_MODULES_SEPARATELY} @@ -957,7 +963,7 @@ build_d_executable( "${LDC_BUILD_RUNTIME_EXE_FULL}" "${PROJECT_BINARY_DIR}/ldc-build-runtime.d" "${DFLAGS_BUILD_TYPE}" - "" + "" # must not be fully static for std.net.curl support (loading libcurl dynamically) "${PROJECT_SOURCE_DIR}/runtime/ldc-build-runtime.d.in" "" ${COMPILE_D_MODULES_SEPARATELY} diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index 3fc3f035d06..df8099e4a55 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -18,7 +18,7 @@ build_d_executable( "${LDCPRUNECACHE_EXE_FULL}" "${LDCPRUNECACHE_D_SRC}" "${DFLAGS_BUILD_TYPE}" - "" + "${FULLY_STATIC_LDFLAG}" "" "" ${COMPILE_D_MODULES_SEPARATELY} @@ -35,7 +35,7 @@ if(LDC_BUNDLE_LLVM_TOOLS) ldc-profdata PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}/bin COMPILE_FLAGS "${LLVM_CXXFLAGS} ${LDC_CXXFLAGS}" - LINK_FLAGS "${SANITIZE_LDFLAGS}" + LINK_FLAGS "${SANITIZE_LDFLAGS} ${FULLY_STATIC_LDFLAG}" ) target_link_libraries(ldc-profdata ${LLVM_LIBRARIES} ${CMAKE_DL_LIBS} ${LLVM_LDFLAGS}) install(TARGETS ldc-profdata DESTINATION ${CMAKE_INSTALL_PREFIX}/bin) @@ -59,7 +59,7 @@ if(LDC_BUNDLE_LLVM_TOOLS) ${ldc_name} PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}/bin COMPILE_FLAGS "${LLVM_CXXFLAGS} ${LDC_CXXFLAGS}" - LINK_FLAGS "${SANITIZE_LDFLAGS}" + LINK_FLAGS "${SANITIZE_LDFLAGS} ${FULLY_STATIC_LDFLAG}" ) target_link_libraries(${ldc_name} ${LLVM_LIBRARIES} ${CMAKE_DL_LIBS} ${LLVM_LDFLAGS}) install(TARGETS ${ldc_name} DESTINATION ${CMAKE_INSTALL_PREFIX}/bin) @@ -91,7 +91,7 @@ build_d_executable( "${TIMETRACE2TXT_EXE_FULL}" "${TIMETRACE2TXT_D_SRC}" "${DFLAGS_BUILD_TYPE}" - "" + "${FULLY_STATIC_LDFLAG}" "" "" ${COMPILE_D_MODULES_SEPARATELY} @@ -111,7 +111,7 @@ if(LDC_ENABLE_PLUGINS) "${LDC_BUILD_PLUGIN_EXE_FULL}" "${PROJECT_BINARY_DIR}/ldc-build-plugin.d" "${DFLAGS_BUILD_TYPE}" - "" + "" # must not be fully static for std.net.curl support (loading libcurl dynamically) "${PROJECT_SOURCE_DIR}/tools/ldc-build-plugin.d.in" "" ${COMPILE_D_MODULES_SEPARATELY}