From 041f6f4496e41f4afbdaa15cc13fee04ca1728ce Mon Sep 17 00:00:00 2001 From: "Sean T. Allen" Date: Tue, 3 Dec 2024 01:56:38 +0000 Subject: [PATCH] Update supported Alpine to 3.20 This commit updates our CI to test using Alpine 3.20 for musl testing. It also updates our release and nightly alpine ponyc images to be based on Alpine 3.20. Due to a change in the version of musl in Alpine 3.20, this required a small patch LLVM 15 to compile. When we move to LLVM 16 or above, the patch shouldn't be needed. The new version of musl doesn't support LSF64 methods like lseek64. In order to keep the standard library working with the new musl, this commit adds a couple new "pony_os" functions for encapsulating lseek and ftruncate behavior that varies by OS and on Linux, between musl and Glibc. --- .../Dockerfile | 4 +- .../x86-64-unknown-linux-musl/Dockerfile | 2 +- .../x86-64-unknown-linux-musl/Dockerfile | 2 +- .github/workflows/nightlies.yml | 2 +- .github/workflows/pr.yml | 2 +- .github/workflows/release.yml | 2 +- .github/workflows/stress-test-runtime.yml | 8 +- .github/workflows/update-lib-cache.yml | 2 +- .release-notes/alpine-320.md | 3 + lib/CMakeLists.txt | 7 +- lib/llvm/patches/llvm15-musl-lseek64.diff | 87 +++++++++++++++++++ packages/files/file.pony | 38 ++------ src/libponyrt/lang/io.c | 34 ++++++++ 13 files changed, 144 insertions(+), 49 deletions(-) create mode 100644 .release-notes/alpine-320.md create mode 100644 lib/llvm/patches/llvm15-musl-lseek64.diff diff --git a/.ci-dockerfiles/x86-64-unknown-linux-musl-builder/Dockerfile b/.ci-dockerfiles/x86-64-unknown-linux-musl-builder/Dockerfile index de941808ce..9f0a536e79 100644 --- a/.ci-dockerfiles/x86-64-unknown-linux-musl-builder/Dockerfile +++ b/.ci-dockerfiles/x86-64-unknown-linux-musl-builder/Dockerfile @@ -1,4 +1,4 @@ -FROM alpine:3.18 +FROM alpine:3.20 RUN apk update \ @@ -19,7 +19,7 @@ RUN apk update \ gdb \ lldb \ py3-lldb \ -&& pip install cloudsmith-cli +&& pip install --break-system-packages cloudsmith-cli # add user pony in order to not run tests as root RUN adduser -D -u 1001 -s /bin/sh -h /home/pony -g root pony diff --git a/.dockerfiles/latest/x86-64-unknown-linux-musl/Dockerfile b/.dockerfiles/latest/x86-64-unknown-linux-musl/Dockerfile index 4ea2e80a48..567d1b8bfa 100644 --- a/.dockerfiles/latest/x86-64-unknown-linux-musl/Dockerfile +++ b/.dockerfiles/latest/x86-64-unknown-linux-musl/Dockerfile @@ -1,4 +1,4 @@ -FROM alpine:3.18 +FROM alpine:3.20 ENV PATH "/root/.local/share/ponyup/bin:$PATH" diff --git a/.dockerfiles/release/x86-64-unknown-linux-musl/Dockerfile b/.dockerfiles/release/x86-64-unknown-linux-musl/Dockerfile index 7099275ab1..36aa25d6c7 100644 --- a/.dockerfiles/release/x86-64-unknown-linux-musl/Dockerfile +++ b/.dockerfiles/release/x86-64-unknown-linux-musl/Dockerfile @@ -1,4 +1,4 @@ -FROM alpine:3.18 +FROM alpine:3.20 ENV PATH "/root/.local/share/ponyup/bin:$PATH" diff --git a/.github/workflows/nightlies.yml b/.github/workflows/nightlies.yml index 74b3d5683b..cc122eb1cb 100644 --- a/.github/workflows/nightlies.yml +++ b/.github/workflows/nightlies.yml @@ -24,7 +24,7 @@ jobs: name: x86-64-unknown-linux-ubuntu20.04 triple-os: linux-ubuntu20.04 triple-vendor: unknown - - image: ghcr.io/ponylang/ponyc-ci-x86-64-unknown-linux-musl-builder:20241202 + - image: ghcr.io/ponylang/ponyc-ci-x86-64-unknown-linux-musl-builder:20241203 name: x86-64-unknown-linux-musl triple-os: linux-musl triple-vendor: unknown diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 782806b17b..f6d414dedb 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -90,7 +90,7 @@ jobs: - image: ghcr.io/ponylang/ponyc-ci-x86-64-unknown-linux-ubuntu24.04-builder:20240425 name: x86-64 Linux glibc debugger: lldb - - image: ghcr.io/ponylang/ponyc-ci-x86-64-unknown-linux-musl-builder:20241202 + - image: ghcr.io/ponylang/ponyc-ci-x86-64-unknown-linux-musl-builder:20241203 name: x86-64 Linux musl debugger: lldb diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 9ad95b335b..f24df57b3a 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -47,7 +47,7 @@ jobs: name: x86-64-unknown-linux-ubuntu20.04 triple-os: linux-ubuntu20.04 triple-vendor: unknown - - image: ghcr.io/ponylang/ponyc-ci-x86-64-unknown-linux-musl-builder:20241202 + - image: ghcr.io/ponylang/ponyc-ci-x86-64-unknown-linux-musl-builder:20241203 name: x86-64-unknown-linux-musl triple-os: linux-musl triple-vendor: unknown diff --git a/.github/workflows/stress-test-runtime.yml b/.github/workflows/stress-test-runtime.yml index 55c4106180..216d8486bc 100644 --- a/.github/workflows/stress-test-runtime.yml +++ b/.github/workflows/stress-test-runtime.yml @@ -28,19 +28,19 @@ jobs: name: x86-64-unknown-linux-ubuntu24.04 [cd] [debug] target: test-stress-with-cd-debug debugger: lldb - - image: ghcr.io/ponylang/ponyc-ci-x86-64-unknown-linux-musl-builder:20241202 + - image: ghcr.io/ponylang/ponyc-ci-x86-64-unknown-linux-musl-builder:20241203 name: x86-64-unknown-linux-musl [release] target: test-stress-release debugger: lldb - - image: ghcr.io/ponylang/ponyc-ci-x86-64-unknown-linux-musl-builder:20241202 + - image: ghcr.io/ponylang/ponyc-ci-x86-64-unknown-linux-musl-builder:20241203 name: x86-64-unknown-linux-musl [debug] target: test-stress-debug debugger: lldb - - image: ghcr.io/ponylang/ponyc-ci-x86-64-unknown-linux-musl-builder:20241202 + - image: ghcr.io/ponylang/ponyc-ci-x86-64-unknown-linux-musl-builder:20241203 name: x86-64-unknown-linux-musl [cd] [release] target: test-stress-with-cd-release debugger: lldb - - image: ghcr.io/ponylang/ponyc-ci-x86-64-unknown-linux-musl-builder:20241202 + - image: ghcr.io/ponylang/ponyc-ci-x86-64-unknown-linux-musl-builder:20241203 name: x86-64-unknown-linux-musl [cd] [debug] target: test-stress-with-cd-debug debugger: lldb diff --git a/.github/workflows/update-lib-cache.yml b/.github/workflows/update-lib-cache.yml index 4deafe3bc4..5f4a4e1776 100644 --- a/.github/workflows/update-lib-cache.yml +++ b/.github/workflows/update-lib-cache.yml @@ -20,7 +20,7 @@ jobs: - image: ghcr.io/ponylang/ponyc-ci-x86-64-unknown-linux-ubuntu24.04-builder:20240425 - image: ghcr.io/ponylang/ponyc-ci-x86-64-unknown-linux-ubuntu22.04-builder:20230924 - image: ghcr.io/ponylang/ponyc-ci-x86-64-unknown-linux-ubuntu20.04-builder:20230830 - - image: ghcr.io/ponylang/ponyc-ci-x86-64-unknown-linux-musl-builder:20241202 + - image: ghcr.io/ponylang/ponyc-ci-x86-64-unknown-linux-musl-builder:20241203 - image: ghcr.io/ponylang/ponyc-ci-cross-arm:20240427 - image: ghcr.io/ponylang/ponyc-ci-cross-armhf:20240427 - image: ghcr.io/ponylang/ponyc-ci-cross-riscv64:20240427 diff --git a/.release-notes/alpine-320.md b/.release-notes/alpine-320.md new file mode 100644 index 0000000000..29cafa52ba --- /dev/null +++ b/.release-notes/alpine-320.md @@ -0,0 +1,3 @@ +## Update Pony musl Docker images to Alpine 3.20 + +We've updated our `ponylang/ponyc:latest-alpine`, `ponylang/ponyc:release-alpine`, and `ponylang/ponyc:x.y.z-alpine` images to be based on Alpine 3.20. Previously, we were using Alpine 3.18 as the base. diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt index 8d853b4b6c..c8732d1c0b 100644 --- a/lib/CMakeLists.txt +++ b/lib/CMakeLists.txt @@ -55,7 +55,7 @@ install(TARGETS blake2 find_package(Git) set(LLVM_DESIRED_HASH "8dfdcc7b7bf66834a761bd8de445840ef68e4d1a") -set(PATCHES_DESIRED_HASH "3e16c097794cb669a8f6a0bd7600b440205ac5c29a6135750c2e83263eb16a95") +set(PATCHES_DESIRED_HASH "99d46d25e96861de9847c055aeda1fef13fa0219524b2b7074c9acedb5f7a1c4") if(GIT_FOUND) if(EXISTS "${PROJECT_SOURCE_DIR}/../.git") @@ -93,9 +93,8 @@ if(GIT_FOUND) set(PATCHES_ACTUAL_HASH "needed_if_no_patches") foreach (PATCH ${PONY_LLVM_PATCHES}) file(STRINGS ${PATCH} patch_file NEWLINE_CONSUME) - string(REPLACE "\n" " " patch_file ${patch_file}) - string(SHA256 patch_file_hash ${patch_file}) - # message("${PATCH}: '${patch_file_hash}'") + string(REPLACE "\n" " " patch_file "${patch_file}") + string(SHA256 patch_file_hash "${patch_file}") string(CONCAT PATCHES_ACTUAL_HASH ${PATCHES_ACTUAL_HASH} ${patch_file_hash}) # message("concat is '${PATCHES_ACTUAL_HASH}'") endforeach() diff --git a/lib/llvm/patches/llvm15-musl-lseek64.diff b/lib/llvm/patches/llvm15-musl-lseek64.diff new file mode 100644 index 0000000000..584f4756cb --- /dev/null +++ b/lib/llvm/patches/llvm15-musl-lseek64.diff @@ -0,0 +1,87 @@ +diff --git a/llvm/cmake/config-ix.cmake b/llvm/cmake/config-ix.cmake +index 7e657fd15..54cd1f0d6 100644 +--- a/llvm/cmake/config-ix.cmake ++++ b/llvm/cmake/config-ix.cmake +@@ -284,9 +284,6 @@ check_symbol_exists(futimes sys/time.h HAVE_FUTIMES) + if( HAVE_SIGNAL_H AND NOT LLVM_USE_SANITIZER MATCHES ".*Address.*" AND NOT APPLE ) + check_symbol_exists(sigaltstack signal.h HAVE_SIGALTSTACK) + endif() +-set(CMAKE_REQUIRED_DEFINITIONS "-D_LARGEFILE64_SOURCE") +-check_symbol_exists(lseek64 "sys/types.h;unistd.h" HAVE_LSEEK64) +-set(CMAKE_REQUIRED_DEFINITIONS "") + check_symbol_exists(mallctl malloc_np.h HAVE_MALLCTL) + check_symbol_exists(mallinfo malloc.h HAVE_MALLINFO) + check_symbol_exists(mallinfo2 malloc.h HAVE_MALLINFO2) +@@ -350,6 +347,11 @@ check_symbol_exists(__GLIBC__ stdio.h LLVM_USING_GLIBC) + if( LLVM_USING_GLIBC ) + add_definitions( -D_GNU_SOURCE ) + list(APPEND CMAKE_REQUIRED_DEFINITIONS "-D_GNU_SOURCE") ++ # enable 64bit off_t on 32bit systems using glibc ++ if (CMAKE_SIZEOF_VOID_P EQUAL 4) ++ add_compile_definitions(_FILE_OFFSET_BITS=64) ++ list(APPEND CMAKE_REQUIRED_DEFINITIONS "-D_FILE_OFFSET_BITS=64") ++ endif() + endif() + # This check requires _GNU_SOURCE + if (NOT PURE_WINDOWS) +diff --git a/llvm/include/llvm/Config/config.h.cmake b/llvm/include/llvm/Config/config.h.cmake +index 21ce3a94a..d551ebad5 100644 +--- a/llvm/include/llvm/Config/config.h.cmake ++++ b/llvm/include/llvm/Config/config.h.cmake +@@ -128,9 +128,6 @@ + /* Define to 1 if you have the header file. */ + #cmakedefine HAVE_LINK_H ${HAVE_LINK_H} + +-/* Define to 1 if you have the `lseek64' function. */ +-#cmakedefine HAVE_LSEEK64 ${HAVE_LSEEK64} +- + /* Define to 1 if you have the header file. */ + #cmakedefine HAVE_MACH_MACH_H ${HAVE_MACH_MACH_H} + +diff --git a/llvm/lib/Support/raw_ostream.cpp b/llvm/lib/Support/raw_ostream.cpp +index 651949ad5..0bc71812c 100644 +--- a/llvm/lib/Support/raw_ostream.cpp ++++ b/llvm/lib/Support/raw_ostream.cpp +@@ -804,8 +804,6 @@ uint64_t raw_fd_ostream::seek(uint64_t off) { + flush(); + #ifdef _WIN32 + pos = ::_lseeki64(FD, off, SEEK_SET); +-#elif defined(HAVE_LSEEK64) +- pos = ::lseek64(FD, off, SEEK_SET); + #else + pos = ::lseek(FD, off, SEEK_SET); + #endif +diff --git a/llvm/utils/gn/secondary/llvm/include/llvm/Config/BUILD.gn b/llvm/utils/gn/secondary/llvm/include/llvm/Config/BUILD.gn +index 897364f00..e4f49e5e2 100644 +--- a/llvm/utils/gn/secondary/llvm/include/llvm/Config/BUILD.gn ++++ b/llvm/utils/gn/secondary/llvm/include/llvm/Config/BUILD.gn +@@ -139,7 +139,6 @@ write_cmake_config("config") { + values += [ + "HAVE_FUTIMENS=1", + "HAVE_LINK_H=1", +- "HAVE_LSEEK64=1", + "HAVE_MALLINFO=1", + "HAVE_STRUCT_STAT_ST_MTIM_TV_NSEC=1", + ] +@@ -147,7 +146,6 @@ write_cmake_config("config") { + values += [ + "HAVE_FUTIMENS=", + "HAVE_LINK_H=", +- "HAVE_LSEEK64=", + "HAVE_MALLINFO=", + "HAVE_STRUCT_STAT_ST_MTIM_TV_NSEC=", + ] +diff --git a/utils/bazel/llvm_configs/config.h.cmake b/utils/bazel/llvm_configs/config.h.cmake +index 21ce3a94a..d551ebad5 100644 +--- a/utils/bazel/llvm_configs/config.h.cmake ++++ b/utils/bazel/llvm_configs/config.h.cmake +@@ -128,9 +128,6 @@ + /* Define to 1 if you have the header file. */ + #cmakedefine HAVE_LINK_H ${HAVE_LINK_H} + +-/* Define to 1 if you have the `lseek64' function. */ +-#cmakedefine HAVE_LSEEK64 ${HAVE_LSEEK64} +- + /* Define to 1 if you have the header file. */ + #cmakedefine HAVE_MACH_MACH_H ${HAVE_MACH_MACH_H} + diff --git a/packages/files/file.pony b/packages/files/file.pony index e6dfda6533..c2cb8ccbf6 100644 --- a/packages/files/file.pony +++ b/packages/files/file.pony @@ -6,16 +6,12 @@ use @read[ISize](fd: I32, buffer: Pointer[None], bytes_to_read: USize) use @_write[I32](fd: I32, buffer: Pointer[None], bytes_to_send: I32) if windows use @writev[ISize](fd: I32, buffer: Pointer[None], num_to_send: I32) if not windows -use @_lseeki64[I64](fd: I32, offset: I64, base: I32) if windows -use @lseek64[I64](fd: I32, offset: I64, base: I32) if linux -use @lseek[I64](fd: I32, offset: I64, base: I32) if not windows and not linux +use @pony_os_lseek[I64](fd: I32, offset: I64, base: I32) use @FlushFileBuffers[Bool](file_handle: Pointer[None]) if windows use @_get_osfhandle[Pointer[None]](fd: I32) if windows use @fsync[I32](fd: I32) if not windows use @fdatasync[I32](fd: I32) if not windows -use @_chsize_s[I32](fd: I32, len: I64) if windows -use @ftruncate64[I32](fd: I32, len: I64) if linux -use @ftruncate[I32](fd: I32, len: I64) if not windows and not linux +use @pony_os_ftruncate[I32](fd: I32, len: I64) use @_close[I32](fd: I32) if windows use @close[I32](fd: I32) if not windows use @pony_os_writev_max[I32]() @@ -438,15 +434,7 @@ class File if _fd != -1 then let o: I64 = 0 let b: I32 = 1 - let r = ifdef windows then - @_lseeki64(_fd, o, b) - else - ifdef linux then - @lseek64(_fd, o, b) - else - @lseek(_fd, o, b) - end - end + let r = @pony_os_lseek(_fd, o, b) if r < 0 then _errno = _get_error() @@ -535,15 +523,7 @@ class File """ if path.caps(FileTruncate) and writeable and (_fd != -1) then let pos = position() - let result = ifdef windows then - @_chsize_s(_fd, len.i64()) - else - ifdef linux then - @ftruncate64(_fd, len.i64()) - else - @ftruncate(_fd, len.i64()) - end - end + let result = @pony_os_ftruncate(_fd, len.i64()) if pos >= len then _seek(0, 2) @@ -627,15 +607,7 @@ class File Move the cursor position. """ if _fd != -1 then - let r = ifdef windows then - @_lseeki64(_fd, offset, base) - else - ifdef linux then - @lseek64(_fd, offset, base) - else - @lseek(_fd, offset, base) - end - end + let r = @pony_os_lseek(_fd, offset, base) if r < 0 then _errno = _get_error() end diff --git a/src/libponyrt/lang/io.c b/src/libponyrt/lang/io.c index 0ff6072603..bde869e344 100644 --- a/src/libponyrt/lang/io.c +++ b/src/libponyrt/lang/io.c @@ -7,8 +7,12 @@ PONY_EXTERN_C_BEGIN #if defined(PLATFORM_IS_POSIX_BASED) #include +#include +#elif defined(PLATFORM_IS_WINDOWS) +#include #endif + PONY_API int pony_os_writev_max() { #if defined(PLATFORM_IS_POSIX_BASED) @@ -18,4 +22,34 @@ PONY_API int pony_os_writev_max() #endif } +PONY_API int64_t pony_os_lseek(int fd, int64_t offset, int whence) +{ +#ifdef PLATFORM_IS_LINUX +#ifdef __GLIBC__ + return lseek64(fd, offset, whence); +#else + return lseek(fd, offset, whence); +#endif +#elif defined(PLATFORM_IS_MACOSX) || defined(PLATFORM_IS_BSD) + return lseek(fd, offset, whence); +#elif defined(PLATFORM_IS_WINDOWS) + return _lseeki64(fd, offset, whence); +#endif +} + +PONY_API int pony_os_ftruncate(int fd, int64_t length) +{ +#ifdef PLATFORM_IS_LINUX +#ifdef __GLIBC__ + return ftruncate64(fd, length); +#else + return ftruncate(fd, length); +#endif +#elif defined(PLATFORM_IS_MACOSX) || defined(PLATFORM_IS_BSD) + return ftruncate(fd, length); +#elif defined(PLATFORM_IS_WINDOWS) + return _chsize_s(fd, length); +#endif +} + PONY_EXTERN_C_END