From b481312f1bdd397508f058d5de7db81275e0da07 Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Thu, 9 Nov 2023 12:43:38 +0200 Subject: [PATCH 01/21] Add triplets for ASAN. --- CMakeLists.txt | 4 +++- cmake/Options/TileDBToolchain.cmake | 18 +++++++++++++----- ports/triplets/arm64-osx-asan.cmake | 9 +++++++++ ports/triplets/x64-linux-asan.cmake | 8 ++++++++ ports/triplets/x64-osx-asan.cmake | 9 +++++++++ ports/triplets/x64-windows-asan.cmake | 9 +++++++++ 6 files changed, 51 insertions(+), 6 deletions(-) create mode 100644 ports/triplets/arm64-osx-asan.cmake create mode 100644 ports/triplets/x64-linux-asan.cmake create mode 100644 ports/triplets/x64-osx-asan.cmake create mode 100644 ports/triplets/x64-windows-asan.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index df428fc1589..3d1a5a065d8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -264,7 +264,9 @@ if (SANITIZER) message(STATUS "The TileDB library is compiled with sanitizer ${SANITIZER} enabled") endif() add_compile_options(-DTILEDB_SANITIZER=${SANITIZER}) - add_compile_options(-g -fno-omit-frame-pointer -fno-optimize-sibling-calls -fsanitize=${SANITIZER}) + add_compile_options( + "$<$>:-g;-fno-omit-frame-pointer;-fno-optimize-sibling-calls>" + -fsanitize=${SANITIZER}) add_link_options(-fsanitize=${SANITIZER}) endif() diff --git a/cmake/Options/TileDBToolchain.cmake b/cmake/Options/TileDBToolchain.cmake index 73dab473f50..470737ddd1a 100644 --- a/cmake/Options/TileDBToolchain.cmake +++ b/cmake/Options/TileDBToolchain.cmake @@ -33,11 +33,19 @@ if (NOT DEFINED CMAKE_TOOLCHAIN_FILE) endif() endif() -if(APPLE AND NOT DEFINED VCPKG_TARGET_TRIPLET) - if (CMAKE_OSX_ARCHITECTURES STREQUAL x86_64 OR CMAKE_SYSTEM_PROCESSOR MATCHES "(x86_64)|(AMD64|amd64)|(^i.86$)") - set(VCPKG_TARGET_TRIPLET "x64-macos") - elseif (CMAKE_OSX_ARCHITECTURES STREQUAL arm64 OR CMAKE_SYSTEM_PROCESSOR MATCHES "^aarch64" OR CMAKE_SYSTEM_PROCESSOR MATCHES "^arm") - set(VCPKG_TARGET_TRIPLET "arm64-macos") +if(NOT DEFINED VCPKG_TARGET_TRIPLET AND SANITIZER STREQUAL "address") + if(APPLE) + if (CMAKE_OSX_ARCHITECTURES STREQUAL x86_64 OR CMAKE_SYSTEM_PROCESSOR MATCHES "(x86_64)|(AMD64|amd64)|(^i.86$)") + set(VCPKG_TARGET_TRIPLET "x64-osx-asan") + elseif (CMAKE_OSX_ARCHITECTURES STREQUAL arm64 OR CMAKE_SYSTEM_PROCESSOR MATCHES "^aarch64" OR CMAKE_SYSTEM_PROCESSOR MATCHES "^arm") + set(VCPKG_TARGET_TRIPLET "arm64-osx-asan") + endif() + elseif(CMAKE_SYSTEM_NAME STREQUAL "Linux") + set(VCPKG_TARGET_TRIPLET "x64-linux-asan") + elseif(WIN32) + set(VCPKG_TARGET_TRIPLET "x64-windows-asan") + else() + message(WARNING "Unrecognized platform for sanitizer build. Dependencies will not be compiled with ASAN.") endif() endif() diff --git a/ports/triplets/arm64-osx-asan.cmake b/ports/triplets/arm64-osx-asan.cmake new file mode 100644 index 00000000000..4aa2d22af54 --- /dev/null +++ b/ports/triplets/arm64-osx-asan.cmake @@ -0,0 +1,9 @@ +set(VCPKG_TARGET_ARCHITECTURE arm64) +set(VCPKG_CRT_LINKAGE dynamic) +set(VCPKG_LIBRARY_LINKAGE static) + +set(VCPKG_CMAKE_SYSTEM_NAME Darwin) +set(VCPKG_OSX_ARCHITECTURES arm64) + +set(VCPKG_C_FLAGS "-fsanitize=address") +set(VCPKG_CXX_FLAGS "-fsanitize=address") diff --git a/ports/triplets/x64-linux-asan.cmake b/ports/triplets/x64-linux-asan.cmake new file mode 100644 index 00000000000..747d7d62020 --- /dev/null +++ b/ports/triplets/x64-linux-asan.cmake @@ -0,0 +1,8 @@ +set(VCPKG_TARGET_ARCHITECTURE x64) +set(VCPKG_CRT_LINKAGE dynamic) +set(VCPKG_LIBRARY_LINKAGE static) + +set(VCPKG_CMAKE_SYSTEM_NAME Linux) + +set(VCPKG_C_FLAGS "-fsanitize=address") +set(VCPKG_CXX_FLAGS "-fsanitize=address") diff --git a/ports/triplets/x64-osx-asan.cmake b/ports/triplets/x64-osx-asan.cmake new file mode 100644 index 00000000000..aac0688e2c7 --- /dev/null +++ b/ports/triplets/x64-osx-asan.cmake @@ -0,0 +1,9 @@ +set(VCPKG_TARGET_ARCHITECTURE x64) +set(VCPKG_CRT_LINKAGE dynamic) +set(VCPKG_LIBRARY_LINKAGE static) + +set(VCPKG_CMAKE_SYSTEM_NAME Darwin) +set(VCPKG_OSX_ARCHITECTURES x86_64) + +set(VCPKG_C_FLAGS "-fsanitize=address") +set(VCPKG_CXX_FLAGS "-fsanitize=address") diff --git a/ports/triplets/x64-windows-asan.cmake b/ports/triplets/x64-windows-asan.cmake new file mode 100644 index 00000000000..d19486d2cfe --- /dev/null +++ b/ports/triplets/x64-windows-asan.cmake @@ -0,0 +1,9 @@ +set(VCPKG_TARGET_ARCHITECTURE x64) +set(VCPKG_CRT_LINKAGE dynamic) +set(VCPKG_LIBRARY_LINKAGE static) + +set(X_VCPKG_APPLOCAL_DEPS_INSTALL ON) + +# bigobj is needed for capnp. +set(VCPKG_C_FLAGS "/fsanitize=address /bigobj") +set(VCPKG_CXX_FLAGS "/fsanitize=address /bigobj") From a2dc1caf99f60b966e87ed033f6d9d4869b60c13 Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Mon, 13 Nov 2023 22:20:50 +0200 Subject: [PATCH 02/21] Rename `macos` to `osx` in the triplet names. --- ports/triplets/{arm64-macos.cmake => arm64-osx.cmake} | 0 ports/triplets/{x64-macos.cmake => x64-osx.cmake} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename ports/triplets/{arm64-macos.cmake => arm64-osx.cmake} (100%) rename ports/triplets/{x64-macos.cmake => x64-osx.cmake} (100%) diff --git a/ports/triplets/arm64-macos.cmake b/ports/triplets/arm64-osx.cmake similarity index 100% rename from ports/triplets/arm64-macos.cmake rename to ports/triplets/arm64-osx.cmake diff --git a/ports/triplets/x64-macos.cmake b/ports/triplets/x64-osx.cmake similarity index 100% rename from ports/triplets/x64-macos.cmake rename to ports/triplets/x64-osx.cmake From 385c2fcc3ee662ed96307c798e6514db9b2fa110 Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Mon, 13 Nov 2023 22:30:46 +0200 Subject: [PATCH 03/21] Bump vcpkg to a version that has the Apple architecture detection fixed. See microsoft/vcpkg#34054. --- cmake/Options/TileDBToolchain.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/Options/TileDBToolchain.cmake b/cmake/Options/TileDBToolchain.cmake index 470737ddd1a..3f141882853 100644 --- a/cmake/Options/TileDBToolchain.cmake +++ b/cmake/Options/TileDBToolchain.cmake @@ -20,7 +20,7 @@ if (NOT DEFINED CMAKE_TOOLCHAIN_FILE) # Inspired from https://github.com/Azure/azure-sdk-for-cpp/blob/azure-core_1.10.3/cmake-modules/AzureVcpkg.cmake message("TILEDB_DISABLE_AUTO_VCPKG is not defined. Fetch a local copy of vcpkg.") # To help with resolving conflicts, when you update the commit, also update its date. - set(VCPKG_COMMIT_STRING 1b4d69f3028d74401a001aa316986a670ca6289a) # 2023-09-27 + set(VCPKG_COMMIT_STRING 421579b6e6cac42d9a5a5935e53e218ce94a347f) # 2023-11-13 message("Vcpkg commit string used: ${VCPKG_COMMIT_STRING}") include(FetchContent) FetchContent_Declare( From 8ae406acb3b85d95a69fc8d93e5dbefe210c6b38 Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Tue, 14 Nov 2023 13:34:20 +0200 Subject: [PATCH 04/21] Fix OS detection. --- cmake/Options/TileDBToolchain.cmake | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/cmake/Options/TileDBToolchain.cmake b/cmake/Options/TileDBToolchain.cmake index 3f141882853..d64fb0b4e41 100644 --- a/cmake/Options/TileDBToolchain.cmake +++ b/cmake/Options/TileDBToolchain.cmake @@ -34,13 +34,14 @@ if (NOT DEFINED CMAKE_TOOLCHAIN_FILE) endif() if(NOT DEFINED VCPKG_TARGET_TRIPLET AND SANITIZER STREQUAL "address") - if(APPLE) + # CMAKE_SYSTEM_NAME might not be set at this point, so we need to also check CMAKE_HOST_SYSTEM_NAME. + if(CMAKE_SYSTEM_NAME STREQUAL "Darwin" OR (NOT CMAKE_SYSTEM_NAME AND CMAKE_HOST_SYSTEM_NAME STREQUAL "Darwin")) if (CMAKE_OSX_ARCHITECTURES STREQUAL x86_64 OR CMAKE_SYSTEM_PROCESSOR MATCHES "(x86_64)|(AMD64|amd64)|(^i.86$)") set(VCPKG_TARGET_TRIPLET "x64-osx-asan") elseif (CMAKE_OSX_ARCHITECTURES STREQUAL arm64 OR CMAKE_SYSTEM_PROCESSOR MATCHES "^aarch64" OR CMAKE_SYSTEM_PROCESSOR MATCHES "^arm") set(VCPKG_TARGET_TRIPLET "arm64-osx-asan") endif() - elseif(CMAKE_SYSTEM_NAME STREQUAL "Linux") + elseif(CMAKE_SYSTEM_NAME STREQUAL "Linux" OR (NOT CMAKE_SYSTEM_NAME AND CMAKE_HOST_SYSTEM_NAME STREQUAL "Linux")) set(VCPKG_TARGET_TRIPLET "x64-linux-asan") elseif(WIN32) set(VCPKG_TARGET_TRIPLET "x64-windows-asan") From fe5643e41596d19ded167bbafb11ee90e60ce449 Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Fri, 17 Nov 2023 13:53:25 +0200 Subject: [PATCH 05/21] Add a license header to TileDBToolchain.cmake. --- cmake/Options/TileDBToolchain.cmake | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/cmake/Options/TileDBToolchain.cmake b/cmake/Options/TileDBToolchain.cmake index 0ef0106461c..f07db9375ed 100644 --- a/cmake/Options/TileDBToolchain.cmake +++ b/cmake/Options/TileDBToolchain.cmake @@ -1,3 +1,28 @@ +# +# cmake/Options/TileDBToolchain.cmake +# +# The MIT License +# +# Copyright (c) 2023 TileDB, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# ############################################################ # TileDB Toolchain Setup ############################################################ From a53ab27f0cd7f168d5d71e12822d12594e289096 Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Fri, 17 Nov 2023 13:54:21 +0200 Subject: [PATCH 06/21] Import the Sanitizer module from #2392. Co-authored-by: Eric Hughes --- cmake/Modules/Sanitizer.cmake | 137 ++++++++++++++++++++++++++++++++++ 1 file changed, 137 insertions(+) create mode 100644 cmake/Modules/Sanitizer.cmake diff --git a/cmake/Modules/Sanitizer.cmake b/cmake/Modules/Sanitizer.cmake new file mode 100644 index 00000000000..58450b23132 --- /dev/null +++ b/cmake/Modules/Sanitizer.cmake @@ -0,0 +1,137 @@ +# +# cmake/Modules/Sanitizers.cmake +# +# The MIT License +# +# Copyright (c) 2021 TileDB, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# +# Sanitizers +# https://github.com/google/sanitizers +# https://github.com/google/sanitizers/wiki/AddressSanitizer +# +# GCC +# all: https://gcc.gnu.org/onlinedocs/gcc/Instrumentation-Options.html +# All the sanitizer options are here, together with all the similar options gcc offers. +# +# Clang +# address: https://clang.llvm.org/docs/AddressSanitizer.html +# Recommends linking with `clang++`. No warning if otherwise at this time. +# undefined behavior: https://clang.llvm.org/docs/UndefinedBehaviorSanitizer.html +# memory: https://clang.llvm.org/docs/MemorySanitizer.html +# +# MSVC +# Only supports the address sanitizer +# https://docs.microsoft.com/en-us/cpp/sanitizers/asan +# https://docs.microsoft.com/en-us/cpp/sanitizers/asan-building +# https://devblogs.microsoft.com/cppblog/addresssanitizer-asan-for-windows-with-msvc/ +# https://devblogs.microsoft.com/cppblog/address-sanitizer-for-msvc-now-generally-available/ + +include_guard() + +# +# target_sanitizer: compile and link a target with a single sanitizer +# +# Syntax mimics that of CMake project commands `target_compile_features` etc. +# Sanitization requires compile and link options to be set, so its name is a +# bit different. +# +# Arguments +# the_target: CMake target +# scope: PUBLIC INTERFACE PRIVATE, passed through for all option-setting +# sanitizer: a single sanitizer use on the target +# +function(target_sanitizer the_target scope sanitizer) + # The basic sanitizer option is standard across compilers + target_compile_options(${the_target} ${scope} -fsanitize=${SANITIZER}) + + # Check that the sanitizer name is well-formed + if (NOT sanitizer MATCHES "^[-A-Za-z]*$") + message(FATAL_ERROR "target_sanitizer: bad sanitizer specification \"${sanitizer}\";" + " permissible characters are only alphabetic and hyphen") + endif() + + # Verify that the sanitizer is one that some compiler supports + string(TOLOWER ${sanitizer} sanitizer) + if (NOT sanitizer MATCHES "^(address|memory|leak|thread|undefined)$") + message(FATAL_ERROR "target_sanitizer: unsupported sanitizer ${sanitizer}") + endif() + + # Validate the scope + if (NOT scope MATCHES "^(PUBLIC|PRIVATE|INTERFACE)$") + message(FATAL_ERROR "target_sanitizer: scope \"${scope}\" is not one of PUBLIC, PRIVATE, or INTERFACE.") + endif() + + # For known compilers, check that the sanitizer is supported. + # If we know it's not supported, we'll fail so that we avoid false confidence. + # If we don't know, we'll warn that it might not work. + if (CMAKE_CXX_COMPILER_ID MATCHES "MSVC") + if (sanitizer MATCHES "^address$") + # MSVC support for the address sanitizer began with Visual Studio 2019 Version 16.4 + # and was announced as "fully supported" in version 16.9 + if (MSVC_VERSION LESS 1924) + message(FATAL_ERROR "MSVC version ${MSVC_VERSION} too early to support address sanitizer." ) + endif() + if (MSVC_VERSION LESS 1929) + message(WARNING "MSVC version ${MSVC_VERSION} may only partially support address sanitizer." ) + endif() + # Catch has a conflict with ASAN on Windows. Disable the SEH handler in Catch to avoid the conflict. + target_compile_definitions(${the_target} ${scope} CATCH_CONFIG_NO_WINDOWS_SEH) + else() + # MSVC support only the address sanitizer + message(FATAL_ERROR "MSVC only supports sanitizer \"address\"") + endif() + # Certain compile options are incompatible with ASAN + # Microsoft suppresses /INCREMENTAL, but emits a warning, so silence it. + target_link_options(${the_target} ${scope} /INCREMENTAL:NO) + + # May also need to explicitly remove /RTC flags + + elseif (CMAKE_CXX_COMPILER_ID MATCHES "Clang") # also matches AppleClang, ARMClang, etc. + # Ordinary gcc behavior. Factor this out into a subroutine when we need more than twice. + target_compile_options(${the_target} ${scope} -g -fno-omit-frame-pointer -fno-optimize-sibling-calls) + + # Clang recommends a linker flag as well as a compiler flag + target_link_options(${the_target} ${scope} -fsanitize=${SANITIZER}) + if (sanitizer MATCHES "^address$") + # There may be problems if clang tries to link the ASAN library statically + target_link_options(${the_target} ${scope} -shared-libasan) + endif() + + elseif (CMAKE_CXX_COMPILER_ID MATCHES "GNU") + # Ordinary gcc behavior + target_compile_options(${the_target} ${scope} -g -fno-omit-frame-pointer -fno-optimize-sibling-calls) + + else() + message(WARN "Compiler \"${CMAKE_CXX_COMPILER_ID}\" not explicitly supported; behaving as if GNU") + endif() +endfunction() + +# +# target_sanitizer_options +# +# There are a number of options for the various sanitizers, notably about error recovery +# and trapping on error. +# +# Limitations: Unimplemented at the present time. For future use. +# +function(target_sanitizer_options the_target sanitizer options) + message(FATAL_ERROR "target_sanitizer_options: not yet implemented") +endfunction() From b3698781a9dd4fa8d9f181ef3af903fcf6a81318 Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Fri, 17 Nov 2023 17:02:13 +0200 Subject: [PATCH 07/21] Change the sanitizer module to set compiler options globally. --- CMakeLists.txt | 23 +----- cmake/Modules/Sanitizer.cmake | 135 +++++++++++++--------------------- 2 files changed, 54 insertions(+), 104 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 3d1a5a065d8..b7f5b2d7996 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -39,6 +39,7 @@ list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake/Options") include(CIConfig) include(BuildOptions) include(global-policies NO_POLICY_SCOPE) +include(Sanitizer) include(TileDBToolchain) ############################################################ @@ -248,28 +249,6 @@ if (COMPILER_SUPPORTS_AVX2) add_compile_options(${COMPILER_AVX2_FLAG}) endif() -# HACK: Set the sanitizer configuration globally after the -# superbuild has finished. We want to enable sanitization for -# TileDB, but not our dependencies. -# -# The reasoning for setting this globally is due to false positives -# in the `AddressSanitizerContainerOverflow` checks [1]. -# -# [1] https://github.com/google/sanitizers/wiki/AddressSanitizerContainerOverflow#false-positives -if (SANITIZER) - string(TOLOWER ${SANITIZER} SANITIZER) - if (NOT SANITIZER MATCHES "^(address|memory|leak|thread|undefined)$") - message(FATAL_ERROR "Unknown clang sanitizer: ${SANITIZER})") - else() - message(STATUS "The TileDB library is compiled with sanitizer ${SANITIZER} enabled") - endif() - add_compile_options(-DTILEDB_SANITIZER=${SANITIZER}) - add_compile_options( - "$<$>:-g;-fno-omit-frame-pointer;-fno-optimize-sibling-calls>" - -fsanitize=${SANITIZER}) - add_link_options(-fsanitize=${SANITIZER}) -endif() - ####################################################### # Header Files ####################################################### diff --git a/cmake/Modules/Sanitizer.cmake b/cmake/Modules/Sanitizer.cmake index 58450b23132..6d6fffcfc5a 100644 --- a/cmake/Modules/Sanitizer.cmake +++ b/cmake/Modules/Sanitizer.cmake @@ -3,7 +3,7 @@ # # The MIT License # -# Copyright (c) 2021 TileDB, Inc. +# Copyright (c) 2023 TileDB, Inc. # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -44,94 +44,65 @@ # https://devblogs.microsoft.com/cppblog/addresssanitizer-asan-for-windows-with-msvc/ # https://devblogs.microsoft.com/cppblog/address-sanitizer-for-msvc-now-generally-available/ -include_guard() +if(NOT SANITIZER) + return() +endif() -# -# target_sanitizer: compile and link a target with a single sanitizer -# -# Syntax mimics that of CMake project commands `target_compile_features` etc. -# Sanitization requires compile and link options to be set, so its name is a -# bit different. -# -# Arguments -# the_target: CMake target -# scope: PUBLIC INTERFACE PRIVATE, passed through for all option-setting -# sanitizer: a single sanitizer use on the target -# -function(target_sanitizer the_target scope sanitizer) - # The basic sanitizer option is standard across compilers - target_compile_options(${the_target} ${scope} -fsanitize=${SANITIZER}) - - # Check that the sanitizer name is well-formed - if (NOT sanitizer MATCHES "^[-A-Za-z]*$") - message(FATAL_ERROR "target_sanitizer: bad sanitizer specification \"${sanitizer}\";" - " permissible characters are only alphabetic and hyphen") - endif() +# The basic sanitizer option is standard across compilers +add_compile_options(-fsanitize=${SANITIZER}) - # Verify that the sanitizer is one that some compiler supports - string(TOLOWER ${sanitizer} sanitizer) - if (NOT sanitizer MATCHES "^(address|memory|leak|thread|undefined)$") - message(FATAL_ERROR "target_sanitizer: unsupported sanitizer ${sanitizer}") - endif() +# Check that the sanitizer name is well-formed +if (NOT SANITIZER MATCHES "^[-A-Za-z]*$") + message(FATAL_ERROR "Bad sanitizer specification \"${sanitizer}\";" + " permissible characters are only alphabetic and hyphen") +endif() - # Validate the scope - if (NOT scope MATCHES "^(PUBLIC|PRIVATE|INTERFACE)$") - message(FATAL_ERROR "target_sanitizer: scope \"${scope}\" is not one of PUBLIC, PRIVATE, or INTERFACE.") - endif() +# Verify that the sanitizer is one that some compiler supports +string(TOLOWER ${SANITIZER} SANITIZER) +if (NOT SANITIZER MATCHES "^(address|memory|leak|thread|undefined)$") + message(FATAL_ERROR "Unsupported sanitizer ${sanitizer}") +endif() - # For known compilers, check that the sanitizer is supported. - # If we know it's not supported, we'll fail so that we avoid false confidence. - # If we don't know, we'll warn that it might not work. - if (CMAKE_CXX_COMPILER_ID MATCHES "MSVC") - if (sanitizer MATCHES "^address$") - # MSVC support for the address sanitizer began with Visual Studio 2019 Version 16.4 - # and was announced as "fully supported" in version 16.9 - if (MSVC_VERSION LESS 1924) - message(FATAL_ERROR "MSVC version ${MSVC_VERSION} too early to support address sanitizer." ) - endif() - if (MSVC_VERSION LESS 1929) - message(WARNING "MSVC version ${MSVC_VERSION} may only partially support address sanitizer." ) - endif() - # Catch has a conflict with ASAN on Windows. Disable the SEH handler in Catch to avoid the conflict. - target_compile_definitions(${the_target} ${scope} CATCH_CONFIG_NO_WINDOWS_SEH) - else() - # MSVC support only the address sanitizer - message(FATAL_ERROR "MSVC only supports sanitizer \"address\"") +# For known compilers, check that the sanitizer is supported. +# If we know it's not supported, we'll fail so that we avoid false confidence. +# If we don't know, we'll warn that it might not work. +if (CMAKE_CXX_COMPILER_ID MATCHES "MSVC") + if (SANITIZER STREQUAL "address") + # MSVC support for the address sanitizer began with Visual Studio 2019 Version 16.4 + # and was announced as "fully supported" in version 16.9 + if (MSVC_VERSION LESS 1924) + message(FATAL_ERROR "MSVC version ${MSVC_VERSION} too early to support address sanitizer." ) endif() - # Certain compile options are incompatible with ASAN - # Microsoft suppresses /INCREMENTAL, but emits a warning, so silence it. - target_link_options(${the_target} ${scope} /INCREMENTAL:NO) - - # May also need to explicitly remove /RTC flags - - elseif (CMAKE_CXX_COMPILER_ID MATCHES "Clang") # also matches AppleClang, ARMClang, etc. - # Ordinary gcc behavior. Factor this out into a subroutine when we need more than twice. - target_compile_options(${the_target} ${scope} -g -fno-omit-frame-pointer -fno-optimize-sibling-calls) - - # Clang recommends a linker flag as well as a compiler flag - target_link_options(${the_target} ${scope} -fsanitize=${SANITIZER}) - if (sanitizer MATCHES "^address$") - # There may be problems if clang tries to link the ASAN library statically - target_link_options(${the_target} ${scope} -shared-libasan) + if (MSVC_VERSION LESS 1929) + message(WARNING "MSVC version ${MSVC_VERSION} may only partially support address sanitizer." ) endif() + # Catch has a conflict with ASAN on Windows. Disable the SEH handler in Catch to avoid the conflict. + add_compile_definitions(CATCH_CONFIG_NO_WINDOWS_SEH) + else() + # MSVC support only the address sanitizer + message(FATAL_ERROR "MSVC only supports sanitizer \"address\"") + endif() + # Certain compile options are incompatible with ASAN + # Microsoft suppresses /INCREMENTAL, but emits a warning, so silence it. + add_link_options(/INCREMENTAL:NO) - elseif (CMAKE_CXX_COMPILER_ID MATCHES "GNU") - # Ordinary gcc behavior - target_compile_options(${the_target} ${scope} -g -fno-omit-frame-pointer -fno-optimize-sibling-calls) + # May also need to explicitly remove /RTC flags - else() - message(WARN "Compiler \"${CMAKE_CXX_COMPILER_ID}\" not explicitly supported; behaving as if GNU") +elseif (CMAKE_CXX_COMPILER_ID MATCHES "Clang") # also matches AppleClang, ARMClang, etc. + # Ordinary gcc behavior. Factor this out into a subroutine when we need more than twice. + add_compile_options(-g -fno-omit-frame-pointer -fno-optimize-sibling-calls) + + # Clang recommends a linker flag as well as a compiler flag + add_link_options(-fsanitize=${SANITIZER}) + if (SANITIZER STREQUAL "address") + # There may be problems if clang tries to link the ASAN library statically + add_link_options(-shared-libasan) endif() -endfunction() -# -# target_sanitizer_options -# -# There are a number of options for the various sanitizers, notably about error recovery -# and trapping on error. -# -# Limitations: Unimplemented at the present time. For future use. -# -function(target_sanitizer_options the_target sanitizer options) - message(FATAL_ERROR "target_sanitizer_options: not yet implemented") -endfunction() +elseif (CMAKE_CXX_COMPILER_ID MATCHES "GNU") + # Ordinary gcc behavior + add_compile_options(-g -fno-omit-frame-pointer -fno-optimize-sibling-calls) + +else() + message(WARN "Compiler \"${CMAKE_CXX_COMPILER_ID}\" not explicitly supported; behaving as if GNU") +endif() From 5c7e0daa4eb14bbcaf6a2b57a0811e899fbea32b Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Fri, 17 Nov 2023 17:23:18 +0200 Subject: [PATCH 08/21] Allow only address sanitizer. --- cmake/Modules/Sanitizer.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/Modules/Sanitizer.cmake b/cmake/Modules/Sanitizer.cmake index 6d6fffcfc5a..dd6d07776e8 100644 --- a/cmake/Modules/Sanitizer.cmake +++ b/cmake/Modules/Sanitizer.cmake @@ -59,7 +59,7 @@ endif() # Verify that the sanitizer is one that some compiler supports string(TOLOWER ${SANITIZER} SANITIZER) -if (NOT SANITIZER MATCHES "^(address|memory|leak|thread|undefined)$") +if (NOT SANITIZER MATCHES "^address$") message(FATAL_ERROR "Unsupported sanitizer ${sanitizer}") endif() From 2c7bf018389b2d8379d5af6e4061b7559d8bba26 Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Thu, 23 Nov 2023 11:55:46 +0200 Subject: [PATCH 09/21] Refactor the sanitizer module to use generator expressions, and move the validation code in a macro to run later. --- CMakeLists.txt | 4 ++ cmake/Modules/Sanitizer.cmake | 76 +++++++++++++++++------------------ 2 files changed, 41 insertions(+), 39 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b7f5b2d7996..c8b2a51eb7d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -249,6 +249,10 @@ if (COMPILER_SUPPORTS_AVX2) add_compile_options(${COMPILER_AVX2_FLAG}) endif() +if(SANITIZER) + validate_sanitizer_options() +endif() + ####################################################### # Header Files ####################################################### diff --git a/cmake/Modules/Sanitizer.cmake b/cmake/Modules/Sanitizer.cmake index dd6d07776e8..d8d82fd685d 100644 --- a/cmake/Modules/Sanitizer.cmake +++ b/cmake/Modules/Sanitizer.cmake @@ -63,46 +63,44 @@ if (NOT SANITIZER MATCHES "^address$") message(FATAL_ERROR "Unsupported sanitizer ${sanitizer}") endif() -# For known compilers, check that the sanitizer is supported. -# If we know it's not supported, we'll fail so that we avoid false confidence. -# If we don't know, we'll warn that it might not work. -if (CMAKE_CXX_COMPILER_ID MATCHES "MSVC") - if (SANITIZER STREQUAL "address") - # MSVC support for the address sanitizer began with Visual Studio 2019 Version 16.4 - # and was announced as "fully supported" in version 16.9 - if (MSVC_VERSION LESS 1924) - message(FATAL_ERROR "MSVC version ${MSVC_VERSION} too early to support address sanitizer." ) - endif() - if (MSVC_VERSION LESS 1929) - message(WARNING "MSVC version ${MSVC_VERSION} may only partially support address sanitizer." ) - endif() - # Catch has a conflict with ASAN on Windows. Disable the SEH handler in Catch to avoid the conflict. - add_compile_definitions(CATCH_CONFIG_NO_WINDOWS_SEH) - else() - # MSVC support only the address sanitizer - message(FATAL_ERROR "MSVC only supports sanitizer \"address\"") - endif() - # Certain compile options are incompatible with ASAN - # Microsoft suppresses /INCREMENTAL, but emits a warning, so silence it. - add_link_options(/INCREMENTAL:NO) +# Catch has a conflict with ASAN on Windows. Disable the SEH handler in Catch to avoid the conflict. +add_compile_definitions("$<$:CATCH_CONFIG_NO_WINDOWS_SEH>") + # Microsoft suppresses /INCREMENTAL, but emits a warning, so silence it. +add_link_options("$<$:/INCREMENTAL:NO>") - # May also need to explicitly remove /RTC flags - -elseif (CMAKE_CXX_COMPILER_ID MATCHES "Clang") # also matches AppleClang, ARMClang, etc. - # Ordinary gcc behavior. Factor this out into a subroutine when we need more than twice. - add_compile_options(-g -fno-omit-frame-pointer -fno-optimize-sibling-calls) +# Ordinary gcc/clang behavior. +add_compile_options("$<$>:-g;-fno-omit-frame-pointer;-fno-optimize-sibling-calls>") +# Clang recommends a linker flag as well as a compiler flag +add_link_options("$<$:-fsanitize=${SANITIZER}>") +if(SANITIZER STREQUAL "address") + # There may be problems if clang tries to link the ASAN library statically + add_link_options("$<$:-shared-libasan>") +endif() - # Clang recommends a linker flag as well as a compiler flag - add_link_options(-fsanitize=${SANITIZER}) - if (SANITIZER STREQUAL "address") - # There may be problems if clang tries to link the ASAN library statically - add_link_options(-shared-libasan) - endif() +# Validate sanitizer options. +# This must be called after the project() command, where the compiler is known. +macro(validate_sanitizer_options) + # For known compilers, check that the sanitizer is supported. + # If we know it's not supported, we'll fail so that we avoid false confidence. + # If we don't know, we'll warn that it might not work. + if (CMAKE_CXX_COMPILER_ID MATCHES "MSVC") + if (SANITIZER STREQUAL "address") + # MSVC support for the address sanitizer began with Visual Studio 2019 Version 16.4 + # and was announced as "fully supported" in version 16.9 + if (MSVC_VERSION LESS 1924) + message(FATAL_ERROR "MSVC version ${MSVC_VERSION} too early to support address sanitizer." ) + endif() + if (MSVC_VERSION LESS 1929) + message(WARNING "MSVC version ${MSVC_VERSION} may only partially support address sanitizer." ) + endif() + else() + # MSVC support only the address sanitizer + message(FATAL_ERROR "MSVC only supports sanitizer \"address\"") + endif() -elseif (CMAKE_CXX_COMPILER_ID MATCHES "GNU") - # Ordinary gcc behavior - add_compile_options(-g -fno-omit-frame-pointer -fno-optimize-sibling-calls) + # May also need to explicitly remove /RTC flags -else() - message(WARN "Compiler \"${CMAKE_CXX_COMPILER_ID}\" not explicitly supported; behaving as if GNU") -endif() + elseif (NOT (CMAKE_CXX_COMPILER_ID MATCHES "Clang" OR CMAKE_CXX_COMPILER_ID MATCHES "GNU")) + message(WARN "Compiler \"${CMAKE_CXX_COMPILER_ID}\" not explicitly supported; behaving as if GNU") + endif() +endmacro() From 7869ffc640a62ad648c2b7fd6fd0f585428dce71 Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Wed, 29 Nov 2023 01:43:57 +0200 Subject: [PATCH 10/21] Try finding an ASAN-compatible triplet. --- cmake/Options/TileDBToolchain.cmake | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/cmake/Options/TileDBToolchain.cmake b/cmake/Options/TileDBToolchain.cmake index f07db9375ed..36cc9cfc621 100644 --- a/cmake/Options/TileDBToolchain.cmake +++ b/cmake/Options/TileDBToolchain.cmake @@ -58,20 +58,16 @@ if (NOT DEFINED CMAKE_TOOLCHAIN_FILE) endif() endif() -if(NOT DEFINED VCPKG_TARGET_TRIPLET AND SANITIZER STREQUAL "address") - # CMAKE_SYSTEM_NAME might not be set at this point, so we need to also check CMAKE_HOST_SYSTEM_NAME. - if(CMAKE_SYSTEM_NAME STREQUAL "Darwin" OR (NOT CMAKE_SYSTEM_NAME AND CMAKE_HOST_SYSTEM_NAME STREQUAL "Darwin")) - if (CMAKE_OSX_ARCHITECTURES STREQUAL x86_64 OR CMAKE_SYSTEM_PROCESSOR MATCHES "(x86_64)|(AMD64|amd64)|(^i.86$)") - set(VCPKG_TARGET_TRIPLET "x64-osx-asan") - elseif (CMAKE_OSX_ARCHITECTURES STREQUAL arm64 OR CMAKE_SYSTEM_PROCESSOR MATCHES "^aarch64" OR CMAKE_SYSTEM_PROCESSOR MATCHES "^arm") - set(VCPKG_TARGET_TRIPLET "arm64-osx-asan") +if(SANITIZER STREQUAL "address") + if(NOT DEFINED VCPKG_TARGET_TRIPLET) + message(FATAL_ERROR "VCPKG_TARGET_TRIPLET must be defined when building with ASAN.") + elseif(NOT VCPKG_TARGET_TRIPLET MATCHES "-asan$") + if(EXISTS "${CMAKE_CURRENT_LIST_DIR}/../../ports/triplets/${VCPKG_TARGET_TRIPLET}-asan.cmake") + message("Overriding vcpkg triplet to ${VCPKG_TARGET_TRIPLET}-asan") + set(VCPKG_TARGET_TRIPLET "${VCPKG_TARGET_TRIPLET}-asan") + else() + message(FATAL_ERROR "Incompatible vcpkg triplet. It must end in -asan, or a triplet with the name ${VCPKG_TARGET_TRIPLET}-asan must exist in ports/triplets.") endif() - elseif(CMAKE_SYSTEM_NAME STREQUAL "Linux" OR (NOT CMAKE_SYSTEM_NAME AND CMAKE_HOST_SYSTEM_NAME STREQUAL "Linux")) - set(VCPKG_TARGET_TRIPLET "x64-linux-asan") - elseif(WIN32) - set(VCPKG_TARGET_TRIPLET "x64-windows-asan") - else() - message(WARNING "Unrecognized platform for sanitizer build. Dependencies will not be compiled with ASAN.") endif() endif() From 9dd3bf900cd6185a5540fc1377a8259431e0d147 Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Wed, 29 Nov 2023 01:44:23 +0200 Subject: [PATCH 11/21] Add options in the bootstrap script to specify the vcpkg triplet. --- bootstrap | 6 ++++++ bootstrap.ps1 | 7 ++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/bootstrap b/bootstrap index 8ade2249dab..bb4b7f94561 100755 --- a/bootstrap +++ b/bootstrap @@ -34,6 +34,8 @@ Configuration: --help print this message --prefix=PREFIX install files in tree rooted at PREFIX ['"${default_prefix}"'] + --vcpkg-target-triplet=TRIPLET vcpkg triplet to use for downloading and building dependencies + [auto-detected] --enable-vcpkg use vcpkg for downloading and building dependencies --dependency=DIRs specify the dependencies at DIRs, separated by colon ['"${default_dependency}"'] @@ -50,6 +52,7 @@ Configuration: --enable-static-tiledb enables building TileDB as a static library --enable-sanitizer=SAN enable sanitizer (clang only) (address|memory|leak|thread|undefined) + Must manually specify a vcpkg triplet. --enable-debug enable debug build --enable-assertions enable assertions in compiled code --enable-release-symbols enable create symbols for release build @@ -88,6 +91,7 @@ Example: # Parse arguments prefix_dirs="${default_prefix}" +vcpkg_target_triplet="" tiledb_vcpkg="ON" dependency_dir="${default_dependency}" sanitizer="" @@ -121,6 +125,7 @@ while test $# != 0; do case "$1" in --prefix=*) dir=`arg "$1"` prefix_dirs="$dir";; + --vcpkg-target-triplet=*) vcpkg_target_triplet=`arg "$1"`;; --enable-vcpkg) echo "Argument '--enable-vcpkg' is obsolete and will be removed in a future version. Vcpkg is now enabled by default." tiledb_vcpkg="ON";; --dependency=*) dir=`arg "$1"` @@ -228,6 +233,7 @@ fi # Configure ${cmake} -DCMAKE_BUILD_TYPE=${build_type} \ -DCMAKE_INSTALL_PREFIX="${prefix_dirs}" \ + -DVCPKG_TARGET_TRIPLET=${vcpkg_target_triplet} \ -DCMAKE_C_COMPILER="${c_compiler}" \ -DCMAKE_CXX_COMPILER="${cxx_compiler}" \ -DCMAKE_PREFIX_PATH="${dependency_dir}" \ diff --git a/bootstrap.ps1 b/bootstrap.ps1 index 6c3c0358406..756381b5641 100644 --- a/bootstrap.ps1 +++ b/bootstrap.ps1 @@ -13,6 +13,10 @@ develop TileDB. .PARAMETER Prefix Installs files in tree rooted at PREFIX (defaults to TileDB\dist). +.PARAMETER VcpkgTargetTriplet +Optionally specify the vcpkg target triplet, e.g. "x64-windows-release". +Defaults to automatically detecting it from the environment. + .PARAMETER Dependency Semicolon separated list to binary dependencies. @@ -118,6 +122,7 @@ https://github.com/TileDB-Inc/TileDB [CmdletBinding()] Param( [string]$Prefix, + [string]$VcpkgTargetTriplet, [string]$Dependency, [string]$CMakeGenerator, [switch]$EnableAssert, @@ -326,7 +331,7 @@ if ($CMakeGenerator -eq $null) { # Run CMake. # We use Invoke-Expression so we can echo the command to the user. -$CommandString = "cmake -A X64 -DTILEDB_VCPKG=$UseVcpkg -DCMAKE_BUILD_TYPE=$BuildType -DCMAKE_INSTALL_PREFIX=""$InstallPrefix"" -DCMAKE_PREFIX_PATH=""$DependencyDir"" -DMSVC_MP_FLAG=""/MP$BuildProcesses"" -DTILEDB_ASSERTIONS=$AssertionMode -DTILEDB_VERBOSE=$Verbosity -DTILEDB_AZURE=$UseAzure -DTILEDB_S3=$UseS3 -DTILEDB_GCS=$UseGcs -DTILEDB_SERIALIZATION=$UseSerialization -DTILEDB_WERROR=$Werror -DTILEDB_CPP_API=$CppApi -DTILEDB_TESTS=$Tests -DTILEDB_STATS=$Stats -DTILEDB_STATIC=$TileDBStatic -DTILEDB_FORCE_ALL_DEPS=$TileDBBuildDeps -DTILEDB_REMOVE_DEPRECATIONS=$RemoveDeprecations -DTILEDB_TOOLS=$TileDBTools -DTILEDB_EXPERIMENTAL_FEATURES=$TileDBExperimentalFeatures -DTILEDB_WEBP=$BuildWebP -DTILEDB_CRC32=$BuildCrc32 -DTILEDB_ARROW_TESTS=$ArrowTests -DTILEDB_TESTS_ENABLE_REST=$RestTests -DTILEDB_TESTS_AWS_S3_CONFIG=$ConfigureS3 $GeneratorFlag ""$SourceDirectory""" +$CommandString = "cmake -A X64 -DTILEDB_VCPKG=$UseVcpkg -DCMAKE_BUILD_TYPE=$BuildType -DCMAKE_INSTALL_PREFIX=""$InstallPrefix"" -DVCPKG_TARGET_TRIPLET=$VcpkgTargetTriplet -DCMAKE_PREFIX_PATH=""$DependencyDir"" -DMSVC_MP_FLAG=""/MP$BuildProcesses"" -DTILEDB_ASSERTIONS=$AssertionMode -DTILEDB_VERBOSE=$Verbosity -DTILEDB_AZURE=$UseAzure -DTILEDB_S3=$UseS3 -DTILEDB_GCS=$UseGcs -DTILEDB_SERIALIZATION=$UseSerialization -DTILEDB_WERROR=$Werror -DTILEDB_CPP_API=$CppApi -DTILEDB_TESTS=$Tests -DTILEDB_STATS=$Stats -DTILEDB_STATIC=$TileDBStatic -DTILEDB_FORCE_ALL_DEPS=$TileDBBuildDeps -DTILEDB_REMOVE_DEPRECATIONS=$RemoveDeprecations -DTILEDB_TOOLS=$TileDBTools -DTILEDB_EXPERIMENTAL_FEATURES=$TileDBExperimentalFeatures -DTILEDB_WEBP=$BuildWebP -DTILEDB_CRC32=$BuildCrc32 -DTILEDB_ARROW_TESTS=$ArrowTests -DTILEDB_TESTS_ENABLE_REST=$RestTests -DTILEDB_TESTS_AWS_S3_CONFIG=$ConfigureS3 $GeneratorFlag ""$SourceDirectory""" Write-Host $CommandString Write-Host Invoke-Expression "$CommandString" From cc1d88dad01e640c705676372b283c2db293f677 Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Wed, 29 Nov 2023 02:26:54 +0200 Subject: [PATCH 12/21] Add instructions in the contributor's guide to build with sanitizers. --- CONTRIBUTING.md | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 606481f12c4..e36c29a0e95 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -61,6 +61,19 @@ Formatting conventions: - comments are good, TileDB uses [doxygen](http://www.stack.nl/~dimitri/doxygen/manual/docblocks.html) for class doc strings. - format code using [clang-format](https://clang.llvm.org/docs/ClangFormat.html) +### Building with sanitizers + +TileDB can be built with [clang sanitizers](https://clang.llvm.org/docs/AddressSanitizer.html) enabled. To enable them, you have to bootstrap with the `--enable-sanitizer` flag, as well as specify a vcpkg triplet compatible with ASAN (that ends with `-asan`). You can view the list of supported triplets [here](https://github.com/TileDB-Inc/TileDB/tree/dev/ports/triplets). + +> [!NOTE] +> Currently only the `address` sanitizer is supported. + +```bash +cd TileDB && mkdir build-asan && cd build-asan +../bootstrap --enable-sanitizer=address --vcpkg-target-triplet=x64-linux-asan +make && make check +``` + ### Pull Requests: - `dev` is the development branch, all PR’s should be rebased on top of the latest `dev` commit. @@ -78,7 +91,7 @@ Formatting conventions: - Submit a PR, writing a descriptive message. If a PR closes an open issue, reference the issue in the PR message (ex. If an issue closes issue number 10, you would write `closes #10`) -- Make sure CI (continuous integration) is passing for your PR -- click `Show all checks` in the pull request status box at the bottom of each PR page. The continous integration project pages will also list all recently-built PRs: +- Make sure CI (continuous integration) is passing for your PR -- click `Show all checks` in the pull request status box at the bottom of each PR page. The continuous integration project pages will also list all recently-built PRs: - [Azure Pipelines](https://dev.azure.com/TileDB-Inc/CI/_build) ### Documentation Pull Requests: @@ -107,8 +120,8 @@ This will install all the required packages in a Python virtual environment, and - [Organization](https://github.com/TileDB-Inc/) -* Github / Git +* GitHub / Git - [Git cheatsheet](https://services.github.com/on-demand/downloads/github-git-cheat-sheet/) - - [Github Documentation](https://help.github.com/) + - [GitHub Documentation](https://help.github.com/) - [Forking a Repo](https://help.github.com/articles/fork-a-repo/) - [More Learning Resources](https://help.github.com/articles/git-and-github-learning-resources/) From 2e847eb48121a875b1b1300f7c3b70f23ddf83b4 Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Wed, 29 Nov 2023 13:41:48 +0200 Subject: [PATCH 13/21] Set the triplet on the ASAN CI job. --- .github/workflows/full-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/full-ci.yml b/.github/workflows/full-ci.yml index a710d9761fb..790dd7edfd6 100644 --- a/.github/workflows/full-ci.yml +++ b/.github/workflows/full-ci.yml @@ -101,7 +101,7 @@ jobs: matrix_compiler_cc: 'gcc-10' matrix_compiler_cxx: 'g++-10' timeout: 120 - bootstrap_args: '--enable-serialization' + bootstrap_args: '--enable-serialization --vcpkg-target-triplet=x64-linux-asan' asan: true ci10: From 31048637bf13a6d4bda9224078dac49543d02bbd Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Wed, 29 Nov 2023 13:50:39 +0200 Subject: [PATCH 14/21] Fix CI. --- bootstrap | 5 +++-- bootstrap.ps1 | 7 ++++++- cmake/Options/TileDBToolchain.cmake | 2 +- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/bootstrap b/bootstrap index bb4b7f94561..78e239c7354 100755 --- a/bootstrap +++ b/bootstrap @@ -125,7 +125,8 @@ while test $# != 0; do case "$1" in --prefix=*) dir=`arg "$1"` prefix_dirs="$dir";; - --vcpkg-target-triplet=*) vcpkg_target_triplet=`arg "$1"`;; + --vcpkg-target-triplet=*) vcpkg_target_triplet=`arg "$1"` + vcpkg_target_triplet="-DVCPKG_TARGET_TRIPLET=${vcpkg_target_triplet}";; --enable-vcpkg) echo "Argument '--enable-vcpkg' is obsolete and will be removed in a future version. Vcpkg is now enabled by default." tiledb_vcpkg="ON";; --dependency=*) dir=`arg "$1"` @@ -233,7 +234,6 @@ fi # Configure ${cmake} -DCMAKE_BUILD_TYPE=${build_type} \ -DCMAKE_INSTALL_PREFIX="${prefix_dirs}" \ - -DVCPKG_TARGET_TRIPLET=${vcpkg_target_triplet} \ -DCMAKE_C_COMPILER="${c_compiler}" \ -DCMAKE_CXX_COMPILER="${cxx_compiler}" \ -DCMAKE_PREFIX_PATH="${dependency_dir}" \ @@ -262,6 +262,7 @@ ${cmake} -DCMAKE_BUILD_TYPE=${build_type} \ -DTILEDB_TESTS_ENABLE_REST=${tiledb_tests_enable_rest} \ -DTILEDB_TESTS_AWS_S3_CONFIG=${tiledb_tests_aws_s3_config} \ ${tiledb_disable_avx2} \ + ${vcpkg_target_triplet} \ "${source_dir}" || die "failed to configure the project" echo 'bootstrap success. Run "make" to build, "make check" to test, or "make -C tiledb install" to install.' diff --git a/bootstrap.ps1 b/bootstrap.ps1 index 756381b5641..a10362c4f10 100644 --- a/bootstrap.ps1 +++ b/bootstrap.ps1 @@ -170,6 +170,11 @@ $DefaultPrefix = Join-Path $BinaryDirectory "dist" # Choose the default dependency install prefix. $DefaultDependency = $DefaultPrefix +# Set the default vcpkg triplet. +if ($VcpkgTargetTriplet.IsPresent) { + $VcpkgTargetTriplet = "-DVCPKG_TARGET_TRIPLET=$VcpkgTargetTriplet" +} + # Set assertion mode # No-op for a debug build. $AssertionMode = "OFF" @@ -331,7 +336,7 @@ if ($CMakeGenerator -eq $null) { # Run CMake. # We use Invoke-Expression so we can echo the command to the user. -$CommandString = "cmake -A X64 -DTILEDB_VCPKG=$UseVcpkg -DCMAKE_BUILD_TYPE=$BuildType -DCMAKE_INSTALL_PREFIX=""$InstallPrefix"" -DVCPKG_TARGET_TRIPLET=$VcpkgTargetTriplet -DCMAKE_PREFIX_PATH=""$DependencyDir"" -DMSVC_MP_FLAG=""/MP$BuildProcesses"" -DTILEDB_ASSERTIONS=$AssertionMode -DTILEDB_VERBOSE=$Verbosity -DTILEDB_AZURE=$UseAzure -DTILEDB_S3=$UseS3 -DTILEDB_GCS=$UseGcs -DTILEDB_SERIALIZATION=$UseSerialization -DTILEDB_WERROR=$Werror -DTILEDB_CPP_API=$CppApi -DTILEDB_TESTS=$Tests -DTILEDB_STATS=$Stats -DTILEDB_STATIC=$TileDBStatic -DTILEDB_FORCE_ALL_DEPS=$TileDBBuildDeps -DTILEDB_REMOVE_DEPRECATIONS=$RemoveDeprecations -DTILEDB_TOOLS=$TileDBTools -DTILEDB_EXPERIMENTAL_FEATURES=$TileDBExperimentalFeatures -DTILEDB_WEBP=$BuildWebP -DTILEDB_CRC32=$BuildCrc32 -DTILEDB_ARROW_TESTS=$ArrowTests -DTILEDB_TESTS_ENABLE_REST=$RestTests -DTILEDB_TESTS_AWS_S3_CONFIG=$ConfigureS3 $GeneratorFlag ""$SourceDirectory""" +$CommandString = "cmake -A X64 -DTILEDB_VCPKG=$UseVcpkg -DCMAKE_BUILD_TYPE=$BuildType -DCMAKE_INSTALL_PREFIX=""$InstallPrefix"" $VcpkgTargetTriplet -DCMAKE_PREFIX_PATH=""$DependencyDir"" -DMSVC_MP_FLAG=""/MP$BuildProcesses"" -DTILEDB_ASSERTIONS=$AssertionMode -DTILEDB_VERBOSE=$Verbosity -DTILEDB_AZURE=$UseAzure -DTILEDB_S3=$UseS3 -DTILEDB_GCS=$UseGcs -DTILEDB_SERIALIZATION=$UseSerialization -DTILEDB_WERROR=$Werror -DTILEDB_CPP_API=$CppApi -DTILEDB_TESTS=$Tests -DTILEDB_STATS=$Stats -DTILEDB_STATIC=$TileDBStatic -DTILEDB_FORCE_ALL_DEPS=$TileDBBuildDeps -DTILEDB_REMOVE_DEPRECATIONS=$RemoveDeprecations -DTILEDB_TOOLS=$TileDBTools -DTILEDB_EXPERIMENTAL_FEATURES=$TileDBExperimentalFeatures -DTILEDB_WEBP=$BuildWebP -DTILEDB_CRC32=$BuildCrc32 -DTILEDB_ARROW_TESTS=$ArrowTests -DTILEDB_TESTS_ENABLE_REST=$RestTests -DTILEDB_TESTS_AWS_S3_CONFIG=$ConfigureS3 $GeneratorFlag ""$SourceDirectory""" Write-Host $CommandString Write-Host Invoke-Expression "$CommandString" diff --git a/cmake/Options/TileDBToolchain.cmake b/cmake/Options/TileDBToolchain.cmake index 36cc9cfc621..5c22380c227 100644 --- a/cmake/Options/TileDBToolchain.cmake +++ b/cmake/Options/TileDBToolchain.cmake @@ -59,7 +59,7 @@ if (NOT DEFINED CMAKE_TOOLCHAIN_FILE) endif() if(SANITIZER STREQUAL "address") - if(NOT DEFINED VCPKG_TARGET_TRIPLET) + if(NOT VCPKG_TARGET_TRIPLET) message(FATAL_ERROR "VCPKG_TARGET_TRIPLET must be defined when building with ASAN.") elseif(NOT VCPKG_TARGET_TRIPLET MATCHES "-asan$") if(EXISTS "${CMAKE_CURRENT_LIST_DIR}/../../ports/triplets/${VCPKG_TARGET_TRIPLET}-asan.cmake") From 3031337dfa21cb0e1b942a1c9cc8ebd353f4d1c8 Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Wed, 29 Nov 2023 17:15:58 +0200 Subject: [PATCH 15/21] Pass `-fsanitize` in the linker to GCC as well. --- cmake/Modules/Sanitizer.cmake | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cmake/Modules/Sanitizer.cmake b/cmake/Modules/Sanitizer.cmake index d8d82fd685d..e3b79fd633e 100644 --- a/cmake/Modules/Sanitizer.cmake +++ b/cmake/Modules/Sanitizer.cmake @@ -70,8 +70,7 @@ add_link_options("$<$:/INCREMENTAL:NO>") # Ordinary gcc/clang behavior. add_compile_options("$<$>:-g;-fno-omit-frame-pointer;-fno-optimize-sibling-calls>") -# Clang recommends a linker flag as well as a compiler flag -add_link_options("$<$:-fsanitize=${SANITIZER}>") +add_link_options("$<$>:-fsanitize=${SANITIZER}>") if(SANITIZER STREQUAL "address") # There may be problems if clang tries to link the ASAN library statically add_link_options("$<$:-shared-libasan>") From f9d1cc6e51bf31d4de7651c124a0d84a14fcaca9 Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Mon, 18 Dec 2023 18:42:20 +0200 Subject: [PATCH 16/21] Rename the `SANITIZER` variable to `TILEDB_SANITIZER` and add it in the build options. --- .github/workflows/nightly-test.yml | 2 +- CMakeLists.txt | 2 +- bootstrap | 2 +- cmake/Modules/Sanitizer.cmake | 16 ++++++++-------- cmake/Options/BuildOptions.cmake | 1 + cmake/Options/TileDBToolchain.cmake | 2 +- cmake/TileDB-Superbuild.cmake | 2 +- tiledb/CMakeLists.txt | 4 ++-- 8 files changed, 16 insertions(+), 15 deletions(-) diff --git a/.github/workflows/nightly-test.yml b/.github/workflows/nightly-test.yml index eab5a6fc375..bcf125cc3f4 100644 --- a/.github/workflows/nightly-test.yml +++ b/.github/workflows/nightly-test.yml @@ -46,7 +46,7 @@ jobs: EXPERIMENTAL: ${{ matrix.experimental || 'OFF' }} working-directory: ${{ matrix.working_directory || github.workspace }} run: | - cmake -B build ${{ matrix.config != 'Debug' && '-DTILEDB_VCPKG=OFF' }} -DTILEDB_WERROR=ON -DTILEDB_SERIALIZATION=ON -DTILEDB_EXPERIMENTAL_FEATURES=$EXPERIMENTAL -DCMAKE_BUILD_TYPE=${{ matrix.config || 'Release' }} -DSANITIZER=$SANITIZER_ARG + cmake -B build ${{ matrix.config != 'Debug' && '-DTILEDB_VCPKG=OFF' }} -DTILEDB_WERROR=ON -DTILEDB_SERIALIZATION=ON -DTILEDB_EXPERIMENTAL_FEATURES=$EXPERIMENTAL -DCMAKE_BUILD_TYPE=${{ matrix.config || 'Release' }} -DTILEDB_SANITIZER=$SANITIZER_ARG - name: Configure TileDB CMake (Windows) if: contains(matrix.os, 'windows') diff --git a/CMakeLists.txt b/CMakeLists.txt index f79690e65c8..82b7b12fd03 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -251,7 +251,7 @@ if (COMPILER_SUPPORTS_AVX2) add_compile_options(${COMPILER_AVX2_FLAG}) endif() -if(SANITIZER) +if(TILEDB_SANITIZER) validate_sanitizer_options() endif() diff --git a/bootstrap b/bootstrap index 3fde75cff97..f897469614d 100755 --- a/bootstrap +++ b/bootstrap @@ -262,7 +262,7 @@ ${cmake} -DCMAKE_BUILD_TYPE=${build_type} \ -DTILEDB_WEBP=${tiledb_build_webp} \ -DTILEDB_FORCE_ALL_DEPS=${tiledb_force_all_deps} \ -DTILEDB_REMOVE_DEPRECATIONS=${tiledb_remove_deprecations} \ - -DSANITIZER="${sanitizer}" \ + -DTILEDB_SANITIZER="${sanitizer}" \ -DTILEDB_EXPERIMENTAL_FEATURES=${tiledb_experimental_features} \ -DTILEDB_TESTS_ENABLE_REST=${tiledb_tests_enable_rest} \ -DTILEDB_TESTS_AWS_S3_CONFIG=${tiledb_tests_aws_s3_config} \ diff --git a/cmake/Modules/Sanitizer.cmake b/cmake/Modules/Sanitizer.cmake index e3b79fd633e..c4adc5e9fda 100644 --- a/cmake/Modules/Sanitizer.cmake +++ b/cmake/Modules/Sanitizer.cmake @@ -44,22 +44,22 @@ # https://devblogs.microsoft.com/cppblog/addresssanitizer-asan-for-windows-with-msvc/ # https://devblogs.microsoft.com/cppblog/address-sanitizer-for-msvc-now-generally-available/ -if(NOT SANITIZER) +if(NOT TILEDB_SANITIZER) return() endif() # The basic sanitizer option is standard across compilers -add_compile_options(-fsanitize=${SANITIZER}) +add_compile_options(-fsanitize=${TILEDB_SANITIZER}) # Check that the sanitizer name is well-formed -if (NOT SANITIZER MATCHES "^[-A-Za-z]*$") +if (NOT TILEDB_SANITIZER MATCHES "^[-A-Za-z]*$") message(FATAL_ERROR "Bad sanitizer specification \"${sanitizer}\";" " permissible characters are only alphabetic and hyphen") endif() # Verify that the sanitizer is one that some compiler supports -string(TOLOWER ${SANITIZER} SANITIZER) -if (NOT SANITIZER MATCHES "^address$") +string(TOLOWER ${TILEDB_SANITIZER} TILEDB_SANITIZER) +if (NOT TILEDB_SANITIZER MATCHES "^address$") message(FATAL_ERROR "Unsupported sanitizer ${sanitizer}") endif() @@ -70,8 +70,8 @@ add_link_options("$<$:/INCREMENTAL:NO>") # Ordinary gcc/clang behavior. add_compile_options("$<$>:-g;-fno-omit-frame-pointer;-fno-optimize-sibling-calls>") -add_link_options("$<$>:-fsanitize=${SANITIZER}>") -if(SANITIZER STREQUAL "address") +add_link_options("$<$>:-fsanitize=${TILEDB_SANITIZER}>") +if(TILEDB_SANITIZER STREQUAL "address") # There may be problems if clang tries to link the ASAN library statically add_link_options("$<$:-shared-libasan>") endif() @@ -83,7 +83,7 @@ macro(validate_sanitizer_options) # If we know it's not supported, we'll fail so that we avoid false confidence. # If we don't know, we'll warn that it might not work. if (CMAKE_CXX_COMPILER_ID MATCHES "MSVC") - if (SANITIZER STREQUAL "address") + if (TILEDB_SANITIZER STREQUAL "address") # MSVC support for the address sanitizer began with Visual Studio 2019 Version 16.4 # and was announced as "fully supported" in version 16.9 if (MSVC_VERSION LESS 1924) diff --git a/cmake/Options/BuildOptions.cmake b/cmake/Options/BuildOptions.cmake index d7ffcf72db2..e4dc5ece66c 100644 --- a/cmake/Options/BuildOptions.cmake +++ b/cmake/Options/BuildOptions.cmake @@ -10,6 +10,7 @@ include(CMakeDependentOption) option(TILEDB_SUPERBUILD "If true, perform a superbuild (builds all missing dependencies)." ON) option(TILEDB_VCPKG "If true, use vcpkg to download and build dependencies." ON) cmake_dependent_option(TILEDB_FORCE_ALL_DEPS "If true, force superbuild to download and build all dependencies, even those installed on the system." OFF "NOT TILEDB_VCPKG" OFF) +option(TILEDB_SANITIZER "Sets the sanitizers to use. Only address is currently supported." "") option(TILEDB_REMOVE_DEPRECATIONS "If true, do not build deprecated APIs." OFF) option(TILEDB_VERBOSE "Prints TileDB errors with verbosity" OFF) option(TILEDB_S3 "Enables S3/minio support using aws-cpp-sdk" OFF) diff --git a/cmake/Options/TileDBToolchain.cmake b/cmake/Options/TileDBToolchain.cmake index 5c22380c227..f6fa4c8f101 100644 --- a/cmake/Options/TileDBToolchain.cmake +++ b/cmake/Options/TileDBToolchain.cmake @@ -58,7 +58,7 @@ if (NOT DEFINED CMAKE_TOOLCHAIN_FILE) endif() endif() -if(SANITIZER STREQUAL "address") +if(TILEDB_SANITIZER STREQUAL "address") if(NOT VCPKG_TARGET_TRIPLET) message(FATAL_ERROR "VCPKG_TARGET_TRIPLET must be defined when building with ASAN.") elseif(NOT VCPKG_TARGET_TRIPLET MATCHES "-asan$") diff --git a/cmake/TileDB-Superbuild.cmake b/cmake/TileDB-Superbuild.cmake index 5af56ab2dc7..f1802fbcb1a 100644 --- a/cmake/TileDB-Superbuild.cmake +++ b/cmake/TileDB-Superbuild.cmake @@ -45,7 +45,7 @@ set(INHERITED_CMAKE_ARGS -DTILEDB_CPP_API=${TILEDB_CPP_API} -DTILEDB_FORCE_ALL_DEPS=${TILEDB_FORCE_ALL_DEPS} -DTILEDB_REMOVE_DEPRECATIONS=${TILEDB_REMOVE_DEPRECATIONS} - -DSANITIZER=${SANITIZER} + -DTILEDB_SANITIZER=${TILEDB_SANITIZER} -DTILEDB_EP_BASE=${TILEDB_EP_BASE} -DTILEDB_STATS=${TILEDB_STATS} -DTILEDB_TESTS=${TILEDB_TESTS} diff --git a/tiledb/CMakeLists.txt b/tiledb/CMakeLists.txt index 530d2b8c68b..3cc6cea153c 100644 --- a/tiledb/CMakeLists.txt +++ b/tiledb/CMakeLists.txt @@ -598,10 +598,10 @@ if (TILEDB_HDFS) endif() # Sanitizer linker flags -if (SANITIZER) +if (TILEDB_SANITIZER) target_link_libraries(TILEDB_CORE_OBJECTS_ILIB INTERFACE - -fsanitize=${SANITIZER} + -fsanitize=${TILEDB_SANITIZER} ) endif() From d7d51d330d80e2d7c69a391223fabcb56f430de2 Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Mon, 18 Dec 2023 19:17:49 +0200 Subject: [PATCH 17/21] Add a "vcpkg base triplet" option that is required when building with sanitizers. --- .github/workflows/full-ci.yml | 2 +- .github/workflows/nightly-test.yml | 3 ++- bootstrap | 13 ++++++------- bootstrap.ps1 | 12 ++++++------ cmake/Options/BuildOptions.cmake | 1 + cmake/Options/TileDBToolchain.cmake | 12 +++--------- 6 files changed, 19 insertions(+), 24 deletions(-) diff --git a/.github/workflows/full-ci.yml b/.github/workflows/full-ci.yml index 790dd7edfd6..a525a9cf382 100644 --- a/.github/workflows/full-ci.yml +++ b/.github/workflows/full-ci.yml @@ -101,7 +101,7 @@ jobs: matrix_compiler_cc: 'gcc-10' matrix_compiler_cxx: 'g++-10' timeout: 120 - bootstrap_args: '--enable-serialization --vcpkg-target-triplet=x64-linux-asan' + bootstrap_args: '--enable-serialization --vcpkg-base-triplet=x64-linux' asan: true ci10: diff --git a/.github/workflows/nightly-test.yml b/.github/workflows/nightly-test.yml index bcf125cc3f4..db03f78205e 100644 --- a/.github/workflows/nightly-test.yml +++ b/.github/workflows/nightly-test.yml @@ -16,6 +16,7 @@ jobs: - os: macos-latest - os: macos-latest # ASAN build sanitizer: "address" + base_triplet: "x64-osx" - os: macos-latest experimental: ON - os: windows-latest @@ -46,7 +47,7 @@ jobs: EXPERIMENTAL: ${{ matrix.experimental || 'OFF' }} working-directory: ${{ matrix.working_directory || github.workspace }} run: | - cmake -B build ${{ matrix.config != 'Debug' && '-DTILEDB_VCPKG=OFF' }} -DTILEDB_WERROR=ON -DTILEDB_SERIALIZATION=ON -DTILEDB_EXPERIMENTAL_FEATURES=$EXPERIMENTAL -DCMAKE_BUILD_TYPE=${{ matrix.config || 'Release' }} -DTILEDB_SANITIZER=$SANITIZER_ARG + cmake -B build ${{ matrix.config != 'Debug' && '-DTILEDB_VCPKG=OFF' }} -DTILEDB_WERROR=ON -DTILEDB_SERIALIZATION=ON -DTILEDB_EXPERIMENTAL_FEATURES=$EXPERIMENTAL -DCMAKE_BUILD_TYPE=${{ matrix.config || 'Release' }} -DTILEDB_SANITIZER=$SANITIZER_ARG -DTILEDB_VCPKG_BASE_TRIPLET=${{ matrix.base_triplet }} - name: Configure TileDB CMake (Windows) if: contains(matrix.os, 'windows') diff --git a/bootstrap b/bootstrap index f897469614d..c3f03c592c7 100755 --- a/bootstrap +++ b/bootstrap @@ -34,8 +34,7 @@ Configuration: --help print this message --prefix=PREFIX install files in tree rooted at PREFIX ['"${default_prefix}"'] - --vcpkg-target-triplet=TRIPLET vcpkg triplet to use for downloading and building dependencies - [auto-detected] + --vcpkg-base-triplet=TRIPLET vcpkg base triplet to use when building with sanitizers --enable-vcpkg use vcpkg for downloading and building dependencies --dependency=DIRs specify the dependencies at DIRs, separated by colon ['"${default_dependency}"'] @@ -54,7 +53,7 @@ Configuration: --enable-static-tiledb enables building TileDB as a static library (deprecated, use --linkage=static) --enable-sanitizer=SAN enable sanitizer (clang only) (address|memory|leak|thread|undefined) - Must manually specify a vcpkg triplet. + Must manually specify a vcpkg base triplet. --enable-debug enable debug build --enable-assertions enable assertions in compiled code --enable-release-symbols enable create symbols for release build @@ -91,7 +90,7 @@ Example: # Parse arguments prefix_dirs="${default_prefix}" -vcpkg_target_triplet="" +vcpkg_base_triplet="" tiledb_vcpkg="ON" dependency_dir="${default_dependency}" sanitizer="" @@ -123,8 +122,8 @@ while test $# != 0; do case "$1" in --prefix=*) dir=`arg "$1"` prefix_dirs="$dir";; - --vcpkg-target-triplet=*) vcpkg_target_triplet=`arg "$1"` - vcpkg_target_triplet="-DVCPKG_TARGET_TRIPLET=${vcpkg_target_triplet}";; + --vcpkg-base-triplet=*) vcpkg_base_triplet=`arg "$1"` + vcpkg_base_triplet="-DVCPKG_BASE_TRIPLET=${vcpkg_base_triplet}";; --enable-vcpkg) echo "Argument '--enable-vcpkg' is obsolete and will be removed in a future version. Vcpkg is now enabled by default." tiledb_vcpkg="ON";; --dependency=*) dir=`arg "$1"` @@ -267,7 +266,7 @@ ${cmake} -DCMAKE_BUILD_TYPE=${build_type} \ -DTILEDB_TESTS_ENABLE_REST=${tiledb_tests_enable_rest} \ -DTILEDB_TESTS_AWS_S3_CONFIG=${tiledb_tests_aws_s3_config} \ ${tiledb_disable_avx2} \ - ${vcpkg_target_triplet} \ + ${vcpkg_base_triplet} \ "${source_dir}" || die "failed to configure the project" echo 'bootstrap success. Run "make" to build, "make check" to test, or "make -C tiledb install" to install.' diff --git a/bootstrap.ps1 b/bootstrap.ps1 index f84cdc8994b..9bb1ee23158 100644 --- a/bootstrap.ps1 +++ b/bootstrap.ps1 @@ -13,7 +13,7 @@ develop TileDB. .PARAMETER Prefix Installs files in tree rooted at PREFIX (defaults to TileDB\dist). -.PARAMETER VcpkgTargetTriplet +.PARAMETER VcpkgBaseTriplet Optionally specify the vcpkg target triplet, e.g. "x64-windows-release". Defaults to automatically detecting it from the environment. @@ -121,7 +121,7 @@ https://github.com/TileDB-Inc/TileDB [CmdletBinding()] Param( [string]$Prefix, - [string]$VcpkgTargetTriplet, + [string]$VcpkgBaseTriplet, [string]$Dependency, [string]$Linkage = "shared", [string]$CMakeGenerator, @@ -168,9 +168,9 @@ $DefaultPrefix = Join-Path $BinaryDirectory "dist" # Choose the default dependency install prefix. $DefaultDependency = $DefaultPrefix -# Set the default vcpkg triplet. -if ($VcpkgTargetTriplet.IsPresent) { - $VcpkgTargetTriplet = "-DVCPKG_TARGET_TRIPLET=$VcpkgTargetTriplet" +# Set the vcpkg base triplet. +if ($VcpkgBaseTriplet.IsPresent) { + $VcpkgBaseTriplet = "-DTILEDB_VCPKG_BASE_TRIPLET=$VcpkgBaseTriplet" } # Set assertion mode @@ -337,7 +337,7 @@ if ($CMakeGenerator -eq $null) { # Run CMake. # We use Invoke-Expression so we can echo the command to the user. -$CommandString = "cmake -A X64 -DTILEDB_VCPKG=$UseVcpkg -DCMAKE_BUILD_TYPE=$BuildType -DCMAKE_INSTALL_PREFIX=""$InstallPrefix"" $VcpkgTargetTriplet -DCMAKE_PREFIX_PATH=""$DependencyDir"" -DMSVC_MP_FLAG=""/MP$BuildProcesses"" -DTILEDB_ASSERTIONS=$AssertionMode -DTILEDB_VERBOSE=$Verbosity -DTILEDB_AZURE=$UseAzure -DTILEDB_S3=$UseS3 -DTILEDB_GCS=$UseGcs -DTILEDB_SERIALIZATION=$UseSerialization -DTILEDB_WERROR=$Werror -DTILEDB_CPP_API=$CppApi -DTILEDB_TESTS=$Tests -DTILEDB_STATS=$Stats -DBUILD_SHARED_LIBS=$BuildSharedLibs -DTILEDB_FORCE_ALL_DEPS=$TileDBBuildDeps -DTILEDB_REMOVE_DEPRECATIONS=$RemoveDeprecations -DTILEDB_TOOLS=$TileDBTools -DTILEDB_EXPERIMENTAL_FEATURES=$TileDBExperimentalFeatures -DTILEDB_WEBP=$BuildWebP -DTILEDB_CRC32=$BuildCrc32 -DTILEDB_ARROW_TESTS=$ArrowTests -DTILEDB_TESTS_ENABLE_REST=$RestTests -DTILEDB_TESTS_AWS_S3_CONFIG=$ConfigureS3 $GeneratorFlag ""$SourceDirectory""" +$CommandString = "cmake -A X64 -DTILEDB_VCPKG=$UseVcpkg -DCMAKE_BUILD_TYPE=$BuildType -DCMAKE_INSTALL_PREFIX=""$InstallPrefix"" $VcpkgBaseTriplet -DCMAKE_PREFIX_PATH=""$DependencyDir"" -DMSVC_MP_FLAG=""/MP$BuildProcesses"" -DTILEDB_ASSERTIONS=$AssertionMode -DTILEDB_VERBOSE=$Verbosity -DTILEDB_AZURE=$UseAzure -DTILEDB_S3=$UseS3 -DTILEDB_GCS=$UseGcs -DTILEDB_SERIALIZATION=$UseSerialization -DTILEDB_WERROR=$Werror -DTILEDB_CPP_API=$CppApi -DTILEDB_TESTS=$Tests -DTILEDB_STATS=$Stats -DBUILD_SHARED_LIBS=$BuildSharedLibs -DTILEDB_FORCE_ALL_DEPS=$TileDBBuildDeps -DTILEDB_REMOVE_DEPRECATIONS=$RemoveDeprecations -DTILEDB_TOOLS=$TileDBTools -DTILEDB_EXPERIMENTAL_FEATURES=$TileDBExperimentalFeatures -DTILEDB_WEBP=$BuildWebP -DTILEDB_CRC32=$BuildCrc32 -DTILEDB_ARROW_TESTS=$ArrowTests -DTILEDB_TESTS_ENABLE_REST=$RestTests -DTILEDB_TESTS_AWS_S3_CONFIG=$ConfigureS3 $GeneratorFlag ""$SourceDirectory""" Write-Host $CommandString Write-Host Invoke-Expression "$CommandString" diff --git a/cmake/Options/BuildOptions.cmake b/cmake/Options/BuildOptions.cmake index e4dc5ece66c..fd439db4700 100644 --- a/cmake/Options/BuildOptions.cmake +++ b/cmake/Options/BuildOptions.cmake @@ -11,6 +11,7 @@ option(TILEDB_SUPERBUILD "If true, perform a superbuild (builds all missing depe option(TILEDB_VCPKG "If true, use vcpkg to download and build dependencies." ON) cmake_dependent_option(TILEDB_FORCE_ALL_DEPS "If true, force superbuild to download and build all dependencies, even those installed on the system." OFF "NOT TILEDB_VCPKG" OFF) option(TILEDB_SANITIZER "Sets the sanitizers to use. Only address is currently supported." "") +option(TILEDB_VCPKG_BASE_TRIPLET "Sets the base vcpkg triplet when building with sanitizers." "") option(TILEDB_REMOVE_DEPRECATIONS "If true, do not build deprecated APIs." OFF) option(TILEDB_VERBOSE "Prints TileDB errors with verbosity" OFF) option(TILEDB_S3 "Enables S3/minio support using aws-cpp-sdk" OFF) diff --git a/cmake/Options/TileDBToolchain.cmake b/cmake/Options/TileDBToolchain.cmake index f6fa4c8f101..59ed7574c10 100644 --- a/cmake/Options/TileDBToolchain.cmake +++ b/cmake/Options/TileDBToolchain.cmake @@ -59,16 +59,10 @@ if (NOT DEFINED CMAKE_TOOLCHAIN_FILE) endif() if(TILEDB_SANITIZER STREQUAL "address") - if(NOT VCPKG_TARGET_TRIPLET) - message(FATAL_ERROR "VCPKG_TARGET_TRIPLET must be defined when building with ASAN.") - elseif(NOT VCPKG_TARGET_TRIPLET MATCHES "-asan$") - if(EXISTS "${CMAKE_CURRENT_LIST_DIR}/../../ports/triplets/${VCPKG_TARGET_TRIPLET}-asan.cmake") - message("Overriding vcpkg triplet to ${VCPKG_TARGET_TRIPLET}-asan") - set(VCPKG_TARGET_TRIPLET "${VCPKG_TARGET_TRIPLET}-asan") - else() - message(FATAL_ERROR "Incompatible vcpkg triplet. It must end in -asan, or a triplet with the name ${VCPKG_TARGET_TRIPLET}-asan must exist in ports/triplets.") - endif() + if(NOT TILEDB_VCPKG_BASE_TRIPLET) + message(FATAL_ERROR "TILEDB_VCPKG_BASE_TRIPLET must be defined when building with ASAN.") endif() + set(VCPKG_TARGET_TRIPLET "${TILEDB_VCPKG_BASE_TRIPLET}-asan") endif() set(VCPKG_INSTALL_OPTIONS "--no-print-usage") From 3b2e13c1cb21ecc784cde94543b510be9acc3a61 Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Mon, 18 Dec 2023 20:02:49 +0200 Subject: [PATCH 18/21] Update documentation. --- CONTRIBUTING.md | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e36c29a0e95..c4355e36ec8 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -63,14 +63,19 @@ Formatting conventions: ### Building with sanitizers -TileDB can be built with [clang sanitizers](https://clang.llvm.org/docs/AddressSanitizer.html) enabled. To enable them, you have to bootstrap with the `--enable-sanitizer` flag, as well as specify a vcpkg triplet compatible with ASAN (that ends with `-asan`). You can view the list of supported triplets [here](https://github.com/TileDB-Inc/TileDB/tree/dev/ports/triplets). +TileDB can be built with [clang sanitizers](https://clang.llvm.org/docs/AddressSanitizer.html) enabled. To enable them, you have to bootstrap with the `--enable-sanitizer` flag, as well as the vcpkg base triplet corresponding to your platform. The following platforms support sanitizers: + +* `arm64-osx` +* `x64-linux` +* `x64-osx` +* `x64-windows` > [!NOTE] > Currently only the `address` sanitizer is supported. ```bash cd TileDB && mkdir build-asan && cd build-asan -../bootstrap --enable-sanitizer=address --vcpkg-target-triplet=x64-linux-asan +../bootstrap --enable-sanitizer=address --vcpkg-base-triplet=x64-linux make && make check ``` From 425fd620df535b5d319bbb0ca917a0d27c6830c0 Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Mon, 18 Dec 2023 20:11:43 +0200 Subject: [PATCH 19/21] Add a notice that sanitizer builds should be placed in their own build directory. --- CONTRIBUTING.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c4355e36ec8..71d27bd3f95 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -79,6 +79,9 @@ cd TileDB && mkdir build-asan && cd build-asan make && make check ``` +> [!IMPORTANT] +> To avoid errors, building with sanitizers must be done in a separate build directory. + ### Pull Requests: - `dev` is the development branch, all PR’s should be rebased on top of the latest `dev` commit. From 620699cb1b170c77381d7cfa7fa5ed3082ba7931 Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Mon, 18 Dec 2023 20:22:19 +0200 Subject: [PATCH 20/21] Fix typo. --- bootstrap | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bootstrap b/bootstrap index c3f03c592c7..627ca30b593 100755 --- a/bootstrap +++ b/bootstrap @@ -123,7 +123,7 @@ while test $# != 0; do --prefix=*) dir=`arg "$1"` prefix_dirs="$dir";; --vcpkg-base-triplet=*) vcpkg_base_triplet=`arg "$1"` - vcpkg_base_triplet="-DVCPKG_BASE_TRIPLET=${vcpkg_base_triplet}";; + vcpkg_base_triplet="-DTILEDB_VCPKG_BASE_TRIPLET=${vcpkg_base_triplet}";; --enable-vcpkg) echo "Argument '--enable-vcpkg' is obsolete and will be removed in a future version. Vcpkg is now enabled by default." tiledb_vcpkg="ON";; --dependency=*) dir=`arg "$1"` From c94ded238ddc0fb993c2a37492c4a5610064927e Mon Sep 17 00:00:00 2001 From: Theodore Tsirpanis Date: Wed, 20 Dec 2023 19:05:09 +0200 Subject: [PATCH 21/21] Update Sanitizer.cmake --- cmake/Modules/Sanitizer.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/Modules/Sanitizer.cmake b/cmake/Modules/Sanitizer.cmake index c4adc5e9fda..7acf3407021 100644 --- a/cmake/Modules/Sanitizer.cmake +++ b/cmake/Modules/Sanitizer.cmake @@ -65,7 +65,7 @@ endif() # Catch has a conflict with ASAN on Windows. Disable the SEH handler in Catch to avoid the conflict. add_compile_definitions("$<$:CATCH_CONFIG_NO_WINDOWS_SEH>") - # Microsoft suppresses /INCREMENTAL, but emits a warning, so silence it. +# Microsoft suppresses /INCREMENTAL, but emits a warning, so silence it. add_link_options("$<$:/INCREMENTAL:NO>") # Ordinary gcc/clang behavior.