Build the AWS SDK with hidden symbols. (#5223)

[SC-49529](https://app.shortcut.com/tiledb-inc/story/49529/core-abort-trap-due-to-aws-sdk-pyarrow-tiledb-interaction-again) It has been noticed that the `tiledb` PyPI package weirdly interacts with `pyarrow` and causes aborts when `pyarrow` gets `import`ed first and we make an S3 operation with `tiledb`, and conversely when `tiledb` gets `import`ed first and we make an S3 operation with `pyarrow`. After some investigations, it was theorized that the problem would be fixed if the AWS SDK in `pyarrow` was built with `-fvisibility=hidden`, which was verified by building a `tiledb` wheel with such configuration and testing that the converse case does no longer abort. Turns out however, that hiding the symbols from `tiledb` also fixes the _original_ case, which means that we don't have to change Arrow. This PR updates all macOS and Linux[^1] vcpkg triplets to pass the CMake equivalent of `-fvisibility=hidden` when building all ports that start with `aws-`. Other ports remain unchanged. The long-term fix might be to update the AWS SDK itself to build with `-fvisibility=hidden`. I don't think there will be any problems since it explicitly marks all the APIs it needs to export. Validated on macOS-arm64. To validate it yourself, you can `pip install` [one of the wheels I built with this change applied](https://github.com/teo-tsirpanis/TileDB-Py/actions/runs/10171044013), and run the following script: ```python import pyarrow import tiledb a = tiledb.open("s3://<bucket>/debug/a1") ``` [^1]: As a precaution, even though the bug has been observed so far on macOS. Windows is not susceptible, it always hides symbols by default in DLLs. --- TYPE: BUG DESC: Fix symbol clashes between `tiledb` and `pyarrow` by building the AWS SDK with its internal symbols hidden.
TileDB-Inc · Aug 7, 2024 · 535097f · 535097f
1 parent 378c073
commit 535097f
Show file tree

Hide file tree

Showing 12 changed files with 72 additions and 0 deletions.
diff --git a/ports/triplets/arm64-osx-asan.cmake b/ports/triplets/arm64-osx-asan.cmake
@@ -7,3 +7,9 @@ set(VCPKG_OSX_ARCHITECTURES arm64)
 
 set(VCPKG_C_FLAGS "-fsanitize=address")
 set(VCPKG_CXX_FLAGS "-fsanitize=address")
+
+# Hide symbols in the AWS SDK. Fixes symbol collisions with other libraries
+# like arrow (https://github.com/apache/arrow/issues/42154).
+if("${PORT}" MATCHES "^aws-")
+    set(VCPKG_CMAKE_CONFIGURE_OPTIONS "-DCMAKE_CXX_VISIBILITY_PRESET=hidden;-DCMAKE_C_VISIBILITY_PRESET=hidden")
+endif()
diff --git a/ports/triplets/arm64-osx-release.cmake b/ports/triplets/arm64-osx-release.cmake
@@ -7,3 +7,9 @@ set(VCPKG_OSX_ARCHITECTURES arm64)
 set(VCPKG_OSX_DEPLOYMENT_TARGET 11)
 
 set(VCPKG_BUILD_TYPE release)
+
+# Hide symbols in the AWS SDK. Fixes symbol collisions with other libraries
+# like arrow (https://github.com/apache/arrow/issues/42154).
+if("${PORT}" MATCHES "^aws-")
+    set(VCPKG_CMAKE_CONFIGURE_OPTIONS "-DCMAKE_CXX_VISIBILITY_PRESET=hidden;-DCMAKE_C_VISIBILITY_PRESET=hidden")
+endif()
diff --git a/ports/triplets/arm64-osx-relwithdebinfo.cmake b/ports/triplets/arm64-osx-relwithdebinfo.cmake
@@ -8,3 +8,9 @@ set(VCPKG_OSX_DEPLOYMENT_TARGET 11)
 
 set(VCPKG_CXX_FLAGS "-g")
 set(VCPKG_C_FLAGS "-g")
+
+# Hide symbols in the AWS SDK. Fixes symbol collisions with other libraries
+# like arrow (https://github.com/apache/arrow/issues/42154).
+if("${PORT}" MATCHES "^aws-")
+    set(VCPKG_CMAKE_CONFIGURE_OPTIONS "-DCMAKE_CXX_VISIBILITY_PRESET=hidden;-DCMAKE_C_VISIBILITY_PRESET=hidden")
+endif()
diff --git a/ports/triplets/arm64-osx.cmake b/ports/triplets/arm64-osx.cmake
@@ -5,3 +5,9 @@ set(VCPKG_LIBRARY_LINKAGE static)
 set(VCPKG_CMAKE_SYSTEM_NAME Darwin)
 set(VCPKG_OSX_ARCHITECTURES arm64)
 set(VCPKG_OSX_DEPLOYMENT_TARGET 11)
+
+# Hide symbols in the AWS SDK. Fixes symbol collisions with other libraries
+# like arrow (https://github.com/apache/arrow/issues/42154).
+if("${PORT}" MATCHES "^aws-")
+    set(VCPKG_CMAKE_CONFIGURE_OPTIONS "-DCMAKE_CXX_VISIBILITY_PRESET=hidden;-DCMAKE_C_VISIBILITY_PRESET=hidden")
+endif()
diff --git a/ports/triplets/x64-linux-asan.cmake b/ports/triplets/x64-linux-asan.cmake
@@ -6,3 +6,9 @@ set(VCPKG_CMAKE_SYSTEM_NAME Linux)
 
 set(VCPKG_C_FLAGS "-fsanitize=address")
 set(VCPKG_CXX_FLAGS "-fsanitize=address")
+
+# Hide symbols in the AWS SDK. Fixes symbol collisions with other libraries
+# like arrow (https://github.com/apache/arrow/issues/42154).
+if("${PORT}" MATCHES "^aws-")
+    set(VCPKG_CMAKE_CONFIGURE_OPTIONS "-DCMAKE_CXX_VISIBILITY_PRESET=hidden;-DCMAKE_C_VISIBILITY_PRESET=hidden")
+endif()
diff --git a/ports/triplets/x64-linux-release.cmake b/ports/triplets/x64-linux-release.cmake
@@ -5,3 +5,9 @@ set(VCPKG_LIBRARY_LINKAGE static)
 set(VCPKG_CMAKE_SYSTEM_NAME Linux)
 
 set(VCPKG_BUILD_TYPE release)
+
+# Hide symbols in the AWS SDK. Fixes symbol collisions with other libraries
+# like arrow (https://github.com/apache/arrow/issues/42154).
+if("${PORT}" MATCHES "^aws-")
+    set(VCPKG_CMAKE_CONFIGURE_OPTIONS "-DCMAKE_CXX_VISIBILITY_PRESET=hidden;-DCMAKE_C_VISIBILITY_PRESET=hidden")
+endif()
diff --git a/ports/triplets/x64-linux-relwithdebinfo.cmake b/ports/triplets/x64-linux-relwithdebinfo.cmake
@@ -6,3 +6,9 @@ set(VCPKG_CMAKE_SYSTEM_NAME Linux)
 
 set(VCPKG_CXX_FLAGS "-g")
 set(VCPKG_C_FLAGS "-g")
+
+# Hide symbols in the AWS SDK. Fixes symbol collisions with other libraries
+# like arrow (https://github.com/apache/arrow/issues/42154).
+if("${PORT}" MATCHES "^aws-")
+    set(VCPKG_CMAKE_CONFIGURE_OPTIONS "-DCMAKE_CXX_VISIBILITY_PRESET=hidden;-DCMAKE_C_VISIBILITY_PRESET=hidden")
+endif()
diff --git a/ports/triplets/x64-linux.cmake b/ports/triplets/x64-linux.cmake
@@ -7,3 +7,9 @@ set(VCPKG_CMAKE_SYSTEM_NAME Linux)
 # For AWS SDK. Remove after update from 1.74
 set(VCPKG_CXX_FLAGS "-Wno-error=nonnull -Wno-error=deprecated-declarations")
 set(VCPKG_C_FLAGS "-Wno-error=nonnull -Wno-error=deprecated-declarations")
+
+# Hide symbols in the AWS SDK. Fixes symbol collisions with other libraries
+# like arrow (https://github.com/apache/arrow/issues/42154).
+if("${PORT}" MATCHES "^aws-")
+    set(VCPKG_CMAKE_CONFIGURE_OPTIONS "-DCMAKE_CXX_VISIBILITY_PRESET=hidden;-DCMAKE_C_VISIBILITY_PRESET=hidden")
+endif()
diff --git a/ports/triplets/x64-osx-asan.cmake b/ports/triplets/x64-osx-asan.cmake
@@ -7,3 +7,9 @@ set(VCPKG_OSX_ARCHITECTURES x86_64)
 
 set(VCPKG_C_FLAGS "-fsanitize=address")
 set(VCPKG_CXX_FLAGS "-fsanitize=address")
+
+# Hide symbols in the AWS SDK. Fixes symbol collisions with other libraries
+# like arrow (https://github.com/apache/arrow/issues/42154).
+if("${PORT}" MATCHES "^aws-")
+    set(VCPKG_CMAKE_CONFIGURE_OPTIONS "-DCMAKE_CXX_VISIBILITY_PRESET=hidden;-DCMAKE_C_VISIBILITY_PRESET=hidden")
+endif()
diff --git a/ports/triplets/x64-osx-release.cmake b/ports/triplets/x64-osx-release.cmake
@@ -7,3 +7,9 @@ set(VCPKG_OSX_ARCHITECTURES x86_64)
 set(VCPKG_OSX_DEPLOYMENT_TARGET 11)
 
 set(VCPKG_BUILD_TYPE release)
+
+# Hide symbols in the AWS SDK. Fixes symbol collisions with other libraries
+# like arrow (https://github.com/apache/arrow/issues/42154).
+if("${PORT}" MATCHES "^aws-")
+    set(VCPKG_CMAKE_CONFIGURE_OPTIONS "-DCMAKE_CXX_VISIBILITY_PRESET=hidden;-DCMAKE_C_VISIBILITY_PRESET=hidden")
+endif()
diff --git a/ports/triplets/x64-osx-relwithdebinfo.cmake b/ports/triplets/x64-osx-relwithdebinfo.cmake
@@ -8,3 +8,9 @@ set(VCPKG_OSX_DEPLOYMENT_TARGET 11)
 
 set(VCPKG_CXX_FLAGS "-g")
 set(VCPKG_C_FLAGS "-g")
+
+# Hide symbols in the AWS SDK. Fixes symbol collisions with other libraries
+# like arrow (https://github.com/apache/arrow/issues/42154).
+if("${PORT}" MATCHES "^aws-")
+    set(VCPKG_CMAKE_CONFIGURE_OPTIONS "-DCMAKE_CXX_VISIBILITY_PRESET=hidden;-DCMAKE_C_VISIBILITY_PRESET=hidden")
+endif()
diff --git a/ports/triplets/x64-osx.cmake b/ports/triplets/x64-osx.cmake
@@ -5,3 +5,9 @@ set(VCPKG_LIBRARY_LINKAGE static)
 set(VCPKG_CMAKE_SYSTEM_NAME Darwin)
 set(VCPKG_OSX_ARCHITECTURES x86_64)
 set(VCPKG_OSX_DEPLOYMENT_TARGET 11)
+
+# Hide symbols in the AWS SDK. Fixes symbol collisions with other libraries
+# like arrow (https://github.com/apache/arrow/issues/42154).
+if("${PORT}" MATCHES "^aws-")
+    set(VCPKG_CMAKE_CONFIGURE_OPTIONS "-DCMAKE_CXX_VISIBILITY_PRESET=hidden;-DCMAKE_C_VISIBILITY_PRESET=hidden")
+endif()