From b48d42d77d2f795933c5b730ab79847baf5aa22b Mon Sep 17 00:00:00 2001 From: Matej Kenda Date: Fri, 8 Dec 2023 10:36:05 +0100 Subject: [PATCH 1/5] enh(ci): Add macos sanitizers job (#4313) --- .github/workflows/ci.yml | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f8e615bfd1..4c887d336d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -273,6 +273,30 @@ jobs: PWD=`pwd` ctest --output-on-failure -E "(DataMySQL)|(DataODBC)|(PostgreSQL)|(MongoDB)|(Redis)" + macos-clang-cmake-openssl3-tsan: + runs-on: macos-12 + steps: + - uses: actions/checkout@v3 + - run: brew install openssl@3 + - run: CXXFLAGS=-fsanitize=thread cmake -H. -Bcmake-build -DENABLE_CPPPARSER=OFF -DENABLE_DATA_ODBC=OFF -DENABLE_DATA_MYSQL=OFF -DENABLE_DATA_POSTGRESQL=OFF -DENABLE_MONGODB=OFF -DENABLE_PDF=OFF -DENABLE_PAGECOMPILER=OFF -DENABLE_ENCODINGS=OFF -DENABLE_REDIS=OFF -DENABLE_TESTS=ON -DOPENSSL_ROOT_DIR=/usr/local/opt/openssl@3 && cmake --build cmake-build --target all + - uses: ./.github/actions/retry-action + with: + timeout_minutes: 90 + max_attempts: 3 + retry_on: any + command: >- + cd cmake-build && + CPPUNIT_IGNORE=" + CppUnit::TestCaller.testTrySleep, + CppUnit::TestCaller.testTimestamp, + CppUnit::TestCaller.testExpireN, + CppUnit::TestCaller.testAccessExpireN, + CppUnit::TestCaller.testExpireN, + CppUnit::TestCaller.testAccessExpireN, + CppUnit::TestCaller.testPollClosedServer" + PWD=`pwd` + ctest -V + # windows-2019-msvc-cmake: # runs-on: windows-2019 # env: From ab71ee24a8e8d4b2b03635574491b973ac6fe8b4 Mon Sep 17 00:00:00 2001 From: Matej Kenda Date: Fri, 15 Dec 2023 14:07:51 +0100 Subject: [PATCH 2/5] enh(ci): macOS thread sanitizer --- .github/workflows/ci.yml | 43 ++++++++++++++++++++++++++++++++++------ 1 file changed, 37 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4c887d336d..fdd32ba05b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -273,19 +273,25 @@ jobs: PWD=`pwd` ctest --output-on-failure -E "(DataMySQL)|(DataODBC)|(PostgreSQL)|(MongoDB)|(Redis)" - macos-clang-cmake-openssl3-tsan: + macos-clang-make-openssl3-tsan: runs-on: macos-12 steps: - uses: actions/checkout@v3 - run: brew install openssl@3 - - run: CXXFLAGS=-fsanitize=thread cmake -H. -Bcmake-build -DENABLE_CPPPARSER=OFF -DENABLE_DATA_ODBC=OFF -DENABLE_DATA_MYSQL=OFF -DENABLE_DATA_POSTGRESQL=OFF -DENABLE_MONGODB=OFF -DENABLE_PDF=OFF -DENABLE_PAGECOMPILER=OFF -DENABLE_ENCODINGS=OFF -DENABLE_REDIS=OFF -DENABLE_TESTS=ON -DOPENSSL_ROOT_DIR=/usr/local/opt/openssl@3 && cmake --build cmake-build --target all + - run: >- + ./configure --everything --no-prefix --no-samples --omit=CppParser,Encodings,Data/MySQL,Data/ODBC,Data/PostgreSQL,MongoDB,PageCompiler,PDF,PocoDoc,ProGen,Redis,SevenZip + --odbc-include=/usr/local/opt/unixodbc/include --odbc-lib=/usr/local/opt/unixodbc/lib + --mysql-include=/usr/local/opt/mysql-client/include --mysql-lib=/usr/local/opt/mysql-client/lib + --include-path="/usr/local/opt/openssl@3/include" --library-path="/usr/local/opt/openssl@3/lib" && + make all -s -j4 SANITIZEFLAGS=-fsanitize=thread + - uses: ./.github/actions/retry-action with: timeout_minutes: 90 max_attempts: 3 retry_on: any command: >- - cd cmake-build && + sudo -s CPPUNIT_IGNORE=" CppUnit::TestCaller.testTrySleep, CppUnit::TestCaller.testTimestamp, @@ -293,9 +299,34 @@ jobs: CppUnit::TestCaller.testAccessExpireN, CppUnit::TestCaller.testExpireN, CppUnit::TestCaller.testAccessExpireN, - CppUnit::TestCaller.testPollClosedServer" - PWD=`pwd` - ctest -V + CppUnit::TestCaller.testPollClosedServer, + CppUnit::TestCaller.testEncryptDecryptGCM" + EXCLUDE_TESTS="Redis Data/MySQL Data/ODBC Data/PostgreSQL MongoDB PDF" + ./ci/runtests.sh TSAN + +# macos-clang-cmake-openssl3-tsan: +# runs-on: macos-12 +# steps: +# - uses: actions/checkout@v3 +# - run: brew install openssl@3 +# - run: CXXFLAGS=-fsanitize=thread cmake -H. -Bcmake-build -DENABLE_CPPPARSER=OFF -DENABLE_DATA_ODBC=OFF -DENABLE_DATA_MYSQL=OFF -DENABLE_DATA_POSTGRESQL=OFF -DENABLE_MONGODB=OFF -DENABLE_PDF=OFF -DENABLE_PAGECOMPILER=OFF -DENABLE_ENCODINGS=OFF -DENABLE_REDIS=OFF -DENABLE_TESTS=ON -DOPENSSL_ROOT_DIR=/usr/local/opt/openssl@3 && cmake --build cmake-build --target all +# - uses: ./.github/actions/retry-action +# with: +# timeout_minutes: 90 +# max_attempts: 3 +# retry_on: any +# command: >- +# cd cmake-build && +# CPPUNIT_IGNORE=" +# CppUnit::TestCaller.testTrySleep, +# CppUnit::TestCaller.testTimestamp, +# CppUnit::TestCaller.testExpireN, +# CppUnit::TestCaller.testAccessExpireN, +# CppUnit::TestCaller.testExpireN, +# CppUnit::TestCaller.testAccessExpireN, +# CppUnit::TestCaller.testPollClosedServer" +# PWD=`pwd` +# ctest -V # windows-2019-msvc-cmake: # runs-on: windows-2019 From a1f4ba16d7d0a7c333e6853981e6f7f959f5a972 Mon Sep 17 00:00:00 2001 From: Matej Kenda Date: Fri, 15 Dec 2023 16:16:28 +0100 Subject: [PATCH 3/5] enh(ci): macOS sanitize jobs for undefined and address. --- .github/workflows/ci.yml | 84 +++++++++++++++++++++++++++++----------- 1 file changed, 61 insertions(+), 23 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fdd32ba05b..36880e2fc9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -304,29 +304,67 @@ jobs: EXCLUDE_TESTS="Redis Data/MySQL Data/ODBC Data/PostgreSQL MongoDB PDF" ./ci/runtests.sh TSAN -# macos-clang-cmake-openssl3-tsan: -# runs-on: macos-12 -# steps: -# - uses: actions/checkout@v3 -# - run: brew install openssl@3 -# - run: CXXFLAGS=-fsanitize=thread cmake -H. -Bcmake-build -DENABLE_CPPPARSER=OFF -DENABLE_DATA_ODBC=OFF -DENABLE_DATA_MYSQL=OFF -DENABLE_DATA_POSTGRESQL=OFF -DENABLE_MONGODB=OFF -DENABLE_PDF=OFF -DENABLE_PAGECOMPILER=OFF -DENABLE_ENCODINGS=OFF -DENABLE_REDIS=OFF -DENABLE_TESTS=ON -DOPENSSL_ROOT_DIR=/usr/local/opt/openssl@3 && cmake --build cmake-build --target all -# - uses: ./.github/actions/retry-action -# with: -# timeout_minutes: 90 -# max_attempts: 3 -# retry_on: any -# command: >- -# cd cmake-build && -# CPPUNIT_IGNORE=" -# CppUnit::TestCaller.testTrySleep, -# CppUnit::TestCaller.testTimestamp, -# CppUnit::TestCaller.testExpireN, -# CppUnit::TestCaller.testAccessExpireN, -# CppUnit::TestCaller.testExpireN, -# CppUnit::TestCaller.testAccessExpireN, -# CppUnit::TestCaller.testPollClosedServer" -# PWD=`pwd` -# ctest -V + macos-clang-make-openssl3-ubsan: + runs-on: macos-12 + steps: + - uses: actions/checkout@v3 + - run: brew install openssl@3 mysql-client unixodbc libpq + - run: >- + ./configure --everything --no-prefix --no-samples --omit=PDF + --odbc-include=/usr/local/opt/unixodbc/include --odbc-lib=/usr/local/opt/unixodbc/lib + --mysql-include=/usr/local/opt/mysql-client/include --mysql-lib=/usr/local/opt/mysql-client/lib + --include-path="/usr/local/opt/openssl@3/include" --library-path="/usr/local/opt/openssl@3/lib" && + make all -s -j4 SANITIZEFLAGS=-fsanitize=undefined + + - uses: ./.github/actions/retry-action + with: + timeout_minutes: 90 + max_attempts: 3 + retry_on: any + command: >- + sudo -s + CPPUNIT_IGNORE=" + CppUnit::TestCaller.testTrySleep, + CppUnit::TestCaller.testTimestamp, + CppUnit::TestCaller.testExpireN, + CppUnit::TestCaller.testAccessExpireN, + CppUnit::TestCaller.testExpireN, + CppUnit::TestCaller.testAccessExpireN, + CppUnit::TestCaller.testPollClosedServer, + CppUnit::TestCaller.testEncryptDecryptGCM" + EXCLUDE_TESTS="Redis Data/MySQL Data/ODBC Data/PostgreSQL MongoDB PDF" + ./ci/runtests.sh + + macos-clang-make-openssl3-asan: + runs-on: macos-12 + steps: + - uses: actions/checkout@v3 + - run: brew install openssl@3 mysql-client unixodbc libpq + - run: >- + ./configure --everything --no-prefix --no-samples --omit=PDF + --odbc-include=/usr/local/opt/unixodbc/include --odbc-lib=/usr/local/opt/unixodbc/lib + --mysql-include=/usr/local/opt/mysql-client/include --mysql-lib=/usr/local/opt/mysql-client/lib + --include-path="/usr/local/opt/openssl@3/include" --library-path="/usr/local/opt/openssl@3/lib" && + make all -s -j4 SANITIZEFLAGS=-fsanitize=address + + - uses: ./.github/actions/retry-action + with: + timeout_minutes: 90 + max_attempts: 3 + retry_on: any + command: >- + sudo -s + CPPUNIT_IGNORE=" + CppUnit::TestCaller.testTrySleep, + CppUnit::TestCaller.testTimestamp, + CppUnit::TestCaller.testExpireN, + CppUnit::TestCaller.testAccessExpireN, + CppUnit::TestCaller.testExpireN, + CppUnit::TestCaller.testAccessExpireN, + CppUnit::TestCaller.testPollClosedServer, + CppUnit::TestCaller.testEncryptDecryptGCM" + EXCLUDE_TESTS="Redis Data/MySQL Data/ODBC Data/PostgreSQL MongoDB PDF" + ./ci/runtests.sh # windows-2019-msvc-cmake: # runs-on: windows-2019 From ab3582d21be0d8295fc7d6f4a6d1e4e692b26189 Mon Sep 17 00:00:00 2001 From: Matej Kenda Date: Wed, 3 Jan 2024 19:47:57 +0100 Subject: [PATCH 4/5] fix(test): lock std:cerr to prevent data race in TCP server tests (reported by clang thread sanitizer) #4313 --- Net/testsuite/src/TCPServerTest.cpp | 7 ++++++- NetSSL_OpenSSL/testsuite/src/TCPServerTest.cpp | 7 ++++++- NetSSL_Win/testsuite/src/TCPServerTest.cpp | 5 +++++ 3 files changed, 17 insertions(+), 2 deletions(-) diff --git a/Net/testsuite/src/TCPServerTest.cpp b/Net/testsuite/src/TCPServerTest.cpp index 195b4162a0..8e2dc30009 100644 --- a/Net/testsuite/src/TCPServerTest.cpp +++ b/Net/testsuite/src/TCPServerTest.cpp @@ -18,6 +18,7 @@ #include "Poco/Net/StreamSocket.h" #include "Poco/Net/ServerSocket.h" #include "Poco/Thread.h" +#include "Poco/Mutex.h" #include @@ -35,6 +36,8 @@ using Poco::Thread; namespace { + static Poco::FastMutex cerrMutex; + class EchoConnection: public TCPServerConnection { public: @@ -55,8 +58,10 @@ namespace n = ss.receiveBytes(buffer, sizeof(buffer)); } } - catch (Poco::Exception& exc) + catch (const Poco::Exception& exc) { + Poco::FastMutex::ScopedLock l(cerrMutex); + std::cerr << "EchoConnection: " << exc.displayText() << std::endl; } } diff --git a/NetSSL_OpenSSL/testsuite/src/TCPServerTest.cpp b/NetSSL_OpenSSL/testsuite/src/TCPServerTest.cpp index af98055ea5..0da2367d8c 100644 --- a/NetSSL_OpenSSL/testsuite/src/TCPServerTest.cpp +++ b/NetSSL_OpenSSL/testsuite/src/TCPServerTest.cpp @@ -25,6 +25,7 @@ #include "Poco/Util/Application.h" #include "Poco/Util/AbstractConfiguration.h" #include "Poco/Thread.h" +#include "Poco/Mutex.h" #include @@ -46,6 +47,8 @@ using Poco::Util::Application; namespace { + static Poco::FastMutex cerrMutex; + class EchoConnection: public TCPServerConnection { public: @@ -66,8 +69,10 @@ namespace n = ss.receiveBytes(buffer, sizeof(buffer)); } } - catch (Poco::Exception& exc) + catch (const Poco::Exception& exc) { + Poco::FastMutex::ScopedLock l(cerrMutex); + std::cerr << "EchoConnection: " << exc.displayText() << std::endl; } } diff --git a/NetSSL_Win/testsuite/src/TCPServerTest.cpp b/NetSSL_Win/testsuite/src/TCPServerTest.cpp index 75cce0dd7a..950a29b4fb 100644 --- a/NetSSL_Win/testsuite/src/TCPServerTest.cpp +++ b/NetSSL_Win/testsuite/src/TCPServerTest.cpp @@ -23,6 +23,7 @@ #include "Poco/Util/Application.h" #include "Poco/Util/AbstractConfiguration.h" #include "Poco/Thread.h" +#include "Poco/Mutex.h" #include @@ -44,6 +45,8 @@ using Poco::Util::Application; namespace { + static Poco::FastMutex cerrMutex; + class EchoConnection: public TCPServerConnection { public: @@ -66,6 +69,8 @@ namespace } catch (Poco::Exception& exc) { + Poco::FastMutex::ScopedLock l(cerrMutex); + std::cerr << "EchoConnection: " << exc.displayText() << std::endl; } } From 2e58aec82d33518d242096ce2ba264b09cfb1797 Mon Sep 17 00:00:00 2001 From: Matej Kenda Date: Wed, 3 Jan 2024 20:13:57 +0100 Subject: [PATCH 5/5] fix(test): Use 96-bit IV with aes-256-gcm to fix (#4347): I/O error: error:1C800066:Provider routines::cipher operation failed --- .github/workflows/ci.yml | 18 ++++++------------ Crypto/testsuite/src/CryptoTest.cpp | 11 ++++------- 2 files changed, 10 insertions(+), 19 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 36880e2fc9..8ef4fd9734 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -216,8 +216,7 @@ jobs: CppUnit::TestCaller.testExpireN, CppUnit::TestCaller.testAccessExpireN, CppUnit::TestCaller.testOldBSD, - CppUnit::TestCaller.testPollClosedServer, - CppUnit::TestCaller.testEncryptDecryptGCM" + CppUnit::TestCaller.testPollClosedServer" EXCLUDE_TESTS="Redis Data/MySQL Data/ODBC Data/PostgreSQL MongoDB PDF" ./ci/runtests.sh @@ -242,8 +241,7 @@ jobs: CppUnit::TestCaller.testAccessExpireN, CppUnit::TestCaller.testExpireN, CppUnit::TestCaller.testAccessExpireN, - CppUnit::TestCaller.testPollClosedServer, - CppUnit::TestCaller.testEncryptDecryptGCM" + CppUnit::TestCaller.testPollClosedServer" PWD=`pwd` ctest --output-on-failure -E "(DataMySQL)|(DataODBC)|(PostgreSQL)|(MongoDB)|(Redis)" @@ -268,8 +266,7 @@ jobs: CppUnit::TestCaller.testAccessExpireN, CppUnit::TestCaller.testExpireN, CppUnit::TestCaller.testAccessExpireN, - CppUnit::TestCaller.testPollClosedServer, - CppUnit::TestCaller.testEncryptDecryptGCM" + CppUnit::TestCaller.testPollClosedServer" PWD=`pwd` ctest --output-on-failure -E "(DataMySQL)|(DataODBC)|(PostgreSQL)|(MongoDB)|(Redis)" @@ -299,8 +296,7 @@ jobs: CppUnit::TestCaller.testAccessExpireN, CppUnit::TestCaller.testExpireN, CppUnit::TestCaller.testAccessExpireN, - CppUnit::TestCaller.testPollClosedServer, - CppUnit::TestCaller.testEncryptDecryptGCM" + CppUnit::TestCaller.testPollClosedServer" EXCLUDE_TESTS="Redis Data/MySQL Data/ODBC Data/PostgreSQL MongoDB PDF" ./ci/runtests.sh TSAN @@ -330,8 +326,7 @@ jobs: CppUnit::TestCaller.testAccessExpireN, CppUnit::TestCaller.testExpireN, CppUnit::TestCaller.testAccessExpireN, - CppUnit::TestCaller.testPollClosedServer, - CppUnit::TestCaller.testEncryptDecryptGCM" + CppUnit::TestCaller.testPollClosedServer" EXCLUDE_TESTS="Redis Data/MySQL Data/ODBC Data/PostgreSQL MongoDB PDF" ./ci/runtests.sh @@ -361,8 +356,7 @@ jobs: CppUnit::TestCaller.testAccessExpireN, CppUnit::TestCaller.testExpireN, CppUnit::TestCaller.testAccessExpireN, - CppUnit::TestCaller.testPollClosedServer, - CppUnit::TestCaller.testEncryptDecryptGCM" + CppUnit::TestCaller.testPollClosedServer" EXCLUDE_TESTS="Redis Data/MySQL Data/ODBC Data/PostgreSQL MongoDB PDF" ./ci/runtests.sh diff --git a/Crypto/testsuite/src/CryptoTest.cpp b/Crypto/testsuite/src/CryptoTest.cpp index 6aa9a1f8d5..214395b9ba 100644 --- a/Crypto/testsuite/src/CryptoTest.cpp +++ b/Crypto/testsuite/src/CryptoTest.cpp @@ -212,15 +212,12 @@ void CryptoTest::testEncryptDecryptDESECB() void CryptoTest::testEncryptDecryptGCM() { - // - // The test sometimes fails when it is running for longer time - // This conversation perhaps contains a hint: - // https://github.com/openssl/openssl/issues/21119 - // - CipherKey key("aes-256-gcm"); - CipherKey::ByteVec iv(20, 213); + // 96-bit (12 byte) IV is recommended for usage with GCM. + // https://crypto.stackexchange.com/questions/41601/aes-gcm-recommended-iv-size-why-12-bytes + + CipherKey::ByteVec iv(12, 213); key.setIV(iv); Cipher::Ptr pCipher = CipherFactory::defaultFactory().createCipher(key);