From 771e0705bfef7ddf13e189c0aee1f62053aef5c4 Mon Sep 17 00:00:00 2001 From: eifrah-aws Date: Fri, 8 Nov 2024 04:01:37 +0200 Subject: [PATCH] Add CMake build system for valkey (#1196) With this commit, users are able to build valkey using `CMake`. ## Example usage: Build `valkey-server` in Release mode with TLS enabled and using `jemalloc` as the allocator: ```bash mkdir build-release cd $_ cmake .. -DCMAKE_BUILD_TYPE=Release \ -DCMAKE_INSTALL_PREFIX=/tmp/valkey-install \ -DBUILD_MALLOC=jemalloc -DBUILD_TLS=1 make -j$(nproc) install # start valkey /tmp/valkey-install/bin/valkey-server ``` Build `valkey-unit-tests`: ```bash mkdir build-release-ut cd $_ cmake .. -DCMAKE_BUILD_TYPE=Release \ -DBUILD_MALLOC=jemalloc -DBUILD_UNIT_TESTS=1 make -j$(nproc) # Run the tests ./bin/valkey-unit-tests ``` Current features supported by this PR: - Building against different allocators: (`jemalloc`, `tcmalloc`, `tcmalloc_minimal` and `libc`), e.g. to enable `jemalloc` pass `-DBUILD_MALLOC=jemalloc` to `cmake` - OpenSSL builds (to enable TLS, pass `-DBUILD_TLS=1` to `cmake`) - Sanitizier: pass `-DBUILD_SANITIZER=` to `cmake` - Install target + redis symbolic links - Build `valkey-unit-tests` executable - Standard CMake variables are supported. e.g. to install `valkey` under `/home/you/root` pass `-DCMAKE_INSTALL_PREFIX=/home/you/root` Why using `CMake`? To list *some* of the advantages of using `CMake`: - Superior IDE integrations: cmake generates the file `compile_commands.json` which is required by `clangd` to get a compiler accuracy code completion (in other words: your VScode will thank you) - Out of the source build tree: with the current build system, object files are created all over the place polluting the build source tree, the best practice is to build the project on a separate folder - Multiple build types co-existing: with the current build system, it is often hard to have multiple build configurations. With cmake you can do it easily: - It is the de-facto standard for C/C++ project these days More build examples: ASAN build: ```bash mkdir build-asan cd $_ cmake .. -DBUILD_SANITIZER=address -DBUILD_MALLOC=libc make -j$(nproc) ``` ASAN with jemalloc: ```bash mkdir build-asan-jemalloc cd $_ cmake .. -DBUILD_SANITIZER=address -DBUILD_MALLOC=jemalloc make -j$(nproc) ``` As seen by the previous examples, any combination is allowed and co-exist on the same source tree. ## Valkey installation With this new `CMake`, it is possible to install the binary by running `make install` or creating a package `make package` (currently supported on Debian like distros) ### Example 1: build & install using `make install`: ```bash mkdir build-release cd $_ cmake .. -DCMAKE_INSTALL_PREFIX=$HOME/valkey-install -DCMAKE_BUILD_TYPE=Release make -j$(nproc) install # valkey is now installed under $HOME/valkey-install ``` ### Example 2: create a `.deb` installer: ```bash mkdir build-release cd $_ cmake .. -DCMAKE_BUILD_TYPE=Release make -j$(nproc) package # ... CPack deb generation output sudo gdebi -n ./valkey_8.1.0_amd64.deb # valkey is now installed under /opt/valkey ``` ### Example 3: create installer for non Debian systems (e.g. FreeBSD or macOS): ```bash mkdir build-release cd $_ cmake .. -DCMAKE_BUILD_TYPE=Release make -j$(nproc) package mkdir -p /opt/valkey && ./valkey-8.1.0-Darwin.sh --prefix=/opt/valkey --exclude-subdir # valkey-server is now installed under /opt/valkey ``` Signed-off-by: Eran Ifrah --- .cmake-format.yaml | 76 ++++++ .github/workflows/ci.yml | 25 ++ .gitignore | 2 + CMakeLists.txt | 43 ++++ README.md | 124 +++++++--- cmake/Modules/Packaging.cmake | 44 ++++ cmake/Modules/SourceFiles.cmake | 153 ++++++++++++ cmake/Modules/Utils.cmake | 102 ++++++++ cmake/Modules/ValkeySetup.cmake | 381 ++++++++++++++++++++++++++++++ deps/CMakeLists.txt | 26 ++ deps/fpconv/CMakeLists.txt | 4 + deps/hdr_histogram/CMakeLists.txt | 7 + deps/jemalloc/CMakeLists.txt | 23 ++ deps/linenoise/CMakeLists.txt | 4 + deps/lua/CMakeLists.txt | 44 ++++ src/CMakeLists.txt | 77 ++++++ src/modules/CMakeLists.txt | 21 ++ src/server.c | 1 - src/unit/CMakeLists.txt | 58 +++++ tests/CMakeLists.txt | 5 + tests/modules/CMakeLists.txt | 58 +++++ tests/rdma/CMakeLists.txt | 9 + 22 files changed, 1252 insertions(+), 35 deletions(-) create mode 100644 .cmake-format.yaml create mode 100644 CMakeLists.txt create mode 100644 cmake/Modules/Packaging.cmake create mode 100644 cmake/Modules/SourceFiles.cmake create mode 100644 cmake/Modules/Utils.cmake create mode 100644 cmake/Modules/ValkeySetup.cmake create mode 100644 deps/CMakeLists.txt create mode 100644 deps/fpconv/CMakeLists.txt create mode 100644 deps/hdr_histogram/CMakeLists.txt create mode 100644 deps/jemalloc/CMakeLists.txt create mode 100644 deps/linenoise/CMakeLists.txt create mode 100644 deps/lua/CMakeLists.txt create mode 100644 src/CMakeLists.txt create mode 100644 src/modules/CMakeLists.txt create mode 100644 src/unit/CMakeLists.txt create mode 100644 tests/CMakeLists.txt create mode 100644 tests/modules/CMakeLists.txt create mode 100644 tests/rdma/CMakeLists.txt diff --git a/.cmake-format.yaml b/.cmake-format.yaml new file mode 100644 index 0000000000..98ab11753a --- /dev/null +++ b/.cmake-format.yaml @@ -0,0 +1,76 @@ +format: + _help_line_width: + - How wide to allow formatted cmake files + line_width: 120 + _help_tab_size: + - How many spaces to tab for indent + tab_size: 4 + _help_use_tabchars: + - If true, lines are indented using tab characters (utf-8 + - 0x09) instead of space characters (utf-8 0x20). + - In cases where the layout would require a fractional tab + - character, the behavior of the fractional indentation is + - governed by + use_tabchars: false + _help_separate_ctrl_name_with_space: + - If true, separate flow control names from their parentheses + - with a space + separate_ctrl_name_with_space: true + _help_min_prefix_chars: + - If the statement spelling length (including space and + - parenthesis) is smaller than this amount, then force reject + - nested layouts. + min_prefix_chars: 4 + _help_max_prefix_chars: + - If the statement spelling length (including space and + - parenthesis) is larger than the tab width by more than this + - amount, then force reject un-nested layouts. + max_prefix_chars: 10 + _help_max_lines_hwrap: + - If a candidate layout is wrapped horizontally but it exceeds + - this many lines, then reject the layout. + max_lines_hwrap: 2 + _help_line_ending: + - What style line endings to use in the output. + line_ending: unix + _help_command_case: + - Format command names consistently as 'lower' or 'upper' case + command_case: lower + _help_keyword_case: + - Format keywords consistently as 'lower' or 'upper' case + keyword_case: unchanged + _help_always_wrap: + - A list of command names which should always be wrapped + always_wrap: [] + _help_enable_sort: + - If true, the argument lists which are known to be sortable + - will be sorted lexicographicall + enable_sort: true + _help_autosort: + - If true, the parsers may infer whether or not an argument + - list is sortable (without annotation). + autosort: false + _help_require_valid_layout: + - By default, if cmake-format cannot successfully fit + - everything into the desired linewidth it will apply the + - last, most agressive attempt that it made. If this flag is + - True, however, cmake-format will print error, exit with non- + - zero status code, and write-out nothing + require_valid_layout: false + _help_layout_passes: + - A dictionary mapping layout nodes to a list of wrap + - decisions. See the documentation for more information. + layout_passes: {} +encode: + _help_emit_byteorder_mark: + - If true, emit the unicode byte-order mark (BOM) at the start + - of the file + emit_byteorder_mark: false + _help_input_encoding: + - Specify the encoding of the input file. Defaults to utf-8 + input_encoding: utf-8 + _help_output_encoding: + - Specify the encoding of the output file. Defaults to utf-8. + - Note that cmake only claims to support utf-8 so be careful + - when using anything else + output_encoding: utf-8 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 48a94ef984..bc946b7193 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -34,6 +34,31 @@ jobs: run: | ./src/valkey-unit-tests + test-ubuntu-latest-cmake: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + - name: cmake and make + run: | + sudo apt-get install -y cmake libssl-dev + mkdir -p build-release + cd build-release + cmake -DCMAKE_BUILD_TYPE=Release .. -DBUILD_TLS=yes -DBUILD_UNIT_TESTS=yes + make -j$(nproc) + - name: test + run: | + sudo apt-get install -y tcl8.6 tclx + ln -sf $(pwd)/build-release/bin/valkey-server $(pwd)/src/valkey-server + ln -sf $(pwd)/build-release/bin/valkey-cli $(pwd)/src/valkey-cli + ln -sf $(pwd)/build-release/bin/valkey-benchmark $(pwd)/src/valkey-benchmark + ln -sf $(pwd)/build-release/bin/valkey-server $(pwd)/src/valkey-check-aof + ln -sf $(pwd)/build-release/bin/valkey-server $(pwd)/src/valkey-check-rdb + ln -sf $(pwd)/build-release/bin/valkey-server $(pwd)/src/valkey-sentinel + ./runtest --verbose --tags -slow --dump-logs + - name: unit tests + run: | + ./build-release/bin/valkey-unit-tests + test-sanitizer-address: runs-on: ubuntu-latest steps: diff --git a/.gitignore b/.gitignore index e448e23f7e..b108b4bb92 100644 --- a/.gitignore +++ b/.gitignore @@ -49,3 +49,5 @@ nodes*.conf tests/cluster/tmp/* tests/rdma/rdma-test tags +build-debug/ +build-release/ diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 0000000000..ad0bab8896 --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,43 @@ +cmake_minimum_required(VERSION 3.20) + +# Must be done first +if (APPLE) + # Force clang compiler on macOS + find_program(CLANGPP "clang++") + find_program(CLANG "clang") + if (CLANG AND CLANGPP) + message(STATUS "Found ${CLANGPP}, ${CLANG}") + set(CMAKE_CXX_COMPILER ${CLANGPP}) + set(CMAKE_C_COMPILER ${CLANG}) + endif () +endif () + +# Options +option(BUILD_UNIT_TESTS "Build valkey-unit-tests" OFF) +option(BUILD_TEST_MODULES "Build all test modules" OFF) +option(BUILD_EXAMPLE_MODULES "Build example modules" OFF) + +set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake/Modules/") +project("valkey") + +set(CMAKE_C_STANDARD 11) +set(CMAKE_C_STANDARD_REQUIRED ON) +set(CMAKE_C_EXTENSIONS ON) + +include(ValkeySetup) +add_subdirectory(src) +add_subdirectory(tests) + +# Include the packaging module +include(Packaging) + +# Clear cached variables from the cache +unset(BUILD_TESTS CACHE) +unset(CLANGPP CACHE) +unset(CLANG CACHE) +unset(BUILD_RDMA_MODULE CACHE) +unset(BUILD_TLS_MODULE CACHE) +unset(BUILD_UNIT_TESTS CACHE) +unset(BUILD_TEST_MODULES CACHE) +unset(BUILD_EXAMPLE_MODULES CACHE) +unset(USE_TLS CACHE) diff --git a/README.md b/README.md index 1a8ce1a4db..94f38bccf7 100644 --- a/README.md +++ b/README.md @@ -4,13 +4,12 @@ This project was forked from the open source Redis project right before the tran This README is just a fast *quick start* document. More details can be found under [valkey.io](https://valkey.io/) -What is Valkey? --------------- +# What is Valkey? + Valkey is a high-performance data structure server that primarily serves key/value workloads. It supports a wide range of native structures and an extensible plugin system for adding new data structures and access patterns. -Building Valkey --------------- +# Building Valkey using `Makefile` Valkey can be compiled and used on Linux, OSX, OpenBSD, NetBSD, FreeBSD. We support big endian and little endian architectures, and both 32 bit @@ -43,7 +42,7 @@ supports RDMA as connection module mode. Run: % make BUILD_RDMA=module -To build with systemd support, you'll need systemd development libraries (such +To build with systemd support, you'll need systemd development libraries (such as libsystemd-dev on Debian/Ubuntu or systemd-devel on CentOS) and run: % make USE_SYSTEMD=yes @@ -71,8 +70,7 @@ More about running the integration tests can be found in [tests/README.md](tests/README.md) and for unit tests, see [src/unit/README.md](src/unit/README.md). -Fixing build problems with dependencies or cached build options ---------- +## Fixing build problems with dependencies or cached build options Valkey has some dependencies which are included in the `deps` directory. `make` does not automatically rebuild dependencies even if something in @@ -91,8 +89,7 @@ optimizations (for debugging purposes), and other similar build time options, those options are cached indefinitely until you issue a `make distclean` command. -Fixing problems building 32 bit binaries ---------- +## Fixing problems building 32 bit binaries If after building Valkey with a 32 bit target you need to rebuild it with a 64 bit target, or the other way around, you need to perform a @@ -105,8 +102,7 @@ the following steps: * Try using the following command line instead of `make 32bit`: `make CFLAGS="-m32 -march=native" LDFLAGS="-m32"` -Allocator ---------- +## Allocator Selecting a non-default memory allocator when building Valkey is done by setting the `MALLOC` environment variable. Valkey is compiled and linked against libc @@ -122,28 +118,25 @@ To compile against jemalloc on Mac OS X systems, use: % make MALLOC=jemalloc -Monotonic clock ---------------- +## Monotonic clock By default, Valkey will build using the POSIX clock_gettime function as the monotonic clock source. On most modern systems, the internal processor clock -can be used to improve performance. Cautions can be found here: +can be used to improve performance. Cautions can be found here: http://oliveryang.net/2015/09/pitfalls-of-TSC-usage/ To build with support for the processor's internal instruction clock, use: % make CFLAGS="-DUSE_PROCESSOR_CLOCK" -Verbose build -------------- +## Verbose build Valkey will build with a user-friendly colorized output by default. If you want to see a more verbose output, use the following: % make V=1 -Running Valkey -------------- +# Running Valkey To run Valkey with the default configuration, just type: @@ -165,10 +158,10 @@ as options using the command line. Examples: All the options in valkey.conf are also supported as options using the command line, with exactly the same name. -Running Valkey with TLS: ------------------- +# Running Valkey with TLS: + +## Running manually -### Running manually To manually run a Valkey server with TLS mode (assuming `./gen-test-certs.sh` was invoked so sample certificates/keys are available): * TLS built-in mode: @@ -204,8 +197,7 @@ Specifying `--tls-replication yes` makes a replica connect to the primary. Using `--tls-cluster yes` makes Valkey Cluster use TLS across nodes. -Running Valkey with RDMA: ------------------- +# Running Valkey with RDMA: Note that Valkey Over RDMA is an experimental feature. It may be changed or removed in any minor or major version. @@ -236,8 +228,7 @@ Or: % ibv_devices -Playing with Valkey ------------------- +# Playing with Valkey You can use valkey-cli to play with Valkey. Start a valkey-server instance, then in another terminal try the following: @@ -256,8 +247,7 @@ then in another terminal try the following: (integer) 2 valkey> -Installing Valkey ------------------ +# Installing Valkey In order to install Valkey binaries into /usr/local/bin, just use: @@ -289,16 +279,82 @@ system reboots. You'll be able to stop and start Valkey using the script named `/etc/init.d/valkey_`, for instance `/etc/init.d/valkey_6379`. -Code contributions ------------------ +# Building using `CMake` + +In addition to the traditional `Makefile` build, Valkey supports an alternative, **experimental**, build system using `CMake`. + +To build and install `Valkey`, in `Release` mode (an optimized build), type this into your terminal: + +```bash +mkdir build-release +cd $_ +cmake .. -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/opt/valkey +sudo make install +# Valkey is now installed under /opt/valkey +``` + +Other options supported by Valkey's `CMake` build system: + +## Special build flags + +- `-DBUILD_TLS=` enable TLS build for Valkey +- `-DBUILD_RDMA=` enable RDMA module build (only module mode supported) +- `-DBUILD_MALLOC=` choose the allocator to use. Default on Linux: `jemalloc`, for other OS: `libc` +- `-DBUILD_SANITIZER=` build with address sanitizer enabled +- `-DBUILD_UNIT_TESTS=[1|0]` when set, the build will produce the executable `valkey-unit-tests` +- `-DBUILD_TEST_MODULES=[1|0]` when set, the build will include the modules located under the `tests/modules` folder +- `-DBUILD_EXAMPLE_MODULES=[1|0]` when set, the build will include the example modules located under the `src/modules` folder + +## Common flags + +- `-DCMAKE_BUILD_TYPE=` define the build type, see CMake manual for more details +- `-DCMAKE_INSTALL_PREFIX=/installation/path` override this value to define a custom install prefix. Default: `/usr/local` +- `-G` generate build files for "Generator Name". By default, CMake will generate `Makefile`s. + +## Verbose build + +`CMake` generates a user-friendly colorized output by default. +If you want to see a more verbose output, use the following: + +```bash +make VERBOSE=1 +``` + +## Troubleshooting + +During the `CMake` stage, `CMake` caches variables in a local file named `CMakeCache.txt`. All variables generated by Valkey +are removed from the cache once consumed (this is done by calling to `unset(VAR-NAME CACHE)`). However, some variables, +like the compiler path, are kept in cache. To start a fresh build either remove the cache file `CMakeCache.txt` from the +build folder, or delete the build folder completely. + +**It is important to re-run `CMake` when adding new source files.** + +## Integration with IDE + +During the `CMake` stage of the build, `CMake` generates a JSON file named `compile_commands.json` and places it under the +build folder. This file is used by many IDEs and text editors for providing code completion (via `clangd`). + +A small caveat is that these tools will look for `compile_commands.json` under the Valkey's top folder. +A common workaround is to create a symbolic link to it: + +```bash +cd /path/to/valkey/ +# We assume here that your build folder is `build-release` +ln -sf $(pwd)/build-release/compile_commands.json $(pwd)/compile_commands.json +``` + +Restart your IDE and voila + +# Code contributions + Please see the [CONTRIBUTING.md][2]. For security bugs and vulnerabilities, please see [SECURITY.md][3]. -[1]: https://github.com/valkey-io/valkey/blob/unstable/COPYING -[2]: https://github.com/valkey-io/valkey/blob/unstable/CONTRIBUTING.md -[3]: https://github.com/valkey-io/valkey/blob/unstable/SECURITY.md +# Valkey is an open community project under LF Projects -Valkey is an open community project under LF Projects ------------------ Valkey a Series of LF Projects, LLC 2810 N Church St, PMB 57274 Wilmington, Delaware 19802-4447 + +[1]: https://github.com/valkey-io/valkey/blob/unstable/COPYING +[2]: https://github.com/valkey-io/valkey/blob/unstable/CONTRIBUTING.md +[3]: https://github.com/valkey-io/valkey/blob/unstable/SECURITY.md diff --git a/cmake/Modules/Packaging.cmake b/cmake/Modules/Packaging.cmake new file mode 100644 index 0000000000..c7ed5c426b --- /dev/null +++ b/cmake/Modules/Packaging.cmake @@ -0,0 +1,44 @@ +set(CPACK_PACKAGE_NAME "valkey") + +valkey_parse_version(CPACK_PACKAGE_VERSION_MAJOR CPACK_PACKAGE_VERSION_MINOR CPACK_PACKAGE_VERSION_PATCH) + +set(CPACK_PACKAGE_CONTACT "maintainers@lists.valkey.io") +set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "Valkey is an open source (BSD) high-performance key/value datastore") +set(CPACK_RESOURCE_FILE_LICENSE "${CMAKE_SOURCE_DIR}/COPYING") +set(CPACK_RESOURCE_FILE_README "${CMAKE_SOURCE_DIR}/README.md") +set(CPACK_STRIP_FILES TRUE) + +valkey_get_distro_name(DISTRO_NAME) +message(STATUS "Current host distro: ${DISTRO_NAME}") + +if (DISTRO_NAME MATCHES ubuntu + OR DISTRO_NAME MATCHES debian + OR DISTRO_NAME MATCHES mint) + message(STATUS "Adding target package for ${DISTRO_NAME}") + set(CPACK_PACKAGING_INSTALL_PREFIX "/opt/valkey") + # Debian related parameters + set(CPACK_DEBIAN_PACKAGE_MAINTAINER "Valkey contributors") + set(CPACK_DEBIAN_PACKAGE_SHLIBDEPS ON) + set(CPACK_DEBIAN_FILE_NAME DEB-DEFAULT) + set(CPACK_GENERATOR "DEB") +endif () + +include(CPack) +unset(DISTRO_NAME CACHE) + +# --------------------------------------------------- +# Create a helper script for creating symbolic links +# --------------------------------------------------- +write_file( + ${CMAKE_BINARY_DIR}/CreateSymlink.sh + "\ +#!/bin/bash \n\ +if [ -z \${DESTDIR} ]; then \n\ + # Script is called during 'make install' \n\ + PREFIX=${CMAKE_INSTALL_PREFIX}/bin \n\ +else \n\ + # Script is called during 'make package' \n\ + PREFIX=\${DESTDIR}${CPACK_PACKAGING_INSTALL_PREFIX}/bin \n\ +fi \n\ +cd \$PREFIX \n\ +ln -sf \$1 \$2") diff --git a/cmake/Modules/SourceFiles.cmake b/cmake/Modules/SourceFiles.cmake new file mode 100644 index 0000000000..d76f17625e --- /dev/null +++ b/cmake/Modules/SourceFiles.cmake @@ -0,0 +1,153 @@ +# ------------------------------------------------- +# Define the sources to be built +# ------------------------------------------------- + +# valkey-server source files +set(VALKEY_SERVER_SRCS + ${CMAKE_SOURCE_DIR}/src/threads_mngr.c + ${CMAKE_SOURCE_DIR}/src/adlist.c + ${CMAKE_SOURCE_DIR}/src/quicklist.c + ${CMAKE_SOURCE_DIR}/src/ae.c + ${CMAKE_SOURCE_DIR}/src/anet.c + ${CMAKE_SOURCE_DIR}/src/dict.c + ${CMAKE_SOURCE_DIR}/src/kvstore.c + ${CMAKE_SOURCE_DIR}/src/sds.c + ${CMAKE_SOURCE_DIR}/src/zmalloc.c + ${CMAKE_SOURCE_DIR}/src/lzf_c.c + ${CMAKE_SOURCE_DIR}/src/lzf_d.c + ${CMAKE_SOURCE_DIR}/src/pqsort.c + ${CMAKE_SOURCE_DIR}/src/zipmap.c + ${CMAKE_SOURCE_DIR}/src/sha1.c + ${CMAKE_SOURCE_DIR}/src/ziplist.c + ${CMAKE_SOURCE_DIR}/src/release.c + ${CMAKE_SOURCE_DIR}/src/memory_prefetch.c + ${CMAKE_SOURCE_DIR}/src/io_threads.c + ${CMAKE_SOURCE_DIR}/src/networking.c + ${CMAKE_SOURCE_DIR}/src/util.c + ${CMAKE_SOURCE_DIR}/src/object.c + ${CMAKE_SOURCE_DIR}/src/db.c + ${CMAKE_SOURCE_DIR}/src/replication.c + ${CMAKE_SOURCE_DIR}/src/rdb.c + ${CMAKE_SOURCE_DIR}/src/t_string.c + ${CMAKE_SOURCE_DIR}/src/t_list.c + ${CMAKE_SOURCE_DIR}/src/t_set.c + ${CMAKE_SOURCE_DIR}/src/t_zset.c + ${CMAKE_SOURCE_DIR}/src/t_hash.c + ${CMAKE_SOURCE_DIR}/src/config.c + ${CMAKE_SOURCE_DIR}/src/aof.c + ${CMAKE_SOURCE_DIR}/src/pubsub.c + ${CMAKE_SOURCE_DIR}/src/multi.c + ${CMAKE_SOURCE_DIR}/src/debug.c + ${CMAKE_SOURCE_DIR}/src/sort.c + ${CMAKE_SOURCE_DIR}/src/intset.c + ${CMAKE_SOURCE_DIR}/src/syncio.c + ${CMAKE_SOURCE_DIR}/src/cluster.c + ${CMAKE_SOURCE_DIR}/src/cluster_legacy.c + ${CMAKE_SOURCE_DIR}/src/cluster_slot_stats.c + ${CMAKE_SOURCE_DIR}/src/crc16.c + ${CMAKE_SOURCE_DIR}/src/endianconv.c + ${CMAKE_SOURCE_DIR}/src/slowlog.c + ${CMAKE_SOURCE_DIR}/src/eval.c + ${CMAKE_SOURCE_DIR}/src/bio.c + ${CMAKE_SOURCE_DIR}/src/rio.c + ${CMAKE_SOURCE_DIR}/src/rand.c + ${CMAKE_SOURCE_DIR}/src/memtest.c + ${CMAKE_SOURCE_DIR}/src/syscheck.c + ${CMAKE_SOURCE_DIR}/src/crcspeed.c + ${CMAKE_SOURCE_DIR}/src/crccombine.c + ${CMAKE_SOURCE_DIR}/src/crc64.c + ${CMAKE_SOURCE_DIR}/src/bitops.c + ${CMAKE_SOURCE_DIR}/src/sentinel.c + ${CMAKE_SOURCE_DIR}/src/notify.c + ${CMAKE_SOURCE_DIR}/src/setproctitle.c + ${CMAKE_SOURCE_DIR}/src/blocked.c + ${CMAKE_SOURCE_DIR}/src/hyperloglog.c + ${CMAKE_SOURCE_DIR}/src/latency.c + ${CMAKE_SOURCE_DIR}/src/sparkline.c + ${CMAKE_SOURCE_DIR}/src/valkey-check-rdb.c + ${CMAKE_SOURCE_DIR}/src/valkey-check-aof.c + ${CMAKE_SOURCE_DIR}/src/geo.c + ${CMAKE_SOURCE_DIR}/src/lazyfree.c + ${CMAKE_SOURCE_DIR}/src/module.c + ${CMAKE_SOURCE_DIR}/src/evict.c + ${CMAKE_SOURCE_DIR}/src/expire.c + ${CMAKE_SOURCE_DIR}/src/geohash.c + ${CMAKE_SOURCE_DIR}/src/geohash_helper.c + ${CMAKE_SOURCE_DIR}/src/childinfo.c + ${CMAKE_SOURCE_DIR}/src/defrag.c + ${CMAKE_SOURCE_DIR}/src/siphash.c + ${CMAKE_SOURCE_DIR}/src/rax.c + ${CMAKE_SOURCE_DIR}/src/t_stream.c + ${CMAKE_SOURCE_DIR}/src/listpack.c + ${CMAKE_SOURCE_DIR}/src/localtime.c + ${CMAKE_SOURCE_DIR}/src/lolwut.c + ${CMAKE_SOURCE_DIR}/src/lolwut5.c + ${CMAKE_SOURCE_DIR}/src/lolwut6.c + ${CMAKE_SOURCE_DIR}/src/acl.c + ${CMAKE_SOURCE_DIR}/src/tracking.c + ${CMAKE_SOURCE_DIR}/src/socket.c + ${CMAKE_SOURCE_DIR}/src/tls.c + ${CMAKE_SOURCE_DIR}/src/sha256.c + ${CMAKE_SOURCE_DIR}/src/timeout.c + ${CMAKE_SOURCE_DIR}/src/setcpuaffinity.c + ${CMAKE_SOURCE_DIR}/src/monotonic.c + ${CMAKE_SOURCE_DIR}/src/mt19937-64.c + ${CMAKE_SOURCE_DIR}/src/resp_parser.c + ${CMAKE_SOURCE_DIR}/src/call_reply.c + ${CMAKE_SOURCE_DIR}/src/script_lua.c + ${CMAKE_SOURCE_DIR}/src/script.c + ${CMAKE_SOURCE_DIR}/src/functions.c + ${CMAKE_SOURCE_DIR}/src/function_lua.c + ${CMAKE_SOURCE_DIR}/src/commands.c + ${CMAKE_SOURCE_DIR}/src/strl.c + ${CMAKE_SOURCE_DIR}/src/connection.c + ${CMAKE_SOURCE_DIR}/src/unix.c + ${CMAKE_SOURCE_DIR}/src/server.c + ${CMAKE_SOURCE_DIR}/src/logreqres.c) + +# valkey-cli +set(VALKEY_CLI_SRCS + ${CMAKE_SOURCE_DIR}/src/anet.c + ${CMAKE_SOURCE_DIR}/src/adlist.c + ${CMAKE_SOURCE_DIR}/src/dict.c + ${CMAKE_SOURCE_DIR}/src/valkey-cli.c + ${CMAKE_SOURCE_DIR}/src/zmalloc.c + ${CMAKE_SOURCE_DIR}/src/release.c + ${CMAKE_SOURCE_DIR}/src/ae.c + ${CMAKE_SOURCE_DIR}/src/serverassert.c + ${CMAKE_SOURCE_DIR}/src/crcspeed.c + ${CMAKE_SOURCE_DIR}/src/crccombine.c + ${CMAKE_SOURCE_DIR}/src/crc64.c + ${CMAKE_SOURCE_DIR}/src/siphash.c + ${CMAKE_SOURCE_DIR}/src/crc16.c + ${CMAKE_SOURCE_DIR}/src/monotonic.c + ${CMAKE_SOURCE_DIR}/src/cli_common.c + ${CMAKE_SOURCE_DIR}/src/mt19937-64.c + ${CMAKE_SOURCE_DIR}/src/strl.c + ${CMAKE_SOURCE_DIR}/src/cli_commands.c) + +# valkey-benchmark +set(VALKEY_BENCHMARK_SRCS + ${CMAKE_SOURCE_DIR}/src/ae.c + ${CMAKE_SOURCE_DIR}/src/anet.c + ${CMAKE_SOURCE_DIR}/src/valkey-benchmark.c + ${CMAKE_SOURCE_DIR}/src/adlist.c + ${CMAKE_SOURCE_DIR}/src/dict.c + ${CMAKE_SOURCE_DIR}/src/zmalloc.c + ${CMAKE_SOURCE_DIR}/src/serverassert.c + ${CMAKE_SOURCE_DIR}/src/release.c + ${CMAKE_SOURCE_DIR}/src/crcspeed.c + ${CMAKE_SOURCE_DIR}/src/crccombine.c + ${CMAKE_SOURCE_DIR}/src/crc64.c + ${CMAKE_SOURCE_DIR}/src/siphash.c + ${CMAKE_SOURCE_DIR}/src/crc16.c + ${CMAKE_SOURCE_DIR}/src/monotonic.c + ${CMAKE_SOURCE_DIR}/src/cli_common.c + ${CMAKE_SOURCE_DIR}/src/mt19937-64.c + ${CMAKE_SOURCE_DIR}/src/strl.c) + +# valkey-rdma module +set(VALKEY_RDMA_MODULE_SRCS ${CMAKE_SOURCE_DIR}/src/rdma.c) + +# valkey-tls module +set(VALKEY_TLS_MODULE_SRCS ${CMAKE_SOURCE_DIR}/src/tls.c) diff --git a/cmake/Modules/Utils.cmake b/cmake/Modules/Utils.cmake new file mode 100644 index 0000000000..304f39fb2c --- /dev/null +++ b/cmake/Modules/Utils.cmake @@ -0,0 +1,102 @@ +# Return the current host distro name. For example: ubuntu, debian, amzn etc +function (valkey_get_distro_name DISTRO_NAME) + if (LINUX AND NOT APPLE) + execute_process( + COMMAND /bin/bash "-c" "cat /etc/os-release |grep ^ID=|cut -d = -f 2" + OUTPUT_VARIABLE _OUT_VAR + OUTPUT_STRIP_TRAILING_WHITESPACE) + # clean the output + string(REPLACE "\"" "" _OUT_VAR "${_OUT_VAR}") + string(REPLACE "." "" _OUT_VAR "${_OUT_VAR}") + set(${DISTRO_NAME} + "${_OUT_VAR}" + PARENT_SCOPE) + elseif (APPLE) + set(${DISTRO_NAME} + "darwin" + PARENT_SCOPE) + elseif (IS_FREEBSD) + set(${DISTRO_NAME} + "freebsd" + PARENT_SCOPE) + else () + set(${DISTRO_NAME} + "unknown" + PARENT_SCOPE) + endif () +endfunction () + +function (valkey_parse_version OUT_MAJOR OUT_MINOR OUT_PATCH) + # Read and parse package version from version.h file + file(STRINGS ${CMAKE_SOURCE_DIR}/src/version.h VERSION_LINES) + foreach (LINE ${VERSION_LINES}) + string(FIND "${LINE}" "#define VALKEY_VERSION " VERSION_STR_POS) + if (VERSION_STR_POS GREATER -1) + string(REPLACE "#define VALKEY_VERSION " "" LINE "${LINE}") + string(REPLACE "\"" "" LINE "${LINE}") + # Change "." to ";" to make it a list + string(REPLACE "." ";" LINE "${LINE}") + list(GET LINE 0 _MAJOR) + list(GET LINE 1 _MINOR) + list(GET LINE 2 _PATCH) + message(STATUS "Valkey version: ${_MAJOR}.${_MINOR}.${_PATCH}") + # Set the output variables + set(${OUT_MAJOR} + ${_MAJOR} + PARENT_SCOPE) + set(${OUT_MINOR} + ${_MINOR} + PARENT_SCOPE) + set(${OUT_PATCH} + ${_PATCH} + PARENT_SCOPE) + endif () + endforeach () +endfunction () + +# Given input argument `OPTION_VALUE`, check that the `OPTION_VALUE` is from the allowed values (one of: +# module/yes/no/1/0/true/false) +# +# Return value: +# +# If ARG is valid, return its number where: +# +# ~~~ +# - `no` | `0` | `off` => return `0` +# - `yes` | `1` | `on` => return `1` +# - `module` => return `2` +# ~~~ +function (valkey_parse_build_option OPTION_VALUE OUT_ARG_ENUM) + list(APPEND VALID_OPTIONS "yes") + list(APPEND VALID_OPTIONS "1") + list(APPEND VALID_OPTIONS "on") + list(APPEND VALID_OPTIONS "no") + list(APPEND VALID_OPTIONS "0") + list(APPEND VALID_OPTIONS "off") + list(APPEND VALID_OPTIONS "module") + + string(TOLOWER "${OPTION_VALUE}" OPTION_VALUE) + list(FIND VALID_OPTIONS "${ARG}" OPT_INDEX) + if (VERSION_STR_POS GREATER -1) + message(FATAL_ERROR "Invalid value passed ''${OPTION_VALUE}'") + endif () + + if ("${OPTION_VALUE}" STREQUAL "yes" + OR "${OPTION_VALUE}" STREQUAL "1" + OR "${OPTION_VALUE}" STREQUAL "on") + set(${OUT_ARG_ENUM} + 1 + PARENT_SCOPE) + elseif ( + "${OPTION_VALUE}" STREQUAL "no" + OR "${OPTION_VALUE}" STREQUAL "0" + OR "${OPTION_VALUE}" STREQUAL "off") + set(${OUT_ARG_ENUM} + 0 + PARENT_SCOPE) + else () + set(${OUT_ARG_ENUM} + 2 + PARENT_SCOPE) + endif () +endfunction () diff --git a/cmake/Modules/ValkeySetup.cmake b/cmake/Modules/ValkeySetup.cmake new file mode 100644 index 0000000000..e935c3b308 --- /dev/null +++ b/cmake/Modules/ValkeySetup.cmake @@ -0,0 +1,381 @@ +include(CheckIncludeFiles) +include(ProcessorCount) +include(Utils) + +set(CMAKE_LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/lib") +set(CMAKE_RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/bin") +set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/lib") + +# Generate compile_commands.json file for IDEs code completion support +set(CMAKE_EXPORT_COMPILE_COMMANDS 1) + +processorcount(VALKEY_PROCESSOR_COUNT) +message(STATUS "Processor count: ${VALKEY_PROCESSOR_COUNT}") + +# Installed executables will have this permissions +set(VALKEY_EXE_PERMISSIONS + OWNER_EXECUTE + OWNER_WRITE + OWNER_READ + GROUP_EXECUTE + GROUP_READ + WORLD_EXECUTE + WORLD_READ) + +set(VALKEY_SERVER_CFLAGS "") +set(VALKEY_SERVER_LDFLAGS "") + +# ---------------------------------------------------- +# Helper functions & macros +# ---------------------------------------------------- +macro (add_valkey_server_compiler_options value) + set(VALKEY_SERVER_CFLAGS "${VALKEY_SERVER_CFLAGS} ${value}") +endmacro () + +macro (add_valkey_server_linker_option value) + list(APPEND VALKEY_SERVER_LDFLAGS ${value}) +endmacro () + +macro (get_valkey_server_linker_option return_value) + list(JOIN VALKEY_SERVER_LDFLAGS " " ${value} ${return_value}) +endmacro () + +set(IS_FREEBSD 0) +if (CMAKE_SYSTEM_NAME MATCHES "^.*BSD$|DragonFly") + message(STATUS "Building for FreeBSD compatible system") + set(IS_FREEBSD 1) + include_directories("/usr/local/include") + add_valkey_server_compiler_options("-DUSE_BACKTRACE") +endif () + +# Helper function for creating symbolic link so that: link -> source +macro (valkey_create_symlink source link) + install( + CODE "execute_process( \ + COMMAND /bin/bash ${CMAKE_BINARY_DIR}/CreateSymlink.sh \ + ${source} \ + ${link} \ + )" + COMPONENT "valkey") +endmacro () + +# Install a binary +macro (valkey_install_bin target) + # Install cli tool and create a redis symbolic link + install( + TARGETS ${target} + DESTINATION ${CMAKE_INSTALL_BINDIR} + PERMISSIONS ${VALKEY_EXE_PERMISSIONS} + COMPONENT "valkey") +endmacro () + +# Helper function that defines, builds and installs `target` In addition, it creates a symbolic link between the target +# and `link_name` +macro (valkey_build_and_install_bin target sources ld_flags libs link_name) + add_executable(${target} ${sources}) + + if (USE_JEMALLOC) + # Using jemalloc + target_link_libraries(${target} jemalloc) + endif () + + # Place this line last to ensure that ${ld_flags} is placed last on the linker line + target_link_libraries(${target} ${libs} ${ld_flags}) + target_link_libraries(${target} hiredis) + if (USE_TLS) + # Add required libraries needed for TLS + target_link_libraries(${target} OpenSSL::SSL hiredis_ssl) + endif () + + if (IS_FREEBSD) + target_link_libraries(${target} execinfo) + endif () + + # Install cli tool and create a redis symbolic link + valkey_install_bin(${target}) + valkey_create_symlink(${target} ${link_name}) +endmacro () + +# Helper function that defines, builds and installs `target` module. +macro (valkey_build_and_install_module target sources ld_flags libs) + add_library(${target} SHARED ${sources}) + + if (USE_JEMALLOC) + # Using jemalloc + target_link_libraries(${target} jemalloc) + endif () + + # Place this line last to ensure that ${ld_flags} is placed last on the linker line + target_link_libraries(${target} ${libs} ${ld_flags}) + if (USE_TLS) + # Add required libraries needed for TLS + target_link_libraries(${target} OpenSSL::SSL hiredis_ssl) + endif () + + if (IS_FREEBSD) + target_link_libraries(${target} execinfo) + endif () + + # Install cli tool and create a redis symbolic link + valkey_install_bin(${target}) +endmacro () + +# Determine if we are building in Release or Debug mode +if (CMAKE_BUILD_TYPE MATCHES Debug OR CMAKE_BUILD_TYPE MATCHES DebugFull) + set(VALKEY_DEBUG_BUILD 1) + set(VALKEY_RELEASE_BUILD 0) + message(STATUS "Building in debug mode") +else () + set(VALKEY_DEBUG_BUILD 0) + set(VALKEY_RELEASE_BUILD 1) + message(STATUS "Building in release mode") +endif () + +# ---------------------------------------------------- +# Helper functions - end +# ---------------------------------------------------- + +# ---------------------------------------------------- +# Build options (allocator, tls, rdma et al) +# ---------------------------------------------------- + +if (NOT BUILD_MALLOC) + if (APPLE) + set(BUILD_MALLOC "libc") + elseif (UNIX) + set(BUILD_MALLOC "jemalloc") + endif () +endif () + +# User may pass different allocator library. Using -DBUILD_MALLOC=, make sure it is a valid value +if (BUILD_MALLOC) + if ("${BUILD_MALLOC}" STREQUAL "jemalloc") + set(MALLOC_LIB "jemalloc") + add_valkey_server_compiler_options("-DUSE_JEMALLOC") + set(USE_JEMALLOC 1) + elseif ("${BUILD_MALLOC}" STREQUAL "libc") + set(MALLOC_LIB "libc") + elseif ("${BUILD_MALLOC}" STREQUAL "tcmalloc") + set(MALLOC_LIB "tcmalloc") + add_valkey_server_compiler_options("-DUSE_TCMALLOC") + elseif ("${BUILD_MALLOC}" STREQUAL "tcmalloc_minimal") + set(MALLOC_LIB "tcmalloc_minimal") + add_valkey_server_compiler_options("-DUSE_TCMALLOC") + else () + message(FATAL_ERROR "BUILD_MALLOC can be one of: jemalloc, libc, tcmalloc or tcmalloc_minimal") + endif () +endif () + +message(STATUS "Using ${MALLOC_LIB}") + +# TLS support +if (BUILD_TLS) + valkey_parse_build_option(${BUILD_TLS} USE_TLS) + if (USE_TLS EQUAL 1) + # Only search for OpenSSL if needed + find_package(OpenSSL REQUIRED) + message(STATUS "OpenSSL include dir: ${OPENSSL_INCLUDE_DIR}") + message(STATUS "OpenSSL libraries: ${OPENSSL_LIBRARIES}") + include_directories(${OPENSSL_INCLUDE_DIR}) + endif () + + if (USE_TLS EQUAL 1) + add_valkey_server_compiler_options("-DUSE_OPENSSL=1") + add_valkey_server_compiler_options("-DBUILD_TLS_MODULE=0") + else () + # Build TLS as a module RDMA can only be built as a module. So disable it + message(WARNING "BUILD_TLS can be one of: [ON | OFF | 1 | 0], but '${BUILD_TLS}' was provided") + message(STATUS "TLS support is disabled") + set(USE_TLS 0) + endif () +else () + # By default, TLS is disabled + message(STATUS "TLS is disabled") + set(USE_TLS 0) +endif () + +if (BUILD_RDMA) + set(BUILD_RDMA_MODULE 0) + # RDMA support (Linux only) + if (LINUX AND NOT APPLE) + valkey_parse_build_option(${BUILD_RDMA} USE_RDMA) + if (USE_RDMA EQUAL 2) # Module + message(STATUS "Building RDMA as module") + add_valkey_server_compiler_options("-DUSE_RDMA=2") + find_package(PkgConfig REQUIRED) + + # Locate librdmacm & libibverbs, fail if we can't find them + pkg_check_modules(RDMACM REQUIRED librdmacm) + pkg_check_modules(IBVERBS REQUIRED libibverbs) + + message(STATUS "${RDMACM_LINK_LIBRARIES};${IBVERBS_LINK_LIBRARIES}") + list(APPEND RDMA_LIBS "${RDMACM_LIBRARIES};${IBVERBS_LIBRARIES}") + unset(RDMACM_LINK_LIBRARIES CACHE) + unset(IBVERBS_LINK_LIBRARIES CACHE) + set(BUILD_RDMA_MODULE 1) + elseif (USE_RDMA EQUAL 1) + # RDMA can only be built as a module. So disable it + message(WARNING "BUILD_RDMA can be one of: [NO | 0 | MODULE], but '${BUILD_RDMA}' was provided") + message(STATUS "RDMA build is disabled") + set(USE_RDMA 0) + endif () + else () + message(WARNING "RDMA is only supported on Linux platforms") + endif () +endif () + +set(BUILDING_ARM64 0) +set(BUILDING_ARM32 0) + +if ("${CMAKE_SYSTEM_PROCESSOR}" STREQUAL "arm64") + set(BUILDING_ARM64 1) +endif () + +if ("${CMAKE_SYSTEM_PROCESSOR}" STREQUAL "arm") + set(BUILDING_ARM32 1) +endif () + +message(STATUS "Building on ${CMAKE_HOST_SYSTEM_NAME}") +if (BUILDING_ARM64) + message(STATUS "Compiling valkey for ARM64") + add_valkey_server_linker_option("-funwind-tables") +endif () + +if (APPLE) + add_valkey_server_linker_option("-rdynamic") + add_valkey_server_linker_option("-ldl") +elseif (UNIX) + add_valkey_server_linker_option("-rdynamic") + add_valkey_server_linker_option("-pthread") + add_valkey_server_linker_option("-ldl") + add_valkey_server_linker_option("-lm") +endif () + +if (VALKEY_DEBUG_BUILD) + # Debug build, use enable "-fno-omit-frame-pointer" + add_valkey_server_compiler_options("-fno-omit-frame-pointer") +endif () + +# Check for Atomic +check_include_files(stdatomic.h HAVE_C11_ATOMIC) +if (HAVE_C11_ATOMIC) + add_valkey_server_compiler_options("-std=gnu11") +else () + add_valkey_server_compiler_options("-std=c99") +endif () + +# Sanitizer +if (BUILD_SANITIZER) + # For best results, force libc + set(MALLOC_LIB, "libc") + if ("${BUILD_SANITIZER}" STREQUAL "address") + add_valkey_server_compiler_options("-fsanitize=address -fno-sanitize-recover=all -fno-omit-frame-pointer") + add_valkey_server_linker_option("-fsanitize=address") + elseif ("${BUILD_SANITIZER}" STREQUAL "thread") + add_valkey_server_compiler_options("-fsanitize=thread -fno-sanitize-recover=all -fno-omit-frame-pointer") + add_valkey_server_linker_option("-fsanitize=thread") + elseif ("${BUILD_SANITIZER}" STREQUAL "undefined") + add_valkey_server_compiler_options("-fsanitize=undefined -fno-sanitize-recover=all -fno-omit-frame-pointer") + add_valkey_server_linker_option("-fsanitize=undefined") + else () + message(FATAL_ERROR "Unknown sanitizer: ${BUILD_SANITIZER}") + endif () +endif () + +include_directories("${CMAKE_SOURCE_DIR}/deps/hiredis") +include_directories("${CMAKE_SOURCE_DIR}/deps/linenoise") +include_directories("${CMAKE_SOURCE_DIR}/deps/lua/src") +include_directories("${CMAKE_SOURCE_DIR}/deps/hdr_histogram") +include_directories("${CMAKE_SOURCE_DIR}/deps/fpconv") + +add_subdirectory("${CMAKE_SOURCE_DIR}/deps") + +# Update linker flags for the allocator +if (USE_JEMALLOC) + include_directories("${CMAKE_SOURCE_DIR}/deps/jemalloc/include") +endif () + +# Common compiler flags +add_valkey_server_compiler_options("-pedantic") + +# ---------------------------------------------------- +# Build options (allocator, tls, rdma et al) - end +# ---------------------------------------------------- + +# ------------------------------------------------- +# Code Generation section +# ------------------------------------------------- +find_program(PYTHON_EXE python3) +if (PYTHON_EXE) + # Python based code generation + message(STATUS "Found python3: ${PYTHON_EXE}") + # Rule for generating commands.def file from json files + message(STATUS "Adding target generate_commands_def") + file(GLOB COMMAND_FILES_JSON "${CMAKE_SOURCE_DIR}/src/commands/*.json") + add_custom_command( + OUTPUT ${CMAKE_BINARY_DIR}/commands_def_generated + DEPENDS ${COMMAND_FILES_JSON} + COMMAND ${PYTHON_EXE} ${CMAKE_SOURCE_DIR}/utils/generate-command-code.py + COMMAND touch ${CMAKE_BINARY_DIR}/commands_def_generated + WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}/src") + add_custom_target(generate_commands_def DEPENDS ${CMAKE_BINARY_DIR}/commands_def_generated) + + # Rule for generating fmtargs.h + message(STATUS "Adding target generate_fmtargs_h") + add_custom_command( + OUTPUT ${CMAKE_BINARY_DIR}/fmtargs_generated + DEPENDS ${CMAKE_SOURCE_DIR}/utils/generate-fmtargs.py + COMMAND sed '/Everything/,$$d' fmtargs.h > fmtargs.h.tmp + COMMAND ${PYTHON_EXE} ${CMAKE_SOURCE_DIR}/utils/generate-fmtargs.py >> fmtargs.h.tmp + COMMAND mv fmtargs.h.tmp fmtargs.h + COMMAND touch ${CMAKE_BINARY_DIR}/fmtargs_generated + WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}/src") + add_custom_target(generate_fmtargs_h DEPENDS ${CMAKE_BINARY_DIR}/fmtargs_generated) + + # Rule for generating test_files.h + message(STATUS "Adding target generate_test_files_h") + file(GLOB UNIT_TEST_SRCS "${CMAKE_SOURCE_DIR}/src/unit/*.c") + add_custom_command( + OUTPUT ${CMAKE_BINARY_DIR}/test_files_generated + DEPENDS "${UNIT_TEST_SRCS};${CMAKE_SOURCE_DIR}/utils/generate-unit-test-header.py" + COMMAND ${PYTHON_EXE} ${CMAKE_SOURCE_DIR}/utils/generate-unit-test-header.py + COMMAND touch ${CMAKE_BINARY_DIR}/test_files_generated + WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}/src") + add_custom_target(generate_test_files_h DEPENDS ${CMAKE_BINARY_DIR}/test_files_generated) +else () + # Fake targets + add_custom_target(generate_commands_def) + add_custom_target(generate_fmtargs_h) + add_custom_target(generate_test_files_h) +endif () + +# Generate release.h file (always) +add_custom_target( + release_header + COMMAND sh -c '${CMAKE_SOURCE_DIR}/src/mkreleasehdr.sh' + WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}/src") + +# ------------------------------------------------- +# Code Generation section - end +# ------------------------------------------------- + +# ---------------------------------------------------------- +# All our source files are defined in SourceFiles.cmake file +# ---------------------------------------------------------- +include(SourceFiles) + +# Clear the below variables from the cache +unset(CMAKE_C_FLAGS CACHE) +unset(BUILD_SANITIZER CACHE) +unset(VALKEY_SERVER_LDFLAGS CACHE) +unset(VALKEY_SERVER_CFLAGS CACHE) +unset(PYTHON_EXE CACHE) +unset(HAVE_C11_ATOMIC CACHE) +unset(USE_TLS CACHE) +unset(USE_RDMA CACHE) +unset(BUILD_TLS CACHE) +unset(BUILD_RDMA CACHE) +unset(BUILD_MALLOC CACHE) +unset(USE_JEMALLOC CACHE) +unset(BUILD_TLS_MODULE CACHE) +unset(BUILD_TLS_BUILTIN CACHE) diff --git a/deps/CMakeLists.txt b/deps/CMakeLists.txt new file mode 100644 index 0000000000..c904b94031 --- /dev/null +++ b/deps/CMakeLists.txt @@ -0,0 +1,26 @@ +add_subdirectory(jemalloc) +add_subdirectory(lua) + +# Set hiredis options. We need to disable the defaults set in the OPTION(..) we do this by setting them in the CACHE +set(BUILD_SHARED_LIBS + OFF + CACHE BOOL "Build shared libraries") +set(DISABLE_TESTS + ON + CACHE BOOL "If tests should be compiled or not") +if (USE_TLS) # Module or no module + message(STATUS "Building hiredis_ssl") + set(ENABLE_SSL + ON + CACHE BOOL "Should we test SSL connections") +endif () + +add_subdirectory(hiredis) +add_subdirectory(linenoise) +add_subdirectory(fpconv) +add_subdirectory(hdr_histogram) + +# Clear any cached variables passed to hiredis from the cache +unset(BUILD_SHARED_LIBS CACHE) +unset(DISABLE_TESTS CACHE) +unset(ENABLE_SSL CACHE) diff --git a/deps/fpconv/CMakeLists.txt b/deps/fpconv/CMakeLists.txt new file mode 100644 index 0000000000..c586aa650a --- /dev/null +++ b/deps/fpconv/CMakeLists.txt @@ -0,0 +1,4 @@ +project(fpconv) + +set(SRCS "${CMAKE_CURRENT_LIST_DIR}/fpconv_dtoa.c" "${CMAKE_CURRENT_LIST_DIR}/fpconv_dtoa.h") +add_library(fpconv STATIC ${SRCS}) diff --git a/deps/hdr_histogram/CMakeLists.txt b/deps/hdr_histogram/CMakeLists.txt new file mode 100644 index 0000000000..7b45bd76ba --- /dev/null +++ b/deps/hdr_histogram/CMakeLists.txt @@ -0,0 +1,7 @@ +project(hdr_histogram) + +set(SRCS "${CMAKE_CURRENT_LIST_DIR}/hdr_histogram.c" "${CMAKE_CURRENT_LIST_DIR}/hdr_histogram.h" + "${CMAKE_CURRENT_LIST_DIR}/hdr_atomic.h" "${CMAKE_CURRENT_LIST_DIR}/hdr_redis_malloc.h") + +add_library(hdr_histogram STATIC ${SRCS}) +target_compile_definitions(hdr_histogram PRIVATE HDR_MALLOC_INCLUDE=\"hdr_redis_malloc.h\") diff --git a/deps/jemalloc/CMakeLists.txt b/deps/jemalloc/CMakeLists.txt new file mode 100644 index 0000000000..e79e960ec2 --- /dev/null +++ b/deps/jemalloc/CMakeLists.txt @@ -0,0 +1,23 @@ +project(jemalloc) + +# Build jemalloc using configure && make install +set(JEMALLOC_INSTALL_DIR ${CMAKE_BINARY_DIR}/jemalloc-build) +set(JEMALLOC_SRC_DIR ${CMAKE_CURRENT_LIST_DIR}) +if (NOT EXISTS ${JEMALLOC_INSTALL_DIR}/lib/libjemalloc.a) + message(STATUS "Building jemalloc (custom build)") + message(STATUS "JEMALLOC_SRC_DIR = ${JEMALLOC_SRC_DIR}") + message(STATUS "JEMALLOC_INSTALL_DIR = ${JEMALLOC_INSTALL_DIR}") + + execute_process( + COMMAND sh -c "${JEMALLOC_SRC_DIR}/configure --disable-cxx \ + --with-version=5.3.0-0-g0 --with-lg-quantum=3 --disable-cache-oblivious --with-jemalloc-prefix=je_ \ + --enable-static --disable-shared --prefix=${JEMALLOC_INSTALL_DIR}" + WORKING_DIRECTORY ${JEMALLOC_SRC_DIR} COMMAND_ERROR_IS_FATAL ANY) + execute_process(COMMAND make -j${VALKEY_PROCESSOR_COUNT} lib/libjemalloc.a install + WORKING_DIRECTORY "${JEMALLOC_SRC_DIR}") +endif () + +# Import the compiled library as a CMake target +add_library(jemalloc STATIC IMPORTED GLOBAL) +set_target_properties(jemalloc PROPERTIES IMPORTED_LOCATION "${JEMALLOC_INSTALL_DIR}/lib/libjemalloc.a" + INCLUDE_DIRECTORIES "${JEMALLOC_INSTALL_DIR}/include") diff --git a/deps/linenoise/CMakeLists.txt b/deps/linenoise/CMakeLists.txt new file mode 100644 index 0000000000..f801e4abf1 --- /dev/null +++ b/deps/linenoise/CMakeLists.txt @@ -0,0 +1,4 @@ +project(linenoise) + +set(SRCS "${CMAKE_CURRENT_LIST_DIR}/linenoise.c" "${CMAKE_CURRENT_LIST_DIR}/linenoise.h") +add_library(linenoise STATIC ${SRCS}) diff --git a/deps/lua/CMakeLists.txt b/deps/lua/CMakeLists.txt new file mode 100644 index 0000000000..e911de9232 --- /dev/null +++ b/deps/lua/CMakeLists.txt @@ -0,0 +1,44 @@ +project(lualib) + +set(LUA_SRC_DIR "${CMAKE_CURRENT_LIST_DIR}/src") +set(LUA_SRCS + ${LUA_SRC_DIR}/fpconv.c + ${LUA_SRC_DIR}/lbaselib.c + ${LUA_SRC_DIR}/lmathlib.c + ${LUA_SRC_DIR}/lstring.c + ${LUA_SRC_DIR}/lparser.c + ${LUA_SRC_DIR}/ldo.c + ${LUA_SRC_DIR}/lzio.c + ${LUA_SRC_DIR}/lmem.c + ${LUA_SRC_DIR}/strbuf.c + ${LUA_SRC_DIR}/lstrlib.c + ${LUA_SRC_DIR}/lundump.c + ${LUA_SRC_DIR}/lua_cmsgpack.c + ${LUA_SRC_DIR}/loslib.c + ${LUA_SRC_DIR}/lua_struct.c + ${LUA_SRC_DIR}/ldebug.c + ${LUA_SRC_DIR}/lobject.c + ${LUA_SRC_DIR}/ldump.c + ${LUA_SRC_DIR}/lua_cjson.c + ${LUA_SRC_DIR}/ldblib.c + ${LUA_SRC_DIR}/ltm.c + ${LUA_SRC_DIR}/ltable.c + ${LUA_SRC_DIR}/lstate.c + ${LUA_SRC_DIR}/lua_bit.c + ${LUA_SRC_DIR}/lua.c + ${LUA_SRC_DIR}/loadlib.c + ${LUA_SRC_DIR}/lcode.c + ${LUA_SRC_DIR}/lapi.c + ${LUA_SRC_DIR}/lgc.c + ${LUA_SRC_DIR}/lvm.c + ${LUA_SRC_DIR}/lfunc.c + ${LUA_SRC_DIR}/lauxlib.c + ${LUA_SRC_DIR}/ltablib.c + ${LUA_SRC_DIR}/linit.c + ${LUA_SRC_DIR}/lopcodes.c + ${LUA_SRC_DIR}/llex.c + ${LUA_SRC_DIR}/liolib.c) + +add_library(lualib STATIC "${LUA_SRCS}") +target_include_directories(lualib PUBLIC "${LUA_SRC_DIR}") +target_compile_definitions(lualib PRIVATE ENABLE_CJSON_GLOBAL) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt new file mode 100644 index 0000000000..b7e328163b --- /dev/null +++ b/src/CMakeLists.txt @@ -0,0 +1,77 @@ +project(valkey-server) + +set(INSTALL_BIN_PATH ${CMAKE_INSTALL_PREFIX}/bin) +set_directory_properties(PROPERTIES CLEAN_NO_CUSTOM 1) + +# Target: valkey-server +set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${VALKEY_SERVER_CFLAGS}") +message(STATUS "CFLAGS: ${CMAKE_C_FLAGS}") + +get_valkey_server_linker_option(VALKEY_SERVER_LDFLAGS) +list(APPEND SERVER_LIBS "fpconv") +list(APPEND SERVER_LIBS "lualib") +list(APPEND SERVER_LIBS "hdr_histogram") +valkey_build_and_install_bin(valkey-server "${VALKEY_SERVER_SRCS}" "${VALKEY_SERVER_LDFLAGS}" "${SERVER_LIBS}" + "redis-server") +add_dependencies(valkey-server generate_commands_def) +add_dependencies(valkey-server generate_fmtargs_h) +add_dependencies(valkey-server release_header) + +if (VALKEY_RELEASE_BUILD) + # Enable LTO for Release build + set_property(TARGET valkey-server PROPERTY INTERPROCEDURAL_OPTIMIZATION TRUE) +endif () + +# Target: valkey-cli +list(APPEND CLI_LIBS "linenoise") +valkey_build_and_install_bin(valkey-cli "${VALKEY_CLI_SRCS}" "${VALKEY_SERVER_LDFLAGS}" "${CLI_LIBS}" "redis-cli") +add_dependencies(valkey-cli generate_commands_def) +add_dependencies(valkey-cli generate_fmtargs_h) + +# Target: valkey-benchmark +list(APPEND BENCH_LIBS "hdr_histogram") +valkey_build_and_install_bin(valkey-benchmark "${VALKEY_BENCHMARK_SRCS}" "${VALKEY_SERVER_LDFLAGS}" "${BENCH_LIBS}" + "redis-benchmark") +add_dependencies(valkey-benchmark generate_commands_def) +add_dependencies(valkey-benchmark generate_fmtargs_h) + +# Targets: valkey-sentinel, valkey-check-aof and valkey-check-rdb are just symbolic links +valkey_create_symlink("valkey-server" "valkey-sentinel") +valkey_create_symlink("valkey-server" "valkey-check-rdb") +valkey_create_symlink("valkey-server" "valkey-check-aof") + +# Target valkey-rdma +if (BUILD_RDMA_MODULE) + set(MODULE_NAME "valkey-rdma") + message(STATUS "Building RDMA module") + add_library(${MODULE_NAME} SHARED "${VALKEY_RDMA_MODULE_SRCS}") + target_compile_options(${MODULE_NAME} PRIVATE -DBUILD_RDMA_MODULE -DUSE_RDMA=1) + target_link_libraries(${MODULE_NAME} "${RDMA_LIBS}") + # remove the "lib" prefix from the module + set_target_properties(${MODULE_NAME} PROPERTIES PREFIX "") + valkey_install_bin(${MODULE_NAME}) +endif () + +# Target valkey-tls (a module) +if (BUILD_TLS_MODULE) + message(STATUS "Building TLS as a module") + set(MODULE_NAME "valkey-tls") + add_library(${MODULE_NAME} SHARED ${VALKEY_TLS_MODULE_SRCS}) + target_compile_options(${MODULE_NAME} PRIVATE -DUSE_OPENSSL=2 -DBUILD_TLS_MODULE=2) + if (APPLE) + # Some symbols can only be resolved during runtime (they exist in the executable) + target_link_options(${MODULE_NAME} PRIVATE -undefined dynamic_lookup) + endif () + target_link_libraries(${MODULE_NAME} hiredis_ssl OpenSSL::SSL) + set_target_properties(${MODULE_NAME} PROPERTIES PREFIX "") +endif () + +if (BUILD_EXAMPLE_MODULES) + # Include the modules ("hello*") + message(STATUS "Building example modules") + add_subdirectory(modules) +endif () + +if (BUILD_UNIT_TESTS) + add_subdirectory(unit) +endif () diff --git a/src/modules/CMakeLists.txt b/src/modules/CMakeLists.txt new file mode 100644 index 0000000000..958796232f --- /dev/null +++ b/src/modules/CMakeLists.txt @@ -0,0 +1,21 @@ +# Build modules +list(APPEND MODULES_LIST "helloacl") +list(APPEND MODULES_LIST "helloblock") +list(APPEND MODULES_LIST "hellocluster") +list(APPEND MODULES_LIST "hellodict") +list(APPEND MODULES_LIST "hellohook") +list(APPEND MODULES_LIST "hellotimer") +list(APPEND MODULES_LIST "hellotype") +list(APPEND MODULES_LIST "helloworld") + +foreach (MODULE_NAME ${MODULES_LIST}) + message(STATUS "Building module: ${MODULE_NAME}") + add_library(${MODULE_NAME} SHARED "${CMAKE_CURRENT_LIST_DIR}/${MODULE_NAME}.c") + target_include_directories(${MODULE_NAME} PRIVATE "${CMAKE_SOURCE_DIR}/src") + set_target_properties(${MODULE_NAME} PROPERTIES PREFIX "") + valkey_install_bin(${MODULE_NAME}) + if (APPLE) + # Some symbols can only be resolved during runtime (they exist in the executable) + target_link_options(${MODULE_NAME} PRIVATE -undefined dynamic_lookup) + endif () +endforeach () diff --git a/src/server.c b/src/server.c index 06a0c8e952..a32cc1ec42 100644 --- a/src/server.c +++ b/src/server.c @@ -7157,5 +7157,4 @@ __attribute__((weak)) int main(int argc, char **argv) { aeDeleteEventLoop(server.el); return 0; } - /* The End */ diff --git a/src/unit/CMakeLists.txt b/src/unit/CMakeLists.txt new file mode 100644 index 0000000000..7d80c533cf --- /dev/null +++ b/src/unit/CMakeLists.txt @@ -0,0 +1,58 @@ +project(valkey-unit-tests) + +file(GLOB UNIT_TEST_SRCS "${CMAKE_CURRENT_LIST_DIR}/*.c") +set(UNIT_TEST_SRCS "${UNIT_TEST_SRCS}") + +get_valkey_server_linker_option(VALKEY_SERVER_LDFLAGS) + +# Build unit tests only +message(STATUS "Building unit tests") +list(APPEND COMPILE_DEFINITIONS "SERVER_TEST=1") +if (USE_TLS) + if (BUILD_TLS_MODULE) + # TLS as a module + list(APPEND COMPILE_DEFINITIONS "USE_OPENSSL=2") + else (BUILD_TLS_MODULE) + # Built-in TLS support + list(APPEND COMPILE_DEFINITIONS "USE_OPENSSL=1") + list(APPEND COMPILE_DEFINITIONS "BUILD_TLS_MODULE=0") + endif () +endif () + +# Build Valkey sources as a static library for the test +add_library(valkeylib STATIC ${VALKEY_SERVER_SRCS}) +target_compile_options(valkeylib PRIVATE "${COMPILE_FLAGS}") +target_compile_definitions(valkeylib PRIVATE "${COMPILE_DEFINITIONS}") + +add_executable(valkey-unit-tests ${UNIT_TEST_SRCS}) +target_compile_options(valkey-unit-tests PRIVATE "${COMPILE_FLAGS}") +target_compile_definitions(valkey-unit-tests PRIVATE "${COMPILE_DEFINITIONS}") +add_dependencies(valkey-unit-tests generate_test_files_h) + +if (UNIX AND NOT APPLE) + # Avoid duplicate symbols on non macOS + target_link_options(valkey-unit-tests PRIVATE "-Wl,--allow-multiple-definition") +endif () + +if (USE_JEMALLOC) + # Using jemalloc + target_link_libraries(valkey-unit-tests jemalloc) +endif () + +if (IS_FREEBSD) + target_link_libraries(valkey-unit-tests execinfo) +endif () + +target_link_libraries( + valkey-unit-tests + valkeylib + fpconv + lualib + hdr_histogram + hiredis + ${VALKEY_SERVER_LDFLAGS}) + +if (USE_TLS) + # Add required libraries needed for TLS + target_link_libraries(valkey-unit-tests OpenSSL::SSL hiredis_ssl) +endif () diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt new file mode 100644 index 0000000000..2a76897bb0 --- /dev/null +++ b/tests/CMakeLists.txt @@ -0,0 +1,5 @@ +add_subdirectory(rdma) + +if (BUILD_TEST_MODULES) + add_subdirectory(modules) +endif () diff --git a/tests/modules/CMakeLists.txt b/tests/modules/CMakeLists.txt new file mode 100644 index 0000000000..0cac0c4cb6 --- /dev/null +++ b/tests/modules/CMakeLists.txt @@ -0,0 +1,58 @@ +# Build test modules +list(APPEND MODULES_LIST "commandfilter") +list(APPEND MODULES_LIST "basics") +list(APPEND MODULES_LIST "testrdb") +list(APPEND MODULES_LIST "fork") +list(APPEND MODULES_LIST "infotest") +list(APPEND MODULES_LIST "propagate") +list(APPEND MODULES_LIST "misc") +list(APPEND MODULES_LIST "hooks") +list(APPEND MODULES_LIST "blockonkeys") +list(APPEND MODULES_LIST "blockonbackground") +list(APPEND MODULES_LIST "scan") +list(APPEND MODULES_LIST "datatype") +list(APPEND MODULES_LIST "datatype2") +list(APPEND MODULES_LIST "auth") +list(APPEND MODULES_LIST "keyspace_events") +list(APPEND MODULES_LIST "blockedclient") +list(APPEND MODULES_LIST "getkeys") +list(APPEND MODULES_LIST "getchannels") +list(APPEND MODULES_LIST "test_lazyfree") +list(APPEND MODULES_LIST "timer") +list(APPEND MODULES_LIST "defragtest") +list(APPEND MODULES_LIST "keyspecs") +list(APPEND MODULES_LIST "hash") +list(APPEND MODULES_LIST "zset") +list(APPEND MODULES_LIST "stream") +list(APPEND MODULES_LIST "mallocsize") +list(APPEND MODULES_LIST "aclcheck") +list(APPEND MODULES_LIST "list") +list(APPEND MODULES_LIST "subcommands") +list(APPEND MODULES_LIST "reply") +list(APPEND MODULES_LIST "cmdintrospection") +list(APPEND MODULES_LIST "eventloop") +list(APPEND MODULES_LIST "moduleconfigs") +list(APPEND MODULES_LIST "moduleconfigstwo") +list(APPEND MODULES_LIST "publish") +list(APPEND MODULES_LIST "usercall") +list(APPEND MODULES_LIST "postnotifications") +list(APPEND MODULES_LIST "moduleauthtwo") +list(APPEND MODULES_LIST "rdbloadsave") +list(APPEND MODULES_LIST "crash") +list(APPEND MODULES_LIST "cluster") + +foreach (MODULE_NAME ${MODULES_LIST}) + message(STATUS "Building test module: ${MODULE_NAME}") + add_library(${MODULE_NAME} SHARED "${CMAKE_SOURCE_DIR}/tests/modules/${MODULE_NAME}.c") + target_include_directories(${MODULE_NAME} PRIVATE "${CMAKE_SOURCE_DIR}/src") + if (LINUX AND NOT APPLE) + # set the std to gnu11 here, to allow crash.c to get compiled + target_compile_options(${MODULE_NAME} PRIVATE "-std=gnu11") + endif () + set_target_properties(${MODULE_NAME} PROPERTIES PREFIX "") + valkey_install_bin(${MODULE_NAME}) + if (APPLE) + # Some symbols can only be resolved during runtime (they exist in the executable) + target_link_options(${MODULE_NAME} PRIVATE -undefined dynamic_lookup) + endif () +endforeach () diff --git a/tests/rdma/CMakeLists.txt b/tests/rdma/CMakeLists.txt new file mode 100644 index 0000000000..f721e9af52 --- /dev/null +++ b/tests/rdma/CMakeLists.txt @@ -0,0 +1,9 @@ +project(rdma-test) + +# Make sure RDMA build is enabled +if (BUILD_RDMA_MODULE) + add_executable(rdma-test "${CMAKE_SOURCE_DIR}/tests/rdma/rdma-test.c") + target_link_libraries(rdma-test "${RDMA_LIBS}") + target_link_options(rdma-test PRIVATE "-pthread") + valkey_install_bin(rdma-test) +endif ()