Merge branch 'valkey-io:unstable' into skolosov/replicate-no-one

valkey-io · Feb 27, 2025 · c636c6b · c636c6b
2 parents c8a713b + e03b3f1
commit c636c6b
Show file tree

Hide file tree

Showing 98 changed files with 4,225 additions and 2,179 deletions.
diff --git a/.config/typos.toml b/.config/typos.toml
@@ -13,6 +13,7 @@ ignore-hidden = false
 exat = "exat"
 optin = "optin"
 smove = "smove"
+Parth = "Parth" # seems like the spellchecker does not like it is similar to "Path"
 
 [type.c]
 extend-ignore-re = [

diff --git a/.github/actions/generate-package-build-matrix/build-config.json b/.github/actions/generate-package-build-matrix/build-config.json
@@ -21,15 +21,15 @@
         },
         {
           "arch": "arm64",
-          "target": "ubuntu20.04",
+          "target": "ubuntu-22.04-arm",
           "type": "deb",
-          "platform": "focal"
+          "platform": "jammy"
         },
         {
           "arch": "arm64",
-          "target": "ubuntu22.04",
+          "target": "ubuntu-24.04-arm",
           "type": "deb",
-          "platform": "jammy"
+          "platform": "noble"
         }
       ]
 }
diff --git a/.github/workflows/call-build-linux-arm-packages.yml b/.github/workflows/call-build-linux-arm-packages.yml
@@ -35,7 +35,7 @@ jobs:
   build-valkey:
     # Capture source tarball and generate checksum for it
     name: Build package ${{ matrix.distro.target }} ${{ matrix.distro.arch }}
-    runs-on: "ubuntu-latest"
+    runs-on: ${{matrix.distro.target}}
     strategy:
       fail-fast: false
       matrix: ${{ fromJSON(inputs.build_matrix) }}
@@ -51,13 +51,11 @@ jobs:
           aws-region: ${{ inputs.region }}
           role-to-assume: ${{ secrets.role_to_assume }}
 
+      - name: Install dependencies
+        run: sudo apt-get update && sudo apt-get install -y build-essential libssl-dev libsystemd-dev
+
       - name: Make Valkey
-        uses: uraimo/run-on-arch-action@v2
-        with:
-          arch: aarch64
-          distro: ${{matrix.distro.target}}
-          install: apt-get update && apt-get install -y build-essential libssl-dev libsystemd-dev
-          run: make -C src all BUILD_TLS=yes USE_SYSTEMD=yes
+        run: make -C src all BUILD_TLS=yes USE_SYSTEMD=yes
 
       - name: Create Tarball and SHA256sums
         run: |

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -70,13 +70,15 @@ jobs:
       - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
       - name: make
         # build with TLS module just for compilation coverage
-        run: make -j4 SANITIZER=address SERVER_CFLAGS='-Werror' BUILD_TLS=module
+        run: make -j4 all-with-unit-tests SANITIZER=address SERVER_CFLAGS='-Werror' BUILD_TLS=module
       - name: testprep
         run: sudo apt-get install tcl8.6 tclx -y
       - name: test
         run: ./runtest --verbose --tags -slow --dump-logs
       - name: module api test
         run: CFLAGS='-Werror' ./runtest-moduleapi --verbose --dump-logs
+      - name: unit tests
+        run: ./src/valkey-unit-tests
 
   test-rdma:
     runs-on: ubuntu-latest

diff --git a/.github/workflows/daily.yml b/.github/workflows/daily.yml
@@ -12,7 +12,7 @@ on:
     inputs:
       skipjobs:
         description: "jobs to skip (delete the ones you wanna keep, do not leave empty)"
-        default: "valgrind,sanitizer,tls,freebsd,macos,alpine,32bit,iothreads,ubuntu,rpm-distros,malloc,specific,fortify,reply-schema"
+        default: "valgrind,sanitizer,tls,freebsd,macos,alpine,32bit,iothreads,ubuntu,rpm-distros,malloc,specific,fortify,reply-schema,arm"
       skiptests:
         description: "tests to skip (delete the ones you wanna keep, do not leave empty)"
         default: "valkey,modules,sentinel,cluster,unittest"
@@ -79,6 +79,48 @@ jobs:
         if: true && !contains(github.event.inputs.skiptests, 'unittest')
         run: ./src/valkey-unit-tests --accurate
 
+  test-ubuntu-arm:
+    runs-on: ubuntu-24.04-arm
+    if: |
+      (github.event_name == 'workflow_dispatch' ||
+        (github.event_name == 'schedule' && github.repository == 'valkey-io/valkey') ||
+        (github.event_name == 'pull_request' && (contains(github.event.pull_request.labels.*.name, 'run-extra-tests') || github.event.pull_request.base.ref != 'unstable'))) &&
+      (!contains(github.event.inputs.skipjobs, 'ubuntu') || !contains(github.event.inputs.skipjobs, 'arm'))
+    timeout-minutes: 14400
+    steps:
+      - name: prep
+        if: github.event_name == 'workflow_dispatch'
+        run: |
+          echo "GITHUB_REPOSITORY=${{github.event.inputs.use_repo}}" >> $GITHUB_ENV
+          echo "GITHUB_HEAD_REF=${{github.event.inputs.use_git_ref}}" >> $GITHUB_ENV
+          echo "skipjobs: ${{github.event.inputs.skipjobs}}"
+          echo "skiptests: ${{github.event.inputs.skiptests}}"
+          echo "test_args: ${{github.event.inputs.test_args}}"
+          echo "cluster_test_args: ${{github.event.inputs.cluster_test_args}}"
+      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+        with:
+          repository: ${{ env.GITHUB_REPOSITORY }}
+          ref: ${{ env.GITHUB_HEAD_REF }}
+      - name: make
+        run: make all-with-unit-tests SERVER_CFLAGS='-Werror'
+      - name: testprep
+        run: sudo apt-get install tcl8.6 tclx
+      - name: test
+        if: true && !contains(github.event.inputs.skiptests, 'valkey')
+        run: ./runtest ${{ github.event_name != 'pull_request' && '--accurate' || '' }} --verbose --dump-logs ${{github.event.inputs.test_args}}
+      - name: module api test
+        if: true && !contains(github.event.inputs.skiptests, 'modules')
+        run: CFLAGS='-Werror' ./runtest-moduleapi --verbose --dump-logs ${{github.event.inputs.test_args}}
+      - name: sentinel tests
+        if: true && !contains(github.event.inputs.skiptests, 'sentinel')
+        run: ./runtest-sentinel ${{github.event.inputs.cluster_test_args}}
+      - name: cluster tests
+        if: true && !contains(github.event.inputs.skiptests, 'cluster')
+        run: ./runtest-cluster ${{github.event.inputs.cluster_test_args}}
+      - name: unittest
+        if: true && !contains(github.event.inputs.skiptests, 'unittest')
+        run: ./src/valkey-unit-tests --accurate
+
   test-ubuntu-jemalloc-fortify:
     runs-on: ubuntu-latest
     if: |
@@ -663,10 +705,10 @@ jobs:
           sudo apt-get install tcl8.6 tclx -y
       - name: test
         if: true && !contains(github.event.inputs.skiptests, 'valkey')
-        run: ./runtest --accurate --verbose --dump-logs ${{github.event.inputs.test_args}}
+        run: ./runtest --accurate --verbose --dump-logs --large-memory ${{github.event.inputs.test_args}}
       - name: module api test
         if: true && !contains(github.event.inputs.skiptests, 'modules')
-        run: CFLAGS='-Werror' ./runtest-moduleapi --verbose --dump-logs ${{github.event.inputs.test_args}}
+        run: CFLAGS='-Werror' ./runtest-moduleapi --verbose --dump-logs --large-memory ${{github.event.inputs.test_args}}
       - name: sentinel tests
         if: true && !contains(github.event.inputs.skiptests, 'sentinel')
         run: ./runtest-sentinel ${{github.event.inputs.cluster_test_args}}
@@ -675,7 +717,7 @@ jobs:
         run: ./runtest-cluster ${{github.event.inputs.cluster_test_args}}
       - name: unittest
         if: true && !contains(github.event.inputs.skiptests, 'unittest')
-        run: ./src/valkey-unit-tests
+        run: ./src/valkey-unit-tests --large-memory
 
   test-sanitizer-undefined:
     runs-on: ubuntu-latest
@@ -713,10 +755,10 @@ jobs:
           sudo apt-get install tcl8.6 tclx -y
       - name: test
         if: true && !contains(github.event.inputs.skiptests, 'valkey')
-        run: ./runtest --accurate --verbose --dump-logs ${{github.event.inputs.test_args}}
+        run: ./runtest --accurate --verbose --dump-logs --large-memory ${{github.event.inputs.test_args}}
       - name: module api test
         if: true && !contains(github.event.inputs.skiptests, 'modules')
-        run: CFLAGS='-Werror' ./runtest-moduleapi --verbose --dump-logs ${{github.event.inputs.test_args}}
+        run: CFLAGS='-Werror' ./runtest-moduleapi --verbose --dump-logs --large-memory ${{github.event.inputs.test_args}}
       - name: sentinel tests
         if: true && !contains(github.event.inputs.skiptests, 'sentinel')
         run: ./runtest-sentinel ${{github.event.inputs.cluster_test_args}}
@@ -725,7 +767,7 @@ jobs:
         run: ./runtest-cluster ${{github.event.inputs.cluster_test_args}}
       - name: unittest
         if: true && !contains(github.event.inputs.skiptests, 'unittest')
-        run: ./src/valkey-unit-tests --accurate
+        run: ./src/valkey-unit-tests --accurate --large-memory
 
   test-sanitizer-force-defrag:
     runs-on: ubuntu-latest
@@ -794,8 +836,8 @@ jobs:
             install_epel: true
           - name: test-fedoralatest-jemalloc
             container: fedora:latest
-          - name: test-fedorarawhide-jemalloc
-            container: fedora:rawhide
+            # - name: test-fedorarawhide-jemalloc
+            #   container: fedora:rawhide
 
     name: ${{ matrix.name }}
     runs-on: ubuntu-latest
@@ -860,8 +902,8 @@ jobs:
             install_epel: true
           - name: test-fedoralatest-tls-module
             container: fedora:latest
-          - name: test-fedorarawhide-tls-module
-            container: fedora:rawhide
+            # - name: test-fedorarawhide-tls-module
+            #   container: fedora:rawhide
 
     name: ${{ matrix.name }}
     runs-on: ubuntu-latest
@@ -932,8 +974,8 @@ jobs:
             install_epel: true
           - name: test-fedoralatest-tls-module-no-tls
             container: fedora:latest
-          - name: test-fedorarawhide-tls-module-no-tls
-            container: fedora:rawhide
+            # - name: test-fedorarawhide-tls-module-no-tls
+            #   container: fedora:rawhide
 
     name: ${{ matrix.name }}
     runs-on: ubuntu-latest
@@ -1262,7 +1304,36 @@ jobs:
   notify-about-job-results:
     runs-on: ubuntu-latest
     if: always() && github.event_name == 'schedule' && github.repository == 'valkey-io/valkey'
-    needs: [test-ubuntu-jemalloc, test-ubuntu-jemalloc-fortify, test-ubuntu-libc-malloc, test-ubuntu-no-malloc-usable-size, test-ubuntu-32bit, test-ubuntu-tls, test-ubuntu-tls-no-tls, test-ubuntu-io-threads, test-ubuntu-tls-io-threads, test-ubuntu-reclaim-cache, test-valgrind-test, test-valgrind-misc, test-valgrind-no-malloc-usable-size-test, test-valgrind-no-malloc-usable-size-misc, test-sanitizer-address, test-sanitizer-undefined, test-sanitizer-force-defrag, test-rpm-distros-jemalloc, test-rpm-distros-tls-module, test-rpm-distros-tls-module-no-tls, test-macos-latest, test-macos-latest-sentinel, test-macos-latest-cluster, build-macos, test-freebsd, test-alpine-jemalloc, test-alpine-libc-malloc, reply-schemas-validator]
+    needs:
+      - test-ubuntu-jemalloc
+      - test-ubuntu-arm
+      - test-ubuntu-jemalloc-fortify
+      - test-ubuntu-libc-malloc
+      - test-ubuntu-no-malloc-usable-size
+      - test-ubuntu-32bit
+      - test-ubuntu-tls
+      - test-ubuntu-tls-no-tls
+      - test-ubuntu-io-threads
+      - test-ubuntu-tls-io-threads
+      - test-ubuntu-reclaim-cache
+      - test-valgrind-test
+      - test-valgrind-misc
+      - test-valgrind-no-malloc-usable-size-test
+      - test-valgrind-no-malloc-usable-size-misc
+      - test-sanitizer-address
+      - test-sanitizer-undefined
+      - test-sanitizer-force-defrag
+      - test-rpm-distros-jemalloc
+      - test-rpm-distros-tls-module
+      - test-rpm-distros-tls-module-no-tls
+      - test-macos-latest
+      - test-macos-latest-sentinel
+      - test-macos-latest-cluster
+      - build-macos
+      - test-freebsd
+      - test-alpine-jemalloc
+      - test-alpine-libc-malloc
+      - reply-schemas-validator
     steps:
       - name: Collect job status
         run: |

diff --git a/COPYING b/COPYING
@@ -1,5 +1,7 @@
 # License 1
 
+SPDX-License-Identifier: BSD-3-Clause
+
 BSD 3-Clause License
 
 Copyright (c) 2024-present, Valkey contributors

diff --git a/cmake/Modules/SourceFiles.cmake b/cmake/Modules/SourceFiles.cmake
@@ -97,11 +97,13 @@ set(VALKEY_SERVER_SRCS
     ${CMAKE_SOURCE_DIR}/src/mt19937-64.c
     ${CMAKE_SOURCE_DIR}/src/resp_parser.c
     ${CMAKE_SOURCE_DIR}/src/call_reply.c
-    ${CMAKE_SOURCE_DIR}/src/script_lua.c
+    ${CMAKE_SOURCE_DIR}/src/lua/script_lua.c
     ${CMAKE_SOURCE_DIR}/src/script.c
     ${CMAKE_SOURCE_DIR}/src/functions.c
     ${CMAKE_SOURCE_DIR}/src/scripting_engine.c
-    ${CMAKE_SOURCE_DIR}/src/function_lua.c
+    ${CMAKE_SOURCE_DIR}/src/lua/function_lua.c
+    ${CMAKE_SOURCE_DIR}/src/lua/engine_lua.c
+    ${CMAKE_SOURCE_DIR}/src/lua/debug_lua.c
     ${CMAKE_SOURCE_DIR}/src/commands.c
     ${CMAKE_SOURCE_DIR}/src/strl.c
     ${CMAKE_SOURCE_DIR}/src/connection.c

diff --git a/deps/fast_float_c_interface/fast_float_strtod.cpp b/deps/fast_float_c_interface/fast_float_strtod.cpp
@@ -1,7 +1,7 @@
 /*
- * Copyright Valkey Contributors.
+ * Copyright (c) Valkey Contributors
  * All rights reserved.
- * SPDX-License-Identifier: BSD 3-Clause
+ * SPDX-License-Identifier: BSD-3-Clause
  */
 
 #include "../fast_float/fast_float.h"

diff --git a/deps/lua/src/lstring.c b/deps/lua/src/lstring.c
@@ -81,9 +81,9 @@ uint32_t murmur32(const uint8_t* key, size_t len, uint32_t seed) {
   static const uint32_t n = 0xe6546b64;
   uint32_t hash = seed;
 
-  const int nblocks = len / 4;
+  const size_t nblocks = len / 4;
   const uint32_t* blocks = (const uint32_t*) key;
-  for (int i = 0; i < nblocks; i++) {
+  for (size_t i = 0; i < nblocks; i++) {
     uint32_t k = blocks[i];
     k *= c1;
     k = (k << r1) | (k >> (32 - r1));
@@ -116,7 +116,7 @@ uint32_t murmur32(const uint8_t* key, size_t len, uint32_t seed) {
     hash ^= (hash >> 16);
 
     return hash;
-  }
+}
 
 TString *luaS_newlstr (lua_State *L, const char *str, size_t l) {
   GCObject *o;

diff --git a/deps/lua/src/lua_cjson.c b/deps/lua/src/lua_cjson.c
@@ -469,23 +469,19 @@ static void json_append_string(lua_State *l, strbuf_t *json, int lindex)
 
     str = lua_tolstring(l, lindex, &len);
 
-    /* Worst case is len * 6 (all unicode escapes).
-     * This buffer is reused constantly for small strings
-     * If there are any excess pages, they won't be hit anyway.
-     * This gains ~5% speedup. */
-    if (len > SIZE_MAX / 6 - 3)
+    if (len > SIZE_MAX - 3)
         abort(); /* Overflow check */
-    strbuf_ensure_empty_length(json, len * 6 + 2);
+    strbuf_ensure_empty_length(json, len + 2);
 
     strbuf_append_char_unsafe(json, '\"');
     for (i = 0; i < len; i++) {
         escstr = char2escape[(unsigned char)str[i]];
         if (escstr)
             strbuf_append_string(json, escstr);
         else
-            strbuf_append_char_unsafe(json, str[i]);
+            strbuf_append_char(json, str[i]);
     }
-    strbuf_append_char_unsafe(json, '\"');
+    strbuf_append_char(json, '\"');
 }
 
 /* Find the size of the array on the top of the Lua stack

diff --git a/src/Makefile b/src/Makefile
@@ -416,7 +416,7 @@ endif
 ENGINE_NAME=valkey
 SERVER_NAME=$(ENGINE_NAME)-server$(PROG_SUFFIX)
 ENGINE_SENTINEL_NAME=$(ENGINE_NAME)-sentinel$(PROG_SUFFIX)
-ENGINE_SERVER_OBJ=threads_mngr.o adlist.o quicklist.o ae.o anet.o dict.o hashtable.o kvstore.o server.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o memory_prefetch.o io_threads.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o cluster.o cluster_legacy.o cluster_slot_stats.o crc16.o endianconv.o commandlog.o eval.o bio.o rio.o rand.o memtest.o syscheck.o crcspeed.o crccombine.o crc64.o bitops.o sentinel.o notify.o setproctitle.o blocked.o hyperloglog.o latency.o sparkline.o valkey-check-rdb.o valkey-check-aof.o geo.o lazyfree.o module.o evict.o expire.o geohash.o geohash_helper.o childinfo.o allocator_defrag.o defrag.o siphash.o rax.o t_stream.o listpack.o localtime.o lolwut.o lolwut5.o lolwut6.o acl.o tracking.o socket.o tls.o sha256.o timeout.o setcpuaffinity.o monotonic.o mt19937-64.o resp_parser.o call_reply.o script_lua.o script.o functions.o function_lua.o commands.o strl.o connection.o unix.o logreqres.o rdma.o scripting_engine.o
+ENGINE_SERVER_OBJ=threads_mngr.o adlist.o quicklist.o ae.o anet.o dict.o hashtable.o kvstore.o server.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o memory_prefetch.o io_threads.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o cluster.o cluster_legacy.o cluster_slot_stats.o crc16.o endianconv.o commandlog.o eval.o bio.o rio.o rand.o memtest.o syscheck.o crcspeed.o crccombine.o crc64.o bitops.o sentinel.o notify.o setproctitle.o blocked.o hyperloglog.o latency.o sparkline.o valkey-check-rdb.o valkey-check-aof.o geo.o lazyfree.o module.o evict.o expire.o geohash.o geohash_helper.o childinfo.o allocator_defrag.o defrag.o siphash.o rax.o t_stream.o listpack.o localtime.o lolwut.o lolwut5.o lolwut6.o acl.o tracking.o socket.o tls.o sha256.o timeout.o setcpuaffinity.o monotonic.o mt19937-64.o resp_parser.o call_reply.o script.o functions.o commands.o strl.o connection.o unix.o logreqres.o rdma.o scripting_engine.o lua/script_lua.o lua/function_lua.o lua/engine_lua.o lua/debug_lua.o
 ENGINE_CLI_NAME=$(ENGINE_NAME)-cli$(PROG_SUFFIX)
 ENGINE_CLI_OBJ=anet.o adlist.o dict.o valkey-cli.o zmalloc.o release.o ae.o serverassert.o crcspeed.o crccombine.o crc64.o siphash.o crc16.o monotonic.o cli_common.o mt19937-64.o strl.o cli_commands.o
 ENGINE_BENCHMARK_NAME=$(ENGINE_NAME)-benchmark$(PROG_SUFFIX)
@@ -537,6 +537,9 @@ DEP = $(ENGINE_SERVER_OBJ:%.o=%.d) $(ENGINE_CLI_OBJ:%.o=%.d) $(ENGINE_BENCHMARK_
 %.o: %.c .make-prerequisites
 	$(SERVER_CC) -MMD -o $@ -c $<
 
+lua/%.o: lua/%.c .make-prerequisites
+	$(SERVER_CC) -MMD -o $@ -c $<
+
 unit/%.o: unit/%.c .make-prerequisites
 	$(SERVER_CC) -MMD -o $@ -c $<
 

diff --git a/src/allocator_defrag.c b/src/allocator_defrag.c
@@ -1,8 +1,8 @@
-/* Copyright 2024- Valkey contributors
+/*
+ * Copyright (c) Valkey Contributors
  * All rights reserved.
  * SPDX-License-Identifier: BSD-3-Clause
  */
-
 /*
  * This file implements allocator-specific defragmentation logic used
  * within the Valkey engine. Below is the relationship between various

diff --git a/src/anet.c b/src/anet.c
@@ -496,6 +496,9 @@ static int anetTcpGenericConnect(char *err, const char *addr, int port, const ch
             continue;
         }
 
+        /* Enable TCP_NODELAY by default */
+        anetEnableTcpNoDelay(NULL, s);
+
         /* If we ended an iteration of the for loop without errors, we
          * have a connected socket. Let's return to the caller. */
         goto end;
@@ -684,6 +687,10 @@ int anetTcpAccept(char *err, int serversock, char *ip, size_t ip_len, int *port)
         if (ip) inet_ntop(AF_INET6, (void *)&(s->sin6_addr), ip, ip_len);
         if (port) *port = ntohs(s->sin6_port);
     }
+
+    /* Enable TCP_NODELAY by default */
+    anetEnableTcpNoDelay(NULL, fd);
+
     return fd;
 }
 

diff --git a/src/asciilogo.h b/src/asciilogo.h
@@ -1,6 +1,5 @@
 /*
  * Copyright (c) 2009-2012, Redis Ltd.
- * Copyright (c) 2024, Valkey contributors
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -27,7 +26,11 @@
  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  * POSSIBILITY OF SUCH DAMAGE.
  */
-
+/*
+ * Copyright (c) Valkey Contributors
+ * All rights reserved.
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
 const char *ascii_logo =
     "                .+^+.                                                \n"
     "            .+#########+.                                            \n"