Make CI faster

IvanNardi · Jan 6, 2025 · 4ef73a2 · 4ef73a2
1 parent c3d19be
commit 4ef73a2
Show file tree

Hide file tree

Showing 6 changed files with 76 additions and 25 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -106,7 +106,7 @@ jobs:
             maxminddb: "--with-maxminddb"
             msan: "--with-sanitizer"
             nBPF: ""
-            lto_gold_linker: "--with-lto-and-gold-linker"
+            lto_gold_linker: ""
           - compiler: "clang-9" # "Oldest" clang easily available
             os: ubuntu-20.04
             arch: "x86_64"
@@ -125,34 +125,46 @@ jobs:
             maxminddb: "--with-maxminddb"
             msan: "--with-sanitizer"
             nBPF: ""
+            lto_gold_linker: ""
+          - compiler: "gcc-14" # "Newest" gcc easily available with lto/gold linker
+            os: ubuntu-24.04
+            arch: "x86_64"
+            gcrypt: ""
+            pcre: "--with-pcre2"
+            maxminddb: "--with-maxminddb"
+            msan: ""
+            nBPF: ""
             lto_gold_linker: "--with-lto-and-gold-linker"
-          - compiler: "cc"
-            os: ubuntu-latest
+          - compiler: "clang-18" # "Newest" clang easily available with lto/gold linker
+            ar: "llvm-ar-18"
+            ranlib: "llvm-ranlib-18"
+            os: ubuntu-24.04
             arch: "x86_64"
             gcrypt: ""
             pcre: "--with-pcre2"
             maxminddb: "--with-maxminddb"
-            msan: "--with-thread-sanitizer"
+            msan: ""
             nBPF: ""
+            lto_gold_linker: "--with-lto-and-gold-linker"
           - compiler: "cc"
             os: ubuntu-latest
             arch: "x86_64"
             gcrypt: ""
             pcre: "--with-pcre2"
             maxminddb: "--with-maxminddb"
-            msan: "--with-sanitizer"
+            msan: ""
             nBPF: "nBPF"
           - compiler: "cc"
             os: ubuntu-latest
             arch: "x86_64"
             gcrypt: ""
             pcre: "--with-pcre2"
             maxminddb: "--with-maxminddb"
-            msan: "--with-sanitizer"
+            msan: ""
             nBPF: ""
             global_context: "--disable-global-context-support"
           - compiler: "clang" #TODO: some issues with masan/clang/ubuntu-24.04
-            os: ubuntu-22.04
+            os: ubuntu-24.04
             arch: "x86_64"
             gcrypt: ""
             pcre: "--with-pcre2"
@@ -222,7 +234,7 @@ jobs:
           sudo apt-get install autoconf automake debhelper libtool pkg-config gettext libjson-c-dev flex bison libpcap-dev
           sudo apt-get install rrdtool librrd-dev parallel
       - name: Install Ubuntu Prerequisites [Mingw-w64] (runs only on ubuntu jobs)
-        if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.arch, 'x86_64') && !startsWith(matrix.msan, '--with-') #Only on a few "standard" builds, without any sanitizers
+        if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.arch, 'x86_64') && !startsWith(matrix.msan, '--with-') && !startsWith(matrix.nBPF, 'nBPF') && !startsWith(matrix.lto_gold_linker, '--with-') #Only on a few "standard" builds
         run: |
           sudo apt-get install gcc-mingw-w64 libc6-dev
       - name: Install Ubuntu Prerequisites (libgcrypt)
@@ -318,15 +330,20 @@ jobs:
           DESTDIR=/tmp/ndpi make install
           ls -alhHR /tmp/ndpi
       - name: Test nDPI [SYMBOLS]
-        if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.arch, 'x86_64') && !startsWith(matrix.msan, '--with-') #Only on a few "standard" builds, without any sanitizers
+        if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.arch, 'x86_64') && !startsWith(matrix.msan, '--with-') && !startsWith(matrix.nBPF, 'nBPF') && !startsWith(matrix.lto_gold_linker, '--with-') #Only on a few "standard" builds
+
         run: |
           ./utils/check_symbols.sh || { FAILED=$?; echo "::error file=${NDPI_LIB}::Unwanted libc symbols found: ${FAILED}. Please make sure to use only ndpi_malloc/ndpi_calloc/ndpi_realloc/ndpi_free wrapper instead of malloc/calloc/realloc/free."; false; }
         env:
           NDPI_LIB: src/lib/libndpi.a
       - name: Test nDPI [DIFF]
-        if: startsWith(matrix.arch, 'x86_64') && !startsWith(matrix.os, 'windows')
+        if: startsWith(matrix.arch, 'x86_64') && !startsWith(matrix.os, 'windows') && !startsWith(matrix.msan, '--with-memory-sanitizer')
         run: |
           NDPI_FORCE_PARALLEL_UTESTS=1 NDPI_SKIP_PARALLEL_BAR=1 ./tests/do.sh
+      - name: Test nDPI (quick) [DIFF]
+        if: startsWith(matrix.arch, 'x86_64') && startsWith(matrix.msan, '--with-memory-sanitizer')
+        run: |
+          NDPI_TEST_ONLY_RECENTLY_UPDATED_PCAPS=1 NDPI_FORCE_PARALLEL_UTESTS=1 NDPI_SKIP_PARALLEL_BAR=1 ./tests/do.sh
       - name: Test nDPI [UNIT]
         #Some issues with masan + json-c. Disable the test as workaround
         if: startsWith(matrix.arch, 'x86_64') && !startsWith(matrix.os, 'windows') && !startsWith(matrix.msan, '--with-memory-sanitizer') && !startsWith(matrix.os, 'macos-14') && !startsWith(matrix.os, 'macos-15')
@@ -361,7 +378,7 @@ jobs:
           make
           cd ../..
       - name: Build nDPI [Mingw-w64] (runs only on ubuntu jobs)
-        if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.arch, 'x86_64') && !startsWith(matrix.msan, '--with-') #Only on a few "standard" builds, without any sanitizers
+        if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.arch, 'x86_64') && !startsWith(matrix.msan, '--with-') && !startsWith(matrix.nBPF, 'nBPF') && !startsWith(matrix.lto_gold_linker, '--with-') #Only on a few "standard" builds
         run: |
           make distclean
           ./autogen.sh --enable-option-checking=fatal --enable-debug-messages --enable-tls-sigs --host=x86_64-w64-mingw32
@@ -387,10 +404,10 @@ jobs:
             apt-get -y install git wdiff colordiff autoconf automake libtool pkg-config gettext libjson-c-dev flex bison libpcap-dev libgcrypt20-dev libpcre3-dev libmaxminddb-dev rrdtool librrd-dev &&
             git config --global --add safe.directory $(realpath .) &&
             env CC=gcc CFLAGS='-Werror' ./autogen.sh --enable-option-checking=fatal --enable-debug-messages ${{ matrix.gcrypt }} ${{ matrix.msan }} ${{ matrix.pcre }} ${{ matrix.maxminddb }} --enable-tls-sigs &&
-            make -j all &&
+            make -j $(nproc) all &&
             make -C example ndpiSimpleIntegration &&
             make -C rrdtool &&
-            make check VERBOSE=1
+            NDPI_TEST_ONLY_RECENTLY_UPDATED_PCAPS=1 make check VERBOSE=1
             "
       - name: Display qemu specified architecture (armhf - little endian)
         if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.arch, 'armhf')
@@ -411,10 +428,10 @@ jobs:
             apt-get -y install git wdiff colordiff autoconf automake libtool pkg-config gettext libjson-c-dev flex bison libpcap-dev libgcrypt20-dev libpcre3-dev libmaxminddb-dev rrdtool librrd-dev &&
             git config --global --add safe.directory $(realpath .) &&
             env CC=gcc CFLAGS='-Werror' ./autogen.sh --enable-option-checking=fatal --enable-debug-messages ${{ matrix.gcrypt }} ${{ matrix.msan }} ${{ matrix.pcre }} ${{ matrix.maxminddb }} --enable-tls-sigs &&
-            make -j all &&
+            make -j $(nproc) all &&
             make -C example ndpiSimpleIntegration &&
             make -C rrdtool &&
-            make check VERBOSE=1
+            NDPI_TEST_ONLY_RECENTLY_UPDATED_PCAPS=1 make check VERBOSE=1
             "
       - name: Display qemu specified architecture (s390x - big endian)
         if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.arch, 's390x')
@@ -435,8 +452,8 @@ jobs:
             apt-get -y install git wdiff colordiff autoconf automake libtool pkg-config gettext libjson-c-dev flex bison libpcap-dev libgcrypt20-dev libpcre3-dev libmaxminddb-dev rrdtool librrd-dev &&
             git config --global --add safe.directory $(realpath .) &&
             env CC=gcc CFLAGS='-Werror' ./autogen.sh --enable-option-checking=fatal --enable-debug-messages ${{ matrix.gcrypt }} ${{ matrix.msan }} ${{ matrix.pcre }} ${{ matrix.maxminddb }} --enable-tls-sigs &&
-            make -j all &&
+            make -j $(nproc) all &&
             make -C example ndpiSimpleIntegration &&
             make -C rrdtool &&
-            make check VERBOSE=1
+            NDPI_TEST_ONLY_RECENTLY_UPDATED_PCAPS=1 make check VERBOSE=1
             "
diff --git a/.github/workflows/build_scheduled.yml b/.github/workflows/build_scheduled.yml
@@ -109,3 +109,26 @@ jobs:
           name: ndpi-performance
           path: ndpi-performance-upload
           retention-days: 7
+
+  thread-sanitizer:
+    name: Thread-Sanitizer (ubuntu-latest)
+    runs-on: ubuntu-latest
+    env:
+      CFLAGS: -Werror
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Install Ubuntu Prerequisites
+        run: |
+          sudo apt-get update
+          sudo apt-get install autoconf automake libtool pkg-config gettext flex bison libpcap-dev libjson-c-dev rrdtool librrd-dev parallel
+      - name: Configure nDPI library
+        run: |
+          ./autogen.sh --enable-option-checking=fatal --with-thread-sanitizer
+      - name: Build nDPI library
+        run: |
+          make -j all
+      - name: Test nDPI
+        run: |
+          NDPI_FORCE_PARALLEL_UTESTS=1 NDPI_SKIP_PARALLEL_BAR=1 ./tests/do.sh
diff --git a/.github/workflows/cifuzz.yml b/.github/workflows/cifuzz.yml
@@ -18,7 +18,7 @@ jobs:
         uses: google/oss-fuzz/infra/cifuzz/actions/run_fuzzers@master
         with:
           oss-fuzz-project-name: 'ndpi'
-          fuzz-seconds: 1200
+          fuzz-seconds: 600
           dry-run: false
           sanitizer: ${{ matrix.sanitizer }}
       - name: Check Crash (fails when a crash is detected)

diff --git a/example/ndpiReader.c b/example/ndpiReader.c
@@ -4015,7 +4015,7 @@ static void printResults(u_int64_t processing_time_usec, u_int64_t setup_time_us
 
     /* LRU caches */
     for(i = 0; i < NDPI_LRUCACHE_MAX; i++) {
-      struct ndpi_lru_cache_stats s;
+      struct ndpi_lru_cache_stats s = { 0 };
       int scope;
       char param[64];
 
@@ -4035,15 +4035,15 @@ static void printResults(u_int64_t processing_time_usec, u_int64_t setup_time_us
 
     /* Automas */
     for(i = 0; i < NDPI_AUTOMA_MAX; i++) {
-      struct ndpi_automa_stats s;
+      struct ndpi_automa_stats s = { 0 };
       ndpi_get_automa_stats(ndpi_thread_info[thread_id].workflow->ndpi_struct, i, &s);
       cumulative_stats.automa_stats[i].n_search += s.n_search;
       cumulative_stats.automa_stats[i].n_found += s.n_found;
     }
 
     /* Patricia trees */
     for(i = 0; i < NDPI_PTREE_MAX; i++) {
-      struct ndpi_patricia_tree_stats s;
+      struct ndpi_patricia_tree_stats s = { 0 };
       ndpi_get_patricia_stats(ndpi_thread_info[thread_id].workflow->ndpi_struct, i, &s);
       cumulative_stats.patricia_stats[i].n_search += s.n_search;
       cumulative_stats.patricia_stats[i].n_found += s.n_found;

diff --git a/tests/do.sh.in b/tests/do.sh.in
@@ -31,6 +31,11 @@ if [ "${NDPI_SKIP_PARALLEL_BAR}" = "1" ]; then
    SKIP_PARALLEL_BAR=1
 fi
 
+TEST_ONLY_RECENTLY_UPDATED_PCAPS=0
+if [ "${NDPI_TEST_ONLY_RECENTLY_UPDATED_PCAPS}" = "1" ]; then
+   TEST_ONLY_RECENTLY_UPDATED_PCAPS=1
+fi
+
 #Remember: valgrind and *SAN are incompatible!
 CMD_PREFIX="${CMD_PREFIX}"
 if [ "${NDPI_TESTS_WINE}" = "1" ]; then
@@ -239,10 +244,16 @@ for d in $(find ./cfgs/* -type d -maxdepth 0 2>/dev/null) ; do
 
     cd ./cfgs/"$(basename "$d")" || exit 1
 
-    if [ "$#" -ne 0 ]; then
-        PCAPS=$*
+    if [ $TEST_ONLY_RECENTLY_UPDATED_PCAPS -eq 1 ]; then
+        # Latest 10 pcaps updatetd recently (to be more precise, pcap which triggered a change
+	# in their result lately)
+        PCAPS=$(cd result || exit 1; find * -type f -print0  | xargs -0 ls -t | head -10 | xargs basename -s .out)
     else
-        PCAPS=$(cd pcap || exit 1; /bin/ls -- *.*cap*)
+	if [ "$#" -ne 0 ]; then
+            PCAPS=$*
+        else
+            PCAPS=$(cd pcap || exit 1; /bin/ls -- *.*cap*)
+        fi
     fi
     FAILURES=()
     READER_EXTRA_PARAM=""

diff --git a/utils/check_symbols.sh b/utils/check_symbols.sh
@@ -28,7 +28,7 @@ for line in $(nm -P -u "${NDPI_LIB}"); do
                     'printf'|'fprintf') SKIP=1 ;;
                 esac
             ;;
-            '[ahocorasick.o]')
+            '[ahocorasick.o]'|'[ndpi_cache.o]')
                 case "${FOUND_SYMBOL}" in
                     'fprintf') SKIP=1 ;;
                 esac