next/350/20240316/v1

.github/workflows/builds.yml

@@ -1471,7 +1471,9 @@ jobs:
   ubuntu-22-04-cov-fuzz:
     name: Ubuntu 22.04 (fuzz corpus coverage)
     runs-on: ubuntu-latest
-    container: ubuntu:22.04
+    container:
+      image: ubuntu:22.04
+      options: --privileged
     needs: [prepare-deps, prepare-cbindgen]
     steps:
       - name: Cache ~/.cargo
@@ -1519,6 +1521,7 @@ jobs:
                 parallel \
                 python3-yaml \
                 software-properties-common \
+                sudo \
                 zlib1g \
                 zlib1g-dev \
                 exuberant-ctags \
@@ -1546,6 +1549,11 @@ jobs:
           cp prep/cbindgen $HOME/.cargo/bin
           chmod 755 $HOME/.cargo/bin/cbindgen
           echo "$HOME/.cargo/bin" >> $GITHUB_PATH
+      - name: Fix kernel mmap rnd bits
+      # Asan in llvm 14 provided in ubuntu 22.04 is incompatible with
+      # high-entropy ASLR in much newer kernels that GitHub runners are
+      # using leading to random crashes: https://github.com/actions/runner-images/issues/9491
+        run: sudo sysctl vm.mmap_rnd_bits=28
       - run: ./autogen.sh
       - run: ./configure --with-gnu-ld --enable-fuzztargets --disable-shared --enable-gccprotect
         env:
@@ -1729,7 +1737,9 @@ jobs:
   ubuntu-22-04-debug-validation:
     name: Ubuntu 22.04 (Debug Validation)
     runs-on: ubuntu-22.04
-    container: ubuntu:22.04
+    container:
+      image: ubuntu:22.04
+      options: --privileged
     needs: [prepare-deps, prepare-cbindgen]
     steps:
 
@@ -1776,6 +1786,7 @@ jobs:
                 python3-yaml \
                 rustc \
                 software-properties-common \
+                sudo \
                 zlib1g \
                 zlib1g-dev \
                 exuberant-ctags
@@ -1795,6 +1806,11 @@ jobs:
           cp prep/cbindgen $HOME/.cargo/bin
           chmod 755 $HOME/.cargo/bin/cbindgen
           echo "$HOME/.cargo/bin" >> $GITHUB_PATH
+      - name: Fix kernel mmap rnd bits
+      # Asan in llvm 14 provided in ubuntu 22.04 is incompatible with
+      # high-entropy ASLR in much newer kernels that GitHub runners are
+      # using leading to random crashes: https://github.com/actions/runner-images/issues/9491
+        run: sudo sysctl vm.mmap_rnd_bits=28
       - run: ./autogen.sh
       - run: ./configure --enable-debug-validation
         env:

doc/userguide/devguide/index.rst

@@ -9,3 +9,4 @@ Suricata Developer Guide
    internals/index.rst
    extending/index.rst
    libsuricata/index.rst
+   upgrading/index.rst

doc/userguide/devguide/upgrading/index.rst

@@ -0,0 +1,21 @@
+Upgrading
+=========
+
+Upgrading 7.0 to 8.0
+--------------------
+
+EVE File Types
+~~~~~~~~~~~~~~
+
+- The ``ThreadInit`` function will now be called when in *threaded*
+  and *non-threaded* modes. This simplifies the initialization for EVE
+  filetypes as they can use the same flow of execution for both
+  modes. To upgrade, either remove the call to ``ThreadInit`` from
+  ``Init``, or move per-thread setup code from ``Init`` to
+  ``ThreadInit``.
+- Many of the function arguments to the callbacks have been made
+  ``const`` where it made sense.
+
+Please see the latest example EVE filetype plugin for an up to date
+example.
+

doxygen.cfg

@@ -829,7 +829,7 @@ WARN_LOGFILE           =
 # spaces. See also FILE_PATTERNS and EXTENSION_MAPPING
 # Note: If this tag is empty the current directory is searched.
 
-INPUT                  = src/ libhtp/htp/
+INPUT                  = src/ libhtp/htp/ examples/
 
 # This tag can be used to specify the character encoding of the source files
 # that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses

examples/plugins/c-json-filetype/filetype.c

@@ -1,4 +1,4 @@
-/* Copyright (C) 2020-2023 Open Information Security Foundation
+/* Copyright (C) 2020-2024 Open Information Security Foundation
  *
  * You can copy, redistribute or modify this Program under the terms of
  * the GNU General Public License version 2 as published by the Free
@@ -17,20 +17,18 @@
 
 #include "suricata-common.h"
 #include "suricata-plugin.h"
+#include "output-eve.h"
 #include "util-mem.h"
 #include "util-debug.h"
 
 #define FILETYPE_NAME "json-filetype-plugin"
 
-static int FiletypeThreadInit(void *ctx, int thread_id, void **thread_data);
-static int FiletypeThreadDeinit(void *ctx, void *thread_data);
-
 /**
  * Per thread context data for each logging thread.
  */
 typedef struct ThreadData_ {
     /** The thread ID, for demonstration purposes only. */
-    int thread_id;
+    ThreadId thread_id;
 
     /** The number of records logged on this thread. */
     uint64_t count;
@@ -42,9 +40,6 @@ typedef struct ThreadData_ {
 typedef struct Context_ {
     /** Verbose, or print to stdout. */
     int verbose;
-
-    /** A thread context to use when not running in threaded mode. */
-    ThreadData *thread;
 } Context;
 
 /**
@@ -58,9 +53,9 @@ typedef struct Context_ {
  * \param data A pointer where context data can be stored relevant to this
  *      output.
  *
- * Eve output plugins need to be thread aware as the threading happens at lower
- * level than the EVE output, so a flag is provided here to notify the plugin if
- * threading is enabled or not.
+ * Eve output plugins need to be thread aware as the threading happens
+ * at a lower level than the EVE output, so a flag is provided here to
+ * notify the plugin if threading is enabled or not.
  *
  * If the plugin does not work with threads disabled, or enabled, this function
  * should return -1.
@@ -69,7 +64,7 @@ typedef struct Context_ {
  * configuration for the eve instance, not just a node named after the plugin.
  * This allows the plugin to get more context about what it is logging.
  */
-static int FiletypeInit(ConfNode *conf, bool threaded, void **data)
+static int FiletypeInit(const ConfNode *conf, const bool threaded, void **data)
 {
     SCLogNotice("Initializing template eve output plugin: threaded=%d", threaded);
     Context *context = SCCalloc(1, sizeof(Context));
@@ -91,15 +86,6 @@ static int FiletypeInit(ConfNode *conf, bool threaded, void **data)
     }
     context->verbose = verbose;
 
-    if (!threaded) {
-        /* We're not running in threaded mode so allocate a thread context here
-         * to avoid duplication of context data such as file pointers, database
-         * connections, etc. */
-        if (FiletypeThreadInit(context, 0, (void **)&context->thread) != 0) {
-            SCFree(context);
-            return -1;
-        }
-    }
     *data = context;
     return 0;
 }
@@ -114,12 +100,9 @@ static int FiletypeInit(ConfNode *conf, bool threaded, void **data)
  */
 static void FiletypeDeinit(void *data)
 {
-    printf("TemplateClose\n");
+    SCLogNotice("data=%p", data);
     Context *ctx = data;
     if (ctx != NULL) {
-        if (ctx->thread) {
-            FiletypeThreadDeinit(ctx, (void *)ctx->thread);
-        }
         SCFree(ctx);
     }
 }
@@ -139,12 +122,12 @@ static void FiletypeDeinit(void *data)
  * of "eve.<thread_id>.json". This plugin may want to do similar, or open
  * multiple connections to whatever the final logging location might be.
  *
- * In the case of non-threaded EVE logging this function is NOT called by
- * Suricata, but instead this plugin chooses to use this method to create a
- * default (single) thread context.
+ * In the case of non-threaded EVE logging this function is called
+ * once with a thread_id of 0.
  */
-static int FiletypeThreadInit(void *ctx, int thread_id, void **thread_data)
+static int FiletypeThreadInit(const void *ctx, const ThreadId thread_id, void **thread_data)
 {
+    SCLogNotice("thread_id=%d", thread_id);
     ThreadData *tdata = SCCalloc(1, sizeof(ThreadData));
     if (tdata == NULL) {
         SCLogError("Failed to allocate thread data");
@@ -163,18 +146,18 @@ static int FiletypeThreadInit(void *ctx, int thread_id, void **thread_data)
  * This is where any cleanup per thread should be done including free'ing of the
  * thread_data if needed.
  */
-static int FiletypeThreadDeinit(void *ctx, void *thread_data)
+static void FiletypeThreadDeinit(const void *ctx, void *thread_data)
 {
+    SCLogNotice("thread_data=%p", thread_data);
     if (thread_data == NULL) {
         // Nothing to do.
-        return 0;
+        return;
     }
 
     ThreadData *tdata = thread_data;
     SCLogNotice(
             "Deinitializing thread %d: records written: %" PRIu64, tdata->thread_id, tdata->count);
     SCFree(tdata);
-    return 0;
 }
 
 /**
@@ -188,16 +171,13 @@ static int FiletypeThreadDeinit(void *ctx, void *thread_data)
  * to any resource that may block it might be best to enqueue the buffers for
  * further processing which will require copying of the provided buffer.
  */
-static int FiletypeWrite(const char *buffer, int buffer_len, void *data, void *thread_data)
+static int FiletypeWrite(
+        const char *buffer, const int buffer_len, const void *data, void *thread_data)
 {
-    Context *ctx = data;
+    const Context *ctx = data;
     ThreadData *thread = thread_data;
 
-    /* The thread_data could be null which is valid, or it could be that we are
-     * in single threaded mode. */
-    if (thread == NULL) {
-        thread = ctx->thread;
-    }
+    SCLogNotice("thread_id=%d, data=%p, thread_data=%p", thread->thread_id, data, thread_data);
 
     thread->count++;
 

rules/http-events.rules

@@ -91,4 +91,6 @@ alert http any any -> any any (msg:"SURICATA HTTP failed protocol change"; flow:
 
 #alert http any any -> any any (msg:"SURICATA HTTP request chunk extension"; flow:established; app-layer-event:http.request_chunk_extension; classtype:protocol-command-decode; sid:2221054; rev:1;)
 
-# next sid 2221055
+alert http any any -> any any (msg:"SURICATA HTTP request missing protocol"; flow:established,to_server; app-layer-event:http.request_line_missing_protocol; classtype:protocol-command-decode; sid:2221055; rev:1;)
+
+# next sid 2221056

src/Makefile.am

@@ -393,6 +393,7 @@ noinst_HEADERS = \
 	log-tcp-data.h \
 	log-tlslog.h \
 	log-tlsstore.h \
+	output-eve.h \
 	output-eve-stream.h \
 	output-eve-null.h \
 	output-filedata.h \
@@ -1058,6 +1059,7 @@ libsuricata_c_a_SOURCES = \
 	output-json-template.c \
 	output-json-tftp.c \
 	output-json-tls.c \
+	output-eve.c \
 	output-eve-syslog.c \
 	output-eve-null.c \
 	output-lua.c \

src/app-layer-htp.c

@@ -167,6 +167,7 @@ SCEnumCharMap http_decoder_event_table[] = {
 
     { "RANGE_INVALID", HTTP_DECODER_EVENT_RANGE_INVALID },
     { "REQUEST_CHUNK_EXTENSION", HTTP_DECODER_EVENT_REQUEST_CHUNK_EXTENSION },
+    { "REQUEST_LINE_MISSING_PROTOCOL", HTTP_DECODER_EVENT_REQUEST_LINE_MISSING_PROTOCOL },
 
     /* suricata warnings/errors */
     { "MULTIPART_GENERIC_ERROR", HTTP_DECODER_EVENT_MULTIPART_GENERIC_ERROR },
@@ -642,6 +643,7 @@ struct {
     { "Ambiguous response C-L value",
             HTTP_DECODER_EVENT_DUPLICATE_CONTENT_LENGTH_FIELD_IN_RESPONSE },
     { "Request chunk extension", HTTP_DECODER_EVENT_REQUEST_CHUNK_EXTENSION },
+    { "Request line: missing protocol", HTTP_DECODER_EVENT_REQUEST_LINE_MISSING_PROTOCOL },
 };
 
 #define HTP_ERROR_MAX (sizeof(htp_errors) / sizeof(htp_errors[0]))

src/app-layer-htp.h

@@ -128,6 +128,7 @@ enum {
 
     HTTP_DECODER_EVENT_RANGE_INVALID,
     HTTP_DECODER_EVENT_REQUEST_CHUNK_EXTENSION,
+    HTTP_DECODER_EVENT_REQUEST_LINE_MISSING_PROTOCOL,
 
     /* suricata errors/warnings */
     HTTP_DECODER_EVENT_MULTIPART_GENERIC_ERROR,

src/decode-erspan.c

@@ -39,6 +39,7 @@
 #include "util-validate.h"
 #include "util-unittest.h"
 #include "util-debug.h"
+#include "conf.h"
 
 /**
  * \brief Functions to decode ERSPAN Type I and II packets

src/defrag-config.c

@@ -26,6 +26,7 @@
 #include "defrag-config.h"
 #include "util-misc.h"
 #include "util-radix-tree.h"
+#include "conf.h"
 
 static SCRadixTree *defrag_tree = NULL;
 

src/detect-engine-mpm.c

@@ -1559,6 +1559,9 @@ static void MpmStoreSetup(const DetectEngineCtx *de_ctx, MpmStore *ms)
 
     MpmInitCtx(ms->mpm_ctx, de_ctx->mpm_matcher);
 
+    const bool mpm_supports_endswith =
+            (mpm_table[de_ctx->mpm_matcher].feature_flags & MPM_FEATURE_FLAG_ENDSWITH) != 0;
+
     /* add the patterns */
     for (sig = 0; sig < (ms->sid_array_size * 8); sig++) {
         if (ms->sid_array[sig / 8] & (1 << (sig % 8))) {
@@ -1585,8 +1588,11 @@ static void MpmStoreSetup(const DetectEngineCtx *de_ctx, MpmStore *ms)
             }
 
             if (!skip) {
-                PopulateMpmHelperAddPattern(ms->mpm_ctx,
-                        cd, s, 0, (cd->flags & DETECT_CONTENT_FAST_PATTERN_CHOP));
+                uint8_t flags = 0;
+                if ((cd->flags & DETECT_CONTENT_ENDS_WITH) && mpm_supports_endswith)
+                    flags = MPM_PATTERN_FLAG_ENDSWITH;
+                PopulateMpmHelperAddPattern(
+                        ms->mpm_ctx, cd, s, flags, (cd->flags & DETECT_CONTENT_FAST_PATTERN_CHOP));
             }
         }
     }

src/detect.c

@@ -729,7 +729,6 @@ static inline void DetectRulePacketRules(
     const DetectRunScratchpad *scratch
 )
 {
-    const Signature *s = NULL;
     const Signature *next_s = NULL;
 
     /* inspect the sigs against the packet */
@@ -760,7 +759,7 @@ static inline void DetectRulePacketRules(
 #ifdef PROFILE_RULES
         bool smatch = false; /* signature match */
 #endif
-        s = next_s;
+        const Signature *s = next_s;
         sflags = next_sflags;
         if (match_cnt) {
             next_s = *match_array++;

src/flow-timeout.c

@@ -334,14 +334,21 @@ int FlowForceReassemblyNeedReassembly(Flow *f)
  *
  *        The function requires flow to be locked beforehand.
  *
+ * Normally, the first thread_id value should be used. This is when the flow is
+ * created on seeing the first packet to the server; sometimes, if the first
+ * packet is determined to be to the client, the second thread_id value should
+ * be used.
+ *
  * \param f Pointer to the flow.
  *
  * \retval 0 This flow doesn't need any reassembly processing; 1 otherwise.
  */
 void FlowForceReassemblyForFlow(Flow *f)
 {
-    const int thread_id = (int)f->thread_id[0];
-    TmThreadsInjectFlowById(f, thread_id);
+    // Have packets traveled to the server? If not,
+    // use the reverse direction
+    int idx = f->todstpktcnt > 0 ? 0 : 1;
+    TmThreadsInjectFlowById(f, (const int)f->thread_id[idx]);
 }
 
 /**
@@ -355,7 +362,7 @@ void FlowForceReassemblyForFlow(Flow *f)
  * - silence complaining profilers
  * - allow us to aggressively check using debug validation assertions
  * - be robust in case of future changes
- * - locking overhead if neglectable when no other thread fights us
+ * - locking overhead is negligible when no other thread fights us
  *
  * \param q The queue to process flows from.
  */

src/flow.c

@@ -291,6 +291,8 @@ void FlowSwap(Flow *f)
     FlowSwapFlags(f);
     FlowSwapFileFlags(f);
 
+    SWAP_VARS(FlowThreadId, f->thread_id[0], f->thread_id[1]);
+
     if (f->proto == IPPROTO_TCP) {
         TcpStreamFlowSwap(f);
     }