detect/threshold: implement tracking 'by_flow'

Add support for 'by_flow' track option. This allows using the various
threshold options in the context of a single flow.

Example:

    alert tcp ... stream-event:pkt_broken_ack; \
        threshold:type limit, track by_flow, count 1, seconds 3600;

The example would limit the number of alerts to once per hour for
packets triggering the 'pkt_broken_ack' stream event.

Implemented as a special "flowvar" holding the threshold entries. This
means no synchronization is required, making this a cheaper option
compared to the other trackers.

Ticket: #6822.

src/detect-engine-threshold.c

@@ -1,4 +1,4 @@
-/* Copyright (C) 2007-2021 Open Information Security Foundation
+/* Copyright (C) 2007-2024 Open Information Security Foundation
  *
  * You can copy, redistribute or modify this Program under the terms of
  * the GNU General Public License version 2 as published by the Free
@@ -69,6 +69,7 @@
 #include "tm-threads.h"
 
 #include "action-globals.h"
+#include "util-validate.h"
 
 static HostStorageId host_threshold_id = { .id = -1 };     /**< host storage id for thresholds */
 static IPPairStorageId ippair_threshold_id = { .id = -1 }; /**< ip pair storage id for thresholds */
@@ -241,6 +242,70 @@ static DetectThresholdEntry *ThresholdHostLookupEntry(Host *h,
     return e;
 }
 
+typedef struct FlowVarThreshold_ {
+    uint8_t type;
+    uint8_t pad[3];
+    uint32_t idx;
+    struct GenericVar_ *next;
+    DetectThresholdEntry *thresholds;
+} FlowVarThreshold;
+
+void FlowThresholdVarFree(void *ptr)
+{
+    FlowVarThreshold *t = ptr;
+    ThresholdListFree(t->thresholds);
+    SCFree(t);
+}
+
+static FlowVarThreshold *FlowThresholdVarGet(Flow *f)
+{
+    if (f == NULL)
+        return NULL;
+
+    for (GenericVar *gv = f->flowvar; gv != NULL; gv = gv->next) {
+        if (gv->type == DETECT_THRESHOLD && gv->idx == 0)
+            return (FlowVarThreshold *)gv;
+    }
+
+    return NULL;
+}
+
+static DetectThresholdEntry *ThresholdFlowLookupEntry(Flow *f, uint32_t sid, uint32_t gid)
+{
+    FlowVarThreshold *t = FlowThresholdVarGet(f);
+    if (t == NULL)
+        return NULL;
+
+    for (DetectThresholdEntry *e = t->thresholds; e != NULL; e = e->next) {
+        if (e->sid == sid && e->gid == gid) {
+            return e;
+        }
+    }
+    return NULL;
+}
+
+static int AddEntryToFlow(Flow *f, DetectThresholdEntry *e, SCTime_t packet_time)
+{
+    DEBUG_VALIDATE_BUG_ON(e == NULL);
+
+    FlowVarThreshold *t = FlowThresholdVarGet(f);
+    if (t == NULL) {
+        t = SCCalloc(1, sizeof(*t));
+        if (t == NULL) {
+            return -1;
+        }
+        t->type = DETECT_THRESHOLD;
+        GenericVarAppend(&f->flowvar, (GenericVar *)t);
+    }
+
+    e->current_count = 1;
+    e->tv1 = packet_time;
+    e->tv_timeout = 0;
+    e->next = t->thresholds;
+    t->thresholds = e;
+    return 0;
+}
+
 static DetectThresholdEntry *ThresholdIPPairLookupEntry(IPPair *pair,
         uint32_t sid, uint32_t gid)
 {
@@ -587,6 +652,28 @@ static int ThresholdHandlePacketRule(DetectEngineCtx *de_ctx, Packet *p,
     return ret;
 }
 
+/**
+ *  \retval 2 silent match (no alert but apply actions)
+ *  \retval 1 normal match
+ *  \retval 0 no match
+ */
+static int ThresholdHandlePacketFlow(Flow *f, Packet *p, const DetectThresholdData *td,
+        uint32_t sid, uint32_t gid, PacketAlert *pa)
+{
+    int ret = 0;
+    DetectThresholdEntry *lookup_tsh = ThresholdFlowLookupEntry(f, sid, gid);
+    SCLogDebug("lookup_tsh %p sid %u gid %u", lookup_tsh, sid, gid);
+
+    DetectThresholdEntry *new_tsh = NULL;
+    ret = ThresholdHandlePacket(p, lookup_tsh, &new_tsh, td, sid, gid, pa);
+    if (new_tsh != NULL) {
+        if (AddEntryToFlow(f, new_tsh, p->ts) == -1) {
+            SCFree(new_tsh);
+        }
+    }
+    return ret;
+}
+
 /**
  * \brief Make the threshold logic for signatures
  *
@@ -633,6 +720,10 @@ int PacketAlertThreshold(DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx
         SCMutexLock(&de_ctx->ths_ctx.threshold_table_lock);
         ret = ThresholdHandlePacketRule(de_ctx,p,td,s,pa);
         SCMutexUnlock(&de_ctx->ths_ctx.threshold_table_lock);
+    } else if (td->track == TRACK_FLOW) {
+        if (p->flow) {
+            ret = ThresholdHandlePacketFlow(p->flow, p, td, s->id, s->gid, pa);
+        }
     }
 
     SCReturnInt(ret);

src/detect-engine-threshold.h

@@ -51,4 +51,6 @@ int ThresholdHostTimeoutCheck(Host *, SCTime_t);
 int ThresholdIPPairTimeoutCheck(IPPair *, SCTime_t);
 void ThresholdListFree(void *ptr);
 
+void FlowThresholdVarFree(void *ptr);
+
 #endif /* __DETECT_ENGINE_THRESHOLD_H__ */

src/detect-threshold.c

@@ -60,7 +60,12 @@
 #include "util-cpu.h"
 #endif
 
-#define PARSE_REGEX "^\\s*(track|type|count|seconds)\\s+(limit|both|threshold|by_dst|by_src|by_both|by_rule|\\d+)\\s*,\\s*(track|type|count|seconds)\\s+(limit|both|threshold|by_dst|by_src|by_both|by_rule|\\d+)\\s*,\\s*(track|type|count|seconds)\\s+(limit|both|threshold|by_dst|by_src|by_both|by_rule|\\d+)\\s*,\\s*(track|type|count|seconds)\\s+(limit|both|threshold|by_dst|by_src|by_both|by_rule|\\d+)\\s*"
+#define PARSE_REGEX                                                                                \
+    "^\\s*(track|type|count|seconds)\\s+(limit|both|threshold|by_dst|by_src|by_both|by_rule|by_"   \
+    "flow|\\d+)\\s*,\\s*(track|type|count|seconds)\\s+(limit|both|threshold|by_dst|by_src|by_"     \
+    "both|by_rule|by_flow|\\d+)\\s*,\\s*(track|type|count|seconds)\\s+(limit|both|threshold|by_"   \
+    "dst|by_src|by_both|by_rule|by_flow|\\d+)\\s*,\\s*(track|type|count|seconds)\\s+(limit|both|"  \
+    "threshold|by_dst|by_src|by_both|by_rule|by_flow|\\d+)\\s*"
 
 static DetectParseRegex parse_regex;
 
@@ -183,6 +188,8 @@ static DetectThresholdData *DetectThresholdParse(const char *rawstr)
             de->track = TRACK_BOTH;
         if (strncasecmp(args[i],"by_rule",strlen("by_rule")) == 0)
             de->track = TRACK_RULE;
+        if (strncasecmp(args[i], "by_flow", strlen("by_flow")) == 0)
+            de->track = TRACK_FLOW;
         if (strncasecmp(args[i],"count",strlen("count")) == 0)
             count_pos = i+1;
         if (strncasecmp(args[i],"seconds",strlen("seconds")) == 0)
@@ -342,6 +349,7 @@ DetectThresholdData *DetectThresholdDataCopy(DetectThresholdData *de)
 #include "util-hashlist.h"
 #include "packet.h"
 #include "action-globals.h"
+
 /**
  * \test ThresholdTestParse01 is a test for a valid threshold options
  *
@@ -360,6 +368,18 @@ static int ThresholdTestParse01(void)
     return 0;
 }
 
+static int ThresholdTestParseByFlow01(void)
+{
+    DetectThresholdData *de = DetectThresholdParse("type limit,track by_flow,count 1,seconds 60");
+    FAIL_IF_NULL(de);
+    FAIL_IF_NOT(de->type == TYPE_LIMIT);
+    FAIL_IF_NOT(de->track == TRACK_FLOW);
+    FAIL_IF_NOT(de->count == 1);
+    FAIL_IF_NOT(de->seconds == 60);
+    DetectThresholdFree(NULL, de);
+    PASS;
+}
+
 /**
  * \test ThresholdTestParse02 is a test for a invalid threshold options
  *
@@ -1692,6 +1712,7 @@ static int DetectThresholdTestSig14(void)
 static void ThresholdRegisterTests(void)
 {
     UtRegisterTest("ThresholdTestParse01", ThresholdTestParse01);
+    UtRegisterTest("ThresholdTestParseByFlow01", ThresholdTestParseByFlow01);
     UtRegisterTest("ThresholdTestParse02", ThresholdTestParse02);
     UtRegisterTest("ThresholdTestParse03", ThresholdTestParse03);
     UtRegisterTest("ThresholdTestParse04", ThresholdTestParse04);

src/detect-threshold.h

@@ -37,6 +37,7 @@
 #define TRACK_RULE     3
 #define TRACK_EITHER   4 /**< either src or dst: only used by suppress */
 #define TRACK_BOTH     5 /* used by rate_filter to match detections by both src and dst addresses */
+#define TRACK_FLOW     6 /**< track by flow */
 
 /* Get the new action to take */
 #define TH_ACTION_ALERT     0x01

src/util-var.c

@@ -1,4 +1,4 @@
-/* Copyright (C) 2007-2013 Open Information Security Foundation
+/* Copyright (C) 2007-2024 Open Information Security Foundation
  *
  * You can copy, redistribute or modify this Program under the terms of
  * the GNU General Public License version 2 as published by the Free
@@ -25,6 +25,7 @@
 
 #include "suricata-common.h"
 #include "detect.h"
+#include "detect-engine-threshold.h"
 
 #include "util-var.h"
 
@@ -67,6 +68,10 @@ void GenericVarFree(GenericVar *gv)
             XBitFree(fb);
             break;
         }
+        case DETECT_THRESHOLD: {
+            FlowThresholdVarFree(gv);
+            break;
+        }
         case DETECT_FLOWVAR:
         {
             FlowVar *fv = (FlowVar *)gv;

src/util-var.h

@@ -35,6 +35,7 @@ enum VarTypes {
     VAR_TYPE_FLOW_BIT,
     VAR_TYPE_FLOW_INT,
     VAR_TYPE_FLOW_VAR,
+    VAR_TYPE_FLOW_THRESHOLD,
 
     VAR_TYPE_HOST_BIT,
     VAR_TYPE_HOST_INT,