Backport of 6553/6875 to main-7.0.x

src/output-json-alert.c

@@ -35,6 +35,7 @@
 #include "tm-threads.h"
 #include "threadvars.h"
 #include "util-debug.h"
+#include "stream-tcp.h"
 
 #include "util-logopenfile.h"
 #include "util-misc.h"
@@ -127,17 +128,6 @@ typedef struct JsonAlertLogThread_ {
     OutputJsonThreadCtx *ctx;
 } JsonAlertLogThread;
 
-/* Callback function to pack payload contents from a stream into a buffer
- * so we can report them in JSON output. */
-static int AlertJsonDumpStreamSegmentCallback(
-        const Packet *p, TcpSegment *seg, void *data, const uint8_t *buf, uint32_t buflen)
-{
-    MemBuffer *payload = (MemBuffer *)data;
-    MemBufferWriteRaw(payload, buf, buflen);
-
-    return 1;
-}
-
 static void AlertJsonTls(const Flow *f, const uint32_t sig_flags, JsonBuilder *js)
 {
     SSLState *ssl_state = (SSLState *)FlowGetAppState(f);
@@ -637,7 +627,8 @@ static void AlertAddFiles(const Packet *p, JsonBuilder *jb, const uint64_t tx_id
     }
 }
 
-static void AlertAddFrame(const Packet *p, JsonBuilder *jb, const int64_t frame_id)
+static void AlertAddFrame(
+        const Packet *p, const int64_t frame_id, JsonBuilder *jb, MemBuffer *buffer)
 {
     if (p->flow == NULL || (p->proto == IPPROTO_TCP && p->flow->protoctx == NULL))
         return;
@@ -659,7 +650,7 @@ static void AlertAddFrame(const Packet *p, JsonBuilder *jb, const int64_t frame_
         }
         Frame *frame = FrameGetById(frames, frame_id);
         if (frame != NULL) {
-            FrameJsonLogOneFrame(IPPROTO_TCP, frame, p->flow, stream, p, jb);
+            FrameJsonLogOneFrame(IPPROTO_TCP, frame, p->flow, stream, p, jb, buffer);
         }
     } else if (p->proto == IPPROTO_UDP) {
         if (PKT_IS_TOSERVER(p)) {
@@ -669,7 +660,7 @@ static void AlertAddFrame(const Packet *p, JsonBuilder *jb, const int64_t frame_
         }
         Frame *frame = FrameGetById(frames, frame_id);
         if (frame != NULL) {
-            FrameJsonLogOneFrame(IPPROTO_UDP, frame, p->flow, NULL, p, jb);
+            FrameJsonLogOneFrame(IPPROTO_UDP, frame, p->flow, NULL, p, jb, buffer);
         }
     }
 }
@@ -725,9 +716,64 @@ void EveAddVerdict(JsonBuilder *jb, const Packet *p)
     jb_close(jb);
 }
 
+struct AlertJsonStreamDataCallbackData {
+    MemBuffer *payload;
+    uint64_t last_re;
+};
+
+static int AlertJsonStreamDataCallback(
+        void *cb_data, const uint8_t *input, const uint32_t input_len, const uint64_t input_offset)
+{
+    struct AlertJsonStreamDataCallbackData *cbd = cb_data;
+    if (input_offset > cbd->last_re) {
+        MemBufferWriteString(
+                cbd->payload, "[%" PRIu64 " bytes missing]", input_offset - cbd->last_re);
+    }
+
+    int done = 0;
+    uint32_t written = MemBufferWriteRaw(cbd->payload, input, input_len);
+    if (written < input_len)
+        done = 1;
+    cbd->last_re = input_offset + input_len;
+    return done;
+}
+
+/** \internal
+ *  \brief try to log stream data into payload/payload_printable
+ *  \retval true stream data logged
+ *  \retval false stream data not logged
+ */
+static bool AlertJsonStreamData(const AlertJsonOutputCtx *json_output_ctx, JsonAlertLogThread *aft,
+        Flow *f, const Packet *p, JsonBuilder *jb)
+{
+    TcpSession *ssn = f->protoctx;
+    TcpStream *stream = (PKT_IS_TOSERVER(p)) ? &ssn->client : &ssn->server;
+
+    MemBufferReset(aft->payload_buffer);
+    struct AlertJsonStreamDataCallbackData cbd = { .payload = aft->payload_buffer,
+        .last_re = STREAM_BASE_OFFSET(stream) };
+    uint64_t unused = 0;
+    StreamReassembleLog(ssn, stream, AlertJsonStreamDataCallback, &cbd, STREAM_BASE_OFFSET(stream),
+            &unused, false);
+    if (cbd.payload->offset) {
+        if (json_output_ctx->flags & LOG_JSON_PAYLOAD_BASE64) {
+            jb_set_base64(jb, "payload", cbd.payload->buffer, cbd.payload->offset);
+        }
+
+        if (json_output_ctx->flags & LOG_JSON_PAYLOAD) {
+            uint8_t printable_buf[cbd.payload->offset + 1];
+            uint32_t offset = 0;
+            PrintStringsToBuffer(printable_buf, &offset, sizeof(printable_buf), cbd.payload->buffer,
+                    cbd.payload->offset);
+            jb_set_string(jb, "payload_printable", (char *)printable_buf);
+        }
+        return true;
+    }
+    return false;
+}
+
 static int AlertJson(ThreadVars *tv, JsonAlertLogThread *aft, const Packet *p)
 {
-    MemBuffer *payload = aft->payload_buffer;
     AlertJsonOutputCtx *json_output_ctx = aft->json_output_ctx;
 
     if (p->alerts.cnt == 0 && !(p->flags & PKT_HAS_TAG))
@@ -835,36 +881,14 @@ static int AlertJson(ThreadVars *tv, JsonAlertLogThread *aft, const Packet *p)
             int stream = (p->proto == IPPROTO_TCP) ?
                          (pa->flags & (PACKET_ALERT_FLAG_STATE_MATCH | PACKET_ALERT_FLAG_STREAM_MATCH) ?
                          1 : 0) : 0;
+            // should be impossible, as stream implies flow
+            DEBUG_VALIDATE_BUG_ON(stream && p->flow == NULL);
 
             /* Is this a stream?  If so, pack part of it into the payload field */
-            if (stream) {
-                uint8_t flag;
-
-                MemBufferReset(payload);
-
-                if (p->flowflags & FLOW_PKT_TOSERVER) {
-                    flag = STREAM_DUMP_TOCLIENT;
-                } else {
-                    flag = STREAM_DUMP_TOSERVER;
-                }
-
-                StreamSegmentForEach((const Packet *)p, flag,
-                                    AlertJsonDumpStreamSegmentCallback,
-                                    (void *)payload);
-                if (payload->offset) {
-                    if (json_output_ctx->flags & LOG_JSON_PAYLOAD_BASE64) {
-                        jb_set_base64(jb, "payload", payload->buffer, payload->offset);
-                    }
-
-                    if (json_output_ctx->flags & LOG_JSON_PAYLOAD) {
-                        uint8_t printable_buf[payload->offset + 1];
-                        uint32_t offset = 0;
-                        PrintStringsToBuffer(printable_buf, &offset,
-                                sizeof(printable_buf),
-                                payload->buffer, payload->offset);
-                        jb_set_string(jb, "payload_printable", (char *)printable_buf);
-                    }
-                } else if (p->payload_len) {
+            if (stream && p->flow != NULL) {
+                const bool stream_data_logged =
+                        AlertJsonStreamData(json_output_ctx, aft, p->flow, p, jb);
+                if (!stream_data_logged && p->payload_len) {
                     /* Fallback on packet payload */
                     AlertAddPayload(json_output_ctx, jb, p);
                 }
@@ -877,7 +901,7 @@ static int AlertJson(ThreadVars *tv, JsonAlertLogThread *aft, const Packet *p)
         }
 
         if (pa->flags & PACKET_ALERT_FLAG_FRAME) {
-            AlertAddFrame(p, jb, pa->frame_id);
+            AlertAddFrame(p, pa->frame_id, jb, aft->payload_buffer);
         }
 
         /* base64-encoded full packet */
@@ -1117,14 +1141,13 @@ static void JsonAlertLogSetupMetadata(AlertJsonOutputCtx *json_output_ctx,
                 warn_no_meta = true;
             }
         }
-
-        json_output_ctx->payload_buffer_size = payload_buffer_size;
     }
 
     if (flags & LOG_JSON_RULE_METADATA) {
         DetectEngineSetParseMetadata();
     }
 
+    json_output_ctx->payload_buffer_size = payload_buffer_size;
     json_output_ctx->flags |= flags;
 }
 

src/output-json-frame.c

@@ -125,61 +125,90 @@ static void PayloadAsHex(const uint8_t *data, uint32_t data_len, char *str, size
 }
 #endif
 
-static void FrameAddPayloadTCP(JsonBuilder *js, const TcpStream *stream, const Frame *frame)
+struct FrameJsonStreamDataCallbackData {
+    MemBuffer *payload;
+    const Frame *frame;
+    uint64_t last_re; /**< used to detect gaps */
+};
+
+static int FrameJsonStreamDataCallback(
+        void *cb_data, const uint8_t *input, const uint32_t input_len, const uint64_t input_offset)
 {
-    uint32_t sb_data_len = 0;
-    const uint8_t *data = NULL;
-    uint64_t data_offset = 0;
+    struct FrameJsonStreamDataCallbackData *cbd = cb_data;
+    const Frame *frame = cbd->frame;
 
-    // TODO consider ACK'd
+    uint32_t write_size = input_len;
+    int done = 0;
 
-    if (frame->offset < STREAM_BASE_OFFSET(stream)) {
-        if (StreamingBufferGetData(&stream->sb, &data, &sb_data_len, &data_offset) == 0) {
-            SCLogDebug("NO DATA1");
-            return;
+    if (frame->len >= 0) {
+        const uint64_t data_re = input_offset + input_len;
+        const uint64_t frame_re = frame->offset + (uint64_t)frame->len;
+
+        /* data entirely after frame, we're done */
+        if (input_offset >= frame_re) {
+            return 1;
         }
-    } else {
-        data_offset = (uint64_t)frame->offset;
-        SCLogDebug("data_offset %" PRIu64, data_offset);
-        if (StreamingBufferGetDataAtOffset(
-                    &stream->sb, &data, &sb_data_len, (uint64_t)data_offset) == 0) {
-            SCLogDebug("NO DATA1");
-            return;
+        /* make sure to only log data belonging to the frame */
+        if (data_re >= frame_re) {
+            const uint64_t to_write = frame_re - input_offset;
+            if (to_write < (uint64_t)write_size) {
+                write_size = (uint32_t)to_write;
+            }
+            done = 1;
         }
     }
-    if (data == NULL || sb_data_len == 0) {
-        SCLogDebug("NO DATA2");
-        return;
+    if (input_offset > cbd->last_re) {
+        MemBufferWriteString(
+                cbd->payload, "[%" PRIu64 " bytes missing]", input_offset - cbd->last_re);
     }
 
-    if (frame->len >= 0) {
-        sb_data_len = MIN(frame->len, (int32_t)sb_data_len);
+    if (write_size > 0) {
+        uint32_t written = MemBufferWriteRaw(cbd->payload, input, write_size);
+        if (written < write_size)
+            done = 1;
     }
-    SCLogDebug("frame data_offset %" PRIu64 ", data_len %u frame len %" PRIi64, data_offset,
-            sb_data_len, frame->len);
+    cbd->last_re = input_offset + write_size;
+    return done;
+}
+
+/** \internal
+ *  \brief try to log frame's stream data into payload/payload_printable
+ */
+static void FrameAddPayloadTCP(Flow *f, const TcpSession *ssn, const TcpStream *stream,
+        const Frame *frame, JsonBuilder *jb, MemBuffer *buffer)
+{
+    MemBufferReset(buffer);
 
+    /* consider all data, ACK'd and non-ACK'd */
+    const uint64_t stream_data_re = StreamDataRightEdge(stream, true);
     bool complete = false;
-    if (frame->len > 0) {
-        const uint64_t frame_re = frame->offset + (uint64_t)frame->len;
-        const uint64_t data_re = data_offset + sb_data_len;
-        complete = frame_re <= data_re;
+    if (frame->len >= 0 && frame->offset + (uint64_t)frame->len <= stream_data_re) {
+        complete = true;
     }
-    jb_set_bool(js, "complete", complete);
 
-    uint32_t data_len = MIN(sb_data_len, 256);
-    jb_set_base64(js, "payload", data, data_len);
+    struct FrameJsonStreamDataCallbackData cbd = {
+        .payload = buffer, .frame = frame, .last_re = frame->offset
+    };
+    uint64_t unused = 0;
+    StreamReassembleLog(
+            ssn, stream, FrameJsonStreamDataCallback, &cbd, frame->offset, &unused, false);
+    /* if we have all data, but didn't log until the end of the frame, we have a gap at the
+     * end of the frame
+     * TODO what about not logging due to buffer full? */
+    if (complete && frame->len >= 0 && cbd.last_re < frame->offset + (uint64_t)frame->len) {
+        MemBufferWriteString(cbd.payload, "[%" PRIu64 " bytes missing]",
+                (frame->offset + (uint64_t)frame->len) - cbd.last_re);
+    }
 
-    uint8_t printable_buf[data_len + 1];
-    uint32_t o = 0;
-    PrintStringsToBuffer(printable_buf, &o, data_len + 1, data, data_len);
-    printable_buf[data_len] = '\0';
-    jb_set_string(js, "payload_printable", (char *)printable_buf);
-#if 0
-    char pretty_buf[data_len * 4 + 1];
-    pretty_buf[0] = '\0';
-    PayloadAsHex(data, data_len, pretty_buf, data_len * 4 + 1);
-    jb_set_string(js, "payload_hex", pretty_buf);
-#endif
+    if (cbd.payload->offset) {
+        jb_set_base64(jb, "payload", cbd.payload->buffer, cbd.payload->offset);
+        uint8_t printable_buf[cbd.payload->offset + 1];
+        uint32_t offset = 0;
+        PrintStringsToBuffer(printable_buf, &offset, sizeof(printable_buf), cbd.payload->buffer,
+                cbd.payload->offset);
+        jb_set_string(jb, "payload_printable", (char *)printable_buf);
+        jb_set_bool(jb, "complete", complete);
+    }
 }
 
 static void FrameAddPayloadUDP(JsonBuilder *js, const Packet *p, const Frame *frame)
@@ -223,8 +252,8 @@ static void FrameAddPayloadUDP(JsonBuilder *js, const Packet *p, const Frame *fr
 /** \brief log a single frame
  *  \note ipproto argument is passed to assist static code analyzers
  */
-void FrameJsonLogOneFrame(const uint8_t ipproto, const Frame *frame, const Flow *f,
-        const TcpStream *stream, const Packet *p, JsonBuilder *jb)
+void FrameJsonLogOneFrame(const uint8_t ipproto, const Frame *frame, Flow *f,
+        const TcpStream *stream, const Packet *p, JsonBuilder *jb, MemBuffer *buffer)
 {
     DEBUG_VALIDATE_BUG_ON(ipproto != p->proto);
     DEBUG_VALIDATE_BUG_ON(ipproto != f->proto);
@@ -249,7 +278,7 @@ void FrameJsonLogOneFrame(const uint8_t ipproto, const Frame *frame, const Flow
         } else {
             jb_set_uint(jb, "length", frame->len);
         }
-        FrameAddPayloadTCP(jb, stream, frame);
+        FrameAddPayloadTCP(f, f->protoctx, stream, frame, jb, buffer);
     } else {
         jb_set_uint(jb, "length", frame->len);
         FrameAddPayloadUDP(jb, p, frame);
@@ -287,7 +316,7 @@ static int FrameJsonUdp(
             return TM_ECODE_OK;
 
         jb_set_string(jb, "app_proto", AppProtoToString(f->alproto));
-        FrameJsonLogOneFrame(IPPROTO_UDP, frame, p->flow, NULL, p, jb);
+        FrameJsonLogOneFrame(IPPROTO_UDP, frame, p->flow, NULL, p, jb, aft->payload_buffer);
         OutputJsonBuilderBuffer(jb, aft->ctx);
         jb_free(jb);
         frame->flags |= FRAME_FLAG_LOGGED;
@@ -359,7 +388,7 @@ static int FrameJson(ThreadVars *tv, JsonFrameLogThread *aft, const Packet *p)
                 return TM_ECODE_OK;
 
             jb_set_string(jb, "app_proto", AppProtoToString(p->flow->alproto));
-            FrameJsonLogOneFrame(IPPROTO_TCP, frame, p->flow, stream, p, jb);
+            FrameJsonLogOneFrame(IPPROTO_TCP, frame, p->flow, stream, p, jb, aft->payload_buffer);
             OutputJsonBuilderBuffer(jb, aft->ctx);
             jb_free(jb);
             frame->flags |= FRAME_FLAG_LOGGED;
@@ -483,8 +512,22 @@ static OutputInitResult JsonFrameLogInitCtxSub(ConfNode *conf, OutputCtx *parent
     }
     memset(json_output_ctx, 0, sizeof(FrameJsonOutputCtx));
 
+    uint32_t payload_buffer_size = 4096;
+    if (conf != NULL) {
+        const char *payload_buffer_value = ConfNodeLookupChildValue(conf, "payload-buffer-size");
+        if (payload_buffer_value != NULL) {
+            uint32_t value;
+            if (ParseSizeStringU32(payload_buffer_value, &value) < 0) {
+                SCLogError("Error parsing payload-buffer-size \"%s\"", payload_buffer_value);
+                goto error;
+            }
+            payload_buffer_size = value;
+        }
+    }
+
     json_output_ctx->file_ctx = ajt->file_ctx;
     json_output_ctx->eve_ctx = ajt;
+    json_output_ctx->payload_buffer_size = payload_buffer_size;
 
     output_ctx->data = json_output_ctx;
     output_ctx->DeInit = JsonFrameLogDeInitCtxSub;

src/output-json-frame.h

@@ -30,8 +30,8 @@
 #include "app-layer-frames.h"
 #include "stream-tcp-private.h"
 
-void FrameJsonLogOneFrame(const uint8_t ipproto, const Frame *frame, const Flow *f,
-        const TcpStream *stream, const Packet *p, JsonBuilder *jb);
+void FrameJsonLogOneFrame(const uint8_t ipproto, const Frame *frame, Flow *f,
+        const TcpStream *stream, const Packet *p, JsonBuilder *jb, MemBuffer *);
 void JsonFrameLogRegister(void);
 
 #endif /* __OUTPUT_JSON_FRAME_H__ */