From 4f053d338f014efb0f7ec5bf3ecb180aeb785313 Mon Sep 17 00:00:00 2001
From: Louis <lspgn@users.noreply.github.com>
Date: Sat, 9 Dec 2023 11:56:17 -0800
Subject: [PATCH] decoders: do not stop reading sets (#239)

---
 decoders/netflow/ipfix.go   |  3 +++
 decoders/netflow/netflow.go | 15 ++++++++++++---
 decoders/netflow/nfv9.go    |  3 +++
 decoders/netflow/packet.go  | 15 +++++++++++++++
 4 files changed, 33 insertions(+), 3 deletions(-)

diff --git a/decoders/netflow/ipfix.go b/decoders/netflow/ipfix.go
index bb1cbbe3..7ff11c79 100644
--- a/decoders/netflow/ipfix.go
+++ b/decoders/netflow/ipfix.go
@@ -977,6 +977,9 @@ func (p IPFIXPacket) String() string {
 		case DataFlowSet:
 			str += fmt.Sprintf("    - DataFlowSet %v:\n", i)
 			str += flowSet.String(IPFIXTypeToString)
+		case RawFlowSet:
+			str += fmt.Sprintf("    - RawFlowSet %v:\n", i)
+			str += flowSet.String()
 		case OptionsDataFlowSet:
 			str += fmt.Sprintf("    - OptionsDataFlowSet %v:\n", i)
 			str += flowSet.String(IPFIXTypeToString, IPFIXTypeToString)
diff --git a/decoders/netflow/netflow.go b/decoders/netflow/netflow.go
index 8c083610..c4c4125f 100644
--- a/decoders/netflow/netflow.go
+++ b/decoders/netflow/netflow.go
@@ -3,6 +3,7 @@ package netflow
 import (
 	"bytes"
 	"encoding/binary"
+	"errors"
 	"fmt"
 
 	"github.com/netsampler/goflow2/v2/decoders/utils"
@@ -286,10 +287,13 @@ func DecodeMessageCommon(payload *bytes.Buffer, templates NetFlowTemplateSystem,
 	var read int
 	startSize := payload.Len()
 	for i := 0; ((i < int(size) && version == 9) || (uint16(read) < size && version == 10)) && payload.Len() > 0; i++ {
-		if flowSet, err := DecodeMessageCommonFlowSet(payload, templates, obsDomainId, version); err != nil {
-			return flowSets, err
+		if flowSet, lerr := DecodeMessageCommonFlowSet(payload, templates, obsDomainId, version); lerr != nil && !errors.Is(lerr, ErrorTemplateNotFound) {
+			return flowSets, lerr
 		} else {
 			flowSets = append(flowSets, flowSet)
+			if lerr != nil {
+				err = errors.Join(err, lerr)
+			}
 		}
 		read = startSize - payload.Len()
 	}
@@ -392,7 +396,12 @@ func DecodeMessageCommonFlowSet(payload *bytes.Buffer, templates NetFlowTemplate
 		}
 
 	} else if fsheader.Id >= 256 {
-		dataReader := bytes.NewBuffer(payload.Next(nextrelpos))
+		rawfs := RawFlowSet{
+			FlowSetHeader: fsheader,
+			Records:       payload.Next(nextrelpos),
+		}
+		flowSet = rawfs
+		dataReader := bytes.NewBuffer(rawfs.Records)
 
 		if templates == nil {
 			return flowSet, &FlowError{version, "Templates", obsDomainId, fsheader.Id, fmt.Errorf("No templates")}
diff --git a/decoders/netflow/nfv9.go b/decoders/netflow/nfv9.go
index b086da12..814c5cc2 100644
--- a/decoders/netflow/nfv9.go
+++ b/decoders/netflow/nfv9.go
@@ -306,6 +306,9 @@ func (p NFv9Packet) String() string {
 		case DataFlowSet:
 			str += fmt.Sprintf("    - DataFlowSet %v:\n", i)
 			str += flowSet.String(NFv9TypeToString)
+		case RawFlowSet:
+			str += fmt.Sprintf("    - RawFlowSet %v:\n", i)
+			str += flowSet.String()
 		case OptionsDataFlowSet:
 			str += fmt.Sprintf("    - OptionsDataFlowSet %v:\n", i)
 			str += flowSet.String(NFv9TypeToString, NFv9ScopeToString)
diff --git a/decoders/netflow/packet.go b/decoders/netflow/packet.go
index 4b890ca9..eaf48bbd 100644
--- a/decoders/netflow/packet.go
+++ b/decoders/netflow/packet.go
@@ -34,6 +34,13 @@ type DataFlowSet struct {
 	Records []DataRecord `json:"records"`
 }
 
+// RawFlowSet is a a set that could not be decoded due to the absence of a template
+type RawFlowSet struct {
+	FlowSetHeader
+
+	Records []byte `json:"records"`
+}
+
 type OptionsDataFlowSet struct {
 	FlowSetHeader
 
@@ -98,6 +105,14 @@ type DataField struct {
 	//Value []byte
 }
 
+func (flowSet RawFlowSet) String() string {
+	str := fmt.Sprintf("       Id %v\n", flowSet.Id)
+	str += fmt.Sprintf("       Length: %v\n", len(flowSet.Records))
+	str += fmt.Sprintf("       Records: %v\n", flowSet.Records)
+
+	return str
+}
+
 func (flowSet OptionsDataFlowSet) String(TypeToString func(uint16) string, ScopeToString func(uint16) string) string {
 	str := fmt.Sprintf("       Id %v\n", flowSet.Id)
 	str += fmt.Sprintf("       Length: %v\n", flowSet.Length)