[New Integration] Goflow2 integration

.github/CODEOWNERS

@@ -205,6 +205,7 @@
 /packages/github @elastic/security-service-integrations
 /packages/gitlab @elastic/security-service-integrations
 /packages/golang @elastic/obs-infraobs-integrations
+/packages/goflow2 @elastic/sec-deployment-and-devices
 /packages/google_cloud_storage @elastic/security-service-integrations
 /packages/google_scc @elastic/security-service-integrations
 /packages/google_workspace @elastic/security-service-integrations

packages/goflow2/_dev/build/build.yml

@@ -0,0 +1,4 @@
+dependencies:
+  ecs:
+    reference: [email protected]
+    import_mappings: true

packages/goflow2/_dev/build/docs/README.md

@@ -0,0 +1,59 @@
+# GoFlow2
+
+The GoFlow2 integration allows you to import logs generated by goflow2.
+
+The only supported protocol is sflow, since there are already existing integrations for netflow/IPFIX.
+
+## Data streams
+### sflow
+The Goflow2 sFlow integration collects one type of data streams: logs
+
+#### Sample Event
+{{ event "sflow" }}
+
+## Requirements
+
+You need Elasticsearch for storing and searching your data and Kibana for visualizing and managing it.
+You can use our hosted Elasticsearch Service on Elastic Cloud, which is recommended, or self-manage the Elastic Stack on your own hardware.
+
+You need GoFlow2 to create log files for sFlow traffic.
+https://github.com/netsampler/goflow2
+
+## Setup
+
+- Install integration and role out elastic agent
+- Install GoFlow2 for sFlow logging
+
+Please use the following GoFlow2 mapping.yaml file:
+
+```
+# File: /etc/goflow2/mapping.yaml
+formatter:
+    fields: # list of fields to format in JSON
+        - type
+        - time_flow_start_ns
+        - sampler_address
+        - sequence_num
+        - in_if
+        - out_if
+        - src_addr
+        - dst_addr
+        - etype
+        - proto
+        - src_port
+        - dst_port
+        - src_vlan
+        - dst_vlan
+        - sampling_rate
+        - bytes
+```
+
+The output sflow transport files must be stored in the directory ```/var/log/sflow/goflow2/```
+
+Full command to run GoFlow2 for sflow traffic:
+```shell
+goflow2 -format json -listen "sflow://:6343" -mapping /etc/goflow2/mapping.yaml -transport.file /var/log/sflow/goflow2/goflow2.log
+```
+
+## Fields
+{{ fields "sflow" }}

packages/goflow2/_dev/deploy/docker/docker-compose.yml

@@ -0,0 +1,7 @@
+services:
+  goflow2-sflow-logfile:
+    image: alpine
+    volumes:
+      - ./sample_logs:/sample_logs:ro
+      - ${SERVICE_LOGS_DIR}:/var/log/sflow/goflow2/
+    command: /bin/sh -c "cp /sample_logs/* /var/log/sflow/goflow2/"

packages/goflow2/_dev/deploy/docker/sample_logs/test-goflow2-sflow-sample.log

@@ -0,0 +1,9 @@
+{"type":"SFLOW_5","time_flow_start_ns":1722384059314899647,"sampler_address":"67.43.156.1","sequence_num":44555,"in_if":563,"out_if":573,"src_addr":"216.160.83.57","dst_addr":"216.160.83.58","etype":"IPv4","proto":"TCP","src_port":10876,"dst_port":443,"src_vlan":1500,"dst_vlan":1500,"sampling_rate":1000,"bytes":70}
+{"type":"SFLOW_5","time_flow_start_ns":1722384059333197201,"sampler_address":"89.160.20.129","sequence_num":27481,"in_if":637,"out_if":742,"src_addr":"81.2.69.193","dst_addr":"81.2.69.194","etype":"IPv4","proto":"TCP","src_port":80,"dst_port":55319,"src_vlan":500,"dst_vlan":500,"sampling_rate":2000,"bytes":1518}
+{"type":"SFLOW_5","time_flow_start_ns":1722384059333197201,"sampler_address":"67.43.156.1","sequence_num":27481,"in_if":637,"out_if":609,"src_addr":"216.160.83.59","dst_addr":"216.160.83.60","etype":"IPv4","proto":"ESP","src_port":0,"dst_port":0,"src_vlan":500,"dst_vlan":500,"sampling_rate":500,"bytes":142}
+{"type":"SFLOW_5","time_flow_start_ns":1722384059483524068,"sampler_address":"67.43.156.1","sequence_num":1022,"in_if":0,"out_if":561,"src_addr":"216.160.83.60","dst_addr":"216.160.83.59","etype":"IPv4","proto":"TCP","src_port":19156,"dst_port":443,"src_vlan":0,"dst_vlan":1500,"sampling_rate":2000,"bytes":1518}
+{"type":"SFLOW_5","time_flow_start_ns":1722384059483524068,"sampler_address":"67.43.156.1","sequence_num":1022,"in_if":0,"out_if":561,"src_addr":"216.160.83.59","dst_addr":"216.160.83.58","etype":"IPv4","proto":"TCP","src_port":19156,"dst_port":443,"src_vlan":0,"dst_vlan":1500,"sampling_rate":2000,"bytes":1518}
+{"type":"SFLOW_5","time_flow_start_ns":1722384059483524068,"sampler_address":"67.43.156.1","sequence_num":1022,"in_if":531,"out_if":561,"src_addr":"216.160.83.59","dst_addr":"216.160.83.58","etype":"IPv4","proto":"UDP","src_port":1122,"dst_port":6097,"src_vlan":1500,"dst_vlan":1500,"sampling_rate":2000,"bytes":1518}
+{"type":"SFLOW_5","time_flow_start_ns":1722384059483524068,"sampler_address":"89.160.20.129","sequence_num":1022,"in_if":0,"out_if":561,"src_addr":"81.2.69.193","dst_addr":"81.2.69.194","etype":"IPv4","proto":"TCP","src_port":49031,"dst_port":443,"src_vlan":0,"dst_vlan":1500,"sampling_rate":2000,"bytes":1518}
+{"type":"SFLOW_5","time_flow_start_ns":1722384059483524068,"sampler_address":"89.160.20.129","sequence_num":1022,"in_if":0,"out_if":561,"src_addr":"81.2.69.193","dst_addr":"81.2.69.194","etype":"IPv4","proto":"TCP","src_port":31385,"dst_port":443,"src_vlan":0,"dst_vlan":1500,"sampling_rate":2000,"bytes":1518}
+{"type":"SFLOW_5","time_flow_start_ns":1722384059483524068,"sampler_address":"89.160.20.129","sequence_num":1022,"in_if":561,"out_if":531,"src_addr":"81.2.69.193","dst_addr":"81.2.69.194","etype":"IPv4","proto":"TCP","src_port":6097,"dst_port":443,"src_vlan":1500,"dst_vlan":1500,"sampling_rate":2000,"bytes":70}

packages/goflow2/changelog.yml

@@ -0,0 +1,71 @@
+# newer versions go on top
+- version: "0.0.14"
+  changes:
+    - description: changed event.kind from metric to event
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/10561
+- version: "0.0.13"
+  changes:
+    - description: changed event.kind from metric to event
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/10561
+- version: "0.0.12"
+  changes:
+    - description: changed event.kind from metric to event
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/10561
+- version: "0.0.11"
+  changes:
+    - description: changed event.kind from metric to event
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/10561
+- version: "0.0.10"
+  changes:
+    - description: changed event.kind from metric to event
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/10561
+- version: "0.0.9"
+  changes:
+    - description: changed event.kind from metric to event
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/10561
+- version: "0.0.8"
+  changes:
+    - description: changed event.kind from metric to event
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/10561
+- version: "0.0.7"
+  changes:
+    - description: changed event.kind from metric to event
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/10561
+- version: "0.0.6"
+  changes:
+    - description: changed event.kind from metric to event
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/10561
+- version: "0.0.5"
+  changes:
+    - description: changed event.kind from metric to event
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/10561
+- version: "0.0.4"
+  changes:
+    - description: fixed small problems
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/10561
+- version: "0.0.3"
+  changes:
+    - description: fixed small problems
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/10561
+- version: "0.0.2"
+  changes:
+    - description: fixed small problems
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/10561
+- version: "0.0.1"
+  changes:
+    - description: Initial version of the package
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/10561

packages/goflow2/data_stream/sflow/_dev/test/pipeline/test-goflow2-sflow-sample.log

@@ -0,0 +1,9 @@
+{"type":"SFLOW_5","time_flow_start_ns":1722384059314899647,"sampler_address":"67.43.156.1","sequence_num":44555,"in_if":563,"out_if":573,"src_addr":"216.160.83.57","dst_addr":"216.160.83.58","etype":"IPv4","proto":"TCP","src_port":10876,"dst_port":443,"src_vlan":1500,"dst_vlan":1500,"sampling_rate":1000,"bytes":70}
+{"type":"SFLOW_5","time_flow_start_ns":1722384059333197201,"sampler_address":"89.160.20.129","sequence_num":27481,"in_if":637,"out_if":742,"src_addr":"81.2.69.193","dst_addr":"81.2.69.194","etype":"IPv4","proto":"TCP","src_port":80,"dst_port":55319,"src_vlan":500,"dst_vlan":500,"sampling_rate":2000,"bytes":1518}
+{"type":"SFLOW_5","time_flow_start_ns":1722384059333197201,"sampler_address":"67.43.156.1","sequence_num":27481,"in_if":637,"out_if":609,"src_addr":"216.160.83.59","dst_addr":"216.160.83.60","etype":"IPv4","proto":"ESP","src_port":0,"dst_port":0,"src_vlan":500,"dst_vlan":500,"sampling_rate":500,"bytes":142}
+{"type":"SFLOW_5","time_flow_start_ns":1722384059483524068,"sampler_address":"67.43.156.1","sequence_num":1022,"in_if":0,"out_if":561,"src_addr":"216.160.83.60","dst_addr":"216.160.83.59","etype":"IPv4","proto":"TCP","src_port":19156,"dst_port":443,"src_vlan":0,"dst_vlan":1500,"sampling_rate":2000,"bytes":1518}
+{"type":"SFLOW_5","time_flow_start_ns":1722384059483524068,"sampler_address":"67.43.156.1","sequence_num":1022,"in_if":0,"out_if":561,"src_addr":"216.160.83.59","dst_addr":"216.160.83.58","etype":"IPv4","proto":"TCP","src_port":19156,"dst_port":443,"src_vlan":0,"dst_vlan":1500,"sampling_rate":2000,"bytes":1518}
+{"type":"SFLOW_5","time_flow_start_ns":1722384059483524068,"sampler_address":"67.43.156.1","sequence_num":1022,"in_if":531,"out_if":561,"src_addr":"216.160.83.59","dst_addr":"216.160.83.58","etype":"IPv4","proto":"UDP","src_port":1122,"dst_port":6097,"src_vlan":1500,"dst_vlan":1500,"sampling_rate":2000,"bytes":1518}
+{"type":"SFLOW_5","time_flow_start_ns":1722384059483524068,"sampler_address":"89.160.20.129","sequence_num":1022,"in_if":0,"out_if":561,"src_addr":"81.2.69.193","dst_addr":"81.2.69.194","etype":"IPv4","proto":"TCP","src_port":49031,"dst_port":443,"src_vlan":0,"dst_vlan":1500,"sampling_rate":2000,"bytes":1518}
+{"type":"SFLOW_5","time_flow_start_ns":1722384059483524068,"sampler_address":"89.160.20.129","sequence_num":1022,"in_if":0,"out_if":561,"src_addr":"81.2.69.193","dst_addr":"81.2.69.194","etype":"IPv4","proto":"TCP","src_port":31385,"dst_port":443,"src_vlan":0,"dst_vlan":1500,"sampling_rate":2000,"bytes":1518}
+{"type":"SFLOW_5","time_flow_start_ns":1722384059483524068,"sampler_address":"89.160.20.129","sequence_num":1022,"in_if":561,"out_if":531,"src_addr":"81.2.69.193","dst_addr":"81.2.69.194","etype":"IPv4","proto":"TCP","src_port":6097,"dst_port":443,"src_vlan":1500,"dst_vlan":1500,"sampling_rate":2000,"bytes":70}

packages/goflow2/data_stream/sflow/_dev/test/pipeline/test-goflow2-sflow-sample.log-config.yml

@@ -0,0 +1,7 @@
+fields:
+  tags:
+    - preserve_original_event
+    - forwarded
+    - sflow
+  event:
+    timezone: "+00:00"