[O11y][Kubernetes] Rally benchmark kubernetes.state_container (#9106)

* rally benchmark kubernetes.state_container

* add container name

* address review comments

* add last_terminated_reason

* rename yml file

* rename yml files for aws package

* rename yml files for mysql package

* remove host.ip and host.mac

packages/kubernetes/_dev/benchmark/rally/state_container-benchmark.yml

@@ -0,0 +1,14 @@
+---
+description: Benchmark 20000 kubernetes.state_container events ingested
+data_stream:
+  name: state_container
+corpora:
+  generator:
+    total_events: 20000
+    template:
+      type: gotext
+      path: ./statecontainer-benchmark/template.ndjson
+    config:
+      path: ./statecontainer-benchmark/config.yml
+    fields:
+      path: ./statecontainer-benchmark/fields.yml

packages/kubernetes/_dev/benchmark/rally/statecontainer-benchmark/config.yml

@@ -0,0 +1,49 @@
+fields:
+  - name: timestamp
+    period: 60m
+  - name: event_duration
+    range:
+      min: 1
+      max: 1000000
+  - name: status_phase
+    enum: ["running", "waiting", "terminated"]
+  - name: status_ready
+    enum: ["true", "false"]
+  - name: restarts
+    range:
+      min: 1
+      max: 100
+  - name: reason_waiting
+    enum: ["ContainerCreating", "CrashLoopBackoff", "ErrImagePull", "ImagePullBackoff"]
+  - name: reason_terminated
+    enum: ["Completed", "ContainerCannotRun", "Error", "OOMKilled"]
+  - name: cpu_limit_cores
+    range:
+      min: 4
+      max: 6
+  - name: cpu_request_cores
+    range:
+      min: 0
+      max: 3
+  - name: memory_limit_bytes
+    range:
+      min: 1000000000
+      max: 16000000000
+  - name: memory_request_bytes
+    range:
+      min: 500000000
+      max: 999000000
+  - name: rangeofid
+    range:
+      min: 0
+      max: 10000
+  - name: cpu_limit_nanocores
+    range:
+      min: 10000000
+      max: 30000000
+  - name: cpu_request_nanocores
+    range:
+      min: 100000
+      max: 300000
+  - name: container_name
+    enum: ["web", "default-http-backend", "dnsmasq", "csi-driver", "web", "web", "web", "prometheus", "konnectivity-agent", "sidecar", "kubedns", "metrics-server-nanny", "web", "web", "fluentbit", "autoscaler", "gke-metrics-agent", "elastic-agent", "web", "kube-state-metrics", "metrics-server", "fluentbit", "elastic-agent", "web", "prometheus-to-sd-exporter"]

packages/kubernetes/_dev/benchmark/rally/statecontainer-benchmark/fields.yml

@@ -0,0 +1,32 @@
+- name: Ip
+  type: ip
+- name: timestamp
+  type: date
+- name: rangeofid
+  type: integer
+- name: event_duration
+  type: long
+- name: container_name
+  type: keyword
+- name: status_phase
+  type: keyword
+- name: status_ready
+  type: boolean
+- name: restarts
+  type: integer
+- name: reason_waiting
+  type: keyword
+- name: reason_terminated
+  type: keyword
+- name: cpu_limit_cores
+  type: float
+- name: cpu_request_cores
+  type: float
+- name: memory_limit_bytes
+  type: long
+- name: memory_request_bytes
+  type: long
+- name: cpu_limit_nanocores
+  type: long
+- name: cpu_request_nanocores
+  type: long

packages/kubernetes/_dev/benchmark/rally/statecontainer-benchmark/template.ndjson

@@ -0,0 +1,174 @@
+{{- $timestamp := generate "timestamp" }}
+{{- $event_duration := generate "event_duration" }}
+{{- $rangeofid := generate "rangeofid" -}}
+{{- $nodeid := div $rangeofid 110 -}}
+{{- $container_name := generate "container_name" }}
+{{- $status_phase := generate "status_phase" }}
+{{- $status_ready := generate "status_ready" }}
+{{- $restarts := generate "restarts" }}
+{{- $reason_waiting := generate "reason_waiting" }}
+{{- $reason_terminated := generate "reason_terminated" }}
+{{- $cpu_limit_cores := generate "cpu_limit_cores" }}
+{{- $cpu_request_cores := generate "cpu_request_cores" }}
+{{- $memory_limit_bytes := generate "memory_limit_bytes" }}
+{{- $memory_request_bytes := generate "memory_request_bytes" }}
+{{- $cpu_limit_nanocores := generate "cpu_limit_nanocores" }}
+{{- $cpu_request_nanocores := generate "cpu_request_nanocores" }}
+{
+    "@timestamp": "{{$timestamp.Format "2006-01-02T15:04:05.999999Z07:00"}}",
+    "container": {
+        "image": {
+            "name": "k8s.gcr.io/coredns/coredns:v1.8.0"
+        },
+        "runtime": "containerd",
+        "id": "container-{{ $rangeofid }}"
+    },
+    "kubernetes": {
+        "container": {
+            "memory": {
+                "request": {
+                    "bytes": {{ $memory_request_bytes }}
+                },
+                "limit": {
+                    "bytes": {{ $memory_limit_bytes }}
+                }
+            },
+            "name": "{{ $container_name }}",
+            "cpu": {
+                "request": {
+                    "cores": {{ $cpu_request_cores }},
+                    "nanocores": {{ $cpu_request_nanocores }}
+                },
+                "limit": {
+                    "cores": {{ $cpu_limit_cores }},
+                    "nanocores": {{ $cpu_limit_nanocores }}
+                }
+            },
+            "id": "container-{{ $rangeofid }}",
+            "status": {
+                "phase": "{{ $status_phase }}",
+            {{- if eq $status_phase "running"}}
+                "ready": "true",
+                "last_terminated_reason": "{{ $reason_terminated }}",
+            {{- else if eq $status_phase "terminated"}}
+                "ready": "false",
+                "reason": "{{ $reason_terminated }}",
+            {{- else if eq $status_phase "waiting"}}
+                "ready": "{{ $status_ready }}",
+                "reason": "{{ $reason_waiting }}",
+            {{- end}}
+                "restarts": {{ $restarts }}
+            }
+        },
+        "node": {
+            "uid": "host-{{ $nodeid }}",
+            "hostname": "host-{{ $nodeid }}",
+            "name": "host-{{ $nodeid }}",
+            "labels": {
+                "kubernetes_io/hostname": "kubernetes-scale-123456",
+                "beta_kubernetes_io/os": "linux",
+                "kubernetes_io/arch": "amd64",
+                "kubernetes_io/os": "linux",
+                "beta_kubernetes_io/arch": "amd64"
+            }
+        },
+        "pod": {
+            "uid": "demo-pod-{{ $rangeofid }}",
+            "ip": "{{generate `Ip`}}",
+            "name": "demo-pod-{{ $rangeofid }}"
+        },
+        "namespace": "demo-{{ $rangeofid }}",
+        "namespace_uid": "demo-{{ $rangeofid }}",
+        "replicaset": {
+            "name": "demo-deployment-{{ $rangeofid }}"
+        },
+        "namespace_labels": {
+            "kubernetes_io/metadata_name": "demo-{{ $rangeofid }}"
+        },
+        "labels": {
+            "app":"demo",
+            "pod-template-hash":"{{ $rangeofid }}",
+            "app-2":"demo-2",
+            "app-1":"demo-1"
+        },
+        "deployment": {
+            "name": "demo-deployment-{{ $rangeofid }}"
+        }
+    },
+    "agent": {
+        "name": "kubernetes-scale-123456",
+        "id": "de42127b-4db8-4471-824e-a7b14f478663",
+        "ephemeral_id": "22ed892c-43bd-408a-9121-65e2f5b6a56e",
+        "type": "metricbeat",
+        "version": "8.8.0"
+    },
+    "elastic_agent": {
+        "id": "de42127b-4db8-4471-824e-a7b14f478663",
+        "version": "8.8.0",
+        "snapshot": true
+    },
+    "cloud": {
+         "provider": "gcp",
+         "availability_zone": "europe-west1-d",
+         "instance":{
+         "name":  "kubernetes-scale-123456" ,
+         "id": "de42127b-4db8-4471-824e-a7b14f478663"
+         },
+         "machine":{
+            "type":"e2-standard-4"
+         },
+         "service":{
+            "name":"GCE"
+         },
+         "project":{
+            "id":"elastic-obs-integrations-dev"
+         },
+         "account":{
+            "id":"elastic-obs-integrations-dev"
+         }
+      },
+      "orchestrator":{
+         "cluster":{
+            "name":"kubernetes-scale",
+            "url":"https://{{ generate `Ip` }}"
+         }
+      },
+    "ecs": {
+        "version": "8.0.0"
+    },
+    "data_stream": {
+        "namespace": "ep",
+        "type": "metrics",
+        "dataset": "kubernetes.state_container"
+    },
+    "service": {
+        "address": "http://kubernetes-scale-123456:8080/metrics",
+        "type": "kubernetes"
+    },
+    "host": {
+        "hostname": "kubernetes-scale-123456",
+        "os": {
+            "kernel": "5.10.47-linuxkit",
+            "codename": "Core",
+            "name": "CentOS Linux",
+            "type": "linux",
+            "family": "redhat",
+            "version": "7 (Core)",
+            "platform": "centos"
+        },
+        "containerized": true,
+        "name": "kubernetes-scale-123456",
+        "id": "85e35c2b5e1b39ba72393a6baf6ee7cd",
+        "architecture": "x86_64"
+    },
+    "metricset": {
+        "period": 10000,
+        "name": "state_container"
+    },
+    "event": {
+        "duration": {{ $event_duration }},
+        "agent_id_status": "verified",
+        "module": "kubernetes",
+        "dataset": "kubernetes.state_container"
+    }
+}