From 11b39ae020a51c220089ecd40f52190eb48ddd21 Mon Sep 17 00:00:00 2001 From: Jonada Hoxha Date: Thu, 30 May 2024 13:02:02 +0200 Subject: [PATCH 1/4] Add Icinga states to pod --- pkg/schema/v1/pod.go | 124 ++++++++++++++++++++++++++++++++++++++++ schema/mysql/schema.sql | 2 + 2 files changed, 126 insertions(+) diff --git a/pkg/schema/v1/pod.go b/pkg/schema/v1/pod.go index d17f6035..955cf626 100644 --- a/pkg/schema/v1/pod.go +++ b/pkg/schema/v1/pod.go @@ -2,6 +2,7 @@ package v1 import ( "database/sql" + "fmt" "github.com/icinga/icinga-go-library/types" "github.com/icinga/icinga-kubernetes/pkg/database" "github.com/icinga/icinga-kubernetes/pkg/strcase" @@ -13,8 +14,11 @@ import ( ktypes "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes" "strings" + "time" ) +const prolongedInitializationThreshold = 10 * time.Minute + type PodFactory struct { clientset *kubernetes.Clientset } @@ -25,6 +29,8 @@ type Pod struct { NominatedNodeName string Ip string Phase string + IcingaState IcingaState + IcingaStateReason string CpuLimits int64 CpuRequests int64 MemoryLimits int64 @@ -116,6 +122,7 @@ func (p *Pod) Obtain(k8s kmetav1.Object) { p.NominatedNodeName = pod.Status.NominatedNodeName p.Ip = pod.Status.PodIP p.Phase = strcase.Snake(string(pod.Status.Phase)) + p.IcingaState, p.IcingaStateReason = p.getIcingaState(pod) p.Reason = pod.Status.Reason p.Message = pod.Status.Message p.Qos = strcase.Snake(string(pod.Status.QOSClass)) @@ -306,6 +313,123 @@ func (p *Pod) Obtain(k8s kmetav1.Object) { p.Yaml = string(output) } +func (p *Pod) getIcingaState(pod *kcorev1.Pod) (IcingaState, string) { + readyContainers := 0 + state := Unknown + reason := string(pod.Status.Phase) + + if pod.Status.Reason != "" { + reason = fmt.Sprintf("Pod %s/%s is in %s state with reason: %s", pod.Namespace, pod.Name, pod.Status.Phase, pod.Status.Reason) + } + + if pod.DeletionTimestamp != nil { + reason = fmt.Sprintf("Pod %s/%s is being deleted", pod.Namespace, pod.Name) + + return Ok, reason + } + + // If the Pod carries {type:PodScheduled, reason:SchedulingGated}, set reason to 'SchedulingGated'. + for _, condition := range pod.Status.Conditions { + if condition.Type == kcorev1.PodScheduled && condition.Reason == kcorev1.PodReasonSchedulingGated { + state = Critical + reason = fmt.Sprintf("Pod %s/%s scheduling skipped because one or more scheduling gates are still present: %s", pod.Namespace, pod.Name, kcorev1.PodReasonSchedulingGated) + } + } + + initializing := false + for i, container := range pod.Status.InitContainerStatuses { + switch { + case container.State.Terminated != nil && container.State.Terminated.ExitCode == 0: + continue + case container.State.Terminated != nil: + // Initialization failed + if len(container.State.Terminated.Reason) == 0 { + if container.State.Terminated.Signal != 0 { + reason = fmt.Sprintf("Init container %s is terminated with signal: %d", container.Name, container.State.Terminated.Signal) + } else { + reason = fmt.Sprintf("Init container %s is terminated with non-zero exit code %d: %s", container.Name, container.State.Terminated.ExitCode, container.State.Terminated.Reason) + } + } else { + reason = fmt.Sprintf("Init container %s is terminated. Reason %s: ", container.Name, container.State.Terminated.Reason) + } + state = Critical + initializing = true + case container.State.Waiting != nil && len(container.State.Waiting.Reason) > 0 && container.State.Waiting.Reason != "PodInitializing": + state = Critical + reason = fmt.Sprintf("Init container %s is waiting: %s", container.Name, container.State.Waiting.Reason) + initializing = true + default: + initializing = true + if container.State.Running != nil { + duration := time.Since(container.State.Running.StartedAt.Time).Round(time.Second) + if duration > prolongedInitializationThreshold { + state = Critical + reason = fmt.Sprintf("Init container %s has been initializing for too long (%d/%d, %s elapsed)", container.Name, i+1, len(pod.Spec.InitContainers), duration) + } else { + reason = fmt.Sprintf("Init container %s is currently initializing (%d/%d)", container.Name, i+1, len(pod.Spec.InitContainers)) + } + } + } + break + } + + if !initializing { + hasRunning := false + for _, container := range pod.Status.ContainerStatuses { + if container.State.Waiting != nil && container.State.Waiting.Reason != "" && container.RestartCount >= 3 { + state = Critical + reason = fmt.Sprintf("Container %s is waiting and has restarted %d times: %s", container.Name, container.RestartCount, container.State.Waiting.Reason) + continue + } else if container.State.Terminated != nil { + if container.State.Terminated.Reason == "Completed" { + state = Ok + reason = fmt.Sprintf("Container %s has completed successfully", container.Name) + } else if container.State.Terminated.Reason != "" { + state = Critical + reason = fmt.Sprintf("Container %s is terminated. Reason: %s", container.Name, container.State.Terminated.Reason) + } else { + if container.State.Terminated.Signal != 0 { + state = Critical + reason = fmt.Sprintf("Container %s is terminated with signal: %d", container.Name, container.State.Terminated.Signal) + } else if container.State.Terminated.ExitCode != 0 { + state = Critical + reason = fmt.Sprintf("Container %s is terminated with non-zero exit code %d: %s", container.Name, container.State.Terminated.ExitCode, container.State.Terminated.Reason) + } else { + state = Ok + reason = fmt.Sprintf("Container %s is terminated normally", container.Name) + } + } + } else if container.State.Running != nil && container.Ready { + readyContainers++ + hasRunning = true + state = Ok + reason = fmt.Sprintf("Container %s is running", container.Name) + } + } + + if reason == "Completed" && hasRunning { + for _, condition := range pod.Status.Conditions { + if pod.Status.Phase == kcorev1.PodRunning { + if condition.Type == kcorev1.PodReady && condition.Status == kcorev1.ConditionTrue { + state = Ok + reason = fmt.Sprintf("Pod %s/%s is %s", pod.Namespace, pod.Name, string(kcorev1.PodRunning)) + } else { + state = Critical + reason = fmt.Sprintf("Pod %s/%s is not ready", pod.Namespace, pod.Name) + } + } + } + } + } + + if readyContainers == len(pod.Spec.Containers) { + state = Ok + reason = "All containers are ready" + } + + return state, reason +} + func (p *Pod) Relations() []database.Relation { fk := database.WithForeignKey("pod_uuid") diff --git a/schema/mysql/schema.sql b/schema/mysql/schema.sql index 6ca7d77a..93a8a76e 100644 --- a/schema/mysql/schema.sql +++ b/schema/mysql/schema.sql @@ -87,6 +87,8 @@ CREATE TABLE pod ( memory_limits bigint unsigned NOT NULL, memory_requests bigint unsigned NOT NULL, phase enum('pending', 'running', 'succeeded', 'failed') COLLATE utf8mb4_unicode_ci NOT NULL, + icinga_state enum('ok', 'warning', 'critical', 'unknown') COLLATE utf8mb4_unicode_ci NOT NULL, + icinga_state_reason text NULL DEFAULT NULL, reason varchar(255) NULL DEFAULT NULL, message varchar(255) NULL DEFAULT NULL, qos enum('guaranteed', 'burstable', 'best_effort') COLLATE utf8mb4_unicode_ci NOT NULL, From e91fc25043e82980d57f8f701c88c34602fd7861 Mon Sep 17 00:00:00 2001 From: Jonada Hoxha Date: Fri, 31 May 2024 21:46:43 +0200 Subject: [PATCH 2/4] Introduce `isPodInitializedConditionTrue` function --- pkg/schema/v1/pod.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/pkg/schema/v1/pod.go b/pkg/schema/v1/pod.go index 955cf626..ebe6e6f2 100644 --- a/pkg/schema/v1/pod.go +++ b/pkg/schema/v1/pod.go @@ -430,6 +430,17 @@ func (p *Pod) getIcingaState(pod *kcorev1.Pod) (IcingaState, string) { return state, reason } +func isPodInitializedConditionTrue(status *kcorev1.PodStatus) bool { + for _, condition := range status.Conditions { + if condition.Type != kcorev1.PodInitialized { + continue + } + + return condition.Status == kcorev1.ConditionTrue + } + return false +} + func (p *Pod) Relations() []database.Relation { fk := database.WithForeignKey("pod_uuid") From 8a5d03c38ee7d7e072459cd953321cf7e1e56f56 Mon Sep 17 00:00:00 2001 From: Jonada Hoxha Date: Fri, 31 May 2024 21:48:32 +0200 Subject: [PATCH 3/4] Enhance state evaluation by considering pod initialization status --- pkg/schema/v1/pod.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/schema/v1/pod.go b/pkg/schema/v1/pod.go index ebe6e6f2..cb6c4598 100644 --- a/pkg/schema/v1/pod.go +++ b/pkg/schema/v1/pod.go @@ -373,7 +373,7 @@ func (p *Pod) getIcingaState(pod *kcorev1.Pod) (IcingaState, string) { break } - if !initializing { + if !initializing || isPodInitializedConditionTrue(&pod.Status) { hasRunning := false for _, container := range pod.Status.ContainerStatuses { if container.State.Waiting != nil && container.State.Waiting.Reason != "" && container.RestartCount >= 3 { From cfd2a74347f858e38d237426e2246ea8c0d9fc2b Mon Sep 17 00:00:00 2001 From: Eric Lippmann Date: Thu, 13 Jun 2024 10:19:31 +0200 Subject: [PATCH 4/4] Introduce Icinga states for containers and use them for pod states --- go.mod | 50 ++--- go.sum | 132 ++++++------- pkg/schema/v1/container.go | 299 +++++++++++++++++++++++++--- pkg/schema/v1/daemon_set.go | 8 +- pkg/schema/v1/deployment.go | 10 +- pkg/schema/v1/icinga_state.go | 4 +- pkg/schema/v1/image_pull_policy.go | 19 ++ pkg/schema/v1/node.go | 8 +- pkg/schema/v1/pod.go | 304 +++++++++++------------------ pkg/schema/v1/replica_set.go | 10 +- pkg/schema/v1/stateful_set.go | 6 +- schema/mysql/schema.sql | 17 +- 12 files changed, 519 insertions(+), 348 deletions(-) create mode 100644 pkg/schema/v1/image_pull_policy.go diff --git a/go.mod b/go.mod index ed7eb0a3..2f6b0c79 100644 --- a/go.mod +++ b/go.mod @@ -1,60 +1,62 @@ module github.com/icinga/icinga-kubernetes -go 1.22 +go 1.22.0 + +toolchain go1.22.3 require ( github.com/creasty/defaults v1.7.0 github.com/go-co-op/gocron v1.37.0 - github.com/go-logr/logr v1.4.1 + github.com/go-logr/logr v1.4.2 github.com/go-sql-driver/mysql v1.8.1 github.com/goccy/go-yaml v1.11.3 github.com/google/uuid v1.6.0 - github.com/icinga/icinga-go-library v0.1.0 + github.com/icinga/icinga-go-library v0.2.0 github.com/jmoiron/sqlx v1.4.0 github.com/lib/pq v1.10.9 github.com/pkg/errors v0.9.1 - golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 + golang.org/x/exp v0.0.0-20240531132922-fd00a4e0eefc golang.org/x/sync v0.7.0 - k8s.io/api v0.29.2 - k8s.io/apimachinery v0.29.2 - k8s.io/client-go v0.29.2 + k8s.io/api v0.30.1 + k8s.io/apimachinery v0.30.1 + k8s.io/client-go v0.30.1 k8s.io/klog/v2 v2.120.1 - k8s.io/utils v0.0.0-20240310230437-4693a0247e57 + k8s.io/utils v0.0.0-20240502163921-fe8a2dddb1d0 ) require ( filippo.io/edwards25519 v1.1.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect - github.com/emicklei/go-restful/v3 v3.12.0 // indirect - github.com/fatih/color v1.16.0 // indirect - github.com/go-openapi/jsonpointer v0.21.0 // indirect - github.com/go-openapi/jsonreference v0.21.0 // indirect - github.com/go-openapi/swag v0.23.0 // indirect + github.com/emicklei/go-restful/v3 v3.11.0 // indirect + github.com/fatih/color v1.10.0 // indirect + github.com/go-openapi/jsonpointer v0.19.6 // indirect + github.com/go-openapi/jsonreference v0.20.2 // indirect + github.com/go-openapi/swag v0.22.3 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/protobuf v1.5.4 // indirect - github.com/google/gnostic-models v0.6.9-0.20230804172637-c7be7c783f49 // indirect + github.com/google/gnostic-models v0.6.8 // indirect github.com/google/go-cmp v0.6.0 // indirect github.com/google/gofuzz v1.2.0 // indirect - github.com/imdario/mergo v0.3.16 // indirect + github.com/imdario/mergo v0.3.6 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/mailru/easyjson v0.7.7 // indirect - github.com/mattn/go-colorable v0.1.13 // indirect - github.com/mattn/go-isatty v0.0.20 // indirect + github.com/mattn/go-colorable v0.1.8 // indirect + github.com/mattn/go-isatty v0.0.12 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/robfig/cron/v3 v3.0.1 // indirect github.com/spf13/pflag v1.0.5 // indirect - go.uber.org/atomic v1.11.0 // indirect - golang.org/x/net v0.22.0 // indirect - golang.org/x/oauth2 v0.18.0 // indirect + go.uber.org/atomic v1.9.0 // indirect + golang.org/x/net v0.23.0 // indirect + golang.org/x/oauth2 v0.10.0 // indirect golang.org/x/sys v0.18.0 // indirect golang.org/x/term v0.18.0 // indirect golang.org/x/text v0.14.0 // indirect - golang.org/x/time v0.5.0 // indirect - golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect - google.golang.org/appengine v1.6.8 // indirect + golang.org/x/time v0.3.0 // indirect + golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect + google.golang.org/appengine v1.6.7 // indirect google.golang.org/protobuf v1.33.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect @@ -62,5 +64,5 @@ require ( k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect - sigs.k8s.io/yaml v1.4.0 // indirect + sigs.k8s.io/yaml v1.3.0 // indirect ) diff --git a/go.sum b/go.sum index 8fdf5309..49b9168c 100644 --- a/go.sum +++ b/go.sum @@ -6,20 +6,20 @@ github.com/creasty/defaults v1.7.0/go.mod h1:iGzKe6pbEHnpMPtfDXZEr0NVxWnPTjb1bbD github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/emicklei/go-restful/v3 v3.12.0 h1:y2DdzBAURM29NFF94q6RaY4vjIH1rtwDapwQtU84iWk= -github.com/emicklei/go-restful/v3 v3.12.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= -github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= -github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= +github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= +github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/fatih/color v1.10.0 h1:s36xzo75JdqLaaWoiEHk767eHiwo0598uUxyfiPkDsg= +github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= github.com/go-co-op/gocron v1.37.0 h1:ZYDJGtQ4OMhTLKOKMIch+/CY70Brbb1dGdooLEhh7b0= github.com/go-co-op/gocron v1.37.0/go.mod h1:3L/n6BkO7ABj+TrfSVXLRzsP26zmikL4ISkLQ0O8iNY= -github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= -github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= -github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ= -github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY= -github.com/go-openapi/jsonreference v0.21.0 h1:Rs+Y7hSXT83Jacb7kFyjn4ijOuVGSvOdF2+tg1TRrwQ= -github.com/go-openapi/jsonreference v0.21.0/go.mod h1:LmZmgsrTkVg9LG4EaHeY8cBDslNPMo06cago5JNLkm4= -github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE= -github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE= +github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= +github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= +github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= +github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/g= +github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q= github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD876Lmtgy7VtROAbHHXk8no= @@ -34,13 +34,11 @@ github.com/goccy/go-yaml v1.11.3 h1:B3W9IdWbvrUu2OYQGwvU1nZtvMQJPBKgBUuweJjLj6I= github.com/goccy/go-yaml v1.11.3/go.mod h1:wKnAMd44+9JAAnGQpWVEgBzGt3YuTaQ4uXoHvE4m7WU= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= -github.com/google/gnostic-models v0.6.9-0.20230804172637-c7be7c783f49 h1:0VpGH+cDhbDtdcweoyCVsF3fhN8kejK6rFe/2FFX2nU= -github.com/google/gnostic-models v0.6.9-0.20230804172637-c7be7c783f49/go.mod h1:BkkQ4L1KS1xMt2aWSPStnn55ChGC0DPOn2FQYj+f25M= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= +github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= @@ -52,10 +50,10 @@ github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLe github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/icinga/icinga-go-library v0.1.0 h1:CKnEBsxl65Ik0F0yHn2N7dX77mIFanPye927lsqu9yI= -github.com/icinga/icinga-go-library v0.1.0/go.mod h1:YN7XJN3W0FodD+j4kirO89zk2tgvanXWt1RMV8UgOLo= -github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= -github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= +github.com/icinga/icinga-go-library v0.2.0 h1:1or5s3KMEJGdhFbMzlN8NPw1NCd/3ntsKLw5et4/9XI= +github.com/icinga/icinga-go-library v0.2.0/go.mod h1:YN7XJN3W0FodD+j4kirO89zk2tgvanXWt1RMV8UgOLo= +github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28= +github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/jmoiron/sqlx v1.4.0 h1:1PLqN7S1UYp5t4SrVVnt4nUVNemrDAtxlulVe+Qgm3o= github.com/jmoiron/sqlx v1.4.0/go.mod h1:ZrZ7UsYB/weZdl2Bxg6jCRO9c3YHl8r3ahlKmRT4JLY= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= @@ -79,11 +77,10 @@ github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= -github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= -github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= -github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= -github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= -github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8= +github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -93,10 +90,10 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/onsi/ginkgo/v2 v2.13.0 h1:0jY9lJquiL8fcf3M4LAXN5aMlS/b2BV86HFFPCPMgE4= -github.com/onsi/ginkgo/v2 v2.13.0/go.mod h1:TE309ZR8s5FsKKpuB1YAQYBzCaAfUgatB/xlT/ETL/o= -github.com/onsi/gomega v1.29.0 h1:KIA/t2t5UBzoirT4H9tsML45GEbo3ouUnBHsCfD2tVg= -github.com/onsi/gomega v1.29.0/go.mod h1:9sxs+SwGrKI0+PWe4Fxa9tFQQBG5xSsSbMXOI8PPpoQ= +github.com/onsi/ginkgo/v2 v2.15.0 h1:79HwNRBAZHOEwrczrgSOPy+eFTTlIGELKy5as+ClttY= +github.com/onsi/ginkgo/v2 v2.15.0/go.mod h1:HlxMHtYF57y6Dpf+mc5529KKmSq9h2FpCF+/ZkwUxKM= +github.com/onsi/gomega v1.31.0 h1:54UJxxj6cPInHS3a35wm6BK/F9nHYueZ1NVujHDrnXE= +github.com/onsi/gomega v1.31.0/go.mod h1:DW9aCi7U6Yi40wNVAvT6kzFnEVEI5n3DloYBiKiT6zk= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -106,8 +103,8 @@ github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.8.1/go.mod h1:JeRgkft04UBgHMgCIwADu4Pn6Mtm5d4nPKWu0nJ5d+o= -github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= -github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -116,82 +113,67 @@ github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpE github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= -go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= -golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 h1:vr/HnozRka3pE4EsMEg1lgkXJkTFJCVUX+S/ZT6wYzM= -golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc= +golang.org/x/exp v0.0.0-20240531132922-fd00a4e0eefc h1:O9NuF4s+E/PvMIy+9IUZB9znFwUIXEWSstNjek6VpVg= +golang.org/x/exp v0.0.0-20240531132922-fd00a4e0eefc/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.22.0 h1:9sGLhx7iRIHEiX0oAJ3MRZMUCElJgy7Br1nO+AMN3Tc= -golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= -golang.org/x/oauth2 v0.18.0 h1:09qnuIAgzdx1XplqJvW6CQqMCtGZykZWcXzPMPUusvI= -golang.org/x/oauth2 v0.18.0/go.mod h1:Wf7knwG0MPoWIMMBgFlEaSUDaKskp0dCfrlJRJXbBi8= +golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= +golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= +golang.org/x/oauth2 v0.10.0 h1:zHCpF2Khkwy4mMB4bv0U37YtJdTGW8jI0glAApi0Kh8= +golang.org/x/oauth2 v0.10.0/go.mod h1:kTpgurOux7LqtuxjuyZa4Gj2gdezIt/jQtGnNFfypQI= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8= golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= -golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= +golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.21.0 h1:qc0xYgIbsSDt9EyWz05J5wfa7LOVW0YTLOXrqdLAWIw= golang.org/x/tools v0.21.0/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 h1:+cNy6SZtPcJQH3LJVLOSmiC7MMxXNOb3PU/VUEz+EhU= -golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= -google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= -google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk= +golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= +google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -207,21 +189,21 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -k8s.io/api v0.29.2 h1:hBC7B9+MU+ptchxEqTNW2DkUosJpp1P+Wn6YncZ474A= -k8s.io/api v0.29.2/go.mod h1:sdIaaKuU7P44aoyyLlikSLayT6Vb7bvJNCX105xZXY0= -k8s.io/apimachinery v0.29.2 h1:EWGpfJ856oj11C52NRCHuU7rFDwxev48z+6DSlGNsV8= -k8s.io/apimachinery v0.29.2/go.mod h1:6HVkd1FwxIagpYrHSwJlQqZI3G9LfYWRPAkUvLnXTKU= -k8s.io/client-go v0.29.2 h1:FEg85el1TeZp+/vYJM7hkDlSTFZ+c5nnK44DJ4FyoRg= -k8s.io/client-go v0.29.2/go.mod h1:knlvFZE58VpqbQpJNbCbctTVXcd35mMyAAwBdpt4jrA= +k8s.io/api v0.30.1 h1:kCm/6mADMdbAxmIh0LBjS54nQBE+U4KmbCfIkF5CpJY= +k8s.io/api v0.30.1/go.mod h1:ddbN2C0+0DIiPntan/bye3SW3PdwLa11/0yqwvuRrJM= +k8s.io/apimachinery v0.30.1 h1:ZQStsEfo4n65yAdlGTfP/uSHMQSoYzU/oeEbkmF7P2U= +k8s.io/apimachinery v0.30.1/go.mod h1:iexa2somDaxdnj7bha06bhb43Zpa6eWH8N8dbqVjTUc= +k8s.io/client-go v0.30.1 h1:uC/Ir6A3R46wdkgCV3vbLyNOYyCJ8oZnjtJGKfytl/Q= +k8s.io/client-go v0.30.1/go.mod h1:wrAqLNs2trwiCH/wxxmT/x3hKVH9PuV0GGW0oDoHVqc= k8s.io/klog/v2 v2.120.1 h1:QXU6cPEOIslTGvZaXvFWiP9VKyeet3sawzTOvdXb4Vw= k8s.io/klog/v2 v2.120.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 h1:BZqlfIlq5YbRMFko6/PM7FjZpUb45WallggurYhKGag= k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340/go.mod h1:yD4MZYeKMBwQKVht279WycxKyM84kkAx2DPrTXaeb98= -k8s.io/utils v0.0.0-20240310230437-4693a0247e57 h1:gbqbevonBh57eILzModw6mrkbwM0gQBEuevE/AaBsHY= -k8s.io/utils v0.0.0-20240310230437-4693a0247e57/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +k8s.io/utils v0.0.0-20240502163921-fe8a2dddb1d0 h1:jgGTlFYnhF1PM1Ax/lAlxUPE+KfCIXHaathvJg1C3ak= +k8s.io/utils v0.0.0-20240502163921-fe8a2dddb1d0/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4= sigs.k8s.io/structured-merge-diff/v4 v4.4.1/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08= -sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= -sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= +sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= +sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= diff --git a/pkg/schema/v1/container.go b/pkg/schema/v1/container.go index 249a479d..49866b8a 100644 --- a/pkg/schema/v1/container.go +++ b/pkg/schema/v1/container.go @@ -4,6 +4,7 @@ import ( "context" "database/sql" "errors" + "fmt" "github.com/go-co-op/gocron" "github.com/icinga/icinga-go-library/types" "github.com/icinga/icinga-kubernetes/pkg/com" @@ -29,6 +30,14 @@ var ( const ( MaxConcurrentJobs int = 60 ScheduleInterval = 5 * time.Minute + + PodInitializing = "PodInitializing" // https://github.com/kubernetes/kubernetes/blob/v1.30.1/pkg/kubelet/kubelet_pods.go#L80 + ContainerCreating = "ContainerCreating" + + ErrImagePull = "ErrImagePull" // https://github.com/kubernetes/kubernetes/blob/v1.30.1/pkg/kubelet/images/types.go#L27 + ErrImagePullBackOff = "ImagePullBackOff" + + // https://github.com/kubernetes/kubernetes/blob/master/pkg/kubelet/container/sync_result.go#L37 ) type ContainerMeta struct { @@ -36,24 +45,66 @@ type ContainerMeta struct { PodUuid types.UUID `db:"pod_uuid"` } -type Container struct { - ContainerMeta - Name string - Image string - CpuLimits int64 - CpuRequests int64 - MemoryLimits int64 - MemoryRequests int64 - State sql.NullString - StateDetails string - Ready types.Bool - Started types.Bool - RestartCount int32 - Devices []ContainerDevice `db:"-"` - Mounts []ContainerMount `db:"-"` +type ContainerCommon struct { + Name string + Image string + ImagePullPolicy ImagePullPolicy + State sql.NullString + StateDetails sql.NullString + IcingaState IcingaState + IcingaStateReason string + Devices []ContainerDevice `db:"-"` + Mounts []ContainerMount `db:"-"` +} + +func (c *ContainerCommon) Obtain(meta ContainerMeta, container kcorev1.Container, status kcorev1.ContainerStatus) { + c.Name = container.Name + c.Image = container.Image + c.ImagePullPolicy = ImagePullPolicy(container.ImagePullPolicy) + + state, stateDetails, err := MarshalFirstNonNilStructFieldToJSON(status.State) + if err != nil { + panic(err) + } + + if state != "" { + c.State.String = state + c.State.Valid = true + c.StateDetails.String = stateDetails + c.StateDetails.Valid = true + } + + c.IcingaState, c.IcingaStateReason = GetContainerState(container, status) + + for _, device := range container.VolumeDevices { + c.Devices = append(c.Devices, ContainerDevice{ + ContainerMeta: meta, + Name: device.Name, + Path: device.DevicePath, + }) + } + + for _, mount := range container.VolumeMounts { + m := ContainerMount{ + ContainerMeta: meta, + VolumeName: mount.Name, + Path: mount.MountPath, + ReadOnly: types.Bool{ + Bool: mount.ReadOnly, + Valid: true, + }, + } + + if mount.SubPath != "" { + m.SubPath.String = mount.SubPath + m.SubPath.Valid = true + } + + c.Mounts = append(c.Mounts, m) + } } -func (c *Container) Relations() []database.Relation { +func (c *ContainerCommon) Relations() []database.Relation { fk := database.WithForeignKey("container_id") return []database.Relation{ @@ -67,20 +118,101 @@ func (c *Container) Relations() []database.Relation { } } +type ContainerResources struct { + CpuLimits int64 + CpuRequests int64 + MemoryLimits int64 + MemoryRequests int64 +} + +func (c *ContainerResources) Obtain(container kcorev1.Container) { + c.CpuLimits = container.Resources.Limits.Cpu().MilliValue() + c.CpuRequests = container.Resources.Requests.Cpu().MilliValue() + c.MemoryLimits = container.Resources.Limits.Memory().MilliValue() + c.MemoryRequests = container.Resources.Requests.Memory().MilliValue() +} + +type ContainerRestartable struct { + Ready types.Bool + Started types.Bool + RestartCount int32 +} + +func (c *ContainerRestartable) Obtain(status kcorev1.ContainerStatus) { + var started bool + if status.Started != nil { + started = *status.Started + } + + c.Ready = types.Bool{ + Bool: status.Ready, + Valid: true, + } + c.Started = types.Bool{ + Bool: started, + Valid: true, + } + c.RestartCount = status.RestartCount +} + +type InitContainer struct { + ContainerMeta + ContainerCommon + ContainerResources +} + +func NewInitContainer(meta ContainerMeta, container kcorev1.Container, status kcorev1.ContainerStatus) *InitContainer { + c := &InitContainer{ContainerMeta: meta} + c.ContainerCommon.Obtain(meta, container, status) + c.ContainerResources.Obtain(container) + + return c +} + +type SidecarContainer struct { + ContainerMeta + ContainerCommon + ContainerResources + ContainerRestartable +} + +func NewSidecarContainer(meta ContainerMeta, container kcorev1.Container, status kcorev1.ContainerStatus) *SidecarContainer { + c := &SidecarContainer{ContainerMeta: meta} + c.ContainerCommon.Obtain(meta, container, status) + c.ContainerResources.Obtain(container) + c.ContainerRestartable.Obtain(status) + + return c +} + +type Container struct { + ContainerMeta + ContainerCommon + ContainerResources + ContainerRestartable +} + +func NewContainer(meta ContainerMeta, container kcorev1.Container, status kcorev1.ContainerStatus) *Container { + c := &Container{ContainerMeta: meta} + c.ContainerCommon.Obtain(meta, container, status) + c.ContainerResources.Obtain(container) + c.ContainerRestartable.Obtain(status) + + return c +} + type ContainerDevice struct { - ContainerUuid types.UUID - PodUuid types.UUID - Name string - Path string + ContainerMeta + Name string + Path string } type ContainerMount struct { - ContainerUuid types.UUID - PodUuid types.UUID - VolumeName string - Path string - SubPath sql.NullString - ReadOnly types.Bool + ContainerMeta + VolumeName string + Path string + SubPath sql.NullString + ReadOnly types.Bool } type ContainerLogMeta struct { @@ -132,6 +264,121 @@ func (cl *ContainerLog) syncContainerLogs(ctx context.Context, clientset *kubern return db.UpsertStreamed(ctx, entities) } +func GetContainerState(container kcorev1.Container, status kcorev1.ContainerStatus) (IcingaState, string) { + if status.State.Terminated != nil { + if status.State.Terminated.ExitCode == 0 { + return Ok, fmt.Sprintf( + "Container %s terminated successfully at %s.", container.Name, status.State.Terminated.FinishedAt) + } + + if status.State.Terminated.Signal != 0 { + return Critical, fmt.Sprintf( + "Container %s terminated with signal %d at %s: %s: %s.", + container.Name, + status.State.Terminated.Signal, + status.State.Terminated.FinishedAt, + status.State.Terminated.Reason, + status.State.Terminated.Message) + } + + return Critical, fmt.Sprintf( + "Container %s terminated with non-zero exit code %d at %s: %s: %s.", + container.Name, + status.State.Terminated.ExitCode, + status.State.Terminated.FinishedAt, + status.State.Terminated.Reason, + status.State.Terminated.Message) + } + + if status.State.Running != nil { + var probe string + + if status.Started == nil || !*status.Started { + probe = "startup" + } + + if !status.Ready { + probe = "liveness" + } + + if probe != "" { + if status.LastTerminationState.Terminated != nil { + return Warning, fmt.Sprintf( + "Container %s is running since %s but not ready due to failing %s probes."+ + " Last terminal with non-zero exit code %d and signal %d was at %s: %s: %s.", + container.Name, + status.State.Running.StartedAt, + probe, + status.LastTerminationState.Terminated.ExitCode, + status.LastTerminationState.Terminated.Signal, + status.LastTerminationState.Terminated.FinishedAt, + status.LastTerminationState.Terminated.Reason, + status.LastTerminationState.Terminated.Message) + } + + return Warning, fmt.Sprintf( + "Container %s is running since %s but not ready due to failing probes.", + container.Name, status.State.Running.StartedAt) + } + + return Ok, fmt.Sprintf( + "Container %s is running since %s.", container.Name, status.State.Running.StartedAt) + } + + if status.State.Waiting != nil { + // TODO(el): Add Kubernetes code ref. + if status.State.Waiting.Reason == "" { + return Pending, fmt.Sprintf("Container %s is pending as it's waiting to be started.", + container.Name) + } + + if status.State.Waiting.Reason == PodInitializing { + return Pending, fmt.Sprintf("Container %s is pending as the Pod it's running on is initializing.", + container.Name) + } + + if status.State.Waiting.Reason == ContainerCreating { + return Pending, fmt.Sprintf("Container %s is pending as it's still being created.", container.Name) + } + + if status.State.Waiting.Reason == ErrImagePull { + // Don't flap. + if status.LastTerminationState.Terminated != nil && + status.LastTerminationState.Terminated.Reason == ErrImagePullBackOff { + return Critical, fmt.Sprintf( + "Container %s can't start: %s: %s", + container.Name, + status.LastTerminationState.Terminated.Reason, + status.LastTerminationState.Terminated.Message) + } + + return Warning, fmt.Sprintf("Container %s is waiting to start as its image can't be pulled: %s.", + container.Name, status.State.Waiting.Message) + } + + return Critical, fmt.Sprintf( + "Container %s can't start: %s: %s.", + container.Name, + status.State.Waiting.Reason, + status.State.Waiting.Message) + } + + var reason string + field, _json, err := MarshalFirstNonNilStructFieldToJSON(status.State) + if err != nil { + reason = err.Error() + } else if field == "" { + reason = "No state provided" + } else { + reason = fmt.Sprintf("%s: %s", field, _json) + + } + + return Unknown, fmt.Sprintf( + "Container %s is unknown as its state could not be obtained: %s.", + container.Name, reason) +} + // SyncContainers consumes from the `upsertPods` and `deletePods` chans concurrently and schedules a job for // each of the containers (drawn from `upsertPods`) that periodically syncs the container logs with the database. // When pods are deleted, their IDs are streamed through the `deletePods` chan, and this fetches all the container diff --git a/pkg/schema/v1/daemon_set.go b/pkg/schema/v1/daemon_set.go index 37654d31..8c0069ba 100644 --- a/pkg/schema/v1/daemon_set.go +++ b/pkg/schema/v1/daemon_set.go @@ -119,7 +119,7 @@ func (d *DaemonSet) Obtain(k8s kmetav1.Object) { func (d *DaemonSet) getIcingaState() (IcingaState, string) { if d.DesiredNumberScheduled < 1 { - reason := fmt.Sprintf("DaemonSet %s/%s has an invalid desired node count: %d", d.Namespace, d.Name, d.DesiredNumberScheduled) + reason := fmt.Sprintf("DaemonSet %s/%s has an invalid desired node count: %d.", d.Namespace, d.Name, d.DesiredNumberScheduled) return Unknown, reason } @@ -130,15 +130,15 @@ func (d *DaemonSet) getIcingaState() (IcingaState, string) { switch { case d.NumberAvailable == 0: - reason := fmt.Sprintf("DaemonSet %s/%s does not have a single pod available which should run on %d desired nodes", d.Namespace, d.Name, d.DesiredNumberScheduled) + reason := fmt.Sprintf("DaemonSet %s/%s does not have a single pod available which should run on %d desired nodes.", d.Namespace, d.Name, d.DesiredNumberScheduled) return Critical, reason case d.NumberAvailable < d.DesiredNumberScheduled: - reason := fmt.Sprintf("DaemonSet %s/%s pods are only available on %d out of %d desired nodes", d.Namespace, d.Name, d.NumberAvailable, d.DesiredNumberScheduled) + reason := fmt.Sprintf("DaemonSet %s/%s pods are only available on %d out of %d desired nodes.", d.Namespace, d.Name, d.NumberAvailable, d.DesiredNumberScheduled) return Warning, reason default: - reason := fmt.Sprintf("DaemonSet %s/%s has pods available on all %d desired nodes", d.Namespace, d.Name, d.DesiredNumberScheduled) + reason := fmt.Sprintf("DaemonSet %s/%s has pods available on all %d desired nodes.", d.Namespace, d.Name, d.DesiredNumberScheduled) return Ok, reason } diff --git a/pkg/schema/v1/deployment.go b/pkg/schema/v1/deployment.go index 70f910f9..2095ee96 100644 --- a/pkg/schema/v1/deployment.go +++ b/pkg/schema/v1/deployment.go @@ -140,12 +140,12 @@ func (d *Deployment) getIcingaState() (IcingaState, string) { for _, condition := range d.Conditions { if condition.Type == string(kappsv1.DeploymentAvailable) && condition.Status != string(kcorev1.ConditionTrue) { - reason := fmt.Sprintf("Deployment %s/%s is not available: %s", d.Namespace, d.Name, condition.Message) + reason := fmt.Sprintf("Deployment %s/%s is not available: %s.", d.Namespace, d.Name, condition.Message) return Critical, reason } if condition.Type == string(kappsv1.ReplicaSetReplicaFailure) && condition.Status != string(kcorev1.ConditionTrue) { - reason := fmt.Sprintf("Deployment %s/%s has replica failure: %s", d.Namespace, d.Name, condition.Message) + reason := fmt.Sprintf("Deployment %s/%s has replica failure: %s.", d.Namespace, d.Name, condition.Message) return Critical, reason } @@ -153,15 +153,15 @@ func (d *Deployment) getIcingaState() (IcingaState, string) { switch { case d.UnavailableReplicas > 0: - reason := fmt.Sprintf("Deployment %s/%s has %d unavailable replicas", d.Namespace, d.Name, d.UnavailableReplicas) + reason := fmt.Sprintf("Deployment %s/%s has %d unavailable replicas.", d.Namespace, d.Name, d.UnavailableReplicas) return Critical, reason case d.AvailableReplicas < d.DesiredReplicas: - reason := fmt.Sprintf("Deployment %s/%s only has %d out of %d desired replicas available", d.Namespace, d.Name, d.AvailableReplicas, d.DesiredReplicas) + reason := fmt.Sprintf("Deployment %s/%s only has %d out of %d desired replicas available.", d.Namespace, d.Name, d.AvailableReplicas, d.DesiredReplicas) return Warning, reason default: - reason := fmt.Sprintf("Deployment %s/%s has all %d desired replicas available", d.Namespace, d.Name, d.DesiredReplicas) + reason := fmt.Sprintf("Deployment %s/%s has all %d desired replicas available.", d.Namespace, d.Name, d.DesiredReplicas) return Ok, reason } diff --git a/pkg/schema/v1/icinga_state.go b/pkg/schema/v1/icinga_state.go index f37cdf99..210a1739 100644 --- a/pkg/schema/v1/icinga_state.go +++ b/pkg/schema/v1/icinga_state.go @@ -9,10 +9,10 @@ type IcingaState uint8 const ( Ok IcingaState = iota + Pending + Unknown Warning Critical - Unknown - Pending IcingaState = 99 ) func (s IcingaState) String() string { diff --git a/pkg/schema/v1/image_pull_policy.go b/pkg/schema/v1/image_pull_policy.go new file mode 100644 index 00000000..99cbbbc9 --- /dev/null +++ b/pkg/schema/v1/image_pull_policy.go @@ -0,0 +1,19 @@ +package v1 + +import ( + "database/sql/driver" + "github.com/icinga/icinga-kubernetes/pkg/strcase" + kcorev1 "k8s.io/api/core/v1" +) + +type ImagePullPolicy kcorev1.PullPolicy + +// Value implements the [driver.Valuer] interface. +func (p ImagePullPolicy) Value() (driver.Value, error) { + return strcase.Snake(string(p)), nil +} + +// Assert interface compliance. +var ( + _ driver.Valuer = (*IcingaState)(nil) +) diff --git a/pkg/schema/v1/node.go b/pkg/schema/v1/node.go index eaf9912e..f2784bc0 100644 --- a/pkg/schema/v1/node.go +++ b/pkg/schema/v1/node.go @@ -192,11 +192,11 @@ func (n *Node) Obtain(k8s kmetav1.Object) { func (n *Node) getIcingaState(node *kcorev1.Node) (IcingaState, string) { if node.Status.Phase == kcorev1.NodePending { - return Pending, fmt.Sprintf("Node %s is pending", node.Name) + return Pending, fmt.Sprintf("Node %s is pending.", node.Name) } if node.Status.Phase == kcorev1.NodeTerminated { - return Ok, fmt.Sprintf("Node %s is terminated", node.Name) + return Ok, fmt.Sprintf("Node %s is terminated.", node.Name) } var state IcingaState @@ -224,11 +224,11 @@ func (n *Node) getIcingaState(node *kcorev1.Node) (IcingaState, string) { } if state != Ok { - return state, strings.Join(reason, ". ") + return state, strings.Join(reason, ". ") + "." } } - return Ok, fmt.Sprintf("Node %s is healthy", n.Name) + return Ok, fmt.Sprintf("Node %s is ok.", n.Name) } func (n *Node) Relations() []database.Relation { diff --git a/pkg/schema/v1/pod.go b/pkg/schema/v1/pod.go index cb6c4598..173a36c6 100644 --- a/pkg/schema/v1/pod.go +++ b/pkg/schema/v1/pod.go @@ -1,7 +1,6 @@ package v1 import ( - "database/sql" "fmt" "github.com/icinga/icinga-go-library/types" "github.com/icinga/icinga-kubernetes/pkg/database" @@ -14,11 +13,8 @@ import ( ktypes "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes" "strings" - "time" ) -const prolongedInitializationThreshold = 10 * time.Minute - type PodFactory struct { clientset *kubernetes.Clientset } @@ -41,7 +37,7 @@ type Pod struct { RestartPolicy string Yaml string Conditions []PodCondition `db:"-"` - Containers []Container `db:"-"` + Containers []*Container `db:"-"` Owners []PodOwner `db:"-"` Labels []Label `db:"-"` PodLabels []PodLabel `db:"-"` @@ -122,7 +118,6 @@ func (p *Pod) Obtain(k8s kmetav1.Object) { p.NominatedNodeName = pod.Status.NominatedNodeName p.Ip = pod.Status.PodIP p.Phase = strcase.Snake(string(pod.Status.Phase)) - p.IcingaState, p.IcingaStateReason = p.getIcingaState(pod) p.Reason = pod.Status.Reason p.Message = pod.Status.Message p.Qos = strcase.Snake(string(pod.Status.QOSClass)) @@ -140,83 +135,25 @@ func (p *Pod) Obtain(k8s kmetav1.Object) { }) } - containerStatuses := make(map[string]kcorev1.ContainerStatus, len(pod.Spec.Containers)) - for _, containerStatus := range pod.Status.ContainerStatuses { - containerStatuses[containerStatus.Name] = containerStatus - } - for _, k8sContainer := range pod.Spec.Containers { - var started bool - if containerStatuses[k8sContainer.Name].Started != nil { - started = *containerStatuses[k8sContainer.Name].Started - } - state, stateDetails, err := MarshalFirstNonNilStructFieldToJSON(containerStatuses[k8sContainer.Name].State) - if err != nil { - panic(err) - } - var containerState sql.NullString - if state != "" { - containerState.String = strcase.Snake(state) - containerState.Valid = true - } + p.Containers = NewContainers[Container](p, pod.Spec.Containers, pod.Status.ContainerStatuses, NewContainer) - container := Container{ - ContainerMeta: ContainerMeta{ - Uuid: NewUUID(p.Uuid, k8sContainer.Name), - PodUuid: p.Uuid, - }, - Name: k8sContainer.Name, - Image: k8sContainer.Image, - CpuLimits: k8sContainer.Resources.Limits.Cpu().MilliValue(), - CpuRequests: k8sContainer.Resources.Requests.Cpu().MilliValue(), - MemoryLimits: k8sContainer.Resources.Limits.Memory().MilliValue(), - MemoryRequests: k8sContainer.Resources.Requests.Memory().MilliValue(), - Ready: types.Bool{ - Bool: containerStatuses[k8sContainer.Name].Ready, - Valid: true, - }, - Started: types.Bool{ - Bool: started, - Valid: true, - }, - RestartCount: containerStatuses[k8sContainer.Name].RestartCount, - State: containerState, - StateDetails: stateDetails, - } - - p.CpuLimits += k8sContainer.Resources.Limits.Cpu().MilliValue() - p.CpuRequests += k8sContainer.Resources.Requests.Cpu().MilliValue() - p.MemoryLimits += k8sContainer.Resources.Limits.Memory().MilliValue() - p.MemoryRequests += k8sContainer.Resources.Requests.Memory().MilliValue() - - for _, device := range k8sContainer.VolumeDevices { - container.Devices = append(container.Devices, ContainerDevice{ - ContainerUuid: container.Uuid, - PodUuid: p.Uuid, - Name: device.Name, - Path: device.DevicePath, - }) - } + p.IcingaState, p.IcingaStateReason = p.getIcingaState(pod) - for _, mount := range k8sContainer.VolumeMounts { - var subPath sql.NullString - if mount.SubPath != "" { - subPath.String = mount.SubPath - subPath.Valid = true - } - container.Mounts = append(container.Mounts, ContainerMount{ - ContainerUuid: container.Uuid, - PodUuid: p.Uuid, - VolumeName: mount.Name, - Path: mount.MountPath, - SubPath: subPath, - ReadOnly: types.Bool{ - Bool: mount.ReadOnly, - Valid: true, - }, - }) - } + for _, container := range pod.Spec.Containers { + p.CpuLimits += container.Resources.Limits.Cpu().MilliValue() + p.CpuRequests += container.Resources.Requests.Cpu().MilliValue() + p.MemoryLimits += container.Resources.Limits.Memory().MilliValue() + p.MemoryRequests += container.Resources.Requests.Memory().MilliValue() + } - p.Containers = append(p.Containers, container) + // https://kubernetes.io/docs/concepts/workloads/pods/init-containers/#resources + for _, container := range pod.Spec.InitContainers { + // Init container must complete successfully before the next one starts, + // so we don't have to sum their resources. + p.CpuLimits = MaxInt(p.CpuLimits, container.Resources.Limits.Cpu().MilliValue()) + p.CpuRequests = MaxInt(p.CpuRequests, container.Resources.Requests.Cpu().MilliValue()) + p.MemoryLimits = MaxInt(p.MemoryLimits, container.Resources.Limits.Memory().MilliValue()) + p.MemoryRequests = MaxInt(p.MemoryRequests, container.Resources.Requests.Memory().MilliValue()) } for labelName, labelValue := range pod.Labels { @@ -270,16 +207,6 @@ func (p *Pod) Obtain(k8s kmetav1.Object) { }) } - // https://kubernetes.io/docs/concepts/workloads/pods/init-containers/#resources - for _, container := range pod.Spec.InitContainers { - // Init container must complete successfully before the next one starts, - // so we don't have to sum their resources. - p.CpuLimits = MaxInt(p.CpuLimits, container.Resources.Limits.Cpu().MilliValue()) - p.CpuRequests = MaxInt(p.CpuRequests, container.Resources.Requests.Cpu().MilliValue()) - p.MemoryLimits = MaxInt(p.MemoryLimits, container.Resources.Limits.Memory().MilliValue()) - p.MemoryRequests = MaxInt(p.MemoryRequests, container.Resources.Requests.Memory().MilliValue()) - } - for _, volume := range pod.Spec.Volumes { if volume.PersistentVolumeClaim != nil { p.Pvcs = append(p.Pvcs, PodPvc{ @@ -314,131 +241,122 @@ func (p *Pod) Obtain(k8s kmetav1.Object) { } func (p *Pod) getIcingaState(pod *kcorev1.Pod) (IcingaState, string) { - readyContainers := 0 - state := Unknown - reason := string(pod.Status.Phase) - - if pod.Status.Reason != "" { - reason = fmt.Sprintf("Pod %s/%s is in %s state with reason: %s", pod.Namespace, pod.Name, pod.Status.Phase, pod.Status.Reason) - } + // TODO(el): Account eviction. if pod.DeletionTimestamp != nil { - reason = fmt.Sprintf("Pod %s/%s is being deleted", pod.Namespace, pod.Name) + if pod.Status.Reason == "NodeLost" { + return Unknown, "" + } + // TODO(el): Return Critical if pod.DeletionTimestamp + pod.DeletionGracePeriodSeconds > now. + return Ok, fmt.Sprintf("Pod %s/%s is being deleted.", pod.Namespace, pod.Name) + } - return Ok, reason + if pod.Status.Phase == kcorev1.PodSucceeded { + return Ok, fmt.Sprintf( + "Pod %s/%s is succeeded as all its containers have been terminated successfully and"+ + " will not be restarted.", + pod.Namespace, pod.Name) } - // If the Pod carries {type:PodScheduled, reason:SchedulingGated}, set reason to 'SchedulingGated'. + var initialized bool for _, condition := range pod.Status.Conditions { - if condition.Type == kcorev1.PodScheduled && condition.Reason == kcorev1.PodReasonSchedulingGated { - state = Critical - reason = fmt.Sprintf("Pod %s/%s scheduling skipped because one or more scheduling gates are still present: %s", pod.Namespace, pod.Name, kcorev1.PodReasonSchedulingGated) + if condition.Type == kcorev1.PodScheduled && condition.Status == kcorev1.ConditionFalse { + return Critical, fmt.Sprintf( + "Pod %s/%s can't be scheduled: %s: %s.", pod.Namespace, pod.Name, condition.Reason, condition.Message) } - } - initializing := false - for i, container := range pod.Status.InitContainerStatuses { - switch { - case container.State.Terminated != nil && container.State.Terminated.ExitCode == 0: - continue - case container.State.Terminated != nil: - // Initialization failed - if len(container.State.Terminated.Reason) == 0 { - if container.State.Terminated.Signal != 0 { - reason = fmt.Sprintf("Init container %s is terminated with signal: %d", container.Name, container.State.Terminated.Signal) - } else { - reason = fmt.Sprintf("Init container %s is terminated with non-zero exit code %d: %s", container.Name, container.State.Terminated.ExitCode, container.State.Terminated.Reason) - } - } else { - reason = fmt.Sprintf("Init container %s is terminated. Reason %s: ", container.Name, container.State.Terminated.Reason) - } - state = Critical - initializing = true - case container.State.Waiting != nil && len(container.State.Waiting.Reason) > 0 && container.State.Waiting.Reason != "PodInitializing": - state = Critical - reason = fmt.Sprintf("Init container %s is waiting: %s", container.Name, container.State.Waiting.Reason) - initializing = true - default: - initializing = true - if container.State.Running != nil { - duration := time.Since(container.State.Running.StartedAt.Time).Round(time.Second) - if duration > prolongedInitializationThreshold { - state = Critical - reason = fmt.Sprintf("Init container %s has been initializing for too long (%d/%d, %s elapsed)", container.Name, i+1, len(pod.Spec.InitContainers), duration) - } else { - reason = fmt.Sprintf("Init container %s is currently initializing (%d/%d)", container.Name, i+1, len(pod.Spec.InitContainers)) - } - } + if condition.Type == kcorev1.PodReadyToStartContainers && condition.Status == kcorev1.ConditionFalse { + return Critical, fmt.Sprintf( + "Pod %s/%s is not ready to start containers.", pod.Namespace, pod.Name) + } + + if condition.Type == kcorev1.DisruptionTarget && condition.Status == kcorev1.ConditionTrue { + return Critical, fmt.Sprintf( + "Pod %s/%s is about to be terminated: %s: %s.", pod.Namespace, pod.Name, condition.Reason, condition.Message) + } + + if condition.Type == kcorev1.PodInitialized && condition.Status == kcorev1.ConditionTrue { + initialized = true } - break } - if !initializing || isPodInitializedConditionTrue(&pod.Status) { - hasRunning := false - for _, container := range pod.Status.ContainerStatuses { - if container.State.Waiting != nil && container.State.Waiting.Reason != "" && container.RestartCount >= 3 { - state = Critical - reason = fmt.Sprintf("Container %s is waiting and has restarted %d times: %s", container.Name, container.RestartCount, container.State.Waiting.Reason) - continue - } else if container.State.Terminated != nil { - if container.State.Terminated.Reason == "Completed" { - state = Ok - reason = fmt.Sprintf("Container %s has completed successfully", container.Name) - } else if container.State.Terminated.Reason != "" { - state = Critical - reason = fmt.Sprintf("Container %s is terminated. Reason: %s", container.Name, container.State.Terminated.Reason) - } else { - if container.State.Terminated.Signal != 0 { - state = Critical - reason = fmt.Sprintf("Container %s is terminated with signal: %d", container.Name, container.State.Terminated.Signal) - } else if container.State.Terminated.ExitCode != 0 { - state = Critical - reason = fmt.Sprintf("Container %s is terminated with non-zero exit code %d: %s", container.Name, container.State.Terminated.ExitCode, container.State.Terminated.Reason) - } else { - state = Ok - reason = fmt.Sprintf("Container %s is terminated normally", container.Name) - } - } - } else if container.State.Running != nil && container.Ready { - readyContainers++ - hasRunning = true - state = Ok - reason = fmt.Sprintf("Container %s is running", container.Name) - } + if pod.Status.Phase == kcorev1.PodFailed { + // TODO(el): For each container, check its state to provide more context about the termination. + return Critical, fmt.Sprintf( + "Pod %s/%s has failed as all its containers have been terminated, but at least one container has failed.", + pod.Namespace, pod.Name) + } + + if !initialized { + initContainers := NewContainers[InitContainer](p, pod.Spec.InitContainers, pod.Status.InitContainerStatuses, NewInitContainer) + state := Ok + reasons := make([]string, 0, len(initContainers)) + for _, c := range initContainers { + state = max(state, c.IcingaState) + reasons = append(reasons, fmt.Sprintf( + "[%s] %s", strings.ToUpper(c.IcingaState.String()), c.IcingaStateReason)) } - if reason == "Completed" && hasRunning { - for _, condition := range pod.Status.Conditions { - if pod.Status.Phase == kcorev1.PodRunning { - if condition.Type == kcorev1.PodReady && condition.Status == kcorev1.ConditionTrue { - state = Ok - reason = fmt.Sprintf("Pod %s/%s is %s", pod.Namespace, pod.Name, string(kcorev1.PodRunning)) - } else { - state = Critical - reason = fmt.Sprintf("Pod %s/%s is not ready", pod.Namespace, pod.Name) - } - } - } + if len(reasons) == 1 { + // Remove square brackets state information. + _, reason, _ := strings.Cut(reasons[0], " ") + + return state, reason } + + return state, fmt.Sprintf("%s/%s is %s.\n%s", pod.Namespace, pod.Name, state, strings.Join(reasons, "\n")) + } + + var notRunning int + state := Ok + reasons := make([]string, 0, len(p.Containers)) + for _, c := range p.Containers { + if c.IcingaState != Ok { + state = max(state, c.IcingaState) + notRunning++ + } + reasons = append(reasons, fmt.Sprintf( + "[%s] %s", strings.ToUpper(c.IcingaState.String()), c.IcingaStateReason)) } - if readyContainers == len(pod.Spec.Containers) { - state = Ok - reason = "All containers are ready" + if len(reasons) == 1 { + // Remove square brackets state information. + _, reason, _ := strings.Cut(reasons[0], " ") + + return state, reason } - return state, reason + return state, fmt.Sprintf( + "%s/%s is %s with %d out %d containers running.\n%s", + pod.Namespace, + pod.Name, + state, + len(p.Containers)-notRunning, + len(p.Containers), + strings.Join(reasons, "\n")) } -func isPodInitializedConditionTrue(status *kcorev1.PodStatus) bool { - for _, condition := range status.Conditions { - if condition.Type != kcorev1.PodInitialized { - continue - } +func NewContainers[T any]( + p *Pod, + containers []kcorev1.Container, + statuses []kcorev1.ContainerStatus, + factory func(ContainerMeta, kcorev1.Container, kcorev1.ContainerStatus) *T, +) []*T { + obtained := make([]*T, 0, len(containers)) + + statusesIdx := make(map[string]kcorev1.ContainerStatus, len(containers)) + for _, status := range statuses { + statusesIdx[status.Name] = status + } - return condition.Status == kcorev1.ConditionTrue + for _, container := range containers { + obtained = append(obtained, factory(ContainerMeta{ + Uuid: NewUUID(p.Uuid, container.Name), + PodUuid: p.Uuid, + }, container, statusesIdx[container.Name])) } - return false + + return obtained } func (p *Pod) Relations() []database.Relation { diff --git a/pkg/schema/v1/replica_set.go b/pkg/schema/v1/replica_set.go index 11db9a27..56698775 100644 --- a/pkg/schema/v1/replica_set.go +++ b/pkg/schema/v1/replica_set.go @@ -155,7 +155,7 @@ func (r *ReplicaSet) Obtain(k8s kmetav1.Object) { func (r *ReplicaSet) getIcingaState() (IcingaState, string) { if r.DesiredReplicas < 1 { - reason := fmt.Sprintf("ReplicaSet %s/%s has an invalid desired replica count: %d", r.Namespace, r.Name, r.DesiredReplicas) + reason := fmt.Sprintf("ReplicaSet %s/%s has an invalid desired replica count: %d.", r.Namespace, r.Name, r.DesiredReplicas) return Unknown, reason } @@ -166,7 +166,7 @@ func (r *ReplicaSet) getIcingaState() (IcingaState, string) { for _, condition := range r.Conditions { if condition.Type == string(kappsv1.ReplicaSetReplicaFailure) && condition.Status == string(kcorev1.ConditionTrue) { - reason := fmt.Sprintf("ReplicaSet %s/%s has a failure condition: %s", r.Namespace, r.Name, condition.Message) + reason := fmt.Sprintf("ReplicaSet %s/%s has a failure condition: %s.", r.Namespace, r.Name, condition.Message) return Critical, reason } @@ -174,15 +174,15 @@ func (r *ReplicaSet) getIcingaState() (IcingaState, string) { switch { case r.AvailableReplicas < 1: - reason := fmt.Sprintf("ReplicaSet %s/%s has no replica available from %d desired", r.Namespace, r.Name, r.DesiredReplicas) + reason := fmt.Sprintf("ReplicaSet %s/%s has no replica available from %d desired.", r.Namespace, r.Name, r.DesiredReplicas) return Critical, reason case r.AvailableReplicas < r.DesiredReplicas: - reason := fmt.Sprintf("ReplicaSet %s/%s only has %d out of %d desired replicas available", r.Namespace, r.Name, r.AvailableReplicas, r.DesiredReplicas) + reason := fmt.Sprintf("ReplicaSet %s/%s only has %d out of %d desired replicas available.", r.Namespace, r.Name, r.AvailableReplicas, r.DesiredReplicas) return Warning, reason default: - reason := fmt.Sprintf("ReplicaSet %s/%s has all %d desired replicas available", r.Namespace, r.Name, r.DesiredReplicas) + reason := fmt.Sprintf("ReplicaSet %s/%s has all %d desired replicas available.", r.Namespace, r.Name, r.DesiredReplicas) return Ok, reason } diff --git a/pkg/schema/v1/stateful_set.go b/pkg/schema/v1/stateful_set.go index 8bde846e..a434a912 100644 --- a/pkg/schema/v1/stateful_set.go +++ b/pkg/schema/v1/stateful_set.go @@ -148,15 +148,15 @@ func (s *StatefulSet) getIcingaState() (IcingaState, string) { switch { case s.AvailableReplicas == 0: - reason := fmt.Sprintf("StatefulSet %s/%s has no replica available from %d desired", s.Namespace, s.Name, s.DesiredReplicas) + reason := fmt.Sprintf("StatefulSet %s/%s has no replica available from %d desired.", s.Namespace, s.Name, s.DesiredReplicas) return Critical, reason case s.AvailableReplicas < s.DesiredReplicas: - reason := fmt.Sprintf("StatefulSet %s/%s only has %d out of %d desired replicas available", s.Namespace, s.Name, s.AvailableReplicas, s.DesiredReplicas) + reason := fmt.Sprintf("StatefulSet %s/%s only has %d out of %d desired replicas available.", s.Namespace, s.Name, s.AvailableReplicas, s.DesiredReplicas) return Warning, reason default: - reason := fmt.Sprintf("StatefulSet %s/%s has all %d desired replicas available", s.Namespace, s.Name, s.DesiredReplicas) + reason := fmt.Sprintf("StatefulSet %s/%s has all %d desired replicas available.", s.Namespace, s.Name, s.DesiredReplicas) return Ok, reason } diff --git a/schema/mysql/schema.sql b/schema/mysql/schema.sql index 93a8a76e..7e654ff4 100644 --- a/schema/mysql/schema.sql +++ b/schema/mysql/schema.sql @@ -47,7 +47,7 @@ CREATE TABLE node ( container_runtime_version varchar(255) NOT NULL, kubelet_version varchar(255) NOT NULL, kube_proxy_version varchar(255) NOT NULL, - icinga_state enum('ok', 'warning', 'critical', 'unknown') COLLATE utf8mb4_unicode_ci NOT NULL, + icinga_state enum('unknown', 'ok', 'warning', 'critical') COLLATE utf8mb4_unicode_ci NOT NULL, icinga_state_reason text NOT NULL, created bigint unsigned NOT NULL, PRIMARY KEY (uuid) @@ -87,7 +87,7 @@ CREATE TABLE pod ( memory_limits bigint unsigned NOT NULL, memory_requests bigint unsigned NOT NULL, phase enum('pending', 'running', 'succeeded', 'failed') COLLATE utf8mb4_unicode_ci NOT NULL, - icinga_state enum('ok', 'warning', 'critical', 'unknown') COLLATE utf8mb4_unicode_ci NOT NULL, + icinga_state enum('pending', 'ok', 'warning', 'critical', 'unknown') COLLATE utf8mb4_unicode_ci NOT NULL, icinga_state_reason text NULL DEFAULT NULL, reason varchar(255) NULL DEFAULT NULL, message varchar(255) NULL DEFAULT NULL, @@ -140,15 +140,18 @@ CREATE TABLE container ( pod_uuid binary(16) NOT NULL, name varchar(63) COLLATE utf8mb4_unicode_ci NOT NULL, image varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL, + image_pull_policy enum('always', 'never', 'if_not_present') COLLATE utf8mb4_unicode_ci NULL DEFAULT NULL, cpu_limits bigint unsigned NOT NULL, cpu_requests bigint unsigned NOT NULL, memory_limits bigint unsigned NOT NULL, memory_requests bigint unsigned NOT NULL, state enum('waiting', 'running', 'terminated') COLLATE utf8mb4_unicode_ci NULL DEFAULT NULL, - state_details longtext NOT NULL, + state_details longtext NULL DEFAULT NULL, ready enum('n', 'y') COLLATE utf8mb4_unicode_ci NOT NULL, started enum('n', 'y') COLLATE utf8mb4_unicode_ci NOT NULL, restart_count int unsigned NOT NULL, + icinga_state enum('unknown', 'pending', 'ok', 'warning', 'critical') COLLATE utf8mb4_unicode_ci NOT NULL, + icinga_state_reason text NULL DEFAULT NULL, PRIMARY KEY (uuid) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; @@ -196,7 +199,7 @@ CREATE TABLE deployment ( available_replicas int unsigned NOT NULL, unavailable_replicas int unsigned NOT NULL, yaml mediumblob DEFAULT NULL, - icinga_state enum('ok', 'warning', 'critical', 'unknown') COLLATE utf8mb4_unicode_ci NOT NULL, + icinga_state enum('unknown', 'ok', 'warning', 'critical') COLLATE utf8mb4_unicode_ci NOT NULL, icinga_state_reason text NOT NULL, created bigint unsigned NOT NULL, PRIMARY KEY (uuid) @@ -378,7 +381,7 @@ CREATE TABLE replica_set ( ready_replicas int unsigned NOT NULL, available_replicas int unsigned NOT NULL, yaml mediumblob DEFAULT NULL, - icinga_state enum('ok', 'warning', 'critical', 'unknown') COLLATE utf8mb4_unicode_ci NOT NULL, + icinga_state enum('unknown', 'ok', 'warning', 'critical') COLLATE utf8mb4_unicode_ci NOT NULL, icinga_state_reason text NOT NULL, created bigint unsigned NOT NULL, PRIMARY KEY (uuid) @@ -421,7 +424,7 @@ CREATE TABLE daemon_set ( number_available int unsigned NOT NULL, number_unavailable int unsigned NOT NULL, yaml mediumblob DEFAULT NULL, - icinga_state enum('ok', 'warning', 'critical', 'unknown') COLLATE utf8mb4_unicode_ci NOT NULL, + icinga_state enum('unknown', 'ok', 'warning', 'critical') COLLATE utf8mb4_unicode_ci NOT NULL, icinga_state_reason text NOT NULL, created bigint unsigned NOT NULL, PRIMARY KEY (uuid) @@ -457,7 +460,7 @@ CREATE TABLE stateful_set ( updated_replicas int unsigned NOT NULL, available_replicas int unsigned NOT NULL, yaml mediumblob DEFAULT NULL, - icinga_state enum('ok', 'warning', 'critical', 'unknown') COLLATE utf8mb4_unicode_ci NOT NULL, + icinga_state enum('unknown', 'ok', 'warning', 'critical') COLLATE utf8mb4_unicode_ci NOT NULL, icinga_state_reason text NOT NULL, created bigint unsigned NOT NULL, PRIMARY KEY (uuid)