k8s-stack: added grafana external host management. fixes #1946 (#1947)

VictoriaMetrics · Jan 23, 2025 · 34903ed · 34903ed
1 parent b26141b
commit 34903ed
Show file tree

Hide file tree

Showing 52 changed files with 230 additions and 200 deletions.
diff --git a/charts/victoria-logs-single/files/rules/generated/vlogs.yaml b/charts/victoria-logs-single/files/rules/generated/vlogs.yaml
@@ -1,7 +1,7 @@
 {{- $Values := (.helm).Values | default .Values }}
 {{- $runbookUrl := ($Values.defaultRules).runbookUrl | default "https://runbooks.prometheus-operator.dev/runbooks" }}
 {{- $clusterLabel := ($Values.global).clusterLabel | default "cluster" }}
-{{- $host := index (($Values.grafana).ingress).hosts 0 }}
+{{- $grafanaHost := ternary (index (($Values.grafana).ingress).hosts 0) (($Values.external).grafana).host ($Values.grafana).enabled }}
 concurrency: 2
 condition: '{{ true }}'
 interval: 30s

diff --git a/charts/victoria-metrics-k8s-stack/CHANGELOG.md b/charts/victoria-metrics-k8s-stack/CHANGELOG.md
@@ -1,6 +1,9 @@
 ## Next release
 
-- TODO
+**Update note**: This release contains breaking change. `.Values.externalVM` was renamed to `.Values.external.vm`
+
+- add `.Values.external.grafana.host` to configure grafana host for alerts, when `.Values.grafana.enabled: false`
+- rename `.Values.externalVM` to `.Values.external.vm` for consistency
 
 ## 0.34.0
 

diff --git a/charts/victoria-metrics-k8s-stack/README.md b/charts/victoria-metrics-k8s-stack/README.md
@@ -845,7 +845,7 @@ perReplica: false
   type: prometheus
 - isDefault: false
   name: VictoriaMetrics (DS)
-  type: victoriametrics-datasource
+  type: victoriametrics-metrics-datasource
 </code>
 </pre>
 </td>
@@ -1326,7 +1326,18 @@ vmsingle:
 </td>
     </tr>
     <tr>
-      <td>externalVM</td>
+      <td>external.grafana</td>
+      <td>object</td>
+      <td><pre class="helm-vars-default-value language-yaml" lang="plaintext">
+<code class="language-yaml">host: grafana.external.host
+</code>
+</pre>
+</td>
+      <td><p>External Grafana host</p>
+</td>
+    </tr>
+    <tr>
+      <td>external.vm</td>
       <td>object</td>
       <td><pre class="helm-vars-default-value language-yaml" lang="plaintext">
 <code class="language-yaml">read:
@@ -2475,7 +2486,7 @@ unauthorizedUserAccessSpec:
 </code>
 </pre>
 </td>
-      <td><p>Full spec for VMAuth CRD. Allowed values described <a href="https://docs.victoriametrics.com/operator/api#vmauthspec" target="_blank">here</a> It&rsquo;s possible to use given below predefined variables in spec: * <code>{{ .vm.read }}</code> - parsed vmselect, vmsingle or externalVM.read URL * <code>{{ .vm.write }}</code> - parsed vminsert, vmsingle or externalVM.write URL</p>
+      <td><p>Full spec for VMAuth CRD. Allowed values described <a href="https://docs.victoriametrics.com/operator/api#vmauthspec" target="_blank">here</a> It&rsquo;s possible to use given below predefined variables in spec: * <code>{{ .vm.read }}</code> - parsed vmselect, vmsingle or external.vm.read URL * <code>{{ .vm.write }}</code> - parsed vminsert, vmsingle or external.vm.write URL</p>
 </td>
     </tr>
     <tr>

diff --git a/charts/victoria-metrics-k8s-stack/files/dashboards/generated/controller-manager.yaml b/charts/victoria-metrics-k8s-stack/files/dashboards/generated/controller-manager.yaml
@@ -232,8 +232,8 @@ panels:
   - datasource:
       type: {{ $defaultDatasource }}
       uid: ${datasource}
-    expr: histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{ {{ $clusterLabel }}=~"$cluster", job="kube-controller-manager", instance=~"$instance", verb="POST"}[$__rate_interval])) by (verb, url, le))
-    legendFormat: '{{`{{`}}verb{{`}}`}} {{`{{`}}url{{`}}`}}'
+    expr: histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{ {{ $clusterLabel }}=~"$cluster", job="kube-controller-manager", instance=~"$instance", verb="POST"}[$__rate_interval])) by (verb, le))
+    legendFormat: '{{`{{`}}verb{{`}}`}}'
   title: Post Request Latency 99th Quantile
   type: timeseries
 - datasource:
@@ -268,8 +268,8 @@ panels:
   - datasource:
       type: {{ $defaultDatasource }}
       uid: ${datasource}
-    expr: histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{ {{ $clusterLabel }}=~"$cluster", job="kube-controller-manager", instance=~"$instance", verb="GET"}[$__rate_interval])) by (verb, url, le))
-    legendFormat: '{{`{{`}}verb{{`}}`}} {{`{{`}}url{{`}}`}}'
+    expr: histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{ {{ $clusterLabel }}=~"$cluster", job="kube-controller-manager", instance=~"$instance", verb="GET"}[$__rate_interval])) by (verb, le))
+    legendFormat: '{{`{{`}}verb{{`}}`}}'
   title: Get Request Latency 99th Quantile
   type: timeseries
 - datasource:

diff --git a/charts/victoria-metrics-k8s-stack/files/dashboards/generated/kubelet.yaml b/charts/victoria-metrics-k8s-stack/files/dashboards/generated/kubelet.yaml
@@ -722,8 +722,8 @@ panels:
   - datasource:
       type: {{ $defaultDatasource }}
       uid: ${datasource}
-    expr: histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{ {{ $clusterLabel }}=~"$cluster",job="kubelet", metrics_path="/metrics", instance=~"$instance"}[$__rate_interval])) by (instance, verb, url, le))
-    legendFormat: '{{`{{`}}instance{{`}}`}} {{`{{`}}verb{{`}}`}} {{`{{`}}url{{`}}`}}'
+    expr: histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{ {{ $clusterLabel }}=~"$cluster",job="kubelet", metrics_path="/metrics", instance=~"$instance"}[$__rate_interval])) by (instance, verb, le))
+    legendFormat: '{{`{{`}}instance{{`}}`}} {{`{{`}}verb{{`}}`}}'
   title: Request duration 99th quantile
   type: timeseries
 - datasource:

diff --git a/charts/victoria-metrics-k8s-stack/files/dashboards/generated/proxy.yaml b/charts/victoria-metrics-k8s-stack/files/dashboards/generated/proxy.yaml
@@ -268,8 +268,8 @@ panels:
   - datasource:
       type: {{ $defaultDatasource }}
       uid: ${datasource}
-    expr: histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{ {{ $clusterLabel }}=~"$cluster", job="kube-proxy",instance=~"$instance",verb="POST"}[$__rate_interval])) by (verb, url, le))
-    legendFormat: '{{`{{`}}verb{{`}}`}} {{`{{`}}url{{`}}`}}'
+    expr: histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{ {{ $clusterLabel }}=~"$cluster", job="kube-proxy",instance=~"$instance",verb="POST"}[$__rate_interval])) by (verb, le))
+    legendFormat: '{{`{{`}}verb{{`}}`}}'
   title: Post Request Latency 99th Quantile
   type: timeseries
 - datasource:
@@ -304,8 +304,8 @@ panels:
   - datasource:
       type: {{ $defaultDatasource }}
       uid: ${datasource}
-    expr: histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{ {{ $clusterLabel }}=~"$cluster", job="kube-proxy", instance=~"$instance", verb="GET"}[$__rate_interval])) by (verb, url, le))
-    legendFormat: '{{`{{`}}verb{{`}}`}} {{`{{`}}url{{`}}`}}'
+    expr: histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{ {{ $clusterLabel }}=~"$cluster", job="kube-proxy", instance=~"$instance", verb="GET"}[$__rate_interval])) by (verb, le))
+    legendFormat: '{{`{{`}}verb{{`}}`}}'
   title: Get Request Latency 99th Quantile
   type: timeseries
 - datasource:

diff --git a/charts/victoria-metrics-k8s-stack/files/dashboards/generated/scheduler.yaml b/charts/victoria-metrics-k8s-stack/files/dashboards/generated/scheduler.yaml
@@ -226,8 +226,8 @@ panels:
   - datasource:
       type: {{ $defaultDatasource }}
       uid: ${datasource}
-    expr: histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{ {{ $clusterLabel }}=~"$cluster", job="kube-scheduler", instance=~"$instance", verb="POST"}[$__rate_interval])) by (verb, url, le))
-    legendFormat: '{{`{{`}}verb{{`}}`}} {{`{{`}}url{{`}}`}}'
+    expr: histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{ {{ $clusterLabel }}=~"$cluster", job="kube-scheduler", instance=~"$instance", verb="POST"}[$__rate_interval])) by (verb, le))
+    legendFormat: '{{`{{`}}verb{{`}}`}}'
   title: Post Request Latency 99th Quantile
   type: timeseries
 - datasource:
@@ -262,8 +262,8 @@ panels:
   - datasource:
       type: {{ $defaultDatasource }}
       uid: ${datasource}
-    expr: histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{ {{ $clusterLabel }}=~"$cluster", job="kube-scheduler", instance=~"$instance", verb="GET"}[$__rate_interval])) by (verb, url, le))
-    legendFormat: '{{`{{`}}verb{{`}}`}} {{`{{`}}url{{`}}`}}'
+    expr: histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{ {{ $clusterLabel }}=~"$cluster", job="kube-scheduler", instance=~"$instance", verb="GET"}[$__rate_interval])) by (verb, le))
+    legendFormat: '{{`{{`}}verb{{`}}`}}'
   title: Get Request Latency 99th Quantile
   type: timeseries
 - datasource:

diff --git a/charts/victoria-metrics-k8s-stack/files/rules/generated/alertmanager.rules.yaml b/charts/victoria-metrics-k8s-stack/files/rules/generated/alertmanager.rules.yaml
@@ -1,7 +1,7 @@
 {{- $Values := (.helm).Values | default .Values }}
 {{- $runbookUrl := ($Values.defaultRules).runbookUrl | default "https://runbooks.prometheus-operator.dev/runbooks" }}
 {{- $clusterLabel := ($Values.global).clusterLabel | default "cluster" }}
-{{- $host := index (($Values.grafana).ingress).hosts 0 }}
+{{- $grafanaHost := ternary (index (($Values.grafana).ingress).hosts 0) (($Values.external).grafana).host ($Values.grafana).enabled }}
 condition: '{{ ($Values.alertmanager).enabled }}'
 name: alertmanager.rules
 rules:

diff --git a/charts/victoria-metrics-k8s-stack/files/rules/generated/etcd.yaml b/charts/victoria-metrics-k8s-stack/files/rules/generated/etcd.yaml
@@ -1,7 +1,7 @@
 {{- $Values := (.helm).Values | default .Values }}
 {{- $runbookUrl := ($Values.defaultRules).runbookUrl | default "https://runbooks.prometheus-operator.dev/runbooks" }}
 {{- $clusterLabel := ($Values.global).clusterLabel | default "cluster" }}
-{{- $host := index (($Values.grafana).ingress).hosts 0 }}
+{{- $grafanaHost := ternary (index (($Values.grafana).ingress).hosts 0) (($Values.external).grafana).host ($Values.grafana).enabled }}
 condition: '{{ ($Values.kubeEtcd).enabled }}'
 name: etcd
 rules:

diff --git a/charts/victoria-metrics-k8s-stack/files/rules/generated/general.rules.yaml b/charts/victoria-metrics-k8s-stack/files/rules/generated/general.rules.yaml
@@ -1,7 +1,7 @@
 {{- $Values := (.helm).Values | default .Values }}
 {{- $runbookUrl := ($Values.defaultRules).runbookUrl | default "https://runbooks.prometheus-operator.dev/runbooks" }}
 {{- $clusterLabel := ($Values.global).clusterLabel | default "cluster" }}
-{{- $host := index (($Values.grafana).ingress).hosts 0 }}
+{{- $grafanaHost := ternary (index (($Values.grafana).ingress).hosts 0) (($Values.external).grafana).host ($Values.grafana).enabled }}
 condition: '{{ true }}'
 name: general.rules
 rules:

diff --git a/charts/victoria-metrics-k8s-stack/files/rules/generated/k8s.rules.container_cpu_limits.yaml b/charts/victoria-metrics-k8s-stack/files/rules/generated/k8s.rules.container_cpu_limits.yaml
@@ -1,7 +1,7 @@
 {{- $Values := (.helm).Values | default .Values }}
 {{- $runbookUrl := ($Values.defaultRules).runbookUrl | default "https://runbooks.prometheus-operator.dev/runbooks" }}
 {{- $clusterLabel := ($Values.global).clusterLabel | default "cluster" }}
-{{- $host := index (($Values.grafana).ingress).hosts 0 }}
+{{- $grafanaHost := ternary (index (($Values.grafana).ingress).hosts 0) (($Values.external).grafana).host ($Values.grafana).enabled }}
 condition: '{{ true }}'
 name: k8s.rules.container_cpu_limits
 rules:

diff --git a/...ts/victoria-metrics-k8s-stack/files/rules/generated/k8s.rules.container_cpu_requests.yaml b/...ts/victoria-metrics-k8s-stack/files/rules/generated/k8s.rules.container_cpu_requests.yaml
@@ -1,7 +1,7 @@
 {{- $Values := (.helm).Values | default .Values }}
 {{- $runbookUrl := ($Values.defaultRules).runbookUrl | default "https://runbooks.prometheus-operator.dev/runbooks" }}
 {{- $clusterLabel := ($Values.global).clusterLabel | default "cluster" }}
-{{- $host := index (($Values.grafana).ingress).hosts 0 }}
+{{- $grafanaHost := ternary (index (($Values.grafana).ingress).hosts 0) (($Values.external).grafana).host ($Values.grafana).enabled }}
 condition: '{{ true }}'
 name: k8s.rules.container_cpu_requests
 rules:

diff --git a/...-metrics-k8s-stack/files/rules/generated/k8s.rules.container_cpu_usage_seconds_total.yaml b/...-metrics-k8s-stack/files/rules/generated/k8s.rules.container_cpu_usage_seconds_total.yaml
@@ -1,7 +1,7 @@
 {{- $Values := (.helm).Values | default .Values }}
 {{- $runbookUrl := ($Values.defaultRules).runbookUrl | default "https://runbooks.prometheus-operator.dev/runbooks" }}
 {{- $clusterLabel := ($Values.global).clusterLabel | default "cluster" }}
-{{- $host := index (($Values.grafana).ingress).hosts 0 }}
+{{- $grafanaHost := ternary (index (($Values.grafana).ingress).hosts 0) (($Values.external).grafana).host ($Values.grafana).enabled }}
 condition: '{{ true }}'
 name: k8s.rules.container_cpu_usage_seconds_total
 rules:

diff --git a/...ts/victoria-metrics-k8s-stack/files/rules/generated/k8s.rules.container_memory_cache.yaml b/...ts/victoria-metrics-k8s-stack/files/rules/generated/k8s.rules.container_memory_cache.yaml
@@ -1,7 +1,7 @@
 {{- $Values := (.helm).Values | default .Values }}
 {{- $runbookUrl := ($Values.defaultRules).runbookUrl | default "https://runbooks.prometheus-operator.dev/runbooks" }}
 {{- $clusterLabel := ($Values.global).clusterLabel | default "cluster" }}
-{{- $host := index (($Values.grafana).ingress).hosts 0 }}
+{{- $grafanaHost := ternary (index (($Values.grafana).ingress).hosts 0) (($Values.external).grafana).host ($Values.grafana).enabled }}
 condition: '{{ true }}'
 name: k8s.rules.container_memory_cache
 rules:

diff --git a/...s/victoria-metrics-k8s-stack/files/rules/generated/k8s.rules.container_memory_limits.yaml b/...s/victoria-metrics-k8s-stack/files/rules/generated/k8s.rules.container_memory_limits.yaml
@@ -1,7 +1,7 @@
 {{- $Values := (.helm).Values | default .Values }}
 {{- $runbookUrl := ($Values.defaultRules).runbookUrl | default "https://runbooks.prometheus-operator.dev/runbooks" }}
 {{- $clusterLabel := ($Values.global).clusterLabel | default "cluster" }}
-{{- $host := index (($Values.grafana).ingress).hosts 0 }}
+{{- $grafanaHost := ternary (index (($Values.grafana).ingress).hosts 0) (($Values.external).grafana).host ($Values.grafana).enabled }}
 condition: '{{ true }}'
 name: k8s.rules.container_memory_limits
 rules:

diff --git a/...victoria-metrics-k8s-stack/files/rules/generated/k8s.rules.container_memory_requests.yaml b/...victoria-metrics-k8s-stack/files/rules/generated/k8s.rules.container_memory_requests.yaml
@@ -1,7 +1,7 @@
 {{- $Values := (.helm).Values | default .Values }}
 {{- $runbookUrl := ($Values.defaultRules).runbookUrl | default "https://runbooks.prometheus-operator.dev/runbooks" }}
 {{- $clusterLabel := ($Values.global).clusterLabel | default "cluster" }}
-{{- $host := index (($Values.grafana).ingress).hosts 0 }}
+{{- $grafanaHost := ternary (index (($Values.grafana).ingress).hosts 0) (($Values.external).grafana).host ($Values.grafana).enabled }}
 condition: '{{ true }}'
 name: k8s.rules.container_memory_requests
 rules:

diff --git a/charts/victoria-metrics-k8s-stack/files/rules/generated/k8s.rules.container_memory_rss.yaml b/charts/victoria-metrics-k8s-stack/files/rules/generated/k8s.rules.container_memory_rss.yaml
@@ -1,7 +1,7 @@
 {{- $Values := (.helm).Values | default .Values }}
 {{- $runbookUrl := ($Values.defaultRules).runbookUrl | default "https://runbooks.prometheus-operator.dev/runbooks" }}
 {{- $clusterLabel := ($Values.global).clusterLabel | default "cluster" }}
-{{- $host := index (($Values.grafana).ingress).hosts 0 }}
+{{- $grafanaHost := ternary (index (($Values.grafana).ingress).hosts 0) (($Values.external).grafana).host ($Values.grafana).enabled }}
 condition: '{{ true }}'
 name: k8s.rules.container_memory_rss
 rules:

diff --git a/charts/victoria-metrics-k8s-stack/files/rules/generated/k8s.rules.container_memory_swap.yaml b/charts/victoria-metrics-k8s-stack/files/rules/generated/k8s.rules.container_memory_swap.yaml
@@ -1,7 +1,7 @@
 {{- $Values := (.helm).Values | default .Values }}
 {{- $runbookUrl := ($Values.defaultRules).runbookUrl | default "https://runbooks.prometheus-operator.dev/runbooks" }}
 {{- $clusterLabel := ($Values.global).clusterLabel | default "cluster" }}
-{{- $host := index (($Values.grafana).ingress).hosts 0 }}
+{{- $grafanaHost := ternary (index (($Values.grafana).ingress).hosts 0) (($Values.external).grafana).host ($Values.grafana).enabled }}
 condition: '{{ true }}'
 name: k8s.rules.container_memory_swap
 rules:

diff --git a/...metrics-k8s-stack/files/rules/generated/k8s.rules.container_memory_working_set_bytes.yaml b/...metrics-k8s-stack/files/rules/generated/k8s.rules.container_memory_working_set_bytes.yaml
@@ -1,7 +1,7 @@
 {{- $Values := (.helm).Values | default .Values }}
 {{- $runbookUrl := ($Values.defaultRules).runbookUrl | default "https://runbooks.prometheus-operator.dev/runbooks" }}
 {{- $clusterLabel := ($Values.global).clusterLabel | default "cluster" }}
-{{- $host := index (($Values.grafana).ingress).hosts 0 }}
+{{- $grafanaHost := ternary (index (($Values.grafana).ingress).hosts 0) (($Values.external).grafana).host ($Values.grafana).enabled }}
 condition: '{{ true }}'
 name: k8s.rules.container_memory_working_set_bytes
 rules:

diff --git a/charts/victoria-metrics-k8s-stack/files/rules/generated/k8s.rules.pod_owner.yaml b/charts/victoria-metrics-k8s-stack/files/rules/generated/k8s.rules.pod_owner.yaml
@@ -1,7 +1,7 @@
 {{- $Values := (.helm).Values | default .Values }}
 {{- $runbookUrl := ($Values.defaultRules).runbookUrl | default "https://runbooks.prometheus-operator.dev/runbooks" }}
 {{- $clusterLabel := ($Values.global).clusterLabel | default "cluster" }}
-{{- $host := index (($Values.grafana).ingress).hosts 0 }}
+{{- $grafanaHost := ternary (index (($Values.grafana).ingress).hosts 0) (($Values.external).grafana).host ($Values.grafana).enabled }}
 condition: '{{ true }}'
 name: k8s.rules.pod_owner
 rules:

diff --git a/...s/victoria-metrics-k8s-stack/files/rules/generated/kube-apiserver-availability.rules.yaml b/...s/victoria-metrics-k8s-stack/files/rules/generated/kube-apiserver-availability.rules.yaml
@@ -1,7 +1,7 @@
 {{- $Values := (.helm).Values | default .Values }}
 {{- $runbookUrl := ($Values.defaultRules).runbookUrl | default "https://runbooks.prometheus-operator.dev/runbooks" }}
 {{- $clusterLabel := ($Values.global).clusterLabel | default "cluster" }}
-{{- $host := index (($Values.grafana).ingress).hosts 0 }}
+{{- $grafanaHost := ternary (index (($Values.grafana).ingress).hosts 0) (($Values.external).grafana).host ($Values.grafana).enabled }}
 condition: '{{ ($Values.kubeApiServer).enabled }}'
 interval: 3m
 name: kube-apiserver-availability.rules
@@ -38,22 +38,22 @@ rules:
         # write too slow
         sum by ({{ $clusterLabel }}) (cluster_verb_scope:apiserver_request_sli_duration_seconds_count:increase30d{verb=~"POST|PUT|PATCH|DELETE"})
         -
-        sum by ({{ $clusterLabel }}) (cluster_verb_scope_le:apiserver_request_sli_duration_seconds_bucket:increase30d{verb=~"POST|PUT|PATCH|DELETE",le="1"})
+        sum by ({{ $clusterLabel }}) (cluster_verb_scope_le:apiserver_request_sli_duration_seconds_bucket:increase30d{verb=~"POST|PUT|PATCH|DELETE",le=~"1(\\.0)?"})
       ) +
       (
         # read too slow
         sum by ({{ $clusterLabel }}) (cluster_verb_scope:apiserver_request_sli_duration_seconds_count:increase30d{verb=~"LIST|GET"})
         -
         (
           (
-            sum by ({{ $clusterLabel }}) (cluster_verb_scope_le:apiserver_request_sli_duration_seconds_bucket:increase30d{verb=~"LIST|GET",scope=~"resource|",le="1"})
+            sum by ({{ $clusterLabel }}) (cluster_verb_scope_le:apiserver_request_sli_duration_seconds_bucket:increase30d{verb=~"LIST|GET",scope=~"resource|",le=~"1(\\.0)?"})
             or
             vector(0)
           )
           +
-          sum by ({{ $clusterLabel }}) (cluster_verb_scope_le:apiserver_request_sli_duration_seconds_bucket:increase30d{verb=~"LIST|GET",scope="namespace",le="5"})
+          sum by ({{ $clusterLabel }}) (cluster_verb_scope_le:apiserver_request_sli_duration_seconds_bucket:increase30d{verb=~"LIST|GET",scope="namespace",le=~"5(\\.0)?"})
           +
-          sum by ({{ $clusterLabel }}) (cluster_verb_scope_le:apiserver_request_sli_duration_seconds_bucket:increase30d{verb=~"LIST|GET",scope="cluster",le="30"})
+          sum by ({{ $clusterLabel }}) (cluster_verb_scope_le:apiserver_request_sli_duration_seconds_bucket:increase30d{verb=~"LIST|GET",scope="cluster",le=~"30(\\.0)?"})
         )
       ) +
       # errors
@@ -72,14 +72,14 @@ rules:
       (
         # too slow
         (
-          sum by ({{ $clusterLabel }}) (cluster_verb_scope_le:apiserver_request_sli_duration_seconds_bucket:increase30d{verb=~"LIST|GET",scope=~"resource|",le="1"})
+          sum by ({{ $clusterLabel }}) (cluster_verb_scope_le:apiserver_request_sli_duration_seconds_bucket:increase30d{verb=~"LIST|GET",scope=~"resource|",le=~"1(\\.0)?"})
           or
           vector(0)
         )
         +
-        sum by ({{ $clusterLabel }}) (cluster_verb_scope_le:apiserver_request_sli_duration_seconds_bucket:increase30d{verb=~"LIST|GET",scope="namespace",le="5"})
+        sum by ({{ $clusterLabel }}) (cluster_verb_scope_le:apiserver_request_sli_duration_seconds_bucket:increase30d{verb=~"LIST|GET",scope="namespace",le=~"5(\\.0)?"})
         +
-        sum by ({{ $clusterLabel }}) (cluster_verb_scope_le:apiserver_request_sli_duration_seconds_bucket:increase30d{verb=~"LIST|GET",scope="cluster",le="30"})
+        sum by ({{ $clusterLabel }}) (cluster_verb_scope_le:apiserver_request_sli_duration_seconds_bucket:increase30d{verb=~"LIST|GET",scope="cluster",le=~"30(\\.0)?"})
       )
       +
       # errors
@@ -97,7 +97,7 @@ rules:
         # too slow
         sum by ({{ $clusterLabel }}) (cluster_verb_scope:apiserver_request_sli_duration_seconds_count:increase30d{verb=~"POST|PUT|PATCH|DELETE"})
         -
-        sum by ({{ $clusterLabel }}) (cluster_verb_scope_le:apiserver_request_sli_duration_seconds_bucket:increase30d{verb=~"POST|PUT|PATCH|DELETE",le="1"})
+        sum by ({{ $clusterLabel }}) (cluster_verb_scope_le:apiserver_request_sli_duration_seconds_bucket:increase30d{verb=~"POST|PUT|PATCH|DELETE",le=~"1(\\.0)?"})
       )
       +
       # errors