From 83e0cdb4fe97fc76a6a7cd457e6e16758bc72d29 Mon Sep 17 00:00:00 2001 From: Pranshu Srivastava Date: Mon, 9 Sep 2024 03:20:47 +0530 Subject: [PATCH 1/2] fix: allow opting-into upstream probes Allow users to opt-into upstream probe definitions. Signed-off-by: Pranshu Srivastava --- .../components/kube-rbac-proxy.libsonnet | 16 ++++-- .../components/kube-state-metrics.libsonnet | 57 +++++++++++++------ manifests/kubeStateMetrics-deployment.yaml | 4 +- manifests/kubeStateMetrics-service.yaml | 8 +-- 4 files changed, 58 insertions(+), 27 deletions(-) diff --git a/jsonnet/kube-prometheus/components/kube-rbac-proxy.libsonnet b/jsonnet/kube-prometheus/components/kube-rbac-proxy.libsonnet index 7055c30890..6d92c5323b 100644 --- a/jsonnet/kube-prometheus/components/kube-rbac-proxy.libsonnet +++ b/jsonnet/kube-prometheus/components/kube-rbac-proxy.libsonnet @@ -38,6 +38,13 @@ local defaults = { 'TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305', 'TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305', ], + // Corresponds to KRP's --ignore-paths flag. + // Some components (for e.g., KSM) may utilize the flag to allow for communication with external parties in scenarios + // where the originating request(s) cannot be modified to the proxy's expectations, and thus, are passed through, as + // is, to certain endpoints that they target, without the proxy's intervention. The kubelet, in KSM's case, can thus + // query health probe endpoints without being blocked by KRP, thus allowing for http-based probes over exec-based + // ones. + ignorePaths:: [], }; @@ -50,10 +57,11 @@ function(params) { name: krp._config.name, image: krp._config.image, args: [ - '--secure-listen-address=' + krp._config.secureListenAddress, - '--tls-cipher-suites=' + std.join(',', krp._config.tlsCipherSuites), - '--upstream=' + krp._config.upstream, - ], + '--secure-listen-address=' + krp._config.secureListenAddress, + '--tls-cipher-suites=' + std.join(',', krp._config.tlsCipherSuites), + '--upstream=' + krp._config.upstream, + ] // Optionals. + + if std.length(krp._config.ignorePaths) > 0 then ['--ignore-paths=' + std.join(',', krp._config.ignorePaths)] else defaults.ignorePaths, resources: krp._config.resources, ports: krp._config.ports, securityContext: { diff --git a/jsonnet/kube-prometheus/components/kube-state-metrics.libsonnet b/jsonnet/kube-prometheus/components/kube-state-metrics.libsonnet index 1535877ed8..f0c60c5340 100644 --- a/jsonnet/kube-prometheus/components/kube-state-metrics.libsonnet +++ b/jsonnet/kube-prometheus/components/kube-state-metrics.libsonnet @@ -15,12 +15,18 @@ local defaults = { }, kubeRbacProxyMain:: { + ports: [ + { name: 'http-metrics', containerPort: 8443 }, + ], resources+: { limits+: { cpu: '40m' }, requests+: { cpu: '20m' }, }, }, kubeRbacProxySelf:: { + ports: [ + { name: 'telemetry', containerPort: 9443 }, + ], resources+: { limits+: { cpu: '20m' }, requests+: { cpu: '10m' }, @@ -46,6 +52,8 @@ local defaults = { runbookURLPattern: 'https://runbooks.prometheus-operator.dev/runbooks/kube-state-metrics/%s', }, }, + // `enableProbes` allows users to opt-into upstream definitions for health probes. + enableProbes:: false, }; function(params) (import 'github.com/kubernetes/kube-state-metrics/jsonnet/kube-state-metrics/kube-state-metrics.libsonnet') { @@ -91,14 +99,14 @@ function(params) (import 'github.com/kubernetes/kube-state-metrics/jsonnet/kube- spec+: { ports: [ { - name: 'https-main', - port: 8443, - targetPort: 'https-main', + name: defaults.kubeRbacProxyMain.ports[0].name, + port: defaults.kubeRbacProxyMain.ports[0].containerPort, + targetPort: defaults.kubeRbacProxyMain.ports[0].name, }, { - name: 'https-self', - port: 9443, - targetPort: 'https-self', + name: defaults.kubeRbacProxySelf.ports[0].name, + port: defaults.kubeRbacProxySelf.ports[0].containerPort, + targetPort: defaults.kubeRbacProxySelf.ports[0].name, }, ], }, @@ -107,21 +115,19 @@ function(params) (import 'github.com/kubernetes/kube-state-metrics/jsonnet/kube- local kubeRbacProxyMain = krp(ksm._config.kubeRbacProxyMain { name: 'kube-rbac-proxy-main', upstream: 'http://127.0.0.1:8081/', - secureListenAddress: ':8443', - ports: [ - { name: 'https-main', containerPort: 8443 }, - ], + secureListenAddress: ':' + std.toString(defaults.kubeRbacProxyMain.ports[0].containerPort), image: ksm._config.kubeRbacProxyImage, + // When enabling probes, kube-rbac-proxy needs to always allow the /livez endpoint. + ignorePaths: if ksm._config.enableProbes then ['/livez'] else super.ignorePaths, }), local kubeRbacProxySelf = krp(ksm._config.kubeRbacProxySelf { name: 'kube-rbac-proxy-self', upstream: 'http://127.0.0.1:8082/', - secureListenAddress: ':9443', - ports: [ - { name: 'https-self', containerPort: 9443 }, - ], + secureListenAddress: ':' + std.toString(defaults.kubeRbacProxySelf.ports[0].containerPort), image: ksm._config.kubeRbacProxyImage, + // When enabling probes, kube-rbac-proxy needs to always allow the /readyz endpoint. + ignorePaths: if ksm._config.enableProbes then ['/readyz'] else super.ignorePaths, }), networkPolicy: { @@ -161,14 +167,31 @@ function(params) (import 'github.com/kubernetes/kube-state-metrics/jsonnet/kube- spec+: { automountServiceAccountToken: true, containers: std.map(function(c) c { - ports:: null, - livenessProbe:: null, - readinessProbe:: null, securityContext+: { runAsGroup: 65534, }, args: ['--host=127.0.0.1', '--port=8081', '--telemetry-host=127.0.0.1', '--telemetry-port=8082'], resources: ksm._config.resources, + } + if !ksm._config.enableProbes then { + ports:: null, + livenessProbe:: null, + readinessProbe:: null, + } else { + ports: defaults.kubeRbacProxyMain.ports + defaults.kubeRbacProxySelf.ports, + livenessProbe: { + httpGet: { + path: '/livez', + port: defaults.kubeRbacProxyMain.ports[0].name, + scheme: 'HTTPS', + }, + }, + readinessProbe: { + httpGet: { + path: '/readyz', + port: defaults.kubeRbacProxySelf.ports[0].name, + scheme: 'HTTPS', + }, + }, }, super.containers) + [kubeRbacProxyMain, kubeRbacProxySelf], }, }, diff --git a/manifests/kubeStateMetrics-deployment.yaml b/manifests/kubeStateMetrics-deployment.yaml index e7d14fbb9b..8b3fb03024 100644 --- a/manifests/kubeStateMetrics-deployment.yaml +++ b/manifests/kubeStateMetrics-deployment.yaml @@ -60,7 +60,7 @@ spec: name: kube-rbac-proxy-main ports: - containerPort: 8443 - name: https-main + name: http-metrics resources: limits: cpu: 40m @@ -87,7 +87,7 @@ spec: name: kube-rbac-proxy-self ports: - containerPort: 9443 - name: https-self + name: telemetry resources: limits: cpu: 20m diff --git a/manifests/kubeStateMetrics-service.yaml b/manifests/kubeStateMetrics-service.yaml index 139b59c1ea..b1cd9a4dc8 100644 --- a/manifests/kubeStateMetrics-service.yaml +++ b/manifests/kubeStateMetrics-service.yaml @@ -11,12 +11,12 @@ metadata: spec: clusterIP: None ports: - - name: https-main + - name: http-metrics port: 8443 - targetPort: https-main - - name: https-self + targetPort: http-metrics + - name: telemetry port: 9443 - targetPort: https-self + targetPort: telemetry selector: app.kubernetes.io/component: exporter app.kubernetes.io/name: kube-state-metrics From 8960c79679fd6fc7f98dad78b0f068514a306a88 Mon Sep 17 00:00:00 2001 From: Pranshu Srivastava Date: Mon, 21 Oct 2024 20:02:04 +0530 Subject: [PATCH 2/2] fixup! fix: allow opting-into upstream probes --- .../kube-prometheus/components/kube-state-metrics.libsonnet | 4 ++-- manifests/kubeStateMetrics-serviceMonitor.yaml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/jsonnet/kube-prometheus/components/kube-state-metrics.libsonnet b/jsonnet/kube-prometheus/components/kube-state-metrics.libsonnet index f0c60c5340..2325af7c7a 100644 --- a/jsonnet/kube-prometheus/components/kube-state-metrics.libsonnet +++ b/jsonnet/kube-prometheus/components/kube-state-metrics.libsonnet @@ -209,7 +209,7 @@ function(params) (import 'github.com/kubernetes/kube-state-metrics/jsonnet/kube- }, endpoints: [ { - port: 'https-main', + port: 'http-metrics', scheme: 'https', interval: ksm._config.scrapeInterval, scrapeTimeout: ksm._config.scrapeTimeout, @@ -234,7 +234,7 @@ function(params) (import 'github.com/kubernetes/kube-state-metrics/jsonnet/kube- }, }, { - port: 'https-self', + port: 'telemetry', scheme: 'https', interval: ksm._config.scrapeInterval, bearerTokenFile: '/var/run/secrets/kubernetes.io/serviceaccount/token', diff --git a/manifests/kubeStateMetrics-serviceMonitor.yaml b/manifests/kubeStateMetrics-serviceMonitor.yaml index b19774641b..188858b2bf 100644 --- a/manifests/kubeStateMetrics-serviceMonitor.yaml +++ b/manifests/kubeStateMetrics-serviceMonitor.yaml @@ -18,7 +18,7 @@ spec: regex: kube_endpoint_address_not_ready|kube_endpoint_address_available sourceLabels: - __name__ - port: https-main + port: http-metrics relabelings: - action: labeldrop regex: (pod|service|endpoint|namespace) @@ -28,7 +28,7 @@ spec: insecureSkipVerify: true - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token interval: 30s - port: https-self + port: telemetry scheme: https tlsConfig: insecureSkipVerify: true