From 238b8f3d03bf10b8e5594acb604b84a3e9efb33c Mon Sep 17 00:00:00 2001
From: Francois Ferrand <francois.ferrand@scality.com>
Date: Tue, 20 Aug 2024 10:26:59 +0200
Subject: [PATCH] pra: Add metrics federation

Issue: ZENKO-4766
---
 .../scripts/end2end/configs/prometheus.yaml   | 48 +++++++++++++++++++
 .../end2end/install-kind-dependencies.sh      | 13 ++++-
 .github/scripts/end2end/run-e2e-ctst.sh       |  1 +
 .github/workflows/end2end.yaml                |  1 +
 tests/ctst/steps/dr/drctl.ts                  |  8 ++++
 tests/ctst/steps/pra.ts                       |  4 ++
 tests/ctst/world/Zenko.ts                     |  1 +
 7 files changed, 74 insertions(+), 2 deletions(-)
 create mode 100644 .github/scripts/end2end/configs/prometheus.yaml

diff --git a/.github/scripts/end2end/configs/prometheus.yaml b/.github/scripts/end2end/configs/prometheus.yaml
new file mode 100644
index 0000000000..e885d6325b
--- /dev/null
+++ b/.github/scripts/end2end/configs/prometheus.yaml
@@ -0,0 +1,48 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: ${PROMETHEUS_NAME}
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  - services
+  - endpoints
+  verbs:
+  - '*'
+---
+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: ${PROMETHEUS_NAME}
+subjects:
+- kind: ServiceAccount
+  name: default
+  namespace: default
+roleRef:
+  kind: Role
+  name: ${PROMETHEUS_NAME}
+  apiGroup: rbac.authorization.k8s.io
+---
+apiVersion: monitoring.coreos.com/v1
+kind: Prometheus
+metadata:
+  name: ${PROMETHEUS_NAME}
+  labels:
+    prometheus: ${PROMETHEUS_NAME}
+spec:
+  version: v2.35.0
+  evaluationInterval: 30s
+  logFormat: logfmt
+  logLevel: info
+  podMonitorNamespaceSelector: {}
+  podMonitorSelector:
+    matchLabels:
+      metalk8s.scality.com/monitor: ""
+  probeSelector:
+    matchLabels:
+      metalk8s.scality.com/monitor: ""
+  ruleSelector:
+    matchLabels:
+      metalk8s.scality.com/monitor: ""
diff --git a/.github/scripts/end2end/install-kind-dependencies.sh b/.github/scripts/end2end/install-kind-dependencies.sh
index 0f07f6e1f4..f76c1ef97d 100755
--- a/.github/scripts/end2end/install-kind-dependencies.sh
+++ b/.github/scripts/end2end/install-kind-dependencies.sh
@@ -57,13 +57,22 @@ kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/${IN
 kubectl rollout status -n ingress-nginx deployment/ingress-nginx-controller --timeout=10m
 
 # cert-manager
-kubectl apply --validate=false -f https://github.com/jetstack/cert-manager/releases/download/${CERT_MANAGER_VERSION}/cert-manager.yaml
+kubectl apply --validate=false -f https://github.com/jetstack/cert-manager/releases/download/${CERT_MANAGER_VERSION}/cert-manager.yaml --wait
+# kubectl apply --validate=false -f - <<EOF
+# apiVersion: cert-manager.io/v1
+# kind: ClusterIssuer
+# metadata:
+#   name: artesca-root-ca-issuer
+# spec:
+#   selfSigned: {}
+# EOF
 
 # prometheus
 # last-applied-configuration can end up larger than 256kB  which is too large for an annotation
 # so if apply fails, replace can work
 prom_url=https://raw.githubusercontent.com/coreos/prometheus-operator/${PROMETHEUS_VERSION}/bundle.yaml
-kubectl create -f $prom_url || kubectl replace -f $prom_url
+kubectl create -f $prom_url --wait || kubectl replace -f $prom_url --wait
+envsubst < configs/prometheus.yaml | kubectl apply -f -
 
 # zookeeper
 helm upgrade --install --version ${ZK_OPERATOR_VERSION} -n default zk-operator pravega/zookeeper-operator --set "watchNamespace=default"
diff --git a/.github/scripts/end2end/run-e2e-ctst.sh b/.github/scripts/end2end/run-e2e-ctst.sh
index 4dee26b20a..19a89a8fa5 100755
--- a/.github/scripts/end2end/run-e2e-ctst.sh
+++ b/.github/scripts/end2end/run-e2e-ctst.sh
@@ -68,6 +68,7 @@ WORLD_PARAMETERS="$(jq -c <<EOF
   "NotificationDestinationAlt":"${NOTIF_ALT_DEST_NAME}",
   "NotificationDestinationTopicAlt":"${NOTIF_ALT_DEST_TOPIC}",
   "KafkaExternalIps": "${KAFKA_EXTERNAL_IP:-}",
+  "PrometheusService":"${PROMETHEUS_NAME}-operated.default.svc.cluster.local",
   "KafkaHosts":"${KAFKA_HOST_PORT}",
   "KeycloakPassword":"${KEYCLOAK_TEST_PASSWORD}",
   "KeycloakHost":"${KEYCLOAK_TEST_HOST}",
diff --git a/.github/workflows/end2end.yaml b/.github/workflows/end2end.yaml
index 9d2760133b..58af31a7f9 100644
--- a/.github/workflows/end2end.yaml
+++ b/.github/workflows/end2end.yaml
@@ -42,6 +42,7 @@ env:
   OIDC_HOST: 'keycloak.zenko.local'
   NAVBAR_ENDPOINT: 'http://shell-ui.zenko.local'
   ENABLE_KEYCLOAK_HTTPS: 'false'
+  PROMETHEUS_NAME: "prometheus"
   # mock-env
   AZURE_ACCOUNT_NAME: devstoreaccount1
   AZURE_BACKEND_ENDPOINT: http://devstoreaccount1.blob.azure-mock.zenko.local
diff --git a/tests/ctst/steps/dr/drctl.ts b/tests/ctst/steps/dr/drctl.ts
index 10004488b4..ceb4a57825 100644
--- a/tests/ctst/steps/dr/drctl.ts
+++ b/tests/ctst/steps/dr/drctl.ts
@@ -22,6 +22,14 @@ type InstallConfig = {
     locations?: string;
     s3Bucket?: string;
 
+    prometheusExternalIps?: string;
+    prometheusExternalIpsDiscovery?: boolean;
+    prometheusService?: string;
+    prometheusIngressClass?: string;
+    prometheusHostname?: string;
+    prometheusDisableTls?: boolean;
+    prometheusTlsSecretName?: string;
+
     sourceKubeconfigPath?: string;
     sourceKubeconfigData?: string;
     sinkKubeconfigPath?: string;
diff --git a/tests/ctst/steps/pra.ts b/tests/ctst/steps/pra.ts
index e55bf6e4c9..2d7be377ef 100644
--- a/tests/ctst/steps/pra.ts
+++ b/tests/ctst/steps/pra.ts
@@ -67,6 +67,10 @@ async function installPRA(world: Zenko, sinkS3Endpoint = 'http://s3.zenko.local'
         sourceZenkoNamespace: 'default',
         sourceS3Endpoint: 'http://s3.zenko.local',
         sinkS3Endpoint,
+        prometheusService: world.parameters.PrometheusService,
+        // prometheusHostname: 'prom.dr.zenko.local', // could be any name, cert will be auto-generated
+        prometheusExternalIpsDiscovery: true,
+        prometheusDisableTls: true,
         ...kafkaExternalIpOption,
     });
 }
diff --git a/tests/ctst/world/Zenko.ts b/tests/ctst/world/Zenko.ts
index 292c1aba49..efdcb0d75d 100644
--- a/tests/ctst/world/Zenko.ts
+++ b/tests/ctst/world/Zenko.ts
@@ -55,6 +55,7 @@ export interface ZenkoWorldParameters extends ClientOptions {
     NotificationDestinationTopicAlt: string;
     KafkaExternalIps: string;
     KafkaHosts: string;
+    PrometheusService: string;
     KeycloakUsername: string;
     KeycloakPassword: string;
     KeycloakHost: string;