pra: Add metrics federation

Issue: ZENKO-4766

.github/scripts/end2end/configs/prometheus.yaml

@@ -0,0 +1,48 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: ${PROMETHEUS_NAME}
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  - services
+  - endpoints
+  verbs:
+  - '*'
+---
+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: ${PROMETHEUS_NAME}
+subjects:
+- kind: ServiceAccount
+  name: default
+  namespace: default
+roleRef:
+  kind: Role
+  name: ${PROMETHEUS_NAME}
+  apiGroup: rbac.authorization.k8s.io
+---
+apiVersion: monitoring.coreos.com/v1
+kind: Prometheus
+metadata:
+  name: ${PROMETHEUS_NAME}
+  labels:
+    prometheus: ${PROMETHEUS_NAME}
+spec:
+  version: v2.35.0
+  evaluationInterval: 30s
+  logFormat: logfmt
+  logLevel: info
+  podMonitorNamespaceSelector: {}
+  podMonitorSelector:
+    matchLabels:
+      metalk8s.scality.com/monitor: ""
+  probeSelector:
+    matchLabels:
+      metalk8s.scality.com/monitor: ""
+  ruleSelector:
+    matchLabels:
+      metalk8s.scality.com/monitor: ""

.github/scripts/end2end/install-kind-dependencies.sh

@@ -57,13 +57,20 @@ kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/${IN
 kubectl rollout status -n ingress-nginx deployment/ingress-nginx-controller --timeout=10m
 
 # cert-manager
-kubectl apply --validate=false -f https://github.com/jetstack/cert-manager/releases/download/${CERT_MANAGER_VERSION}/cert-manager.yaml
+kubectl apply --validate=false -f https://github.com/jetstack/cert-manager/releases/download/${CERT_MANAGER_VERSION}/cert-manager.yaml --wait
+kubectl apply -f - <<EOF
+apiVersion: cert-manager.io/v1
+  name: artesca-root-ca-issuer
+spec:
+  selfSigned: {}
+EOF
 
 # prometheus
 # last-applied-configuration can end up larger than 256kB  which is too large for an annotation
 # so if apply fails, replace can work
 prom_url=https://raw.githubusercontent.com/coreos/prometheus-operator/${PROMETHEUS_VERSION}/bundle.yaml
-kubectl create -f $prom_url || kubectl replace -f $prom_url
+kubectl create -f $prom_url --wait || kubectl replace -f $prom_url --wait
+envsubst < configs/prometheus.yaml | kubectl apply -f -
 
 # zookeeper
 helm upgrade --install --version ${ZK_OPERATOR_VERSION} -n default zk-operator pravega/zookeeper-operator --set "watchNamespace=default"

.github/scripts/end2end/run-e2e-ctst.sh

@@ -68,6 +68,7 @@ WORLD_PARAMETERS="$(jq -c <<EOF
   "NotificationDestinationAlt":"${NOTIF_ALT_DEST_NAME}",
   "NotificationDestinationTopicAlt":"${NOTIF_ALT_DEST_TOPIC}",
   "KafkaExternalIps": "${KAFKA_EXTERNAL_IP:-}",
+  "PrometheusService":"${PROMETHEUS_NAME}-operated.default.svc.cluster.local",
   "KafkaHosts":"${KAFKA_HOST_PORT}",
   "KeycloakPassword":"${KEYCLOAK_TEST_PASSWORD}",
   "KeycloakHost":"${KEYCLOAK_TEST_HOST}",

.github/workflows/end2end.yaml

@@ -42,6 +42,7 @@ env:
   OIDC_HOST: 'keycloak.zenko.local'
   NAVBAR_ENDPOINT: 'http://shell-ui.zenko.local'
   ENABLE_KEYCLOAK_HTTPS: 'false'
+  PROMETHEUS_NAME: "prometheus"
   # mock-env
   AZURE_ACCOUNT_NAME: devstoreaccount1
   AZURE_BACKEND_ENDPOINT: http://devstoreaccount1.blob.azure-mock.zenko.local

tests/ctst/steps/dr/drctl.ts

@@ -22,6 +22,14 @@ type InstallConfig = {
     locations?: string;
     s3Bucket?: string;
 
+    prometheusExternalIps?: string;
+    prometheusExternalIpsDiscovery?: boolean;
+    prometheusService?: string;
+    prometheusIngressClass?: string;
+    prometheusHostname?: string;
+    prometheusDisableTls?: boolean;
+    prometheusTlsSecretName?: string;
+
     sourceKubeconfigPath?: string;
     sourceKubeconfigData?: string;
     sinkKubeconfigPath?: string;

tests/ctst/steps/pra.ts

@@ -67,6 +67,9 @@ async function installPRA(world: Zenko, sinkS3Endpoint = 'http://s3.zenko.local'
         sourceZenkoNamespace: 'default',
         sourceS3Endpoint: 'http://s3.zenko.local',
         sinkS3Endpoint,
+        prometheusService: world.parameters.PrometheusService,
+        prometheusHostname: 'prom.dr.zenko.local', // could be any name, cert will be auto-generated
+        prometheusExternalIps: world.parameters.KafkaExternalIps,
         ...kafkaExternalIpOption,
     });
 }

tests/ctst/world/Zenko.ts

@@ -55,6 +55,7 @@ export interface ZenkoWorldParameters extends ClientOptions {
     NotificationDestinationTopicAlt: string;
     KafkaExternalIps: string;
     KafkaHosts: string;
+    PrometheusService: string;
     KeycloakUsername: string;
     KeycloakPassword: string;
     KeycloakHost: string;