From db8d2ff19e48a933b6829b6158deef2e2694feee Mon Sep 17 00:00:00 2001 From: Max Edwards Date: Fri, 9 Feb 2024 15:19:22 +0000 Subject: [PATCH 1/7] added prometheus to logging stack --- src/backends/kubernetes/kubernetes_backend.py | 55 ++++++++++++++++++- src/templates/k8s/grafana/values.yaml | 10 ++++ src/templates/k8s/install_logging.sh | 3 +- src/templates/rpc/namespace.yaml | 7 +++ src/templates/rpc/rbac-config.yaml | 21 ++++--- 5 files changed, 85 insertions(+), 11 deletions(-) create mode 100644 src/templates/k8s/grafana/values.yaml diff --git a/src/backends/kubernetes/kubernetes_backend.py b/src/backends/kubernetes/kubernetes_backend.py index e3fc4a6c4..0e95a09db 100644 --- a/src/backends/kubernetes/kubernetes_backend.py +++ b/src/backends/kubernetes/kubernetes_backend.py @@ -9,6 +9,7 @@ from backends import BackendInterface, ServiceType from cli.image import build_image from kubernetes import client, config +from kubernetes.dynamic import DynamicClient from kubernetes.client.models.v1_pod import V1Pod from kubernetes.client.rest import ApiException from kubernetes.stream import stream @@ -17,12 +18,15 @@ from warnet.tank import Tank from warnet.utils import default_bitcoin_conf_args, parse_raw_messages + DOCKER_REGISTRY_CORE = "bitcoindevproject/bitcoin" DOCKER_REGISTRY_LND = "lightninglabs/lnd:v0.17.0-beta" LOCAL_REGISTRY = "warnet/bitcoin-core" + POD_PREFIX = "tank" BITCOIN_CONTAINER_NAME = "bitcoin" LN_CONTAINER_NAME = "ln" +MAIN_NAMESPACE = "warnet" logger = logging.getLogger("KubernetesBackend") @@ -35,6 +39,7 @@ def __init__(self, config_dir: Path, network_name: str, logs_pod="fluentd") -> N # running inside a k8s cluster as a statefulset config.load_incluster_config() self.client = client.CoreV1Api() + self.dynamic_client = DynamicClient(client.ApiClient()) self.namespace = "warnet" self.logs_pod = logs_pod self.network_name = network_name @@ -415,6 +420,45 @@ def create_bitcoind_container(self, tank) -> client.V1Container: ) return bitcoind_container + def create_prometheus_container(self, tank) -> client.V1Container: + return client.V1Container( + name="prometheus", + image="jvstein/bitcoin-prometheus-exporter:latest", + env=[ + client.V1EnvVar(name="BITCOIN_RPC_HOST", value="localhost"), + client.V1EnvVar(name="BITCOIN_RPC_PORT", value=str(tank.rpc_port)), + client.V1EnvVar(name="BITCOIN_RPC_USER", value=tank.rpc_user), + client.V1EnvVar(name="BITCOIN_RPC_PASSWORD", value=tank.rpc_password), + ], + ) + + def apply_prometheus_service_monitor(self, tanks): + for tank in tanks: + if not tank.exporter: + continue + + service_monitor = { + "apiVersion": "monitoring.coreos.com/v1", + "kind": "ServiceMonitor", + "metadata": { + "name": f"warnet-tank-{tank.index:06d}", + "namespace": MAIN_NAMESPACE, + "labels": { + "app.kubernetes.io/name": "bitcoind-metrics", + "release": "prometheus", + }, + }, + "spec": { + "endpoints": [{"port": "prometheus-metrics"}], + "selector": {"matchLabels": {"app": f"warnet-tank-{tank.index:06d}"}}, + }, + } + # Create the custom resource using the dynamic client + sc_crd = self.dynamic_client.resources.get( + api_version="monitoring.coreos.com/v1", kind="ServiceMonitor" + ) + sc_crd.create(body=service_monitor, namespace=MAIN_NAMESPACE) + def create_lnd_container(self, tank, bitcoind_service_name) -> client.V1Container: # These args are appended to the Dockerfile `ENTRYPOINT ["lnd"]` bitcoind_rpc_host = f"{bitcoind_service_name}.{self.namespace}" @@ -487,8 +531,11 @@ def create_pod_object( ), ) + def get_service_name(self, tank_index: int) -> str: + return f"bitcoind-{POD_PREFIX}-{tank_index:06d}" + def create_bitcoind_service(self, tank) -> client.V1Service: - service_name = f"bitcoind-{POD_PREFIX}-{tank.index:06d}" + service_name = self.get_service_name(tank.index) self.log.debug(f"Creating bitcoind service {service_name} for tank {tank.index}") service = client.V1Service( api_version="v1", @@ -512,6 +559,7 @@ def create_bitcoind_service(self, tank) -> client.V1Service: client.V1ServicePort( port=tank.zmqtxport, target_port=tank.zmqtxport, name="zmqtx" ), + client.V1ServicePort(port=9332, target_port=9332, name="prometheus-metrics"), ], ), ) @@ -557,6 +605,8 @@ def deploy_pods(self, warnet): bitcoind_pod = self.create_pod_object( tank, bitcoind_container, self.get_pod_name(tank.index, ServiceType.BITCOIN) ) + prometheus_container = self.create_prometheus_container(tank) + bitcoind_pod.spec.containers.append(prometheus_container) bitcoind_service = self.create_bitcoind_service(tank) self.client.create_namespaced_pod(namespace=self.namespace, body=bitcoind_pod) # delete the service if it already exists, ignore 404 @@ -588,6 +638,9 @@ def deploy_pods(self, warnet): namespace=self.namespace, body=lightning_service ) + # add metrics scraping for tanks configured to export metrics + self.apply_prometheus_service_monitor(warnet.tanks) + self.log.debug("Containers and services created. Configuring IP addresses") # now that the pods have had a second to create, # get the ips and set them on the tanks diff --git a/src/templates/k8s/grafana/values.yaml b/src/templates/k8s/grafana/values.yaml new file mode 100644 index 000000000..bbcdfe865 --- /dev/null +++ b/src/templates/k8s/grafana/values.yaml @@ -0,0 +1,10 @@ +datasources: + datasources.yaml: + apiVersion: 1 + datasources: + - name: Prometheus + type: prometheus + url: http://prometheus-kube-prometheus-prometheus.warnet-logging:9090 + - name: Loki + type: loki + url: http://loki-gateway.warnet-logging:80 diff --git a/src/templates/k8s/install_logging.sh b/src/templates/k8s/install_logging.sh index a8685c6c6..3ad86b8a9 100755 --- a/src/templates/k8s/install_logging.sh +++ b/src/templates/k8s/install_logging.sh @@ -8,4 +8,5 @@ helm repo update helm upgrade --install --namespace warnet-logging --create-namespace --values "${SCRIPT_DIR}/loki/values.yaml" loki grafana/loki helm upgrade --install --namespace warnet-logging promtail grafana/promtail -helm upgrade --install --namespace warnet-logging loki-grafana grafana/grafana \ No newline at end of file +helm upgrade --install --namespace warnet-logging prometheus prometheus-community/kube-prometheus-stack --namespace warnet-logging --set grafana.enabled=false +helm upgrade --install --namespace warnet-logging loki-grafana grafana/grafana --values "${SCRIPT_DIR}/grafana/values.yaml" \ No newline at end of file diff --git a/src/templates/rpc/namespace.yaml b/src/templates/rpc/namespace.yaml index 0db190e50..f37e67014 100644 --- a/src/templates/rpc/namespace.yaml +++ b/src/templates/rpc/namespace.yaml @@ -4,3 +4,10 @@ metadata: name: warnet labels: name: warnet +--- +apiVersion: v1 +kind: Namespace +metadata: + name: warnet-logging + labels: + name: warnet-logging \ No newline at end of file diff --git a/src/templates/rpc/rbac-config.yaml b/src/templates/rpc/rbac-config.yaml index 0e6aefe2a..7b2e059a4 100644 --- a/src/templates/rpc/rbac-config.yaml +++ b/src/templates/rpc/rbac-config.yaml @@ -5,17 +5,20 @@ metadata: name: pod-reader-exec rules: - apiGroups: [""] - resources: ["pods"] - verbs: ["get", "list", "create", "update", "patch", "delete"] + resources: [pods] + verbs: [get, list, create, update, patch, delete] - apiGroups: [""] - resources: ["services"] - verbs: ["get", "list", "create", "update", "patch", "delete"] + resources: [services] + verbs: [get, list, create, update, patch, delete] - apiGroups: [""] - resources: ["pods/exec"] - verbs: ["create", "get"] + resources: [pods/exec] + verbs: [create, get] - apiGroups: [""] - resources: ["pods/log"] # Granting permission to access pod logs - verbs: ["get"] + resources: [pods/log] # Granting permission to access pod logs + verbs: [get] +- apiGroups: [monitoring.coreos.com] + resources: [servicemonitors] + verbs: [get, list, create, update, patch, delete] --- apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding @@ -29,4 +32,4 @@ subjects: roleRef: kind: Role name: pod-reader-exec - apiGroup: rbac.authorization.k8s.io + apiGroup: rbac.authorization.k8s.io \ No newline at end of file From 7cb38db21b2ea461fde16dc61fc93ecaaeddd7b1 Mon Sep 17 00:00:00 2001 From: Max Edwards Date: Tue, 13 Feb 2024 17:44:04 +0000 Subject: [PATCH 2/7] update admin password on grafana to be password --- src/templates/k8s/connect_logging.sh | 4 +--- src/templates/k8s/grafana/values.yaml | 3 +++ 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/templates/k8s/connect_logging.sh b/src/templates/k8s/connect_logging.sh index 0cd233cea..862381a60 100755 --- a/src/templates/k8s/connect_logging.sh +++ b/src/templates/k8s/connect_logging.sh @@ -3,8 +3,6 @@ set -e POD_NAME=$(kubectl get pods --namespace warnet-logging -l "app.kubernetes.io/name=grafana,app.kubernetes.io/instance=loki-grafana" -o jsonpath="{.items[0].metadata.name}") -GRAFANA_PASSWORD=$(kubectl get secret --namespace warnet-logging loki-grafana -o jsonpath="{.data.admin-password}" | base64 --decode || true) - -echo "Go to http://localhost:3000 and login with the username 'admin' and the password '${GRAFANA_PASSWORD}' to see your logs" +echo "Go to http://localhost:3000 and login with the username 'admin' and the password 'password' to see your logs" kubectl --namespace warnet-logging port-forward "${POD_NAME}" 3000 \ No newline at end of file diff --git a/src/templates/k8s/grafana/values.yaml b/src/templates/k8s/grafana/values.yaml index bbcdfe865..d530b5094 100644 --- a/src/templates/k8s/grafana/values.yaml +++ b/src/templates/k8s/grafana/values.yaml @@ -1,3 +1,6 @@ +adminUser: admin +adminPassword: password +admin: {} datasources: datasources.yaml: apiVersion: 1 From a90305715e8bd54bb7babb51a034492321bc876a Mon Sep 17 00:00:00 2001 From: Max Edwards Date: Tue, 13 Feb 2024 18:01:34 +0000 Subject: [PATCH 3/7] delete service monitors when downing network --- src/backends/kubernetes/kubernetes_backend.py | 21 +++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/src/backends/kubernetes/kubernetes_backend.py b/src/backends/kubernetes/kubernetes_backend.py index 0e95a09db..78384e89a 100644 --- a/src/backends/kubernetes/kubernetes_backend.py +++ b/src/backends/kubernetes/kubernetes_backend.py @@ -68,6 +68,9 @@ def down(self, warnet) -> bool: if tank.lnnode: pod_name = self.get_pod_name(tank.index, ServiceType.LIGHTNING) self.client.delete_namespaced_pod(pod_name, self.namespace) + + self.remove_prometheus_service_monitors(warnet.tanks) + return True def get_file(self, tank_index: int, service: ServiceType, file_path: str): @@ -432,7 +435,7 @@ def create_prometheus_container(self, tank) -> client.V1Container: ], ) - def apply_prometheus_service_monitor(self, tanks): + def apply_prometheus_service_monitors(self, tanks): for tank in tanks: if not tank.exporter: continue @@ -459,6 +462,20 @@ def apply_prometheus_service_monitor(self, tanks): ) sc_crd.create(body=service_monitor, namespace=MAIN_NAMESPACE) + # attempts to delete the service monitors whether they exist or not + def remove_prometheus_service_monitors(self, tanks): + for tank in tanks: + try: + self.dynamic_client.resources.get( + api_version="monitoring.coreos.com/v1", kind="ServiceMonitor" + ).delete( + name=f"warnet-tank-{tank.index:06d}", + namespace=MAIN_NAMESPACE, + ) + except ApiException as e: + if e.status != 404: + raise e + def create_lnd_container(self, tank, bitcoind_service_name) -> client.V1Container: # These args are appended to the Dockerfile `ENTRYPOINT ["lnd"]` bitcoind_rpc_host = f"{bitcoind_service_name}.{self.namespace}" @@ -639,7 +656,7 @@ def deploy_pods(self, warnet): ) # add metrics scraping for tanks configured to export metrics - self.apply_prometheus_service_monitor(warnet.tanks) + self.apply_prometheus_service_monitors(warnet.tanks) self.log.debug("Containers and services created. Configuring IP addresses") # now that the pods have had a second to create, From e92d3100784411eac065b28bdbc0ac22270d8860 Mon Sep 17 00:00:00 2001 From: Max Edwards Date: Tue, 13 Feb 2024 18:06:38 +0000 Subject: [PATCH 4/7] fix ruff --- src/backends/kubernetes/kubernetes_backend.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/backends/kubernetes/kubernetes_backend.py b/src/backends/kubernetes/kubernetes_backend.py index 78384e89a..71b1397b2 100644 --- a/src/backends/kubernetes/kubernetes_backend.py +++ b/src/backends/kubernetes/kubernetes_backend.py @@ -9,16 +9,15 @@ from backends import BackendInterface, ServiceType from cli.image import build_image from kubernetes import client, config -from kubernetes.dynamic import DynamicClient from kubernetes.client.models.v1_pod import V1Pod from kubernetes.client.rest import ApiException +from kubernetes.dynamic import DynamicClient from kubernetes.stream import stream from warnet.lnnode import LNNode from warnet.status import RunningStatus from warnet.tank import Tank from warnet.utils import default_bitcoin_conf_args, parse_raw_messages - DOCKER_REGISTRY_CORE = "bitcoindevproject/bitcoin" DOCKER_REGISTRY_LND = "lightninglabs/lnd:v0.17.0-beta" LOCAL_REGISTRY = "warnet/bitcoin-core" From cfbffa5d1454e150ce5e00909db4180c09597ce9 Mon Sep 17 00:00:00 2001 From: Max Edwards Date: Tue, 13 Feb 2024 18:36:05 +0000 Subject: [PATCH 5/7] add prometheus helm chart --- src/templates/k8s/install_logging.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/src/templates/k8s/install_logging.sh b/src/templates/k8s/install_logging.sh index 3ad86b8a9..358699053 100755 --- a/src/templates/k8s/install_logging.sh +++ b/src/templates/k8s/install_logging.sh @@ -4,6 +4,7 @@ set -e SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) helm repo add grafana https://grafana.github.io/helm-charts +helm repo add prometheus-community https://prometheus-community.github.io/helm-charts helm repo update helm upgrade --install --namespace warnet-logging --create-namespace --values "${SCRIPT_DIR}/loki/values.yaml" loki grafana/loki From c4a4187bee9aa2022ef43987fc4698dfe85f785e Mon Sep 17 00:00:00 2001 From: Max Edwards Date: Tue, 13 Feb 2024 23:23:07 +0000 Subject: [PATCH 6/7] select correct continaer for greping logs --- src/backends/kubernetes/kubernetes_backend.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/backends/kubernetes/kubernetes_backend.py b/src/backends/kubernetes/kubernetes_backend.py index 71b1397b2..70477fd51 100644 --- a/src/backends/kubernetes/kubernetes_backend.py +++ b/src/backends/kubernetes/kubernetes_backend.py @@ -276,6 +276,7 @@ def logs_grep(self, pattern: str, network: str): try: log_stream = self.client.read_namespaced_pod_log( name=pod.metadata.name, + container=BITCOIN_CONTAINER_NAME, namespace=self.namespace, timestamps=True, _preload_content=False, From 1850e31a9ee435ae4ef36256ab7ba758fe41b3e7 Mon Sep 17 00:00:00 2001 From: Max Edwards Date: Wed, 14 Feb 2024 13:05:56 +0000 Subject: [PATCH 7/7] only apply prom exporter if metrics are enabled --- src/backends/kubernetes/kubernetes_backend.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/backends/kubernetes/kubernetes_backend.py b/src/backends/kubernetes/kubernetes_backend.py index 70477fd51..2d2ab04ae 100644 --- a/src/backends/kubernetes/kubernetes_backend.py +++ b/src/backends/kubernetes/kubernetes_backend.py @@ -26,6 +26,7 @@ BITCOIN_CONTAINER_NAME = "bitcoin" LN_CONTAINER_NAME = "ln" MAIN_NAMESPACE = "warnet" +PROMETHEUS_METRICS_PORT = 9332 logger = logging.getLogger("KubernetesBackend") @@ -576,7 +577,11 @@ def create_bitcoind_service(self, tank) -> client.V1Service: client.V1ServicePort( port=tank.zmqtxport, target_port=tank.zmqtxport, name="zmqtx" ), - client.V1ServicePort(port=9332, target_port=9332, name="prometheus-metrics"), + client.V1ServicePort( + port=PROMETHEUS_METRICS_PORT, + target_port=PROMETHEUS_METRICS_PORT, + name="prometheus-metrics", + ), ], ), ) @@ -622,8 +627,11 @@ def deploy_pods(self, warnet): bitcoind_pod = self.create_pod_object( tank, bitcoind_container, self.get_pod_name(tank.index, ServiceType.BITCOIN) ) - prometheus_container = self.create_prometheus_container(tank) - bitcoind_pod.spec.containers.append(prometheus_container) + + if tank.exporter: + prometheus_container = self.create_prometheus_container(tank) + bitcoind_pod.spec.containers.append(prometheus_container) + bitcoind_service = self.create_bitcoind_service(tank) self.client.create_namespaced_pod(namespace=self.namespace, body=bitcoind_pod) # delete the service if it already exists, ignore 404