Merge pull request bitcoin-dev-project#273 from bitcoin-dev-project/k…

…8s-log-aggregation

src/backends/kubernetes/kubernetes_backend.py

@@ -11,6 +11,7 @@
 from kubernetes import client, config
 from kubernetes.client.models.v1_pod import V1Pod
 from kubernetes.client.rest import ApiException
+from kubernetes.dynamic import DynamicClient
 from kubernetes.stream import stream
 from warnet.lnnode import LNNode
 from warnet.status import RunningStatus
@@ -20,9 +21,12 @@
 DOCKER_REGISTRY_CORE = "bitcoindevproject/bitcoin"
 DOCKER_REGISTRY_LND = "lightninglabs/lnd:v0.17.0-beta"
 LOCAL_REGISTRY = "warnet/bitcoin-core"
+
 POD_PREFIX = "tank"
 BITCOIN_CONTAINER_NAME = "bitcoin"
 LN_CONTAINER_NAME = "ln"
+MAIN_NAMESPACE = "warnet"
+PROMETHEUS_METRICS_PORT = 9332
 
 
 logger = logging.getLogger("KubernetesBackend")
@@ -35,6 +39,7 @@ def __init__(self, config_dir: Path, network_name: str, logs_pod="fluentd") -> N
         # running inside a k8s cluster as a statefulset
         config.load_incluster_config()
         self.client = client.CoreV1Api()
+        self.dynamic_client = DynamicClient(client.ApiClient())
         self.namespace = "warnet"
         self.logs_pod = logs_pod
         self.network_name = network_name
@@ -63,6 +68,9 @@ def down(self, warnet) -> bool:
             if tank.lnnode:
                 pod_name = self.get_pod_name(tank.index, ServiceType.LIGHTNING)
                 self.client.delete_namespaced_pod(pod_name, self.namespace)
+
+        self.remove_prometheus_service_monitors(warnet.tanks)
+
         return True
 
     def get_file(self, tank_index: int, service: ServiceType, file_path: str):
@@ -269,6 +277,7 @@ def logs_grep(self, pattern: str, network: str):
             try:
                 log_stream = self.client.read_namespaced_pod_log(
                     name=pod.metadata.name,
+                    container=BITCOIN_CONTAINER_NAME,
                     namespace=self.namespace,
                     timestamps=True,
                     _preload_content=False,
@@ -415,6 +424,59 @@ def create_bitcoind_container(self, tank) -> client.V1Container:
         )
         return bitcoind_container
 
+    def create_prometheus_container(self, tank) -> client.V1Container:
+        return client.V1Container(
+            name="prometheus",
+            image="jvstein/bitcoin-prometheus-exporter:latest",
+            env=[
+                client.V1EnvVar(name="BITCOIN_RPC_HOST", value="localhost"),
+                client.V1EnvVar(name="BITCOIN_RPC_PORT", value=str(tank.rpc_port)),
+                client.V1EnvVar(name="BITCOIN_RPC_USER", value=tank.rpc_user),
+                client.V1EnvVar(name="BITCOIN_RPC_PASSWORD", value=tank.rpc_password),
+            ],
+        )
+
+    def apply_prometheus_service_monitors(self, tanks):
+        for tank in tanks:
+            if not tank.exporter:
+                continue
+
+            service_monitor = {
+                "apiVersion": "monitoring.coreos.com/v1",
+                "kind": "ServiceMonitor",
+                "metadata": {
+                    "name": f"warnet-tank-{tank.index:06d}",
+                    "namespace": MAIN_NAMESPACE,
+                    "labels": {
+                        "app.kubernetes.io/name": "bitcoind-metrics",
+                        "release": "prometheus",
+                    },
+                },
+                "spec": {
+                    "endpoints": [{"port": "prometheus-metrics"}],
+                    "selector": {"matchLabels": {"app": f"warnet-tank-{tank.index:06d}"}},
+                },
+            }
+            # Create the custom resource using the dynamic client
+            sc_crd = self.dynamic_client.resources.get(
+                api_version="monitoring.coreos.com/v1", kind="ServiceMonitor"
+            )
+            sc_crd.create(body=service_monitor, namespace=MAIN_NAMESPACE)
+
+    # attempts to delete the service monitors whether they exist or not
+    def remove_prometheus_service_monitors(self, tanks):
+        for tank in tanks:
+            try:
+                self.dynamic_client.resources.get(
+                    api_version="monitoring.coreos.com/v1", kind="ServiceMonitor"
+                ).delete(
+                    name=f"warnet-tank-{tank.index:06d}",
+                    namespace=MAIN_NAMESPACE,
+                )
+            except ApiException as e:
+                if e.status != 404:
+                    raise e
+
     def create_lnd_container(self, tank, bitcoind_service_name) -> client.V1Container:
         # These args are appended to the Dockerfile `ENTRYPOINT ["lnd"]`
         bitcoind_rpc_host = f"{bitcoind_service_name}.{self.namespace}"
@@ -487,8 +549,11 @@ def create_pod_object(
             ),
         )
 
+    def get_service_name(self, tank_index: int) -> str:
+        return f"bitcoind-{POD_PREFIX}-{tank_index:06d}"
+
     def create_bitcoind_service(self, tank) -> client.V1Service:
-        service_name = f"bitcoind-{POD_PREFIX}-{tank.index:06d}"
+        service_name = self.get_service_name(tank.index)
         self.log.debug(f"Creating bitcoind service {service_name} for tank {tank.index}")
         service = client.V1Service(
             api_version="v1",
@@ -512,6 +577,11 @@ def create_bitcoind_service(self, tank) -> client.V1Service:
                     client.V1ServicePort(
                         port=tank.zmqtxport, target_port=tank.zmqtxport, name="zmqtx"
                     ),
+                    client.V1ServicePort(
+                        port=PROMETHEUS_METRICS_PORT,
+                        target_port=PROMETHEUS_METRICS_PORT,
+                        name="prometheus-metrics",
+                    ),
                 ],
             ),
         )
@@ -557,6 +627,11 @@ def deploy_pods(self, warnet):
             bitcoind_pod = self.create_pod_object(
                 tank, bitcoind_container, self.get_pod_name(tank.index, ServiceType.BITCOIN)
             )
+
+            if tank.exporter:
+                prometheus_container = self.create_prometheus_container(tank)
+                bitcoind_pod.spec.containers.append(prometheus_container)
+
             bitcoind_service = self.create_bitcoind_service(tank)
             self.client.create_namespaced_pod(namespace=self.namespace, body=bitcoind_pod)
             # delete the service if it already exists, ignore 404
@@ -588,6 +663,9 @@ def deploy_pods(self, warnet):
                     namespace=self.namespace, body=lightning_service
                 )
 
+        # add metrics scraping for tanks configured to export metrics
+        self.apply_prometheus_service_monitors(warnet.tanks)
+
         self.log.debug("Containers and services created. Configuring IP addresses")
         # now that the pods have had a second to create,
         # get the ips and set them on the tanks

src/templates/k8s/connect_logging.sh

@@ -3,8 +3,6 @@ set -e
 
 POD_NAME=$(kubectl get pods --namespace warnet-logging -l "app.kubernetes.io/name=grafana,app.kubernetes.io/instance=loki-grafana" -o jsonpath="{.items[0].metadata.name}")
 
-GRAFANA_PASSWORD=$(kubectl get secret --namespace warnet-logging loki-grafana -o jsonpath="{.data.admin-password}" | base64 --decode || true)
-
-echo "Go to http://localhost:3000 and login with the username 'admin' and the password '${GRAFANA_PASSWORD}' to see your logs"
+echo "Go to http://localhost:3000 and login with the username 'admin' and the password 'password' to see your logs"
 
 kubectl --namespace warnet-logging port-forward "${POD_NAME}" 3000

src/templates/k8s/grafana/values.yaml

@@ -0,0 +1,13 @@
+adminUser: admin
+adminPassword: password
+admin: {}
+datasources:
+  datasources.yaml:
+    apiVersion: 1
+    datasources:
+    - name: Prometheus
+      type: prometheus
+      url: http://prometheus-kube-prometheus-prometheus.warnet-logging:9090
+    - name: Loki
+      type: loki
+      url: http://loki-gateway.warnet-logging:80

src/templates/k8s/install_logging.sh

@@ -4,8 +4,10 @@ set -e
 SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
 
 helm repo add grafana https://grafana.github.io/helm-charts
+helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
 helm repo update
 
 helm upgrade --install --namespace warnet-logging --create-namespace --values "${SCRIPT_DIR}/loki/values.yaml" loki grafana/loki
 helm upgrade --install --namespace warnet-logging promtail grafana/promtail
-helm upgrade --install --namespace warnet-logging loki-grafana grafana/grafana
+helm upgrade --install --namespace warnet-logging prometheus prometheus-community/kube-prometheus-stack --namespace warnet-logging --set grafana.enabled=false
+helm upgrade --install --namespace warnet-logging loki-grafana grafana/grafana --values "${SCRIPT_DIR}/grafana/values.yaml"

src/templates/rpc/namespace.yaml

@@ -4,3 +4,10 @@ metadata:
   name: warnet
   labels:
     name: warnet
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: warnet-logging
+  labels:
+    name: warnet-logging

src/templates/rpc/rbac-config.yaml

@@ -5,17 +5,20 @@ metadata:
   name: pod-reader-exec
 rules:
 - apiGroups: [""]
-  resources: ["pods"]
-  verbs: ["get", "list", "create", "update", "patch", "delete"]
+  resources: [pods]
+  verbs: [get, list, create, update, patch, delete]
 - apiGroups: [""]
-  resources: ["services"]
-  verbs: ["get", "list", "create", "update", "patch", "delete"]
+  resources: [services]
+  verbs: [get, list, create, update, patch, delete]
 - apiGroups: [""]
-  resources: ["pods/exec"]
-  verbs: ["create", "get"]
+  resources: [pods/exec]
+  verbs: [create, get]
 - apiGroups: [""]
-  resources: ["pods/log"]  # Granting permission to access pod logs
-  verbs: ["get"]
+  resources: [pods/log]  # Granting permission to access pod logs
+  verbs: [get]
+- apiGroups: [monitoring.coreos.com]
+  resources: [servicemonitors]
+  verbs: [get, list, create, update, patch, delete]
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding
@@ -29,4 +32,4 @@ subjects:
 roleRef:
   kind: Role
   name: pod-reader-exec
-  apiGroup: rbac.authorization.k8s.io
+  apiGroup: rbac.authorization.k8s.io