VEC-519 Allow configuring advertised listeners from within helm chart (…

…#20) feat: allow configuring advertised listeners ( @davi17g ) --------- Co-authored-by: David Gerchikov <[email protected]>
aerospike · Feb 5, 2025 · a59ee99 · a59ee99
1 parent b456edc
commit a59ee99
Show file tree

Hide file tree

Showing 15 changed files with 395 additions and 207 deletions.
diff --git a/avs-init-container/avs-kind-values.yaml b/avs-init-container/avs-kind-values.yaml
@@ -1,37 +1,32 @@
-replicaCount: 1
+replicaCount: 3
 aerospikeVectorSearchConfig:
   cluster:
     cluster-name: "avs-db-1"
   feature-key-file: "/etc/aerospike-vector-search/secrets/features.conf"
   service:
     metadata-namespace: "avs-meta"
     ports:
-      5500:
-        addresses:
-          "0.0.0.0"
-        # tls-id: service-tls
+      5000: { }
   manage:
     ports:
       5040: { }
 
   heartbeat:
     seeds:
-      - address: avs-app-aerospike-vector-search-0.avs-app-aerospike-vector-search.aerospike.svc.cluster.local
-        port: 5500
+      - address: avs-app-aerospike-vector-search-0.avs-app-aerospike-vector-search-internal.aerospike.svc.cluster.local
+        port: 5001
   interconnect:
-    # client-tls-id: interconnect-tls
     ports:
       5001:
         addresses:
           "0.0.0.0"
-        # tls-id: interconnect-tls
+
   storage:
     client-policy: {}
     seeds:
       - aerocluster-0-0.aerocluster.aerospike.svc.cluster.local:
           port: 3000
   logging:
-    #    file: /var/log/aerospike-vector-search/aerospike-vector-search.log
     enable-console-logging: false
     format: simple
     max-history: 30
@@ -46,9 +41,14 @@ securityContext:
 image:
   repository: "aerospike/aerospike-vector-search"
   pullPolicy: "IfNotPresent"
-  # Overrides the image tag whose default is the chart appVersion.
   tag: "1.0.0"
 
+initContainer:
+  image:
+    repository: davi17g/avs-init-container
+    tag: "4.0.5"
+    pullPolicy: Always
+
 aerospikeVectorSearchNodeRoles:
   node-label-1:
     - query
@@ -58,3 +58,12 @@ aerospikeVectorSearchNodeRoles:
 
 serviceAccount:
   create: true
+
+service:
+  enabled: true
+  type: NodePort
+  ports:
+    - name: "svc-5000"
+      port: 5000
+      targetPort: 5000
+      nodePort: 30036
diff --git a/avs-init-container/cmd/main.go b/avs-init-container/cmd/main.go
@@ -1,14 +1,18 @@
 package main
 
 import (
-	"aerospike.com/avs-init-container/v2/util"
 	"context"
 	"encoding/json"
 	"fmt"
+	"os"
+	"sync"
+
+	"aerospike.com/avs-init-container/v2/util"
+	v1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
-	"os"
 	"sigs.k8s.io/yaml"
 )
 
@@ -18,21 +22,197 @@ const (
 	AVS_CONFIG_FILE_PATH = "/etc/aerospike-vector-search/aerospike-vector-search.yml"
 )
 
-func getNodeLabels() (string, error) {
+type NodeInfoSingleton struct {
+	node    *v1.Node
+	service *v1.Service
+	err     error
+}
+
+var (
+	instance *NodeInfoSingleton
+	once     sync.Once
+)
+
+func GetNodeInstance() (*v1.Node, *v1.Service, error) {
+	once.Do(func() {
+		nodeName := os.Getenv("NODE_NAME")
+		if nodeName == "" {
+			instance = &NodeInfoSingleton{
+				node:    nil,
+				service: nil,
+				err:     fmt.Errorf("NODE_NAME environment variable is not set"),
+			}
+			return
+		}
+
+		serviceName := os.Getenv("SERVICE_NAME")
+		if nodeName == "" {
+			instance = &NodeInfoSingleton{
+				node:    nil,
+				service: nil,
+				err:     fmt.Errorf("SERVICE_NAME environment variable is not set"),
+			}
+			return
+		}
+
+		podNamespace := os.Getenv("POD_NAMESPACE")
+		if nodeName == "" {
+			instance = &NodeInfoSingleton{
+				node:    nil,
+				service: nil,
+				err:     fmt.Errorf("POD_NAMESPACE environment variable is not set"),
+			}
+			return
+		}
+
+		config, err := rest.InClusterConfig()
+		if err != nil {
+			instance = &NodeInfoSingleton{
+				node:    nil,
+				service: nil,
+				err:     err,
+			}
+			return
+		}
 
-	nodeName := os.Getenv("NODE_NAME")
+		clientset, err := kubernetes.NewForConfig(config)
+		if err != nil {
+			instance = &NodeInfoSingleton{
+				node:    nil,
+				service: nil,
+				err:     err,
+			}
+			return
+		}
+
+		node, err := clientset.CoreV1().Nodes().Get(context.TODO(), nodeName, metav1.GetOptions{})
+		if err != nil {
+			instance = &NodeInfoSingleton{
+				node:    nil,
+				service: nil,
+				err:     err,
+			}
+			return
+		}
+
+		service, err := clientset.CoreV1().Services(podNamespace).Get(context.TODO(), serviceName, metav1.GetOptions{})
+		if err != nil {
+			if errors.IsNotFound(err) {
+				instance = &NodeInfoSingleton{
+					node:    node,
+					service: nil,
+					err:     nil,
+				}
+			} else {
+				instance = &NodeInfoSingleton{
+					node:    nil,
+					service: nil,
+					err:     err,
+				}
+				return
+			}
+		} else {
+			instance = &NodeInfoSingleton{
+				node:    node,
+				service: service,
+				err:     err,
+			}
+		}
+	})
+
+	return instance.node, instance.service, instance.err
+}
+
+func getNodeIp() (string, int32, error) {
+
+	externalIP, internalIP := "", ""
 
-	config, err := rest.InClusterConfig()
+	node, service, err := GetNodeInstance()
 	if err != nil {
-		return "", err
+		return "", 0, err
+	}
+
+	if service == nil {
+		return "", 0, nil
+	}
+
+	if service.Spec.Type != v1.ServiceTypeNodePort {
+		return "", 0, nil
+	}
+
+	var nodePort int32 = 0
+	for _, port := range service.Spec.Ports {
+		if port.NodePort != 0 {
+			nodePort = port.NodePort
+			break
+		}
+	}
+	if nodePort == 0 {
+		return "", 0, fmt.Errorf("node port is not set")
 	}
 
-	clientset, err := kubernetes.NewForConfig(config)
+	for _, addr := range node.Status.Addresses {
+		if addr.Type == v1.NodeInternalIP {
+			internalIP = addr.Address
+		} else if addr.Type == v1.NodeExternalIP {
+			externalIP = addr.Address
+		}
+	}
+
+	if externalIP != "" {
+		return externalIP, nodePort, nil
+	}
+
+	if internalIP != "" {
+		return internalIP, nodePort, nil
+	}
+
+	return "", 0, nil
+}
+
+func setAdvertisedListeners(aerospikeVectorSearchConfig map[string]interface{}) error {
+
+	nodeIP, nodePort, err := getNodeIp()
 	if err != nil {
-		return "", err
+		return err
+	}
+
+	if nodeIP == "" && nodePort == 0 {
+		return nil
+	}
+
+	fmt.Printf("Node IP: %s Port: %d\n", nodeIP, nodePort)
+	service, ok := aerospikeVectorSearchConfig["service"].(map[string]interface{})
+	if !ok {
+		return fmt.Errorf("Was not able to get service")
+	}
+
+	ports, ok := service["ports"].(map[string]interface{})
+	if !ok {
+		return fmt.Errorf("Was not able to get service.ports")
 	}
 
-	node, err := clientset.CoreV1().Nodes().Get(context.TODO(), nodeName, metav1.GetOptions{})
+	for _, portConfig := range ports {
+		portMap, ok := portConfig.(map[string]interface{})
+		if !ok {
+			continue
+		}
+
+		portMap["advertised-listeners"] = map[string][]map[string]interface{}{
+			"default": {
+				{
+					"address": nodeIP,
+					"port":    nodePort,
+				},
+			},
+		}
+	}
+	return nil
+}
+
+func getNodeLabels() (string, error) {
+
+	node, _, err := GetNodeInstance()
 	if err != nil {
 		return "", err
 	}
@@ -123,7 +303,6 @@ func run() int {
 
 	aerospikeVectorSearchConfigEnvVariable := os.Getenv("AEROSPIKE_VECTOR_SEARCH_CONFIG")
 	if aerospikeVectorSearchConfigEnvVariable == "" {
-		fmt.Println("Aerospike Vector Search configuration is empty")
 		return 1
 	}
 
@@ -141,6 +320,12 @@ func run() int {
 		return util.ToExitVal(err)
 	}
 
+	err = setAdvertisedListeners(aerospikeVectorSearchConfig)
+	if err != nil {
+		fmt.Println(err)
+		return util.ToExitVal(err)
+	}
+
 	err = writeConfig(aerospikeVectorSearchConfig)
 	if err != nil {
 		fmt.Println(err)

diff --git a/charts/aerospike-vector-search/templates/roles.yaml b/charts/aerospike-vector-search/templates/roles.yaml
@@ -5,8 +5,8 @@ metadata:
   name: {{ include "aerospike-vector-search.serviceAccountName" . }}
 rules:
   - apiGroups: [""]
-    resources: ["nodes"]
-    verbs: ["get"]
+    resources: ["nodes", "services", "pods"]
+    verbs: ["get", "list"]
 
 ---
 kind: ClusterRoleBinding