fix(tailscale): make version checks less flaky with retries, fallback…

… to previous version

Signed-off-by: Gyuho Lee <[email protected]>

components/tailscale/component_output.go

@@ -6,11 +6,13 @@ import (
 	"errors"
 	"fmt"
 	"sync"
+	"time"
 
 	"github.com/leptonai/gpud/components"
 	components_metrics "github.com/leptonai/gpud/components/metrics"
 	"github.com/leptonai/gpud/components/query"
 	tailscale_id "github.com/leptonai/gpud/components/tailscale/id"
+	"github.com/leptonai/gpud/log"
 
 	"sigs.k8s.io/yaml"
 )
@@ -113,6 +115,33 @@ func getDefaultPoller() query.Poller {
 	return defaultPoller
 }
 
+var (
+	prevVer   *VersionInfo
+	prevVerMu sync.Mutex
+)
+
+func getPrevVer() *VersionInfo {
+	var copied *VersionInfo = nil
+
+	prevVerMu.Lock()
+	if prevVer != nil {
+		c := *prevVer
+		copied = &c
+	}
+	prevVerMu.Unlock()
+
+	return copied
+}
+
+func setPrevVer(ver *VersionInfo) {
+	prevVerMu.Lock()
+	if ver != nil {
+		c := *ver
+		prevVer = &c
+	}
+	prevVerMu.Unlock()
+}
+
 func Get(ctx context.Context) (_ any, e error) {
 	defer func() {
 		if e != nil {
@@ -122,12 +151,28 @@ func Get(ctx context.Context) (_ any, e error) {
 		}
 	}()
 
-	ver, err := CheckVersion()
-	if err != nil {
-		return nil, err
+	var ver *VersionInfo
+	for i := 0; i < 5; i++ {
+		var err error
+		ver, err = checkVersion()
+		if err != nil {
+			log.Logger.Warnw("failed to get tailscale version", "error", err)
+			time.Sleep(2 * time.Second)
+			continue
+		}
+		break
 	}
+
+	prev := getPrevVer()
+	setPrevVer(ver)
+
 	if ver == nil {
+		if prev != nil {
+			log.Logger.Warnw("tailscale version is nil -- fallback to previous version", "prev", prev)
+			return prev, nil
+		}
 		return nil, errors.New("tailscale version is nil")
 	}
+
 	return &Output{Version: *ver}, nil
 }

components/tailscale/tailscale.go

@@ -16,18 +16,18 @@ func TailscaleExists() bool {
 	return p != ""
 }
 
-// CheckVersion returns the tailscale version by running `tailscale version --json`.
-func CheckVersion() (*VersionInfo, error) {
+// checkVersion returns the tailscale version by running `tailscale version --json`.
+func checkVersion() (*VersionInfo, error) {
 	p, err := exec.LookPath("tailscale")
 	if err != nil {
 		return nil, fmt.Errorf("tailscale version check requires tailscale (%w)", err)
 	}
 
-	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
 	b, err := exec.CommandContext(ctx, p, "version", "--json").CombinedOutput()
 	cancel()
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("%w (%s)", err, string(b))
 	}
 
 	return ParseVersion(b)

components/tailscale/tailscale_test.go

@@ -6,7 +6,7 @@ import (
 )
 
 func TestVersion(t *testing.T) {
-	version, err := CheckVersion()
+	version, err := checkVersion()
 	if err != nil {
 		t.Skip(err) // ci may not have tailscale installed
 	}