Merge pull request #3 from kazhuravlev/ka/gh-1

Background checks

README.md

@@ -12,7 +12,7 @@ feature [Liveness and Readiness](https://kubernetes.io/docs/tasks/configure-pod-
 ## Features
 
 - Logger to log failed probes
-- Automatic and manual checks
+- Automatic, manual and background checks
 - Respond with all healthchecks status in JSON format
 - Callback for integrate with metrics or other systems
 - Integrated web server
@@ -23,6 +23,8 @@ feature [Liveness and Readiness](https://kubernetes.io/docs/tasks/configure-pod-
 go get -u github.com/kazhuravlev/healthckeck
 ```
 
+Check an [examples](./examples/example.go).
+
 ```go
 package main
 
@@ -41,7 +43,7 @@ func main() {
 	// 1. Init healthcheck instance. It will store all our checks.
 	hc, _ := healthcheck.New()
 
-	// 2. Register checks for our redis client
+	// 2. Register checks that will random respond with an error.
 	hc.Register(ctx, healthcheck.NewBasic("redis", time.Second, func(ctx context.Context) error {
 		if rand.Float64() > 0.5 {
 			return errors.New("service is not available")

api.go

@@ -36,6 +36,11 @@ CheckID:
 		}
 	}
 
+	switch check := check.(type) {
+	case *bgCheck:
+		check.run()
+	}
+
 	s.checks = append(s.checks, checkRec{
 		ID:      checkID,
 		CheckFn: check.check,

api_checks.go

@@ -66,3 +66,67 @@ func (c *manualCheck) check(_ context.Context) result {
 
 	return result{Err: c.err}
 }
+
+type bgCheck struct {
+	name   string
+	period time.Duration
+	delay  time.Duration
+	ttl    time.Duration
+	fn     CheckFn
+
+	muErr *sync.RWMutex
+	err   error
+}
+
+// NewBackground will create a check that runs in background. Usually used for slow or expensive checks.
+// Note: period should be greater than timeout.
+//
+//	hc, _ := healthcheck.New(...)
+//	hc.Register(healthcheck.NewBackground("some_subsystem"))
+func NewBackground(name string, initialErr error, delay, period, timeout time.Duration, fn CheckFn) *bgCheck {
+	return &bgCheck{
+		name: name,
+
+		period: period,
+		delay:  delay,
+		ttl:    timeout,
+		fn:     fn,
+		muErr:  new(sync.RWMutex),
+		err:    initialErr,
+	}
+}
+
+func (c *bgCheck) run() {
+	go func() {
+		time.Sleep(c.delay)
+
+		t := time.NewTimer(c.period)
+		defer t.Stop()
+
+		for {
+			func() {
+				ctx, cancel := context.WithTimeout(context.Background(), c.ttl)
+				defer cancel()
+
+				err := c.fn(ctx)
+
+				c.muErr.Lock()
+				c.err = err
+				c.muErr.Unlock()
+			}()
+
+			select {
+			case <-t.C:
+			}
+		}
+	}()
+}
+
+func (c *bgCheck) id() string             { return c.name }
+func (c *bgCheck) timeout() time.Duration { return time.Hour }
+func (c *bgCheck) check(_ context.Context) result {
+	c.muErr.RLock()
+	defer c.muErr.RUnlock()
+
+	return result{Err: c.err}
+}

examples/example.go

@@ -0,0 +1,61 @@
+package examples
+
+import (
+	"context"
+	"errors"
+	"github.com/kazhuravlev/healthcheck"
+	"math/rand/v2"
+	"time"
+)
+
+func example() {
+	ctx := context.TODO()
+
+	// 1. Init healthcheck instance. It will store all our checks.
+	hc, _ := healthcheck.New()
+
+	// 2. Register basic check. It will be called each time when you call `/ready` endpoint.
+	{
+		hc.Register(ctx, healthcheck.NewBasic("postgres", time.Second, func(ctx context.Context) error {
+			if rand.Float64() > 0.5 {
+				return errors.New("service is not available")
+			}
+
+			return nil
+		}))
+	}
+
+	// 3. Register manual check.
+	{
+		cacheWarmUp := healthcheck.NewManual("cache-warmup")
+		cacheWarmUp.SetErr(errors.New("cache did not warmed up yet"))
+
+		time.AfterFunc(5*time.Second, func() {
+			// Mark this check as OK
+			cacheWarmUp.SetErr(nil)
+		})
+
+		hc.Register(ctx, cacheWarmUp)
+	}
+
+	// 4. Register background check. It will be checked in background even nobody call `/ready` endpoint.
+	{
+		hc.Register(ctx, healthcheck.NewBackground(
+			"clickhouse",
+			errors.New("clickhouse not ready"),
+			1*time.Second,
+			30*time.Second,
+			10*time.Second,
+			func(ctx context.Context) error {
+				return nil
+			},
+		))
+	}
+
+	// 3. Init and run a webserver for integration with Kubernetes.
+	sysServer, _ := healthcheck.NewServer(hc, healthcheck.WithPort(8080))
+	_ = sysServer.Run(ctx)
+
+	// 4. Open http://localhost:8080/ready to check the status of your system
+	select {}
+}

healthcheck_test.go

@@ -2,6 +2,7 @@ package healthcheck_test
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"io"
 	"sync"
@@ -192,6 +193,71 @@ func TestService(t *testing.T) { //nolint:funlen
 			}, res)
 		})
 	})
+
+	t.Run("background_check", func(t *testing.T) {
+		t.Parallel()
+
+		errNotReady := errors.New("not ready")
+
+		curErrorMu := new(sync.Mutex)
+		var curError error
+
+		delay := 200 * time.Millisecond
+		bgCheck := hc.NewBackground(
+			"some_system",
+			errNotReady,
+			delay,
+			delay,
+			10*time.Second,
+			func(ctx context.Context) error {
+				curErrorMu.Lock()
+				defer curErrorMu.Unlock()
+
+				return curError
+			},
+		)
+		hcInst := hcWithChecks(t, bgCheck)
+
+		t.Run("initial_error_is_used", func(t *testing.T) {
+			res := hcInst.RunAllChecks(context.Background())
+			requireReportEqual(t, hc.Report{
+				Status: hc.StatusDown,
+				Checks: []hc.CheckStatus{
+					{Name: "some_system", Status: hc.StatusDown, Error: "not ready"},
+				},
+			}, res)
+		})
+
+		// wait for bg check next run
+		time.Sleep(delay)
+
+		t.Run("check_current_error_nil", func(t *testing.T) {
+			res := hcInst.RunAllChecks(context.Background())
+			requireReportEqual(t, hc.Report{
+				Status: hc.StatusUp,
+				Checks: []hc.CheckStatus{
+					{Name: "some_system", Status: hc.StatusUp, Error: ""},
+				},
+			}, res)
+		})
+
+		// set error
+		curErrorMu.Lock()
+		curError = io.EOF
+		curErrorMu.Unlock()
+		// wait for bg check next run
+		time.Sleep(delay)
+
+		t.Run("change_status_after_each_run", func(t *testing.T) {
+			res := hcInst.RunAllChecks(context.Background())
+			requireReportEqual(t, hc.Report{
+				Status: hc.StatusDown,
+				Checks: []hc.CheckStatus{
+					{Name: "some_system", Status: hc.StatusDown, Error: "EOF"},
+				},
+			}, res)
+		})
+	})
 }
 
 func TestServiceMetrics(t *testing.T) { //nolint:paralleltest