-
Notifications
You must be signed in to change notification settings - Fork 63
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #296 from linode/more-robust-healthcheck
[feat] add linode token health check
- Loading branch information
Showing
8 changed files
with
308 additions
and
18 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,63 @@ | ||
| package linode | ||
|
|
||
| import ( | ||
| "context" | ||
| "time" | ||
|
|
||
| "github.com/linode/linode-cloud-controller-manager/cloud/linode/client" | ||
| "k8s.io/apimachinery/pkg/util/wait" | ||
| "k8s.io/klog/v2" | ||
| ) | ||
|
|
||
| type healthChecker struct { | ||
| period time.Duration | ||
| linodeClient client.Client | ||
| stopCh chan<- struct{} | ||
| } | ||
|
|
||
| func newHealthChecker(apiToken string, timeout time.Duration, period time.Duration, stopCh chan<- struct{}) (*healthChecker, error) { | ||
| client, err := client.New(apiToken, timeout) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
|
|
||
| return &healthChecker{ | ||
| period: period, | ||
| linodeClient: client, | ||
| stopCh: stopCh, | ||
| }, nil | ||
| } | ||
|
|
||
| func (r *healthChecker) Run(stopCh <-chan struct{}) { | ||
| ctx := wait.ContextForChannel(stopCh) | ||
| wait.Until(r.worker(ctx), r.period, stopCh) | ||
| } | ||
|
|
||
| func (r *healthChecker) worker(ctx context.Context) func() { | ||
| return func() { | ||
| r.do(ctx) | ||
| } | ||
| } | ||
|
|
||
| func (r *healthChecker) do(ctx context.Context) { | ||
| if r.stopCh == nil { | ||
| klog.Errorf("stop signal already fired. nothing to do") | ||
| return | ||
| } | ||
|
|
||
| authenticated, err := client.CheckClientAuthenticated(ctx, r.linodeClient) | ||
| if err != nil { | ||
| klog.Warningf("unable to determine linode client authentication status: %s", err.Error()) | ||
| return | ||
| } | ||
|
|
||
| if !authenticated { | ||
| klog.Error("detected invalid linode api token: stopping controllers") | ||
|
|
||
| close(r.stopCh) | ||
| r.stopCh = nil | ||
| return | ||
| } | ||
|
|
||
| klog.Info("linode api token is healthy") | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,153 @@ | ||
| package linode | ||
|
|
||
| import ( | ||
| "testing" | ||
| "time" | ||
|
|
||
| "github.com/golang/mock/gomock" | ||
| "github.com/linode/linode-cloud-controller-manager/cloud/linode/client/mocks" | ||
| "github.com/linode/linodego" | ||
| ) | ||
|
|
||
| func TestHealthCheck(t *testing.T) { | ||
| testCases := []struct { | ||
| name string | ||
| f func(*testing.T, *mocks.MockClient) | ||
| }{ | ||
| { | ||
| name: "Test succeeding calls to linode api stop signal is not fired", | ||
| f: testSucceedingCallsToLinodeAPIHappenStopSignalNotFired, | ||
| }, | ||
| { | ||
| name: "Test Unauthorized calls to linode api stop signal is fired", | ||
| f: testFailingCallsToLinodeAPIHappenStopSignalFired, | ||
| }, | ||
| { | ||
| name: "Test failing calls to linode api stop signal is not fired", | ||
| f: testErrorCallsToLinodeAPIHappenStopSignalNotFired, | ||
| }, | ||
| } | ||
|
|
||
| for _, tc := range testCases { | ||
| t.Run(tc.name, func(t *testing.T) { | ||
| ctrl := gomock.NewController(t) | ||
| defer ctrl.Finish() | ||
|
|
||
| client := mocks.NewMockClient(ctrl) | ||
| tc.f(t, client) | ||
| }) | ||
| } | ||
| } | ||
|
|
||
| func testSucceedingCallsToLinodeAPIHappenStopSignalNotFired(t *testing.T, client *mocks.MockClient) { | ||
| writableStopCh := make(chan struct{}) | ||
| readableStopCh := make(chan struct{}) | ||
|
|
||
| client.EXPECT().GetProfile(gomock.Any()).Times(2).Return(&linodego.Profile{}, nil) | ||
|
|
||
| hc, err := newHealthChecker("validToken", 1*time.Second, 1*time.Second, writableStopCh) | ||
| if err != nil { | ||
| t.Fatalf("expected a nil error, got %v", err) | ||
| } | ||
| // inject mocked linodego.Client | ||
| hc.linodeClient = client | ||
|
|
||
| defer close(readableStopCh) | ||
| go hc.Run(readableStopCh) | ||
|
|
||
| // wait for two checks to happen | ||
| time.Sleep(1500 * time.Millisecond) | ||
|
|
||
| select { | ||
| case <-writableStopCh: | ||
| t.Error("healthChecker sent stop signal") | ||
| default: | ||
| } | ||
| } | ||
|
|
||
| func testFailingCallsToLinodeAPIHappenStopSignalFired(t *testing.T, client *mocks.MockClient) { | ||
| writableStopCh := make(chan struct{}) | ||
| readableStopCh := make(chan struct{}) | ||
|
|
||
| client.EXPECT().GetProfile(gomock.Any()).Times(1).Return(&linodego.Profile{}, nil) | ||
|
|
||
| hc, err := newHealthChecker("validToken", 1*time.Second, 1*time.Second, writableStopCh) | ||
| if err != nil { | ||
| t.Fatalf("expected a nil error, got %v", err) | ||
| } | ||
| // inject mocked linodego.Client | ||
| hc.linodeClient = client | ||
|
|
||
| defer close(readableStopCh) | ||
| go hc.Run(readableStopCh) | ||
|
|
||
| // wait for check to happen | ||
| time.Sleep(500 * time.Millisecond) | ||
|
|
||
| select { | ||
| case <-writableStopCh: | ||
| t.Error("healthChecker sent stop signal") | ||
| default: | ||
| } | ||
|
|
||
| // invalidate token | ||
| client.EXPECT().GetProfile(gomock.Any()).Times(1).Return(&linodego.Profile{}, &linodego.Error{Code: 401, Message: "Invalid Token"}) | ||
|
|
||
| // wait for check to happen | ||
| time.Sleep(1 * time.Second) | ||
|
|
||
| select { | ||
| case <-writableStopCh: | ||
| default: | ||
| t.Error("healthChecker did not send stop signal") | ||
| } | ||
| } | ||
|
|
||
| func testErrorCallsToLinodeAPIHappenStopSignalNotFired(t *testing.T, client *mocks.MockClient) { | ||
| writableStopCh := make(chan struct{}) | ||
| readableStopCh := make(chan struct{}) | ||
|
|
||
| client.EXPECT().GetProfile(gomock.Any()).Times(1).Return(&linodego.Profile{}, nil) | ||
|
|
||
| hc, err := newHealthChecker("validToken", 1*time.Second, 1*time.Second, writableStopCh) | ||
| if err != nil { | ||
| t.Fatalf("expected a nil error, got %v", err) | ||
| } | ||
| // inject mocked linodego.Client | ||
| hc.linodeClient = client | ||
|
|
||
| defer close(readableStopCh) | ||
| go hc.Run(readableStopCh) | ||
|
|
||
| // wait for check to happen | ||
| time.Sleep(500 * time.Millisecond) | ||
|
|
||
| select { | ||
| case <-writableStopCh: | ||
| t.Error("healthChecker sent stop signal") | ||
| default: | ||
| } | ||
|
|
||
| // simulate server error | ||
| client.EXPECT().GetProfile(gomock.Any()).Times(1).Return(&linodego.Profile{}, &linodego.Error{Code: 500}) | ||
|
|
||
| // wait for check to happen | ||
| time.Sleep(1 * time.Second) | ||
|
|
||
| select { | ||
| case <-writableStopCh: | ||
| t.Error("healthChecker sent stop signal") | ||
| default: | ||
| } | ||
|
|
||
| client.EXPECT().GetProfile(gomock.Any()).Times(1).Return(&linodego.Profile{}, nil) | ||
|
|
||
| // wait for check to happen | ||
| time.Sleep(1 * time.Second) | ||
|
|
||
| select { | ||
| case <-writableStopCh: | ||
| t.Error("healthChecker sent stop signal") | ||
| default: | ||
| } | ||
| } |
Oops, something went wrong.