From f03d3a586a68374fb85f2514fa6519d28ad35a55 Mon Sep 17 00:00:00 2001 From: Bailin He Date: Tue, 28 May 2024 19:15:10 +0000 Subject: [PATCH] add more files Signed-off-by: Bailin He --- .buildkite/pipeline.yml | 67 ++++++++ .github/CODEOWNERS | 1 + .github/workflows/release.yml | 47 ++++++ .gitignore | 2 + .golangci.yml | 61 ++++++++ Makefile | 34 ++++ README.md | 282 +++++++++++++++++++++++++++++++++- cmd/root.go | 117 ++++++++++++++ docker-compose-ci.yml | 13 ++ go.mod | 1 + go.sum | 1 + main.go | 8 + pkg/erdscli/doc.go | 2 +- renovate.json | 11 ++ 14 files changed, 645 insertions(+), 2 deletions(-) create mode 100644 .buildkite/pipeline.yml create mode 100644 .github/CODEOWNERS create mode 100644 .github/workflows/release.yml create mode 100644 .golangci.yml create mode 100644 Makefile create mode 100644 cmd/root.go create mode 100644 docker-compose-ci.yml create mode 100644 main.go create mode 100644 renovate.json diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml new file mode 100644 index 0000000..0b7a405 --- /dev/null +++ b/.buildkite/pipeline.yml @@ -0,0 +1,67 @@ +env: + APP_NAME: ${BUILDKITE_PIPELINE_SLUG} + IMAGE_REPO: ghcr.io/metal-toolbox/${APP_NAME} + IMAGE_TAG: ${BUILDKITE_BUILD_NUMBER}-${BUILDKITE_COMMIT:0:8} + +steps: +- label: ":golangci-lint: lint :lint-roller:" + key: "lint" + plugins: + - docker#v5.10.0: + environment: + - GOFLAGS=-buildvcs=false + image: "registry.hub.docker.com/golangci/golangci-lint:v1.57-alpine" + command: ["golangci-lint", "run", "-v", "--timeout", "5m"] + +- label: ":test_tube: test" + key: "test" + plugins: + - docker-compose#v4.16.0: + cli-version: 2 + run: go + config: docker-compose-ci.yml + command: ["make", "test"] + +- label: ":golang: build" + key: "gobuild" + artifact_paths: "bin/${APP_NAME}" + plugins: + - docker#v5.10.0: + image: "golang:1.22" + environment: + - CGO_ENABLED=0 + - GOOS=linux + command: ["go", "build", "-buildvcs=false", "-mod=mod", "-a", "-o", "bin/$APP_NAME"] + +- label: ":docker: docker build and publish" + key: "build" + depends_on: ["lint", "test", "gobuild"] + env: + BUILDKITE_PLUGINS_ALWAYS_CLONE_FRESH: "true" + commands: | + #!/bin/bash + echo --- Retrieve Artifacts + buildkite-agent artifact download "bin/${APP_NAME}" . + # move it to where we expect and make sure it is executable + cp bin/${APP_NAME} ${APP_NAME} + chmod +x ${APP_NAME} + plugins: + - docker-login#v2.1.0: + username: metal-buildkite + password-env: SECRET_GHCR_PUBLISH_TOKEN + server: ghcr.io + - equinixmetal-buildkite/docker-metadata#v1.0.0: + images: + - "${IMAGE_REPO}" + extra_tags: + - "${IMAGE_TAG}" + - equinixmetal-buildkite/docker-build#v1.1.0: + push: true + build-args: + - NAME=${APP_NAME} + - equinixmetal-buildkite/trivy#v1.18.5: + severity: CRITICAL,HIGH + ignore-unfixed: true + security-checks: config,secret,vuln + skip-files: 'cosign.key,Dockerfile.dev' + diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000..3a66e64 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1 @@ +* @metal-toolbox/identity-core diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..b773be4 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,47 @@ +# name: goreleaser + +# on: +# push: +# tags: +# - 'v*.*.*' + +# jobs: +# goreleaser: +# runs-on: ubuntu-latest +# permissions: +# contents: write +# id-token: write +# packages: write +# steps: +# - +# name: Login to GHCR +# uses: docker/login-action@v2 +# with: +# registry: ghcr.io +# username: ${{ github.actor }} +# password: ${{ secrets.GITHUB_TOKEN }} +# - +# name: Checkout +# uses: actions/checkout@v4 +# with: +# fetch-depth: 0 +# - +# name: Set up Go +# uses: actions/setup-go@v4 +# with: +# go-version: "1.22" +# - +# name: install cosign +# uses: sigstore/cosign-installer@main +# - +# uses: anchore/sbom-action/download-syft@v0.14.3 +# - +# name: Run GoReleaser +# uses: goreleaser/goreleaser-action@v5 +# with: +# version: latest +# args: release --clean +# env: +# GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} +# COSIGN_EXPERIMENTAL: 1 +# GOVERSION: "1.22" diff --git a/.gitignore b/.gitignore index 303b582..cfb7287 100644 --- a/.gitignore +++ b/.gitignore @@ -62,4 +62,6 @@ Temporary Items go.work go.work.sum +bin tmp +governor-extension-sdk diff --git a/.golangci.yml b/.golangci.yml new file mode 100644 index 0000000..2599b66 --- /dev/null +++ b/.golangci.yml @@ -0,0 +1,61 @@ +linters-settings: + goimports: + local-prefixes: go.metalkube.net/mf-example-microservice + gofumpt: + extra-rules: true + +linters: + enable: + # default linters + - deadcode + - errcheck + - gosimple + - govet + - ineffassign + - staticcheck + - structcheck + - typecheck + - unused + - varcheck + + # additional linters + - bodyclose + - gocritic + - gocyclo + - goerr113 + - gofmt + - gofumpt + - goimports + - gomnd + - govet + - misspell + - noctx + - revive + - stylecheck + - whitespace + - wsl + +issues: + exclude: + # Default excludes from `golangci-lint run --help` with EXC0002 removed + # EXC0001 errcheck: Almost all programs ignore errors on these functions and in most cases it's ok + - Error return value of .((os\.)?std(out|err)\..*|.*Close|.*Flush|os\.Remove(All)?|.*print(f|ln)?|os\.(Un)?Setenv). is not checked + # EXC0002 golint: Annoying issue about not having a comment. The rare codebase has such comments + # - (comment on exported (method|function|type|const)|should have( a package)? comment|comment should be of the form) + # EXC0003 golint: False positive when tests are defined in package 'test' + - func name will be used as test\.Test.* by other packages, and that stutters; consider calling this + # EXC0004 govet: Common false positives + - (possible misuse of unsafe.Pointer|should have signature) + # EXC0005 staticcheck: Developers tend to write in C-style with an explicit 'break' in a 'switch', so it's ok to ignore + - ineffective break statement. Did you mean to break out of the outer loop + # EXC0006 gosec: Too many false-positives on 'unsafe' usage + - Use of unsafe calls should be audited + # EXC0007 gosec: Too many false-positives for parametrized shell calls + - Subprocess launch(ed with variable|ing should be audited) + # EXC0008 gosec: Duplicated errcheck checks + - (G104|G307) + # EXC0009 gosec: Too many issues in popular repos + - (Expect directory permissions to be 0750 or less|Expect file permissions to be 0600 or less) + # EXC0010 gosec: False positive is triggered by 'src, err := ioutil.ReadFile(filename)' + - Potential file inclusion via variable + exclude-use-default: false diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..cc4c565 --- /dev/null +++ b/Makefile @@ -0,0 +1,34 @@ +all: lint test +PHONY: test coverage lint golint clean vendor docker-up docker-down unit-test +GOOS=linux +# use the working dir as the app name, this should be the repo name +APP_NAME=$(shell basename $(CURDIR)) + +test: | unit-test + +unit-test: + @echo Running unit tests... + @go test -cover -short -tags testtools ./... + +lint: golint + +golint: | vendor + @echo Linting Go files... + @golangci-lint run --build-tags "-tags testtools" + +bin/${APP_NAME}: + @go mod download + @CGO_ENABLED=0 GOOS=linux go build -mod=readonly -v -o bin/${APP_NAME} + +build: bin/${APP_NAME} + +clean: docker-clean + @echo Cleaning... + @rm -f app + @rm -rf ./dist/ + @rm -rf coverage.out + @go clean -testcache + +vendor: + @go mod download + @go mod tidy diff --git a/README.md b/README.md index 54fdbf1..04c00c2 100644 --- a/README.md +++ b/README.md @@ -1 +1,281 @@ -# governor-extension-sdk \ No newline at end of file +# Governor Extension SDK + +Governor extension SDK is the SDK for developing Governor extensions. It provides +a set of tools and utilities to help developers to create, test and deploy +[Governor extensions](https://github.com/equinixmetal/napkins/blob/main/napkins/NAP0012.md). + +## CLI + +### ERDs Validation + +The `validate` command validates ERD files in `./erds` + +Usage: + +```console +$ governor-extension-sdk erds validate +2023-11-08T17:44:35.527Z INFO cmd/erds.go:93 validating ERDs +2023-11-08T17:44:35.564Z INFO cmd/erds.go:140 ERDs are valid +``` + +Run `governor-extension-sdk erds validate --help` for more information + +### Create ERD file + +The `new` command creates a new ERD file populated with default place holder +values. + +Usage: + +```console +$ governor-extension-sdk erds new --filename my-erd.yaml +2023-11-08T17:46:49.134Z INFO cmd/erds.go:222 creating new ERD my-erd.yaml +``` + +The default values can be overriden with additional flags: + +```console +Flags: + --description string description of the new ERD (default "some-description") + --enabled enabled status of the new ERD (default true) + --filename string filename of the new ERD, only .json, .yml and .yaml are supported + --name string name of the new ERD (default "hello-world") + --scope string scope of the new ERD (default "user") + --slug-plural string plural slug of the new ERD (default "greetings") + --slug-singular string singular slug of the new ERD (default "greeting") + --version string version of the new ERD (default "v1alpha1") +``` + +## Development + +### Event Router + +The event router listens to +events from the Governor API and dispatches them to the appropriate event +processors. + +#### Event router sample usage + +1. Create a new event processor that implements `pkg/eventprocessor.Processor` + + ```go + import ( + "github.com/metal-toolbox/governor-extension-sdk/pkg/eventprocessor" + "github.com/metal-toolbox/governor-api/pkg/api/v1alpha1" + ) + + type Processor struct { + // add fields here + } + + // Processor implements the eventrouter.EventProcessor interface + var _ eventprocessor.EventProcessor = (*Processor)(nil) + + func (p *Processor) ProcessEvent(ctx context.Context, event *event.Event) error { + // add event processing logic here + return nil + } + + func (p. Processor) Register(router eventrouter.EventRouter, ext *v1alpha1.Extension) error { + router.Create("groups", p.ProcessEvent) + router.Update("groups", p.ProcessEvent) + // ... add more event types here + + return nil + } + + ``` + +1. Initialize a new event router + + ```go + import ( + "github.com/metal-toolbox/governor-extension-sdk/pkg/eventrouter" + ) + + router := eventrouter.NewRouter() + ``` + +1. Register the event processor with the event router + + ```go + processor := &Processor{} // processor instance + + err := processor.Register(roueter, &v1alpha1.Extension{ /* extension metadata */ }) + ``` + +### Server + +The `server` package provides a simple HTTP server that listens to incoming +events from the Governor API. + +#### Server sample usage + +With the event router initialized and event processor registered, the server +can be created as follows: + +```go +import ( + "github.com/metal-toolbox/governor-extension-sdk/pkg/server" +) + +natsclient, err := server.NewNATSClient(/* nats client options */) + +server := server.NewServer( + "0.0.0.0:8080", + "governor-extension-id", + "paht/to/erds", + server.WithEventRouter(router), + server.WithNATSClient(natsclient) +) +``` + +Developers can also provide only the event processor to the server, and the +server will construct a new event router and register the event processor. + +```go +import ( + "github.com/metal-toolbox/governor-extension-sdk/pkg/server" +) + +server := server.NewServer( + "0.0.0.0:8080", + "governor-extension-id", + "paht/to/erds", + server.WithNATSClient(natsclient) + // multiple processors can be added here + server.WithEventProcessor(&Processor{}), + server.WithEventProcessor(/* another processor */), + server.WithEventProcessor(/* and another */), +) +``` + +Now the server can be started: + +```go +err := server.Run(ctx) +``` + +### Tracing + +Tracing should work out of the box. Top level tracer is defined in `server/server.go` +and it can be passed to any event processors. + +Event processors can inherit parent trace context in `event.TraceContext` +populated by the Governor API. + +The trace context can be passed to other HTTP services by using the +`WithTraceContext` roundtripper option. + +```go +import ( + "net/http" + "github.com/metal-toolbox/governor-extension-sdk/pkg/eventrouter" +) + +client := &http.Client{ + Transport: roundtripper.NewGovExtRoundTripper( + http.DefaultTransport.RoundTrip, + roundtripper.WithTraceContext(), + ), +} +``` + +### Correlation IDs + +To prevent infinite update loop that caused by the extension reacting to its +own update, the event router provides a correlation ID processor that can be +used to track the correlation ID of the incoming events, +see [this](https://github.com/equinixmetal/eis/blob/main/napkins/NAP0018.md#correlation-ids) +for more information. + +The correlation ID processor works by extracting the correlation ID from the +incoming event and storing it in the context with a middleware, it also checks +if the correlation ID is already present in the context and if it is, it will +skip the event processing. Finally, it will inject the correlation ID into the +subsequent outgoing API requests to the Governor API. + +For this to work, the event router should be initialized with the correlation ID +processor: + +```go +import ( + "github.com/metal-toolbox/governor-extension-sdk/pkg/eventrouter" +) + +router := eventrouter.NewRouter(eventrouter.WithCorrelationIDProcessor( + eventrouter.NewCorrelationIDProcessor() +)) +``` + +In addition, the http client used to make API requests to the Governor API should +be initialized with the correlation ID Roundtripper: + +```go +import ( + "net/http" + "github.com/metal-toolbox/governor-api/pkg/client" + "github.com/metal-toolbox/governor-extension-sdk/pkg/roundtripper" +) + +client := governor.NewClient( + governor.WithHTTPClient(&http.Client{ + Transport: roundtripper.NewGovExtRoundTripper( + http.DefaultTransport.RoundTrip, + roundtripper.WithCorrelationID(), + ), + }), + /* other options */ +) +``` + +#### Skip Strategy + +The correlation ID processor provides three strategies to skip the event processing: + +1. **Skip All**: Skip the any event processing if the correlation ID already exists + + ```go + import ( + "github.com/metal-toolbox/governor-extension-sdk/pkg/eventrouter" + ) + + cidp := eventrouter.NewCorrelationIDProcessor( + eventrouter.CorrelationIDProcessorWithSkipStrategySkipAll(), + ) + ``` + +1. **Update Only**: Skip the event processing if the correlation ID already exists + and the event type is `update` + + ```go + import ( + "github.com/metal-toolbox/governor-extension-sdk/pkg/eventrouter" + ) + + cidp := eventrouter.NewCorrelationIDProcessor( + eventrouter.CorrelationIDProcessorWithSkipStrategyUpdateOnly(), + ) + ``` + +1. **Custom**: Skip the event processing if the correlation ID already exists and + the routes are provided in the custom skip strategy + + ```go + import ( + "github.com/metal-toolbox/governor-extension-sdk/pkg/eventrouter" + "github.com/metal-toolbox/governor-api/pkg/events/v1alpha1" + ) + + cidp := eventrouter.NewCorrelationIDProcessor( + eventrouter.CorrelationIDProcessorWithSkipStrategyCustom(map[string]map[string]struct{}{ + // skips all updates + govevents.GovernorEventUpdate: {"*": {}}, + // skips create groups and create roles + govevents.GovernorEventCreate: { + "groups": {}, + "roles": {}, + }, + }), + ) + ``` diff --git a/cmd/root.go b/cmd/root.go new file mode 100644 index 0000000..811572b --- /dev/null +++ b/cmd/root.go @@ -0,0 +1,117 @@ +// Package cmd is our cobra/viper cli implementation +package cmd + +import ( + "strings" + + "github.com/metal-toolbox/governor-extension-sdk/pkg/erdscli" + homedir "github.com/mitchellh/go-homedir" + "github.com/spf13/cobra" + "github.com/spf13/pflag" + "github.com/spf13/viper" + "go.uber.org/zap" +) + +const appName = "governor-extension-sdk" + +var ( + cfgFile string + logger *zap.SugaredLogger +) + +// rootCmd represents the base command when called without any subcommands +var rootCmd = &cobra.Command{ + Use: appName, + Short: "governor-extension-sdk", + Long: `Governor Extension SDK`, +} + +// Execute adds all child commands to the root command and sets flags appropriately. +// This is called by main.main(). It only needs to happen once to the rootCmd. +func Execute() { + cobra.CheckErr(rootCmd.Execute()) +} + +func init() { + cobra.OnInitialize(initConfig) + + rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file (default is $HOME/."+appName+".yaml)") + + rootCmd.PersistentFlags().Bool("debug", false, "enable debug logging") + viperBindFlag("logging.debug", rootCmd.PersistentFlags().Lookup("debug")) + + rootCmd.PersistentFlags().Bool("pretty", false, "enable pretty (human readable) logging output") + viperBindFlag("logging.pretty", rootCmd.PersistentFlags().Lookup("pretty")) + + rootCmd.PersistentFlags().Bool("development", false, "enable development settings") + viperBindFlag("development", rootCmd.PersistentFlags().Lookup("development")) + + rootCmd.Flags().String("erds-path", "erds", "path contains erd files") + viperBindFlag("erds.path", rootCmd.Flags().Lookup("erds-path")) + + erdscli.RegisterCobraCommand(rootCmd, func() { + erdscli.SetAppName(appName) + erdscli.SetLogger(logger.Desugar()) + erdscli.SetERDPath(viper.GetString("erds.path")) + }) +} + +// initConfig reads in config file and ENV variables if set. +func initConfig() { + if cfgFile != "" { + // Use config file from the flag. + viper.SetConfigFile(cfgFile) + } else { + // Find home directory. + home, err := homedir.Dir() + cobra.CheckErr(err) + + // Search config in home directory with name ".TODO" (without extension). + viper.AddConfigPath(home) + viper.SetConfigName("." + appName) + } + + viper.SetEnvKeyReplacer(strings.NewReplacer(".", "_", "-", "_")) + // TODO: This needs to match [a-z]+, this may not be true for your app name + viper.SetEnvPrefix("govext") + + viper.AutomaticEnv() // read in environment variables that match + + setupLogging() + + // If a config file is found, read it in. + err := viper.ReadInConfig() + if err == nil { + logger.Infow("using config file", + "file", viper.ConfigFileUsed(), + ) + } +} + +func setupLogging() { + cfg := zap.NewProductionConfig() + if viper.GetBool("logging.pretty") { + cfg = zap.NewDevelopmentConfig() + } + + if viper.GetBool("logging.debug") { + cfg.Level = zap.NewAtomicLevelAt(zap.DebugLevel) + } else { + cfg.Level = zap.NewAtomicLevelAt(zap.InfoLevel) + } + + l, err := cfg.Build() + if err != nil { + panic(err) + } + + logger = l.Sugar().With("app", appName) + defer logger.Sync() //nolint:errcheck +} + +// viperBindFlag provides a wrapper around the viper bindings that handles error checks +func viperBindFlag(name string, flag *pflag.Flag) { + if err := viper.BindPFlag(name, flag); err != nil { + panic(err) + } +} diff --git a/docker-compose-ci.yml b/docker-compose-ci.yml new file mode 100644 index 0000000..da3e4f4 --- /dev/null +++ b/docker-compose-ci.yml @@ -0,0 +1,13 @@ +services: + go: + image: golang:1.22 + volumes: + - type: bind + source: ./ + target: /app + working_dir: /app + networks: + - governor + +networks: + governor: diff --git a/go.mod b/go.mod index 882c912..d3538bc 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ require ( github.com/gin-contrib/zap v1.1.3 github.com/gin-gonic/gin v1.10.0 github.com/metal-toolbox/governor-api v0.2.3 + github.com/mitchellh/go-homedir v1.1.0 github.com/nats-io/nats.go v1.35.0 github.com/spf13/cobra v1.8.0 github.com/spf13/pflag v1.0.5 diff --git a/go.sum b/go.sum index e53008b..0fc59af 100644 --- a/go.sum +++ b/go.sum @@ -482,6 +482,7 @@ github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJys github.com/mitchellh/cli v1.1.0/go.mod h1:xcISNoH86gajksDmfB23e/pu+B+GeFRMYmoHXxx3xhI= github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw= github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= diff --git a/main.go b/main.go new file mode 100644 index 0000000..23e5a58 --- /dev/null +++ b/main.go @@ -0,0 +1,8 @@ +// package main is the main package +package main + +import "github.com/metal-toolbox/governor-extension-sdk/cmd" + +func main() { + cmd.Execute() +} diff --git a/pkg/erdscli/doc.go b/pkg/erdscli/doc.go index c97efa7..1f2838b 100644 --- a/pkg/erdscli/doc.go +++ b/pkg/erdscli/doc.go @@ -1,2 +1,2 @@ -// Package erds is a package creating commands for ERD files +// Package erdscli is a package creating commands for ERD files package erdscli diff --git a/renovate.json b/renovate.json new file mode 100644 index 0000000..55ecf43 --- /dev/null +++ b/renovate.json @@ -0,0 +1,11 @@ +{ + "extends": [ + "config:base" + ], + "packageRules": [ + { + "matchUpdateTypes": ["minor", "patch", "pin", "digest"], + "automerge": true + } + ] +}