Skip to content

Commit

Permalink
Automated cherry pick of #308: Add tagging controller configuration#3…
Browse files Browse the repository at this point in the history
…34: Stop retrying failed workitem after a certain amount of (#358)

* Add tagging controller
* Stop retrying failed workitem after a certain amount of retries Added metrics
* Changes to make tagging controller compatible with K8s 1.21

Co-authored-by: Nguyen Dinh <[email protected]>
Co-authored-by: Nicholas Turner <[email protected]>
  • Loading branch information
3 people authored May 12, 2022
1 parent 315d55a commit 8b6a466
Show file tree
Hide file tree
Showing 16 changed files with 847 additions and 39 deletions.
5 changes: 3 additions & 2 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,8 @@
/ecr-credential-provider.exe
/cloudconfig

.vscode/

docs/book/_book/
site/
.vscode/
e2e.test
.idea/
26 changes: 14 additions & 12 deletions Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -14,41 +14,43 @@
## BUILD ARGS ##
################################################################################
# This build arg allows the specification of a custom Golang image.
ARG GOLANG_IMAGE=golang:1.15.3
ARG GOLANG_IMAGE=golang:1.18.1

# The distroless image on which the CPI manager image is built.
#
# Please do not use "latest". Explicit tags should be used to provide
# deterministic builds. This image doesn't have semantic version tags, but
# the fully-qualified image can be obtained by entering
# "gcr.io/distroless/static:latest" in a browser and then copying the
# fully-qualified image from the web page.
ARG DISTROLESS_IMAGE=gcr.io/distroless/static@sha256:c6d5981545ce1406d33e61434c61e9452dad93ecd8397c41e89036ef977a88f4
# deterministic builds. Follow what kubernetes uses to build
# kube-controller-manager, for example for 1.23.x:
# https://github.com/kubernetes/kubernetes/blob/release-1.24/build/common.sh#L94
ARG DISTROLESS_IMAGE=k8s.gcr.io/build-image/go-runner:v2.3.1-go1.18.1-bullseye.0

################################################################################
## BUILD STAGE ##
################################################################################
# Build the manager as a statically compiled binary so it has no dependencies
# libc, muscl, etc.
FROM ${GOLANG_IMAGE} as builder
FROM --platform=linux/amd64 ${GOLANG_IMAGE} as builder

ARG VERSION
ARG GOPROXY=https://goproxy.io,direct
ARG GOOS=linux
ARG TARGETOS
ARG TARGETARCH
ARG VERSION

WORKDIR /build
COPY go.mod go.sum ./
COPY cmd/ cmd/
COPY pkg/ pkg/
RUN GO111MODULE=on CGO_ENABLED=0 GOOS=${GOOS} GOPROXY=${GOPROXY} go build \
-ldflags="-w -s -X 'main.version=${VERSION}'" \
RUN GO111MODULE=on CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} GOPROXY=${GOPROXY} \
go build \
-trimpath \
-ldflags="-w -s -X k8s.io/component-base/version.gitVersion=${VERSION}" \
-o=aws-cloud-controller-manager \
cmd/aws-cloud-controller-manager/main.go

################################################################################
## MAIN STAGE ##
################################################################################
# Copy the manager into the distroless image.
FROM ${DISTROLESS_IMAGE}
FROM --platform=${TARGETPLATFORM} ${DISTROLESS_IMAGE}
COPY --from=builder /build/aws-cloud-controller-manager /bin/aws-cloud-controller-manager
ENTRYPOINT [ "/bin/aws-cloud-controller-manager" ]
80 changes: 69 additions & 11 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -13,39 +13,70 @@
# limitations under the License.
#

.EXPORT_ALL_VARIABLES:

SHELL := /bin/bash
SOURCES := $(shell find . -name '*.go')
GOOS ?= $(shell go env GOOS)
GOARCH ?= $(shell go env GOARCH)
GOPROXY ?= $(shell go env GOPROXY)
GIT_VERSION := $(shell git describe --match=$(git rev-parse --short=8 HEAD) --always --dirty --abbrev=8)
GIT_VERSION := $(shell git describe --dirty --tags --match='v*')
VERSION ?= $(GIT_VERSION)
IMAGE := amazon/cloud-controller-manager:$(VERSION)
IMAGE ?= amazon/cloud-controller-manager:$(VERSION)
OUTPUT ?= $(shell pwd)/_output
INSTALL_PATH ?= $(OUTPUT)/bin
LDFLAGS ?= -w -s -X k8s.io/component-base/version.gitVersion=$(VERSION)

aws-cloud-controller-manager: $(SOURCES)
GO111MODULE=on CGO_ENABLED=0 GOOS=$(GOOS) GOPROXY=$(GOPROXY) go build \
-ldflags="-w -s -X 'main.version=$(VERSION)'" \
GO111MODULE=on CGO_ENABLED=0 GOOS=$(GOOS) GOARCH=$(GOARCH) GOPROXY=$(GOPROXY) go build \
-trimpath \
-ldflags="$(LDFLAGS)" \
-o=aws-cloud-controller-manager \
cmd/aws-cloud-controller-manager/main.go

ecr-credential-provider: $(shell find ./cmd/ecr-credential-provider -name '*.go')
GO111MODULE=on CGO_ENABLED=0 GOOS=$(GOOS) GOPROXY=$(GOPROXY) go build \
-ldflags="-w -s -X 'main.version=$(VERSION)'" \
GO111MODULE=on CGO_ENABLED=0 GOOS=$(GOOS) GOARCH=$(GOARCH) GOPROXY=$(GOPROXY) go build \
-trimpath \
-ldflags="$(LDFLAGS)" \
-o=ecr-credential-provider \
cmd/ecr-credential-provider/*.go

ecr-credential-provider.exe: $(wildcard ./cmd/ecr-credential-provider/*.go)
GO111MODULE=on CGO_ENABLED=0 GOOS=windows GOPROXY=$(GOPROXY) go build \
-ldflags="-w -s -X 'main.version=$(VERSION)'" \
-trimpath \
-ldflags="$(LDFLAGS)" \
-o=ecr-credential-provider.exe \
cmd/ecr-credential-provider/*.go

.PHONY: docker-build-amd64
docker-build-amd64:
docker buildx build --output=type=docker \
--build-arg VERSION=$(VERSION) \
--build-arg GOPROXY=$(GOPROXY) \
--platform linux/amd64 \
--tag $(IMAGE) .

.PHONY: docker-build-arm64
docker-build-arm64:
docker buildx build --output=type=docker \
--build-arg VERSION=$(VERSION) \
--build-arg GOPROXY=$(GOPROXY) \
--platform linux/arm64 \
--tag $(IMAGE) .

.PHONY: docker-build
docker-build:
docker build \
--build-arg VERSION=$(VERSION) \
--build-arg GOOS=$(GOOS) \
docker buildx build --output=type=registry \
--build-arg LDFLAGS=$(LDFLAGS) \
--build-arg GOPROXY=$(GOPROXY) \
--platform linux/amd64,linux/arm64 \
--tag $(IMAGE) .

e2e.test:
pushd tests/e2e > /dev/null && \
go test -c && popd
mv tests/e2e/e2e.test e2e.test

.PHONY: check
check: verify-fmt verify-lint vet

Expand All @@ -59,7 +90,7 @@ verify-fmt:

.PHONY: verify-lint
verify-lint:
which golint 2>&1 >/dev/null || go get golang.org/x/lint/golint
which golint 2>&1 >/dev/null || go install golang.org/x/lint/golint@latest
golint -set_exit_status $(shell go list ./...)

.PHONY: verify-codegen
Expand All @@ -81,3 +112,30 @@ docs:
.PHONY: publish-docs
publish-docs:
./hack/publish-docs.sh

.PHONY: kops-example
kops-example:
./hack/kops-example.sh

.PHONY: test-e2e
test-e2e: e2e.test docker-build-amd64 install-e2e-tools
AWS_REGION=us-west-2 \
TEST_PATH=./tests/e2e/... \
BUILD_IMAGE=$(IMAGE) \
BUILD_VERSION=$(VERSION) \
INSTALL_PATH=$(INSTALL_PATH) \
GINKGO_FOCUS="\[cloud-provider-aws-e2e\]" \
./hack/e2e/run.sh

# Use `make install-e2e-tools KOPS_ROOT=<local-kops-installation>`
# to skip the kops download, test local changes to the kubetest2-kops
# deployer, etc.
.PHONY: install-e2e-tools
install-e2e-tools:
mkdir -p $(INSTALL_PATH)
INSTALL_PATH=$(INSTALL_PATH) \
./hack/install-e2e-tools.sh

.PHONY: print-image-tag
print-image-tag:
@echo $(IMAGE)
24 changes: 13 additions & 11 deletions cmd/aws-cloud-controller-manager/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,22 +26,21 @@ limitations under the License.
package main

import (
"math/rand"
"os"
"time"

"k8s.io/apimachinery/pkg/util/wait"
cloudprovider "k8s.io/cloud-provider"
"k8s.io/cloud-provider-aws/pkg/controllers/tagging"
awsv1 "k8s.io/cloud-provider-aws/pkg/providers/v1"
awsv2 "k8s.io/cloud-provider-aws/pkg/providers/v2"
"k8s.io/cloud-provider/app"
"k8s.io/cloud-provider/options"
cliflag "k8s.io/component-base/cli/flag"
"k8s.io/component-base/logs"
_ "k8s.io/component-base/metrics/prometheus/clientgo" // for client metric registration
_ "k8s.io/component-base/metrics/prometheus/version" // for version metric registration
"k8s.io/klog/v2"

cloudprovider "k8s.io/cloud-provider"
awsv1 "k8s.io/cloud-provider-aws/pkg/providers/v1"
awsv2 "k8s.io/cloud-provider-aws/pkg/providers/v2"
"math/rand"
"os"
"time"

cloudcontrollerconfig "k8s.io/cloud-provider/app/config"
)
Expand All @@ -50,8 +49,6 @@ const (
enableAlphaV2EnvVar = "ENABLE_ALPHA_V2"
)

var version string

func main() {
rand.Seed(time.Now().UTC().UnixNano())

Expand All @@ -64,11 +61,16 @@ func main() {
}

controllerInitializers := app.DefaultInitFuncConstructors
taggingControllerWrapper := tagging.ControllerWrapper{}
fss := cliflag.NamedFlagSets{}
taggingControllerWrapper.Options.AddFlags(fss.FlagSet("tagging controller"))

controllerInitializers[tagging.TaggingControllerKey] = taggingControllerWrapper.StartTaggingControllerWrapper
app.ControllersDisabledByDefault.Insert(tagging.TaggingControllerKey)
command := app.NewCloudControllerManagerCommand(opts, cloudInitializer, controllerInitializers, fss, wait.NeverStop)

if err := command.Execute(); err != nil {
os.Exit(1)
klog.Fatalf("unable to execute command: %v", err)
}
}

Expand Down
4 changes: 4 additions & 0 deletions docs/TODO.md
Original file line number Diff line number Diff line change
Expand Up @@ -15,3 +15,7 @@
### Kops

* Add a full example (ideally with IAM roles)

### Tagging Controller

* Add e2e testing which enables the controller, and monitors if the resources are tagged properly
7 changes: 7 additions & 0 deletions docs/tagging_controller.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
# The Tagging Controller

The tagging controller is responsible for tagging and untagging node resources when they join and leave the cluster, respectively. It can add and remove tags based on user input. Additionally, if a tag is updated, it would leave the updated tag and reapply the user-provided tag. Unlike the existing controllers, the tagging controller works exclusively with AWS. The AWS APIs it uses are `ec2:CreateTags` and `ec2:DeleteTags`.

| Flag | Valid Values | Default | Description |
|------| --- | --- | --- |
| tags | Comma-separated list of key=value | - | A comma-separated list of key-value pairs which will be recorded as nodes' additional tags. For example: "Key1=Val1,Key2=Val2,KeyNoVal1=,KeyNoVal2" |
2 changes: 2 additions & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ require (
github.com/golang/mock v1.4.4
github.com/google/go-cmp v0.5.2
github.com/spf13/cobra v1.1.1
github.com/spf13/pflag v1.0.5
github.com/stretchr/testify v1.6.1
gopkg.in/gcfg.v1 v1.2.0
k8s.io/api v0.21.0
Expand All @@ -16,6 +17,7 @@ require (
k8s.io/cloud-provider v0.21.0
k8s.io/code-generator v0.21.0
k8s.io/component-base v0.21.0
k8s.io/controller-manager v0.21.0
k8s.io/csi-translation-lib v0.21.0
k8s.io/klog/v2 v2.8.0
k8s.io/kubelet v0.21.0
Expand Down
24 changes: 24 additions & 0 deletions pkg/controllers/options/resources.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
/*
Copyright 2016 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package options

const (
// Instance presenting the string literal "instance"
Instance string = "instance"
)

// SupportedResources contains the resources that can be tagged by the controller at the moment
var SupportedResources = []string{
Instance,
}
53 changes: 53 additions & 0 deletions pkg/controllers/options/tagging_controller.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
/*
Copyright 2016 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package options

import (
"fmt"
"github.com/spf13/pflag"
)

// TaggingControllerOptions contains the inputs that can
// be used in the tagging controller
type TaggingControllerOptions struct {
Tags map[string]string
Resources []string
}

// AddFlags add the additional flags for the controller
func (o *TaggingControllerOptions) AddFlags(fs *pflag.FlagSet) {
fs.StringToStringVar(&o.Tags, "tags", o.Tags, "Tags to apply to AWS resources in the tagging controller, in a form of key=value.")
fs.StringArrayVar(&o.Resources, "resources", o.Resources, "AWS resources name to add/remove tags in the tagging controller.")
}

// Validate checks for errors from user input
func (o *TaggingControllerOptions) Validate() error {
if len(o.Tags) == 0 {
return fmt.Errorf("--tags must not be empty and must be a form of key=value")
}

if len(o.Resources) == 0 {
return fmt.Errorf("--resources must not be empty")
}

for _, r := range o.Resources {
for _, resource := range SupportedResources {
if r != resource {
return fmt.Errorf("%s is not a supported resource. Current supported resources %v", r, SupportedResources)
}
}
}

return nil
}
Loading

0 comments on commit 8b6a466

Please sign in to comment.