Automated cherry pick of #308: Add tagging controller configuration #334: Stop retrying failed workitem after a certain amount of

.gitignore

@@ -1,2 +1,9 @@
 /aws-cloud-controller-manager
 /cloudconfig
+_output/
+/kops-example
+docs/book/_book/
+site/
+.vscode/
+e2e.test
+.idea/

cmd/aws-cloud-controller-manager/main.go

@@ -27,6 +27,7 @@ package main
 
 import (
 	"fmt"
+	"k8s.io/cloud-provider-aws/pkg/controllers/tagging"
 	"math/rand"
 	"net"
 	"net/http"
@@ -49,6 +50,21 @@ import (
 	"k8s.io/kubernetes/cmd/cloud-controller-manager/app/options"
 	"k8s.io/kubernetes/pkg/features" // add the kubernetes feature gates
 	netutils "k8s.io/utils/net"
+	"k8s.io/apimachinery/pkg/util/wait"
+	cloudprovider "k8s.io/cloud-provider"
+	"k8s.io/cloud-provider-aws/pkg/controllers/tagging"
+	awsv1 "k8s.io/cloud-provider-aws/pkg/providers/v1"
+	awsv2 "k8s.io/cloud-provider-aws/pkg/providers/v2"
+	"k8s.io/cloud-provider/app"
+	"k8s.io/cloud-provider/options"
+	cliflag "k8s.io/component-base/cli/flag"
+	"k8s.io/component-base/logs"
+	_ "k8s.io/component-base/metrics/prometheus/clientgo" // for client metric registration
+	_ "k8s.io/component-base/metrics/prometheus/version"  // for version metric registration
+	"k8s.io/klog/v2"
+	"math/rand"
+	"os"
+	"time"
 
 	cloudprovider "k8s.io/cloud-provider"
 	awsv1 "k8s.io/cloud-provider-aws/pkg/providers/v1"

docs/TODO.md

@@ -0,0 +1,21 @@
+# TODO
+
+### Prereqs
+
+* Document required instance tags (i.e. KubernetesCluster:<cluster-name>)
+
+### Load Balancers
+
+* document all available label/annotations to configure ELBs/NLBs for Service Type=LoadBalancer
+
+### Known Limitations
+
+* Document limitation with hostname / private DNS?
+
+### Kops
+
+* Add a full example (ideally with IAM roles)
+
+### Tagging Controller
+
+* Add e2e testing which enables the controller, and monitors if the resources are tagged properly

docs/tagging_controller.md

@@ -0,0 +1,7 @@
+# The Tagging Controller
+
+The tagging controller is responsible for tagging and untagging node resources when they join and leave the cluster, respectively. It can add and remove tags based on user input. Additionally, if a tag is updated, it would leave the updated tag and reapply the user-provided tag. Unlike the existing controllers, the tagging controller works exclusively with AWS. The AWS APIs it uses are `ec2:CreateTags` and `ec2:DeleteTags`.
+
+| Flag | Valid Values | Default | Description |
+|------| --- | --- | --- |
+| tags          | Comma-separated list of key=value | -   | A comma-separated list of key-value pairs which will be recorded as nodes' additional tags. For example: "Key1=Val1,Key2=Val2,KeyNoVal1=,KeyNoVal2" |

pkg/controllers/options/resources.go

@@ -0,0 +1,24 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+    http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package options
+
+const (
+	// Instance presenting the string literal "instance"
+	Instance string = "instance"
+)
+
+// SupportedResources contains the resources that can be tagged by the controller at the moment
+var SupportedResources = []string{
+	Instance,
+}

pkg/controllers/options/tagging_controller.go

@@ -0,0 +1,53 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+    http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package options
+
+import (
+	"fmt"
+	"github.com/spf13/pflag"
+)
+
+// TaggingControllerOptions contains the inputs that can
+// be used in the tagging controller
+type TaggingControllerOptions struct {
+	Tags      map[string]string
+	Resources []string
+}
+
+// AddFlags add the additional flags for the controller
+func (o *TaggingControllerOptions) AddFlags(fs *pflag.FlagSet) {
+	fs.StringToStringVar(&o.Tags, "tags", o.Tags, "Tags to apply to AWS resources in the tagging controller, in a form of key=value.")
+	fs.StringArrayVar(&o.Resources, "resources", o.Resources, "AWS resources name to add/remove tags in the tagging controller.")
+}
+
+// Validate checks for errors from user input
+func (o *TaggingControllerOptions) Validate() error {
+	if len(o.Tags) == 0 {
+		return fmt.Errorf("--tags must not be empty and must be a form of key=value")
+	}
+
+	if len(o.Resources) == 0 {
+		return fmt.Errorf("--resources must not be empty")
+	}
+
+	for _, r := range o.Resources {
+		for _, resource := range SupportedResources {
+			if r != resource {
+				return fmt.Errorf("%s is not a supported resource. Current supported resources %v", r, SupportedResources)
+			}
+		}
+	}
+
+	return nil
+}

pkg/controllers/tagging/metrics.go

@@ -0,0 +1,56 @@
+/*
+Copyright 2020 The Kubernetes Authors.
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+    http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tagging
+
+import (
+	"k8s.io/component-base/metrics"
+	"k8s.io/component-base/metrics/legacyregistry"
+	"sync"
+)
+
+var register sync.Once
+
+var (
+	workItemDuration = metrics.NewHistogramVec(
+		&metrics.HistogramOpts{
+			Name:           "cloudprovider_aws_tagging_controller_work_item_duration_seconds",
+			Help:           "workitem latency of workitem being in the queue and time it takes to process",
+			StabilityLevel: metrics.ALPHA,
+		},
+		[]string{"latency_type"})
+
+	workItemError = metrics.NewCounterVec(
+		&metrics.CounterOpts{
+			Name:           "cloudprovider_aws_tagging_controller_work_item_errors_total",
+			Help:           "any error in dequeueing the work queue and processing workItem",
+			StabilityLevel: metrics.ALPHA,
+		},
+		[]string{"error_type", "instance_id"})
+)
+
+// registerMetrics registers tagging-controller metrics.
+func registerMetrics() {
+	register.Do(func() {
+		legacyregistry.MustRegister(workItemDuration)
+		legacyregistry.MustRegister(workItemError)
+	})
+}
+
+func recordWorkItemLatencyMetrics(latencyType string, timeTaken float64) {
+	workItemDuration.With(metrics.Labels{"latency_type": latencyType}).Observe(timeTaken)
+}
+
+func recordWorkItemErrorMetrics(errorType string, instanceID string) {
+	workItemError.With(metrics.Labels{"error_type": errorType, "instance_id": instanceID}).Inc()
+}