From 449aa0f0c9564da2ef057f7eb1185dad1df42226 Mon Sep 17 00:00:00 2001 From: Nick Turner Date: Tue, 20 Apr 2021 16:02:51 -0700 Subject: [PATCH 01/43] Bump dependencies to v1.21.0 * Move to NewCloudControllerManagerCommand to remove copy/paste template code, we don't do anything too special here that requires duplication of the command creation. --- cmd/aws-cloud-controller-manager/main.go | 277 +++-------------------- go.mod | 28 +-- go.sum | 226 ++++++++++++++++++ 3 files changed, 268 insertions(+), 263 deletions(-) diff --git a/cmd/aws-cloud-controller-manager/main.go b/cmd/aws-cloud-controller-manager/main.go index a559199ac2..1673a74c54 100644 --- a/cmd/aws-cloud-controller-manager/main.go +++ b/cmd/aws-cloud-controller-manager/main.go @@ -26,40 +26,24 @@ limitations under the License. package main import ( - "fmt" "math/rand" - "net" - "net/http" "os" - "strings" "time" - "github.com/spf13/cobra" - - "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/wait" - utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/cloud-provider/app" "k8s.io/cloud-provider/options" cliflag "k8s.io/component-base/cli/flag" - "k8s.io/component-base/cli/globalflag" "k8s.io/component-base/logs" _ "k8s.io/component-base/metrics/prometheus/clientgo" // for client metric registration _ "k8s.io/component-base/metrics/prometheus/version" // for version metric registration - "k8s.io/component-base/term" - "k8s.io/component-base/version/verflag" "k8s.io/klog/v2" - netutils "k8s.io/utils/net" cloudprovider "k8s.io/cloud-provider" awsv1 "k8s.io/cloud-provider-aws/pkg/providers/v1" awsv2 "k8s.io/cloud-provider-aws/pkg/providers/v2" cloudcontrollerconfig "k8s.io/cloud-provider/app/config" - cloudnodecontroller "k8s.io/cloud-provider/controllers/node" - cloudnodelifecyclecontroller "k8s.io/cloud-provider/controllers/nodelifecycle" - routecontroller "k8s.io/cloud-provider/controllers/route" - servicecontroller "k8s.io/cloud-provider/controllers/service" ) const ( @@ -74,260 +58,55 @@ func main() { logs.InitLogs() defer logs.FlushLogs() - s, err := options.NewCloudControllerManagerOptions() + opts, err := options.NewCloudControllerManagerOptions() if err != nil { klog.Fatalf("unable to initialize command options: %v", err) } - var controllerInitializers map[string]app.InitFunc - command := &cobra.Command{ - Use: "aws-cloud-controller-manager", - Long: `aws-cloud-controller-manager manages AWS cloud resources for a Kubernetes cluster.`, - Run: func(cmd *cobra.Command, args []string) { - - // Use our version instead of the Kubernetes formatted version - versionFlag := cmd.Flags().Lookup("version") - if versionFlag.Value.String() == "true" { - fmt.Printf("%s version: %s\n", cmd.Name(), version) - os.Exit(0) - } - - // Default to the v1 provider if not set - cloudProviderFlag := cmd.Flags().Lookup("cloud-provider") - if cloudProviderFlag.Value.String() == "" { - cloudProviderFlag.Value.Set(awsv1.ProviderName) - } - - cloudProvider := cloudProviderFlag.Value.String() - if cloudProvider != awsv1.ProviderName && cloudProvider != awsv2.ProviderName { - klog.Fatalf("unknown cloud provider %s, only 'aws' and 'aws/v2' are supported", cloudProvider) - } - - if cloudProvider == awsv2.ProviderName { - if v2Enabled := os.Getenv(enableAlphaV2EnvVar); v2Enabled != "true" { - klog.Fatalf("aws/v2 cloud provider requires environment variable ENABLE_ALPHA_V2=true to be set") - } - } - - cliflag.PrintFlags(cmd.Flags()) - - c, err := s.Config(KnownControllers(), app.ControllersDisabledByDefault.List()) - if err != nil { - fmt.Fprintf(os.Stderr, "%v\n", err) - os.Exit(1) - } - - cloudConfigFile := c.ComponentConfig.KubeCloudShared.CloudProvider.CloudConfigFile - - // initialize cloud provider with the cloud provider name and config file provided - cloud, err := cloudprovider.InitCloudProvider(cloudProvider, cloudConfigFile) - if err != nil { - klog.Fatalf("Cloud provider could not be initialized: %v", err) - } - if cloud == nil { - klog.Fatalf("Cloud provider is nil") - } - - if !cloud.HasClusterID() { - if c.ComponentConfig.KubeCloudShared.AllowUntaggedCloud { - klog.Warning("detected a cluster without a ClusterID. A ClusterID will be required in the future. Please tag your cluster to avoid any future issues") - } else { - klog.Fatalf("no ClusterID found. A ClusterID is required for the cloud provider to function properly. This check can be bypassed by setting the allow-untagged-cloud option") - } - } - - // Initialize the cloud provider with a reference to the clientBuilder - cloud.Initialize(c.ClientBuilder, make(chan struct{})) - // Set the informer on the user cloud object - if informerUserCloud, ok := cloud.(cloudprovider.InformerUser); ok { - informerUserCloud.SetInformers(c.SharedInformers) - } - - controllerInitializers = app.DefaultControllerInitializers(c.Complete(), cloud) - - if err := app.Run(c.Complete(), controllerInitializers, wait.NeverStop); err != nil { - fmt.Fprintf(os.Stderr, "%v\n", err) - os.Exit(1) - } - }, - Args: func(cmd *cobra.Command, args []string) error { - for _, arg := range args { - if len(arg) > 0 { - return fmt.Errorf("%q does not take any arguments, got %q", cmd.CommandPath(), args) - } - } - return nil - }, - } - - fs := command.Flags() - namedFlagSets := s.Flags(app.KnownControllers(controllerInitializers), app.ControllersDisabledByDefault.List()) - verflag.AddFlags(namedFlagSets.FlagSet("global")) - globalflag.AddGlobalFlags(namedFlagSets.FlagSet("global"), command.Name()) - - for _, f := range namedFlagSets.FlagSets { - fs.AddFlagSet(f) - } - usageFmt := "Usage:\n %s\n" - cols, _, _ := term.TerminalSize(command.OutOrStdout()) - command.SetUsageFunc(func(cmd *cobra.Command) error { - fmt.Fprintf(cmd.OutOrStderr(), usageFmt, cmd.UseLine()) - cliflag.PrintSections(cmd.OutOrStderr(), namedFlagSets, cols) - return nil - }) - command.SetHelpFunc(func(cmd *cobra.Command, args []string) { - fmt.Fprintf(cmd.OutOrStdout(), "%s\n\n"+usageFmt, cmd.Long, cmd.UseLine()) - cliflag.PrintSections(cmd.OutOrStdout(), namedFlagSets, cols) - }) + controllerInitializers := app.DefaultInitFuncConstructors + fss := cliflag.NamedFlagSets{} + command := app.NewCloudControllerManagerCommand(opts, cloudInitializer, controllerInitializers, fss, wait.NeverStop) if err := command.Execute(); err != nil { - fmt.Fprintf(os.Stderr, "error: %v\n", err) os.Exit(1) } } -// initFunc is used to launch a particular controller. It may run additional "should I activate checks". -// Any error returned will cause the controller process to `Fatal` -// The bool indicates whether the controller was enabled. -type initFunc func(ctx *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface, stop <-chan struct{}) (debuggingHandler http.Handler, enabled bool, err error) - -// KnownControllers indicate the default controller we are known. -func KnownControllers() []string { - ret := sets.StringKeySet(newControllerInitializers()) - return ret.List() -} - -// newControllerInitializers is a private map of named controller groups (you can start more than one in an init func) -// paired to their initFunc. This allows for structured downstream composition and subdivision. -func newControllerInitializers() map[string]initFunc { - controllers := map[string]initFunc{} - controllers["cloud-node"] = startCloudNodeController - controllers["cloud-node-lifecycle"] = startCloudNodeLifecycleController - controllers["service"] = startServiceController - controllers["route"] = startRouteController - return controllers -} - -func startCloudNodeController(ctx *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface, stopCh <-chan struct{}) (http.Handler, bool, error) { - // Start the CloudNodeController - nodeController, err := cloudnodecontroller.NewCloudNodeController( - ctx.SharedInformers.Core().V1().Nodes(), - // cloud node controller uses existing cluster role from node-controller - ctx.ClientBuilder.ClientOrDie("node-controller"), - cloud, - ctx.ComponentConfig.NodeStatusUpdateFrequency.Duration, - ) - if err != nil { - klog.Warningf("failed to start cloud node controller: %s", err) - return nil, false, nil - } - - go nodeController.Run(stopCh) - - return nil, true, nil -} - -func startCloudNodeLifecycleController(ctx *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface, stopCh <-chan struct{}) (http.Handler, bool, error) { - // Start the cloudNodeLifecycleController - cloudNodeLifecycleController, err := cloudnodelifecyclecontroller.NewCloudNodeLifecycleController( - ctx.SharedInformers.Core().V1().Nodes(), - // cloud node lifecycle controller uses existing cluster role from node-controller - ctx.ClientBuilder.ClientOrDie("node-controller"), - cloud, - ctx.ComponentConfig.KubeCloudShared.NodeMonitorPeriod.Duration, - ) - if err != nil { - klog.Warningf("failed to start cloud node lifecycle controller: %s", err) - return nil, false, nil - } - - go cloudNodeLifecycleController.Run(stopCh) - - return nil, true, nil -} +func cloudInitializer(config *cloudcontrollerconfig.CompletedConfig) cloudprovider.Interface { + cloudConfig := config.ComponentConfig.KubeCloudShared.CloudProvider + providerName := cloudConfig.Name -func startServiceController(ctx *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface, stopCh <-chan struct{}) (http.Handler, bool, error) { - // Start the service controller - serviceController, err := servicecontroller.New( - cloud, - ctx.ClientBuilder.ClientOrDie("service-controller"), - ctx.SharedInformers.Core().V1().Services(), - ctx.SharedInformers.Core().V1().Nodes(), - ctx.ComponentConfig.KubeCloudShared.ClusterName, - utilfeature.DefaultFeatureGate, - ) - if err != nil { - // This error shouldn't fail. It lives like this as a legacy. - klog.Errorf("Failed to start service controller: %v", err) - return nil, false, nil + // Default to the v1 provider if not set + if providerName == "" { + providerName = awsv1.ProviderName } - go serviceController.Run(stopCh, int(ctx.ComponentConfig.ServiceController.ConcurrentServiceSyncs)) - - return nil, true, nil -} - -func startRouteController(ctx *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface, stopCh <-chan struct{}) (http.Handler, bool, error) { - if !ctx.ComponentConfig.KubeCloudShared.AllocateNodeCIDRs || !ctx.ComponentConfig.KubeCloudShared.ConfigureCloudRoutes { - klog.Infof("Will not configure cloud provider routes for allocate-node-cidrs: %v, configure-cloud-routes: %v.", ctx.ComponentConfig.KubeCloudShared.AllocateNodeCIDRs, ctx.ComponentConfig.KubeCloudShared.ConfigureCloudRoutes) - return nil, false, nil + if providerName != awsv1.ProviderName && providerName != awsv2.ProviderName { + klog.Fatalf("unknown cloud provider %s, only 'aws' and 'aws/v2' are supported", providerName) } - // If CIDRs should be allocated for pods and set on the CloudProvider, then start the route controller - routes, ok := cloud.Routes() - if !ok { - klog.Warning("configure-cloud-routes is set, but cloud provider does not support routes. Will not configure cloud provider routes.") - return nil, false, nil + if providerName == awsv2.ProviderName { + if v2Enabled := os.Getenv(enableAlphaV2EnvVar); v2Enabled != "true" { + klog.Fatalf("aws/v2 cloud provider requires environment variable ENABLE_ALPHA_V2=true to be set") + } } - // failure: bad cidrs in config - clusterCIDRs, dualStack, err := processCIDRs(ctx.ComponentConfig.KubeCloudShared.ClusterCIDR) + // initialize cloud provider with the cloud provider name and config file provided + cloud, err := cloudprovider.InitCloudProvider(providerName, cloudConfig.CloudConfigFile) if err != nil { - return nil, false, err - } - - // failure: more than one cidr and dual stack is not enabled - if len(clusterCIDRs) > 1 && !utilfeature.DefaultFeatureGate.Enabled(app.IPv6DualStack) { - return nil, false, fmt.Errorf("len of ClusterCIDRs==%v and dualstack feature is not enabled", len(clusterCIDRs)) + klog.Fatalf("Cloud provider could not be initialized: %v", err) } - - // failure: more than one cidr but they are not configured as dual stack - if len(clusterCIDRs) > 1 && !dualStack { - return nil, false, fmt.Errorf("len of ClusterCIDRs==%v and they are not configured as dual stack (at least one from each IPFamily", len(clusterCIDRs)) - } - - // failure: more than two cidrs is not allowed even with dual stack - if len(clusterCIDRs) > 2 { - return nil, false, fmt.Errorf("length of clusterCIDRs is:%v more than max allowed of 2", len(clusterCIDRs)) + if cloud == nil { + klog.Fatalf("Cloud provider is nil") } - routeController := routecontroller.New( - routes, - ctx.ClientBuilder.ClientOrDie("route-controller"), - ctx.SharedInformers.Core().V1().Nodes(), - ctx.ComponentConfig.KubeCloudShared.ClusterName, - clusterCIDRs, - ) - go routeController.Run(stopCh, ctx.ComponentConfig.KubeCloudShared.RouteReconciliationPeriod.Duration) - - return nil, true, nil -} - -// processCIDRs is a helper function that works on a comma separated cidrs and returns -// a list of typed cidrs -// a flag if cidrs represents a dual stack -// error if failed to parse any of the cidrs -func processCIDRs(cidrsList string) ([]*net.IPNet, bool, error) { - cidrsSplit := strings.Split(strings.TrimSpace(cidrsList), ",") - - cidrs, err := netutils.ParseCIDRs(cidrsSplit) - if err != nil { - return nil, false, err + if !cloud.HasClusterID() { + if config.ComponentConfig.KubeCloudShared.AllowUntaggedCloud { + klog.Warning("detected a cluster without a ClusterID. A ClusterID will be required in the future. Please tag your cluster to avoid any future issues") + } else { + klog.Fatalf("no ClusterID found. A ClusterID is required for the cloud provider to function properly. This check can be bypassed by setting the allow-untagged-cloud option") + } } - // if cidrs has an error then the previous call will fail - // safe to ignore error checking on next call - dualstack, _ := netutils.IsDualStackCIDRs(cidrs) - - return cidrs, dualstack, nil + return cloud } diff --git a/go.mod b/go.mod index 18512a6772..6854bd4bb8 100644 --- a/go.mod +++ b/go.mod @@ -3,23 +3,23 @@ module k8s.io/cloud-provider-aws go 1.15 require ( - github.com/aws/aws-sdk-go v1.35.24 - github.com/golang/mock v1.4.1 + github.com/aws/aws-sdk-go v1.38.21 + github.com/golang/mock v1.4.4 github.com/google/go-cmp v0.5.2 github.com/spf13/cobra v1.1.1 github.com/stretchr/testify v1.6.1 gopkg.in/gcfg.v1 v1.2.0 - k8s.io/api v0.20.0 - k8s.io/apimachinery v0.20.0 - k8s.io/apiserver v0.20.0 - k8s.io/client-go v0.20.0 - k8s.io/cloud-provider v0.20.0 - k8s.io/code-generator v0.20.0 - k8s.io/component-base v0.20.0 - k8s.io/csi-translation-lib v0.20.0 - k8s.io/klog/v2 v2.5.0 - k8s.io/kubelet v0.20.0 - k8s.io/legacy-cloud-providers v0.20.0 - k8s.io/utils v0.0.0-20201110183641-67b214c5f920 + k8s.io/api v0.21.0 + k8s.io/apimachinery v0.21.0 + k8s.io/apiserver v0.21.0 + k8s.io/client-go v0.21.0 + k8s.io/cloud-provider v0.21.0 + k8s.io/code-generator v0.21.0 + k8s.io/component-base v0.21.0 + k8s.io/csi-translation-lib v0.21.0 + k8s.io/klog/v2 v2.8.0 + k8s.io/kubelet v0.21.0 + k8s.io/legacy-cloud-providers v0.21.0 + k8s.io/utils v0.0.0-20210305010621-2afb4311ab10 // indirect sigs.k8s.io/yaml v1.2.0 ) diff --git a/go.sum b/go.sum index ba09cfba36..04142cf248 100644 --- a/go.sum +++ b/go.sum @@ -9,78 +9,119 @@ cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6T cloud.google.com/go v0.51.0/go.mod h1:hWtGJ6gnXH+KgDv+V0zFGDvpi07n3z8ZNj3T1RW0Gcw= cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= +cloud.google.com/go v0.54.0 h1:3ithwDMr7/3vpAMXiH+ZQnYbuIsh+OPhUPMFC9enmn0= cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= +cloud.google.com/go/bigquery v1.4.0 h1:xE3CPsOgttP4ACBePh79zTKALtXwn/Edhcr16R5hMWU= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/datastore v1.1.0 h1:/May9ojXjRkPBNVrq+oWLqmWCkr4OU5uRY29bu0mRyQ= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= +cloud.google.com/go/firestore v1.1.0 h1:9x7Bx0A9R5/M9jibeJeZWqjeVEIxYW9fZYqB9a70/bY= cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= +cloud.google.com/go/pubsub v1.2.0 h1:Lpy6hKgdcl7a3WGSfJIFmxmcdjSpP6OmBEfcOv1Y680= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= +cloud.google.com/go/storage v1.6.0 h1:UDpwYIwla4jHGzZJaEJYx1tOejbgSoNqsAfHAUYe2r8= cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9 h1:VpgP7xuJadIUuKccphEpTJnWhS2jkQyMt6Y7pJCD7fY= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/Azure/azure-sdk-for-go v43.0.0+incompatible h1:/wSNCu0e6EsHFR4Qa3vBEBbicaprEHMyyga9g8RTULI= github.com/Azure/azure-sdk-for-go v43.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 h1:w+iIsaOQNcT7OZ575w+acHgRric5iCyQh+xv+KJ4HB8= github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= +github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= github.com/Azure/go-autorest/autorest v0.11.1/go.mod h1:JFgpikqFJ/MleTTxwepExTKnFUKKszPS8UavbQYUMuw= +github.com/Azure/go-autorest/autorest v0.11.12 h1:gI8ytXbxMfI+IVbI9mP2JGCTXIuhHLgRlvQ9X4PsnHE= +github.com/Azure/go-autorest/autorest v0.11.12/go.mod h1:eipySxLmqSyC5s5k1CLupqet0PSENBEDP93LQ9a8QYw= github.com/Azure/go-autorest/autorest/adal v0.9.0/go.mod h1:/c022QCutn2P7uY+/oQWWNcK9YU+MH96NgK+jErpbcg= +github.com/Azure/go-autorest/autorest/adal v0.9.5 h1:Y3bBUV4rTuxenJJs41HU3qmqsb+auo+a3Lz+PlJPpL0= github.com/Azure/go-autorest/autorest/adal v0.9.5/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A= +github.com/Azure/go-autorest/autorest/date v0.3.0 h1:7gUk1U5M/CQbp9WoqinNzJar+8KY+LPI6wiWrP/myHw= github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= github.com/Azure/go-autorest/autorest/mocks v0.4.0/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= +github.com/Azure/go-autorest/autorest/mocks v0.4.1 h1:K0laFcLE6VLTOwNgSxaGbUcLPuGXlNkbVvq4cW4nIHk= github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= +github.com/Azure/go-autorest/autorest/to v0.2.0 h1:nQOZzFCudTh+TvquAtCRjM01VEYx85e9qbwt5ncW4L8= github.com/Azure/go-autorest/autorest/to v0.2.0/go.mod h1:GunWKJp1AEqgMaGLV+iocmRAJWqST1wQYhyyjXJ3SJc= +github.com/Azure/go-autorest/autorest/validation v0.1.0 h1:ISSNzGUh+ZSzizJWOWzs8bwpXIePbGLW4z/AmUFGH5A= github.com/Azure/go-autorest/autorest/validation v0.1.0/go.mod h1:Ha3z/SqBeaalWQvokg3NZAlQTalVMtOIAs1aGK7G6u8= +github.com/Azure/go-autorest/logger v0.2.0 h1:e4RVHVZKC5p6UANLJHkM4OfR1UKZPj8Wt8Pcx+3oqrE= github.com/Azure/go-autorest/logger v0.2.0/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= +github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo= github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802 h1:1BDTz0u9nC3//pOCMdNH+CiXJVYJh5UQNCOBG7jbELc= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/GoogleCloudPlatform/k8s-cloud-provider v0.0.0-20200415212048-7901bc822317 h1:JhyuWIqYrstW7KHMjk/fTqU0xtMpBOHuiTA2FVc7L4E= github.com/GoogleCloudPlatform/k8s-cloud-provider v0.0.0-20200415212048-7901bc822317/go.mod h1:DF8FZRxMHMGv/vP2lQP6h+dYzzjpuRn24VeRiYn3qjQ= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46 h1:lsxEuwrXEAokXB9qhlbKWPpo3KMLZQ5WB5WLQRW1uq0= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= +github.com/NYTimes/gziphandler v1.1.1 h1:ZUDjpQae29j0ryrS0u/B8HZfJBtBQHjqw2rQ2cqUQ3I= +github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= +github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI= github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4 h1:Hs82Z41s6SdL1CELW+XaDYmOH4hkBN4/N9og/AsOv7E= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e h1:QEF07wC0T1rKkctt1RINW/+RMTVmiwxETico2l3gxJA= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= +github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da h1:8GUt8eRujhVEGZFFEjBj46YV4rDjvGrNxb0KMWYkL2I= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= +github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310 h1:BUAU3CGlLvorLI26FmByPp2eC2qla6E1Tw+scpcg/to= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a h1:idn718Q4B6AGu/h5Sxe66HYVdqdGu2l9Iebqhi/AEoA= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/aws/aws-sdk-go v1.35.24 h1:U3GNTg8+7xSM6OAJ8zksiSM4bRqxBWmVwwehvOSNG3A= github.com/aws/aws-sdk-go v1.35.24/go.mod h1:tlPOdRjfxPBpNIwqDj61rmsnA85v9jc0Ps9+muhnW+k= +github.com/aws/aws-sdk-go v1.38.21 h1:D08DXWI4QRaawLaW+OtsIEClOI90I6eheJs1GwXTQVI= +github.com/aws/aws-sdk-go v1.38.21/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bgentry/speakeasy v0.1.0 h1:ByYyxL9InA1OWqxJqqp2A5pYHUrCiAL6K3J+LKSsQkY= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= +github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c h1:+0HFd5KSZ/mm3JmhmrDukiId5iR6w4+BdFtfSy4yWIc= github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ= github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= +github.com/census-instrumentation/opencensus-proto v0.2.1 h1:glEXhBS5PSLLv4IXzLA5yPRVX4bilULVyxxbrfOtDAk= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/chzyer/logex v1.1.10 h1:Swpa1K6QvQznwJRcfTfQJmTE72DqScAa40E+fbHEXEE= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e h1:fY5BOSpyZCqRo5OhCuC+XN+r/bBCmeuuJtjz+bCNIf8= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1 h1:q763qf9huN11kDQavWsoZXJNW3xEE4JJyHa5Q25/sd8= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/client9/misspell v0.3.4 h1:ta993UF76GwbvJcIo3Y68y/M3WxlpEHPWIGDkJYwzJI= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa h1:OaNxuTZr7kxeODyLWsRMC+OD03aFUH+mW6r2d+MWa5Y= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= +github.com/coreos/bbolt v1.3.2 h1:wZwiHHUieZCquLkDL0B8UhzreNWsPHooDAG3q34zk0s= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= +github.com/coreos/etcd v3.3.13+incompatible h1:8F3hqu9fGYLBifCmRCJsicFqDx/D68Rt3q1JMazcgBQ= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-oidc v2.1.0+incompatible h1:sdJrfw8akMnCuUlaZU3tE/uYXFgfqom8DBE9so9EBsM= github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= @@ -92,43 +133,60 @@ github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7 github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f h1:lBNOc5arjvs8E5mO2tbpBpLoyyu8B6e44T7hJy6potg= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/cpuguy83/go-md2man/v2 v2.0.0 h1:EoUDS0afbrsXAZ9YQ9jdu/mZ2sXgT1/2yyNng4PGlyM= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/creack/pty v1.1.11 h1:07n33Z8lZxZ2qwegKbObQohDhXDQxiMMz1NOUGYlesw= +github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954 h1:RMLoZVzv4GliuWafOuPuQDKSm1SJph7uCRnnS61JAn4= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= +github.com/dnaeon/go-vcr v1.0.1 h1:r8L/HqC0Hje5AXMu1ooW8oyQyOFv4GxqpL0nRP7SLLY= github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E= github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= +github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815 h1:bWDMxwH3px2JBh6AyO7hdCn/PkvCZXii8TGj7sbtEbQ= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153 h1:yUdfgN0XgIJw7foRItutHYUIhlcKzcSf5vDpdhQAKTc= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633 h1:H2pdYOb3KQ1/YsqVWoWNLQO+fusocsw354rqGTZtAgw= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/emicklei/go-restful v2.9.5+incompatible h1:spTtZBk5DYEvbxMVutUuTyh1Ao2r4iyvLdACqsl/Ljk= github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473 h1:4cmBvAEBNJaGARUEs3/suWRyfyBfhf7I60WBZq+bv2w= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/protoc-gen-validate v0.1.0 h1:EQciDnbrYxy13PgWoY8AqoxGiPrpgBZ1R8UNe3ddc+A= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/evanphx/json-patch v4.9.0+incompatible h1:kLcOMZeuLAJvL2BPWLMIj5oaZQobrkAqrL+WFZwQses= github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/form3tech-oss/jwt-go v3.2.2+incompatible h1:TcekIExNqud5crz4xD2pavyTgWiPvpYe4Xau31I0PRk= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1 h1:QbL/5oDUmRBzO9/Z7Seo6zf912W/a6Sr4Eu0G/3Jho0= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4 h1:WtGNWLvXpe6ZudgnXrq0barxBImvnnJoMEhXAzcbM0I= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/kit v0.9.0 h1:wDJmvq38kDhkVxi50ni9ykkdUr1PKgqKOoi01fa0Mdk= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logfmt/logfmt v0.4.0 h1:MP4Eh7ZCb31lleYCFuwm0oe4/YGak+5l1vA2NOE80nA= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= github.com/go-logr/logr v0.2.0 h1:QvGt2nLcHH0WK9orKa+ppBPAxREcH364nPUedEpK0TY= @@ -145,10 +203,13 @@ github.com/go-openapi/jsonreference v0.19.3 h1:5cxNfTy0UVC3X8JL5ymxzyoUZmo8iZb+j github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= github.com/go-openapi/spec v0.19.3 h1:0XRyw8kguri6Yw4SxhsQA/atC88yqrk0+G4YhI2wabc= github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo= +github.com/go-openapi/spec v0.19.5 h1:Xm0Ao53uqnk9QE/LlYV5DEU09UAgpliA85QoT9LzqPw= +github.com/go-openapi/spec v0.19.5/go.mod h1:Hm2Jr4jv8G1ciIAo+frC/Ft+rR2kQDh8JHKHb3gWUSk= github.com/go-openapi/swag v0.19.2 h1:jvO6bCMBEilGwMfHhrd61zIID4oIFdwb76V17SM88dE= github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-openapi/swag v0.19.5 h1:lTz6Ys4CmqqCQmZPBlbQENR1/GucA2bzYTE12Pw4tFY= github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gogo/protobuf v1.1.1 h1:72R+M5VuhED/KujmZVcIquuo8mBgX4oVda//DQb3PXo= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= @@ -156,6 +217,8 @@ github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903 h1:LbsanbbD6LieFkXbj9YNNBupiGHJgFeLpO0j0Fza1h8= @@ -173,6 +236,8 @@ github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFU github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.1 h1:ocYkMQY5RrXTYgXl7ICpV0IXwlEQGwKIsery4gyXa1U= github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.4 h1:l75CXGRSwbaYNpl/Z2X1XIIAMSCquvXgpVZDhwEIJsc= +github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg= @@ -208,12 +273,15 @@ github.com/google/gofuzz v1.0.0 h1:A8PeW59pxE9IoFRqBp37U+mSNaQoZ46F1f0f863XSXw= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3 h1:SRgJV+IoxM5MKyFdlSUeNy6/ycRUF2yBAKdAQswoHUk= github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/renameio v0.1.0 h1:GOZbcHa3HfsPKPlmyPyN2KEohoMXOhdMbHrvbpl2QaA= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.0.0 h1:b4Gk+7WdP/d3HZH8EJsZpvV7EtDOgaZLtnaNGIu1adA= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -222,14 +290,18 @@ github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gnostic v0.4.1 h1:DLJCy1n/vrD4HPjOvYcT8aYQXpPIzoRZONaYwyycI+I= github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg= +github.com/gophercloud/gophercloud v0.1.0 h1:P/nh25+rzXouhytV2pUHBb65fnds26Ghl8/391+sT5o= github.com/gophercloud/gophercloud v0.1.0/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 h1:pdN6V1QBWetyv/0+wjACpqVH+eVULgEjkurDLq3goeM= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4 h1:z53tR0945TRRQO/fLEVPI6SMv7ZflF0TEaTAoU7tOzg= @@ -239,30 +311,48 @@ github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgf github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.9.5 h1:UImYN5qQ8tuGpGE16ZmjvcTtTw24zw1QAp/SlnNrZhI= github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/hashicorp/consul/api v1.1.0 h1:BNQPM9ytxj6jbjjdRPioQ94T6YXriSopn0i8COv6SRA= github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= +github.com/hashicorp/consul/sdk v0.1.1 h1:LnuDWGNsoajlhGyHJvuWW6FVqRl8JOTPqS6CPTsYjhY= github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= +github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-cleanhttp v0.5.1 h1:dH3aiDG9Jvb5r5+bYHsikaOUIpcM0xvgMXVoDkXMzJM= github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-immutable-radix v1.0.0 h1:AKDB1HM5PWEA7i4nhcpwOrO2byshxBjXVn/J/3+z5/0= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-msgpack v0.5.3 h1:zKjpN5BK/P5lMYrLmBHdBULWbJ0XpYR+7NGzqkZzoD4= github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= +github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uPribsnS6o= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= +github.com/hashicorp/go-rootcerts v1.0.0 h1:Rqb66Oo1X/eSV1x66xbDccZjhJigjg0+e82kpwzSwCI= github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= +github.com/hashicorp/go-sockaddr v1.0.0 h1:GeH6tui99pF4NJgfnhp+L6+FfobzVW3Ah46sLo0ICXs= github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= +github.com/hashicorp/go-syslog v1.0.0 h1:KaodqZuhUoZereWVIYmpUgZysurB1kBLX2j0MwMrUAE= github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.1 h1:fv1ep09latC32wFoVwnqcnKJGnMSdBanPczbHAYm1BE= github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go.net v0.0.1 h1:sNCoNyDEvN1xa+X0baata4RdcpKwcMS6DH+xwfqPgjw= github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0 h1:CL2msUPvZTLb5O648aiLNJw3hnBxN2+1Jq8rCOH9wdo= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hashicorp/logutils v1.0.0 h1:dLEQVugN8vlakKOUE3ihGLTZJRB4j+M2cdTm/ORI65Y= github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= +github.com/hashicorp/mdns v1.0.0 h1:WhIgCr5a7AaVH6jPUwjtRuuE7/RDufnUvzIr48smyxs= github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= +github.com/hashicorp/memberlist v0.1.3 h1:EmmoJme1matNzb+hMpDuR/0sbJSUisxyqBGG676r31M= github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= +github.com/hashicorp/serf v0.8.2 h1:YZ7UKsJv+hKjqGVUUbtE3HNj79Eln2oQ75tniF6iPt0= github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6 h1:UDMh68UUwekSh5iP2OMhRRZJiiBccgV7axzUG8vi56c= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.5 h1:JboBksRwiiAJWvIYJVo46AfV+IAIKZpfrSzVKj42R4Q= github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= @@ -280,51 +370,75 @@ github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/u github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jstemmer/go-junit-report v0.9.1 h1:6QPYqodiu3GuPL+7mfx+NwDdp2eTkp9IfEUpgAwUN0o= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/julienschmidt/httprouter v1.2.0 h1:TDTW5Yz1mjftljbcKqRcrYhd4XeOoI98t+9HbQbYf7g= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= +github.com/kisielk/errcheck v1.5.0 h1:e8esj/e4R+SAOwFwN+n3zr0nYeCyeweozKfO23MvHzY= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0 h1:AV2c/EiW3KqPNT9ZKl07ehoAGi4C5/01Cfbblndcapg= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJKvHsGVBgn1xWYf1NbHQhywc8= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515 h1:T+h1c/A9Gawja4Y9mFVWj2vyii2bbUNDw3kt9VxK2EY= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/pty v1.1.5 h1:hyz3dwM5QLc1Rfoz4FuWJQG5BN7tc6K1MndAUnGpQr4= github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63 h1:nTT4s92Dgz2HlrB2NaMgvlfqHH39OgMhA7z3PK7PGD4= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.7.0 h1:aizVhC/NAAcKWb+5QsU1iNOZb4Yws5UO2I+aIprQITM= github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= +github.com/mattn/go-colorable v0.0.9 h1:UVL0vNpWh04HeJXV0KLcaT7r06gOH2l4OW6ddYRUIY4= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-isatty v0.0.4 h1:bnP0vzxcAdeI1zdubAl5PjU6zsERjGZb7raWodagDYs= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-runewidth v0.0.2 h1:UnlwIPBGaTZfPQ6T1IGzPI0EkYAQmT9fAEJ/poFC63o= github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/miekg/dns v1.0.14 h1:9jZdLNd/P4+SfEJ0TNyxYpsK8N4GtfylBLqtbYN1sbA= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= +github.com/mitchellh/cli v1.0.0 h1:iGBIsUe3+HZ/AD/Vd7DErOt5sU9fa8Uj7A2s1aggv1Y= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-testing-interface v1.0.0 h1:fzU/JVNcaqHQEcVFAKeR41fkiLdIPrefOvVG1VZ96U0= github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= +github.com/mitchellh/gox v0.4.0 h1:lfGJxY7ToLJQjHHwi0EX6uYBdK78egf954SQl13PQJc= github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= +github.com/mitchellh/iochan v1.0.0 h1:C+X3KsSTLFVBr/tK1eYN/vs4rJcvsiLU338UhYPJWeY= github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/moby/spdystream v0.2.0 h1:cjW1zVyyoiM0T7b6UoySUFqzXMoqRckQtXwGPiBhOM8= +github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= github.com/moby/term v0.0.0-20200312100748-672ec06f55cd h1:aY7OQNf2XqY/JQ6qREWamhI/81os/agb2BAGpcx5yWI= github.com/moby/term v0.0.0-20200312100748-672ec06f55cd/go.mod h1:DdlQx2hp0Ss5/fLikoLlEeIYiATotOjgB//nb973jeo= +github.com/moby/term v0.0.0-20201216013528-df9cb8a40635 h1:rzf0wL0CHVc8CEsgyygG0Mn9CNCCPZqOPaz8RiiHYQk= +github.com/moby/term v0.0.0-20201216013528-df9cb8a40635/go.mod h1:FBS0z0QWA44HXygs7VXDUOGoN/1TV3RuWkLO04am3wc= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -335,9 +449,15 @@ github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d h1:7PxY7LVfSZm7P github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223 h1:F9x/1yl3T2AeKLr2AMdilSD8+f9bvMnNN8VS5iDtovc= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= +github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= +github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5 h1:58+kh9C6jJVXYjt8IE48G2eWl6BjwU5Gj0gqY84fy78= github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.6.0 h1:Ix8l273rp3QzYgXSR+c8d1fTG7UPgYkOSELPhiY/YGw= @@ -347,7 +467,9 @@ github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+ github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c h1:Lgl0gzECD8GnQ5QCWA8o6BtfL6mDH5rQgM4/fX3avOs= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= @@ -359,7 +481,9 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/posener/complete v1.1.1 h1:ccV59UEOTzVDnDUEFdT95ZzHVZ+5+158q8+SJb2QV5w= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= +github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021 h1:0XM1XL/OFFJjXsYXlG30spTkV/E9+gmd5GD1w2HE8xM= github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= @@ -388,14 +512,23 @@ github.com/prometheus/procfs v0.1.3 h1:F0+tqvhOksq22sc6iCHF5WGlWjdwj92p0udFh1VFB github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.2.0 h1:wH4vA7pcjKuZzjF7lM8awk4fnuJO6idemZXoKnULUx4= github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= +github.com/prometheus/tsdb v0.7.1 h1:YZcsG11NqnK4czYLrWd9mpEuAJIHVQLwdrleYfszMAA= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= +github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af h1:gu+uRPtBe88sKxUCEXRoeCvVG90TJmwhiqRpvdhQFng= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= +github.com/rogpeppe/go-internal v1.3.0 h1:RR9dF3JtopPvtkroDZuVD7qquD0bnHlKSqaQhgwt8yk= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rubiojr/go-vhd v0.0.0-20200706105327-02e210299021 h1:if3/24+h9Sq6eDx8UUz1SO9cT9tizyIsATfB7b4D3tc= github.com/rubiojr/go-vhd v0.0.0-20200706105327-02e210299021/go.mod h1:DM5xW0nvfNNm2uytzsvhI3OnX8uzaRAg8UX/CnDqbto= +github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f h1:UFr9zpz4xgTnIE5yIMtWAMngCdZ9p/+q6lTbgelo80M= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= +github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUtVbo7ada43DJhG55ua/hjS5I= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= +github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0 h1:juTguoYk5qI21pwyTXY3B3Y5cOTH3ZUyZCg1v/mihuo= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= @@ -403,19 +536,26 @@ github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= +github.com/sirupsen/logrus v1.7.0 h1:ShrD1U9pZB12TX0cVy0DtePoCH97K8EtX+mg7ZARUtM= +github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/soheilhy/cmux v0.1.4 h1:0HKaf1o97UwFjHH9o5XsHUOF+tqmdA7KEzXLpiyaw0E= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= +github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.3 h1:ZlrZ4XsMRm04Fr5pSFxBgfND2EBVa1nLpiy1stUsX/8= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v1.1.1 h1:KfztREH0tPxJJ+geloSLaAkaPkr4ki2Er5quFV1TDo4= github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI= +github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.1 h1:aCvUg6QPl3ibpQUxyLkrEkCHtPqYJL4x9AuhqVqFis4= @@ -424,6 +564,7 @@ github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.7.0 h1:xVKxvI7ouOI5I+U9s2eeiUfMaWBVoXA3AWskkrqK0VM= github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A= @@ -438,16 +579,21 @@ github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJy github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8 h1:ndzgwNDnKIqyCvHTXaCqh9KlOWKvBry6nuXMJmonVsE= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 h1:LnC5Kc/wtumK+WB441p7ynQJzVuNRJiqddSIE3IlSEQ= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/urfave/cli v1.20.0 h1:fDqGv3UG/4jbVl/QkFwEdddtEDjh/5Ov6X+0B/3bPaw= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= +github.com/vmware/govmomi v0.20.3 h1:gpw/0Ku+6RgF3jsi7fnCLmlcikBHfKBCUcu1qgc16OU= github.com/vmware/govmomi v0.20.3/go.mod h1:URlwyTFZX72RmxtxuaFL2Uj3fD1JTvZdx59bHWk6aFU= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1 h1:ruQGxdhGHe7FWOJPT0mKs5+pD2Xs1Bm/kdGlHO04FmM= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.3 h1:MUGmc65QhB3pIlaQ5bB4LwqSj6GIonVJXpZiaKNyaKk= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= @@ -458,6 +604,7 @@ go.etcd.io/etcd v0.5.0-alpha.5.0.20200910180754-dd1b699fc489/go.mod h1:yVHk9ub3C go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.3 h1:8sGtKOrtQqkN1bp2AtX+misvLIlOmsEsNd+9NIcPEm8= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.uber.org/atomic v1.3.2 h1:2Oa65PReHzfn29GpvgsYwloV9AVFHPDk8tYxt2c2tr4= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= @@ -481,6 +628,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnk golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0 h1:hb9wdF1z5waM+dSIICn1l0DkLVDT3hqhhQsDNUmHPRE= golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83 h1:/ZScEX8SfEmUGRHs0gxpqteO5nfNW6axyZbBdw9A12g= +golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -490,8 +639,10 @@ golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= +golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6 h1:QE6XYQK6naiK1EPAe1g/ILLxN5RBoH5xkJk3CqlMI/Y= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b h1:+qEpEAPhDZ1o0x3tHzZTQDArnOixOzGD9HUJfcg0mb4= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= @@ -502,8 +653,10 @@ golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b h1:Wh+f8QHJXR411sJR8/vRBTZ7YapZaRvUcLFFJhusH0k= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028 h1:4+4C/Iv2U4fMZBiMCc98MG1In4gJY5YRhtpDNeDeHWs= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= @@ -512,6 +665,8 @@ golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.1-0.20200828183125-ce943fd02449 h1:xUIPaMhvROX9dhPvRCenIJtU78+lbEenGbgqB5hfHCQ= +golang.org/x/mod v0.3.1-0.20200828183125-ce943fd02449/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -540,8 +695,11 @@ golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b h1:uwuIcX0g4Yl1NC5XAz37xsr2lTtcqevgzYNVt49waME= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210224082022-3d97a244fca7 h1:OgUuv8lsRpBibGNbSizVwKWlysjaNzmC9gYMhPVfqFM= +golang.org/x/net v0.0.0-20210224082022-3d97a244fca7/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 h1:SVwTIAaPC2U/AvvLNZ2a7OVsmBpC8L5BlwK1whH3hm0= @@ -558,6 +716,8 @@ golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 h1:SQFwaSi55rU7vdNs9Yr0Z324VNlrF+0wMqRXT4St8ck= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -582,6 +742,7 @@ golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456 h1:ng0gs1AKnRRuEMZoTLLlbOd+C golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -594,9 +755,18 @@ golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201112073958-5cba982894dd h1:5CtCZbICpIOFdgO940moixOPjc0178IU44m4EjOO5IY= golang.org/x/sys v0.0.0-20201112073958-5cba982894dd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073 h1:8qxJSnu+7dRq6upnbntrmriWByIakBuct5OM/MdQC1M= +golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d h1:SZxvLBoTP5yHO3Frd4z4vrF+DBX9vMVanchswa69toE= +golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -616,6 +786,8 @@ golang.org/x/time v0.0.0-20191024005414-555d28b269f0 h1:/5xXl8Y5W96D+TtHSlonuFqG golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e h1:EHBhcS0mlXEAVwNyO2dLfjToGsyY4j24pTs2ScHnX7s= golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba h1:O8mE0/t419eoIwhTFpKVkHiTs/Igowgfkj25AcZrtiE= +golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -655,6 +827,10 @@ golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjs golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200616133436-c1934b75d054 h1:HHeAlu5H9b71C+Fx0K+1dGgVFN1DM1/wz4aoGOA5qS8= golang.org/x/tools v0.0.0-20200616133436-c1934b75d054/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.0 h1:po9/4sTYwZU9lPhi1tOrb4hCv3qrhiQ77LZfGa2OjwY= +golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= @@ -671,6 +847,7 @@ google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsb google.golang.org/api v0.15.1-0.20200106000736-b8fc810ca6b5/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.20.0 h1:jz2KixHX7EcCPiQrySzPdnYT7DbINAypCqKZ1Z7GM40= google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -725,6 +902,7 @@ google.golang.org/protobuf v1.24.0 h1:UhZDfRO8JRQru4/+LlLE0BRKGF8L+PICnvYZmx/fEG google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -732,7 +910,11 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/cheggaaa/pb.v1 v1.0.25 h1:Ev7yu1/f6+d+b3pi5vPdRPc6nNtP1umSfcWiEfRqv6I= gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= +gopkg.in/errgo.v2 v2.1.0 h1:0vLT13EuvQ0hNvakwLuFZ/jYrLp5F3kcWHXdRggjCE8= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= @@ -740,9 +922,11 @@ gopkg.in/gcfg.v1 v1.2.0 h1:0HIbH907iBTAntm+88IJV2qmJALDAh8sPekI9Vc1fm0= gopkg.in/gcfg.v1 v1.2.0/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/ini.v1 v1.51.0 h1:AQvPpx3LzTDM0AjnIRlVFwFFGC+npRopjZxLJj6gdno= gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= +gopkg.in/resty.v1 v1.12.0 h1:CuXP0Pjfw9rOuY6EP+UvtNvt5DSqHpIxILZKT/quCZI= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/square/go-jose.v2 v2.2.2 h1:orlkJ3myw8CN1nVQHBFfloD+L3egixIa4FvUP6RosSA= gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= @@ -760,38 +944,63 @@ gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= +gotest.tools/v3 v3.0.3 h1:4AuOwCGf4lLR9u3YOe2awrHygurzhO/HeQ6laiA6Sx0= +gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +honnef.co/go/tools v0.0.1-2020.1.3 h1:sXmLre5bzIR6ypkjXCDI3jHPssRhc8KD/Ome589sc3U= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= k8s.io/api v0.20.0 h1:WwrYoZNM1W1aQEbyl8HNG+oWGzLpZQBlcerS9BQw9yI= k8s.io/api v0.20.0/go.mod h1:HyLC5l5eoS/ygQYl1BXBgFzWNlkHiAuyNAbevIn+FKg= +k8s.io/api v0.21.0 h1:gu5iGF4V6tfVCQ/R+8Hc0h7H1JuEhzyEi9S4R5LM8+Y= +k8s.io/api v0.21.0/go.mod h1:+YbrhBBGgsxbF6o6Kj4KJPJnBmAKuXDeS3E18bgHNVU= k8s.io/apimachinery v0.20.0 h1:jjzbTJRXk0unNS71L7h3lxGDH/2HPxMPaQY+MjECKL8= k8s.io/apimachinery v0.20.0/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= +k8s.io/apimachinery v0.21.0 h1:3Fx+41if+IRavNcKOz09FwEXDBG6ORh6iMsTSelhkMA= +k8s.io/apimachinery v0.21.0/go.mod h1:jbreFvJo3ov9rj7eWT7+sYiRx+qZuCYXwWT1bcDswPY= k8s.io/apiserver v0.20.0 h1:0MwO4xCoqZwhoLbFyyBSJdu55CScp4V4sAgX6z4oPBY= k8s.io/apiserver v0.20.0/go.mod h1:6gRIWiOkvGvQt12WTYmsiYoUyYW0FXSiMdNl4m+sxY8= +k8s.io/apiserver v0.21.0 h1:1hWMfsz+cXxB77k6/y0XxWxwl6l9OF26PC9QneUVn1Q= +k8s.io/apiserver v0.21.0/go.mod h1:w2YSn4/WIwYuxG5zJmcqtRdtqgW/J2JRgFAqps3bBpg= k8s.io/client-go v0.20.0 h1:Xlax8PKbZsjX4gFvNtt4F5MoJ1V5prDvCuoq9B7iax0= k8s.io/client-go v0.20.0/go.mod h1:4KWh/g+Ocd8KkCwKF8vUNnmqgv+EVnQDK4MBF4oB5tY= +k8s.io/client-go v0.21.0 h1:n0zzzJsAQmJngpC0IhgFcApZyoGXPrDIAD601HD09ag= +k8s.io/client-go v0.21.0/go.mod h1:nNBytTF9qPFDEhoqgEPaarobC8QPae13bElIVHzIglA= k8s.io/cloud-provider v0.20.0 h1:CVPQ66iyfNgeGomUq2jE/TWrfzE77bdCpemhFS8955U= k8s.io/cloud-provider v0.20.0/go.mod h1:Lz/luSVD5BrHDDhtVdjFh0C2qQCRYdf0b9BHQ9L+bXc= +k8s.io/cloud-provider v0.21.0 h1:NSTS+czpv6LQAaIpY/VUghsT4oj62hYmQPErkDKTzKU= +k8s.io/cloud-provider v0.21.0/go.mod h1:z17TQgu3JgUFjcgby8sj5X86YdVK5Pbt+jm/eYMZU9M= k8s.io/code-generator v0.20.0 h1:c8JaABvEEZPDE8MICTOtveHX2axchl+EptM+o4OGvbg= k8s.io/code-generator v0.20.0/go.mod h1:UsqdF+VX4PU2g46NC2JRs4gc+IfrctnwHb76RNbWHJg= +k8s.io/code-generator v0.21.0 h1:LGWJOvkbBNpuRBqBRXUjzfvymUh7F/iR2KDpwLnqCM4= +k8s.io/code-generator v0.21.0/go.mod h1:hUlps5+9QaTrKx+jiM4rmq7YmH8wPOIko64uZCHDh6Q= k8s.io/component-base v0.20.0 h1:BXGL8iitIQD+0NgW49UsM7MraNUUGDU3FBmrfUAtmVQ= k8s.io/component-base v0.20.0/go.mod h1:wKPj+RHnAr8LW2EIBIK7AxOHPde4gme2lzXwVSoRXeA= +k8s.io/component-base v0.21.0 h1:tLLGp4BBjQaCpS/KiuWh7m2xqvAdsxLm4ATxHSe5Zpg= +k8s.io/component-base v0.21.0/go.mod h1:qvtjz6X0USWXbgmbfXR+Agik4RZ3jv2Bgr5QnZzdPYw= k8s.io/controller-manager v0.20.0 h1:OwUoxVhRMp3kDHjEOJ48alqBVZPlFikAZeFCqI4zyss= k8s.io/controller-manager v0.20.0/go.mod h1:nD4qym/pmCz2v1tpqvlEBVlHW9CAZwedloM8GrJTLpg= +k8s.io/controller-manager v0.21.0 h1:60kaloHJ815CDLohPmoa0n5/WbW7J0PXHX+y+dOBSWc= +k8s.io/controller-manager v0.21.0/go.mod h1:Ohy0GRNRKPVjB8C8G+dV+4aPn26m8HYUI6ejloUBvUA= k8s.io/csi-translation-lib v0.20.0 h1:I4wsA4/nW2low2auCUIq/ZCF58/FlCADr1OddBh0AOQ= k8s.io/csi-translation-lib v0.20.0/go.mod h1:M4CdD66GxEI6ev8aTtsA2NkK9kIF9K5VZQMcw/SsoLs= +k8s.io/csi-translation-lib v0.21.0 h1:kzwraa2eenvtMuOAe+xSQgIgG0G88pocorYP8CSitRY= +k8s.io/csi-translation-lib v0.21.0/go.mod h1:edq+UMpgqEx3roTuGF/03uIuSOsI986jtu65+ytLlkA= k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/gengo v0.0.0-20201113003025-83324d819ded h1:JApXBKYyB7l9xx+DK7/+mFjC7A9Bt5A93FPvFD0HIFE= k8s.io/gengo v0.0.0-20201113003025-83324d819ded/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= +k8s.io/gengo v0.0.0-20201214224949-b6c5ce23f027 h1:Uusb3oh8XcdzDF/ndlI4ToKTYVlkCSJP39SRY2mfRAw= +k8s.io/gengo v0.0.0-20201214224949-b6c5ce23f027/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= k8s.io/klog/v2 v2.2.0 h1:XRvcwJozkgZ1UQJmfMGpvRthQHOvihEhYtDfAaxMz/A= k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= @@ -799,22 +1008,39 @@ k8s.io/klog/v2 v2.4.0 h1:7+X0fUguPyrKEC4WjH8iGDg3laWgMo5tMnRTIGTTxGQ= k8s.io/klog/v2 v2.4.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= k8s.io/klog/v2 v2.5.0 h1:8mOnjf1RmUPW6KRqQCfYSZq/K20Unmp3IhuZUhxl8KI= k8s.io/klog/v2 v2.5.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= +k8s.io/klog/v2 v2.8.0 h1:Q3gmuM9hKEjefWFFYF0Mat+YyFJvsUyYuwyNNJ5C9Ts= +k8s.io/klog/v2 v2.8.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd h1:sOHNzJIkytDF6qadMNKhhDRpc6ODik8lVC6nOur7B2c= k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd/go.mod h1:WOJ3KddDSol4tAGcJo0Tvi+dK12EcqSLqcWsryKMpfM= +k8s.io/kube-openapi v0.0.0-20210305001622-591a79e4bda7 h1:vEx13qjvaZ4yfObSSXW7BrMc/KQBBT/Jyee8XtLf4x0= +k8s.io/kube-openapi v0.0.0-20210305001622-591a79e4bda7/go.mod h1:wXW5VT87nVfh/iLV8FpR2uDvrFyomxbtb1KivDbvPTE= k8s.io/kubelet v0.20.0 h1:vtQMgZ7B79vCUyEpoij2Oy6UmcJx0/D2lNXfViVTBMg= k8s.io/kubelet v0.20.0/go.mod h1:lMdjO1NA+JZXSYtxb48pQmNERmC+vVIXIYkJIugVhl0= +k8s.io/kubelet v0.21.0 h1:1VUfM5vKqLPlWFI0zee6fm9kwIZ/UEOGCodVFN+OZrg= +k8s.io/kubelet v0.21.0/go.mod h1:G5ZxMTVev9t4bhmsSxDAWhH6wXDYEVHVVFyYsw4laR4= k8s.io/kubernetes v1.20.0 h1:mnc69esJC3PJgSptxNJomGz2gBthyGLSEy18WiyRH4U= k8s.io/legacy-cloud-providers v0.20.0 h1:bF2WKO3ZWEjBvCv9RJxsxOAGrmQHWSgTiDBjK8uW2iw= k8s.io/legacy-cloud-providers v0.20.0/go.mod h1:1jEkaU7h9+b1EYdfWDBvhFAr+QpRfUjQfK+dGhxPGfA= +k8s.io/legacy-cloud-providers v0.21.0 h1:iWf5xaX9yvYT5mkz8UB96UtISQ5IkrWeuMPMhRp01ZY= +k8s.io/legacy-cloud-providers v0.21.0/go.mod h1:bNxo7gDg+PGkBmT/MFZswLTWdSWK9kAlS1s8DJca5q4= k8s.io/utils v0.0.0-20201110183641-67b214c5f920 h1:CbnUZsM497iRC5QMVkHwyl8s2tB3g7yaSHkYPkpgelw= k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +k8s.io/utils v0.0.0-20210305010621-2afb4311ab10 h1:u5rPykqiCpL+LBfjRkXvnK71gOgIdmq3eHUEkPrbeTI= +k8s.io/utils v0.0.0-20210305010621-2afb4311ab10/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +rsc.io/binaryregexp v0.2.0 h1:HfqmD5MEmC0zvwBuF187nq9mdnXjXsSivRiXN7SmRkE= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/quote/v3 v3.1.0 h1:9JKUTTIUgS6kzR9mK1YuGKv6Nl+DijDNIc0ghT58FaY= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= +rsc.io/sampler v1.3.0 h1:7uVkIFmeBqHfdjD+gZwtXXI+RODJ2Wc4O7MPEh/QiW4= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.14 h1:TihvEz9MPj2u0KWds6E2OBUXfwaL4qRJ33c7HGiJpqk= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.14/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.15 h1:4uqm9Mv+w2MmBYD+F4qf/v6tDFUdPOk29C095RbU5mY= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.15/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= sigs.k8s.io/structured-merge-diff/v4 v4.0.2 h1:YHQV7Dajm86OuqnIR6zAelnDWBRjo+YhYV9PmGrh1s8= sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= +sigs.k8s.io/structured-merge-diff/v4 v4.1.0 h1:C4r9BgJ98vrKnnVCjwCSXcWjWe0NKcUQkmzDXZXGwH8= +sigs.k8s.io/structured-merge-diff/v4 v4.1.0/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= From 4bce33b41a51791ab6e39a252f737fbc515c3b49 Mon Sep 17 00:00:00 2001 From: Nguyen Dinh <92940366+nguyenkndinh@users.noreply.github.com> Date: Tue, 22 Feb 2022 19:04:44 -0800 Subject: [PATCH 02/43] Add tagging controller configuration --- cmd/aws-cloud-controller-manager/main.go | 4 +- .../v1/controllers/aws_controllermanager.go | 50 +++++++++++++++++++ .../controllers/tagging/tagging-controller.go | 43 ++++++++++++++++ 3 files changed, 96 insertions(+), 1 deletion(-) create mode 100644 pkg/providers/v1/controllers/aws_controllermanager.go create mode 100644 pkg/providers/v1/controllers/tagging/tagging-controller.go diff --git a/cmd/aws-cloud-controller-manager/main.go b/cmd/aws-cloud-controller-manager/main.go index 1673a74c54..b0166ff321 100644 --- a/cmd/aws-cloud-controller-manager/main.go +++ b/cmd/aws-cloud-controller-manager/main.go @@ -44,6 +44,8 @@ import ( awsv2 "k8s.io/cloud-provider-aws/pkg/providers/v2" cloudcontrollerconfig "k8s.io/cloud-provider/app/config" + + acm "k8s.io/cloud-provider-aws/pkg/providers/v1/controllers" ) const ( @@ -63,7 +65,7 @@ func main() { klog.Fatalf("unable to initialize command options: %v", err) } - controllerInitializers := app.DefaultInitFuncConstructors + controllerInitializers := acm.BuildControllerInitializers() fss := cliflag.NamedFlagSets{} command := app.NewCloudControllerManagerCommand(opts, cloudInitializer, controllerInitializers, fss, wait.NeverStop) diff --git a/pkg/providers/v1/controllers/aws_controllermanager.go b/pkg/providers/v1/controllers/aws_controllermanager.go new file mode 100644 index 0000000000..c89c41f18a --- /dev/null +++ b/pkg/providers/v1/controllers/aws_controllermanager.go @@ -0,0 +1,50 @@ +package controllers + +import ( + "context" + "k8s.io/klog/v2" + + cloudprovider "k8s.io/cloud-provider" + "k8s.io/cloud-provider/app" + cloudcontrollerconfig "k8s.io/cloud-provider/app/config" + "k8s.io/controller-manager/controller" + genericcontrollermanager "k8s.io/controller-manager/app" + taggingcontroller "k8s.io/cloud-provider-aws/pkg/providers/v1/controllers/tagging" +) + +// BuildControllerInitializers is used to add new controllers built in this package to +// the existing list of controllers from cloud-provider +func BuildControllerInitializers() map[string]app.ControllerInitFuncConstructor { + controllerInitializers := app.DefaultInitFuncConstructors + + taggingControllerInitFuncConstrustor := app.ControllerInitFuncConstructor{ + InitContext: app.ControllerInitContext{ + ClientName: "tagging-controller", + }, + Constructor: startTaggingControllerWrapper, + } + + controllerInitializers["tagging"] = taggingControllerInitFuncConstrustor + + return controllerInitializers +} + +// StartCloudNodeControllerWrapper is used to take cloud cofig as input and start cloud node controller +func startTaggingControllerWrapper(initContext app.ControllerInitContext, completedConfig *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface) app.InitFunc { + return func(ctx context.Context, controllerContext genericcontrollermanager.ControllerContext) (controller.Interface, bool, error) { + return startTaggingController(ctx, initContext, completedConfig, cloud) + } +} + +func startTaggingController(ctx context.Context, initContext app.ControllerInitContext, completedConfig *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface) (controller.Interface, bool, error) { + // Start the CloudNodeController + taggingcontroller, err := taggingcontroller.NewTaggingController() + if err != nil { + klog.Warningf("failed to start tagging controller: %s", err) + return nil, false, nil + } + + go taggingcontroller.Run(ctx.Done()) + + return nil, true, nil +} \ No newline at end of file diff --git a/pkg/providers/v1/controllers/tagging/tagging-controller.go b/pkg/providers/v1/controllers/tagging/tagging-controller.go new file mode 100644 index 0000000000..f63875e901 --- /dev/null +++ b/pkg/providers/v1/controllers/tagging/tagging-controller.go @@ -0,0 +1,43 @@ +/* +Copyright 2016 The Kubernetes Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tagging + +import ( + "k8s.io/klog/v2" +) + +// eksResourceTagPrefix is the prefix for tag to group resources that are used by eks +// for a particular cluster, this tag is added to the existing tags. +// Example: "Key1=Val1,aws:eks:cluster-name:my-cluster=Val2" +const eksResourceTagPrefix = "aws:eks:cluster-name:" + +// TaggingController is the controller implementation for tagging cluster resources +type TaggingController struct { +} + +// NewTaggingController creates a NewTaggingController object +func NewTaggingController() (*TaggingController, error) { + tc := &TaggingController{ + } + + return tc, nil +} + +// Run will start the controller to tag resources attached to a cluster +// at any point in time +func (tc *TaggingController) Run(stopCh <-chan struct{}) { + klog.Infof("Starting the TaggingController, eksResourceTagPrefix is %s.", eksResourceTagPrefix) + + <-stopCh +} \ No newline at end of file From 516038372484faa697c8604a4712a8f04a723759 Mon Sep 17 00:00:00 2001 From: Nguyen Dinh <92940366+nguyenkndinh@users.noreply.github.com> Date: Tue, 22 Feb 2022 19:20:08 -0800 Subject: [PATCH 03/43] add log --- cmd/aws-cloud-controller-manager/main.go | 3 +++ pkg/providers/v1/controllers/aws_controllermanager.go | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/cmd/aws-cloud-controller-manager/main.go b/cmd/aws-cloud-controller-manager/main.go index b0166ff321..8526d6033b 100644 --- a/cmd/aws-cloud-controller-manager/main.go +++ b/cmd/aws-cloud-controller-manager/main.go @@ -65,7 +65,10 @@ func main() { klog.Fatalf("unable to initialize command options: %v", err) } + klog.Info("Nguyen about to add tagging.") controllerInitializers := acm.BuildControllerInitializers() + + klog.Info("Nguyen tagging added.") fss := cliflag.NamedFlagSets{} command := app.NewCloudControllerManagerCommand(opts, cloudInitializer, controllerInitializers, fss, wait.NeverStop) diff --git a/pkg/providers/v1/controllers/aws_controllermanager.go b/pkg/providers/v1/controllers/aws_controllermanager.go index c89c41f18a..5dabf3f975 100644 --- a/pkg/providers/v1/controllers/aws_controllermanager.go +++ b/pkg/providers/v1/controllers/aws_controllermanager.go @@ -29,7 +29,7 @@ func BuildControllerInitializers() map[string]app.ControllerInitFuncConstructor return controllerInitializers } -// StartCloudNodeControllerWrapper is used to take cloud cofig as input and start cloud node controller +// StartTaggingControllerWrapper is used to take cloud config as input and start tagging controller func startTaggingControllerWrapper(initContext app.ControllerInitContext, completedConfig *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface) app.InitFunc { return func(ctx context.Context, controllerContext genericcontrollermanager.ControllerContext) (controller.Interface, bool, error) { return startTaggingController(ctx, initContext, completedConfig, cloud) @@ -37,7 +37,7 @@ func startTaggingControllerWrapper(initContext app.ControllerInitContext, comple } func startTaggingController(ctx context.Context, initContext app.ControllerInitContext, completedConfig *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface) (controller.Interface, bool, error) { - // Start the CloudNodeController + // Start the TaggingController taggingcontroller, err := taggingcontroller.NewTaggingController() if err != nil { klog.Warningf("failed to start tagging controller: %s", err) From fc6280928289fa0baba670ec13528607d61a6b38 Mon Sep 17 00:00:00 2001 From: Nguyen Dinh <92940366+nguyenkndinh@users.noreply.github.com> Date: Wed, 23 Feb 2022 10:22:26 -0800 Subject: [PATCH 04/43] rearrange the controllers --- .../controllers/aws_controllermanager.go | 4 ++-- .../controllers/tagging/tagging-controller.go | 0 cmd/aws-cloud-controller-manager/main.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) rename {pkg/providers/v1 => cmd/aws-cloud-controller-manager}/controllers/aws_controllermanager.go (94%) rename {pkg/providers/v1 => cmd/aws-cloud-controller-manager}/controllers/tagging/tagging-controller.go (100%) diff --git a/pkg/providers/v1/controllers/aws_controllermanager.go b/cmd/aws-cloud-controller-manager/controllers/aws_controllermanager.go similarity index 94% rename from pkg/providers/v1/controllers/aws_controllermanager.go rename to cmd/aws-cloud-controller-manager/controllers/aws_controllermanager.go index 5dabf3f975..16e88cf54a 100644 --- a/pkg/providers/v1/controllers/aws_controllermanager.go +++ b/cmd/aws-cloud-controller-manager/controllers/aws_controllermanager.go @@ -5,11 +5,11 @@ import ( "k8s.io/klog/v2" cloudprovider "k8s.io/cloud-provider" + taggingcontroller "k8s.io/cloud-provider-aws/cmd/aws-cloud-controller-manager/controllers/tagging" "k8s.io/cloud-provider/app" cloudcontrollerconfig "k8s.io/cloud-provider/app/config" - "k8s.io/controller-manager/controller" genericcontrollermanager "k8s.io/controller-manager/app" - taggingcontroller "k8s.io/cloud-provider-aws/pkg/providers/v1/controllers/tagging" + "k8s.io/controller-manager/controller" ) // BuildControllerInitializers is used to add new controllers built in this package to diff --git a/pkg/providers/v1/controllers/tagging/tagging-controller.go b/cmd/aws-cloud-controller-manager/controllers/tagging/tagging-controller.go similarity index 100% rename from pkg/providers/v1/controllers/tagging/tagging-controller.go rename to cmd/aws-cloud-controller-manager/controllers/tagging/tagging-controller.go diff --git a/cmd/aws-cloud-controller-manager/main.go b/cmd/aws-cloud-controller-manager/main.go index 8526d6033b..35725adda9 100644 --- a/cmd/aws-cloud-controller-manager/main.go +++ b/cmd/aws-cloud-controller-manager/main.go @@ -45,7 +45,7 @@ import ( cloudcontrollerconfig "k8s.io/cloud-provider/app/config" - acm "k8s.io/cloud-provider-aws/pkg/providers/v1/controllers" + acm "k8s.io/cloud-provider-aws/cmd/aws-cloud-controller-manager/controllers" ) const ( From 78ae2501fc8a3a0004dfb8868c3cb170e02855f3 Mon Sep 17 00:00:00 2001 From: Nguyen Dinh <92940366+nguyenkndinh@users.noreply.github.com> Date: Wed, 23 Feb 2022 16:07:01 -0800 Subject: [PATCH 05/43] remove debugging log --- .../controllers/tagging/tagging-controller.go | 6 ++++-- cmd/aws-cloud-controller-manager/main.go | 4 +--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/cmd/aws-cloud-controller-manager/controllers/tagging/tagging-controller.go b/cmd/aws-cloud-controller-manager/controllers/tagging/tagging-controller.go index f63875e901..a8256834dd 100644 --- a/cmd/aws-cloud-controller-manager/controllers/tagging/tagging-controller.go +++ b/cmd/aws-cloud-controller-manager/controllers/tagging/tagging-controller.go @@ -31,13 +31,15 @@ func NewTaggingController() (*TaggingController, error) { tc := &TaggingController{ } + klog.Infof("Starting the TaggingController") + return tc, nil } // Run will start the controller to tag resources attached to a cluster -// at any point in time +// and untag resources detached from a cluster. func (tc *TaggingController) Run(stopCh <-chan struct{}) { - klog.Infof("Starting the TaggingController, eksResourceTagPrefix is %s.", eksResourceTagPrefix) + klog.Infof("Running the TaggingController, eksResourceTagPrefix is %s.", eksResourceTagPrefix) <-stopCh } \ No newline at end of file diff --git a/cmd/aws-cloud-controller-manager/main.go b/cmd/aws-cloud-controller-manager/main.go index 35725adda9..c58ea61bba 100644 --- a/cmd/aws-cloud-controller-manager/main.go +++ b/cmd/aws-cloud-controller-manager/main.go @@ -65,14 +65,12 @@ func main() { klog.Fatalf("unable to initialize command options: %v", err) } - klog.Info("Nguyen about to add tagging.") controllerInitializers := acm.BuildControllerInitializers() - - klog.Info("Nguyen tagging added.") fss := cliflag.NamedFlagSets{} command := app.NewCloudControllerManagerCommand(opts, cloudInitializer, controllerInitializers, fss, wait.NeverStop) if err := command.Execute(); err != nil { + klog.Fatalf("unable to execute command: %v", err) os.Exit(1) } } From eaa794b0b3a68570fcff43db195e4aef9e53cd10 Mon Sep 17 00:00:00 2001 From: Nguyen Dinh <92940366+nguyenkndinh@users.noreply.github.com> Date: Wed, 23 Feb 2022 17:32:58 -0800 Subject: [PATCH 06/43] removed route controller --- .../controllers/aws_controllermanager.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cmd/aws-cloud-controller-manager/controllers/aws_controllermanager.go b/cmd/aws-cloud-controller-manager/controllers/aws_controllermanager.go index 16e88cf54a..7808398f72 100644 --- a/cmd/aws-cloud-controller-manager/controllers/aws_controllermanager.go +++ b/cmd/aws-cloud-controller-manager/controllers/aws_controllermanager.go @@ -26,6 +26,9 @@ func BuildControllerInitializers() map[string]app.ControllerInitFuncConstructor controllerInitializers["tagging"] = taggingControllerInitFuncConstrustor + // for testing only, remove when raise a PR + delete(controllerInitializers, "route") + return controllerInitializers } From e665663de8dce9768fef130e6c86d75d94251dff Mon Sep 17 00:00:00 2001 From: Nguyen Dinh <92940366+nguyenkndinh@users.noreply.github.com> Date: Wed, 23 Feb 2022 19:30:13 -0800 Subject: [PATCH 07/43] added a blank test file for the tagging controller --- .../controllers/aws_controllermanager.go | 5 +---- .../controllers/tagging/tagging-controller-test.go | 14 ++++++++++++++ .../controllers/tagging/tagging-controller.go | 4 ++-- 3 files changed, 17 insertions(+), 6 deletions(-) create mode 100644 cmd/aws-cloud-controller-manager/controllers/tagging/tagging-controller-test.go diff --git a/cmd/aws-cloud-controller-manager/controllers/aws_controllermanager.go b/cmd/aws-cloud-controller-manager/controllers/aws_controllermanager.go index 7808398f72..6e4ee514ac 100644 --- a/cmd/aws-cloud-controller-manager/controllers/aws_controllermanager.go +++ b/cmd/aws-cloud-controller-manager/controllers/aws_controllermanager.go @@ -26,13 +26,10 @@ func BuildControllerInitializers() map[string]app.ControllerInitFuncConstructor controllerInitializers["tagging"] = taggingControllerInitFuncConstrustor - // for testing only, remove when raise a PR - delete(controllerInitializers, "route") - return controllerInitializers } -// StartTaggingControllerWrapper is used to take cloud config as input and start tagging controller +// StartTaggingControllerWrapper is used to take cloud config as input and start the tagging controller func startTaggingControllerWrapper(initContext app.ControllerInitContext, completedConfig *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface) app.InitFunc { return func(ctx context.Context, controllerContext genericcontrollermanager.ControllerContext) (controller.Interface, bool, error) { return startTaggingController(ctx, initContext, completedConfig, cloud) diff --git a/cmd/aws-cloud-controller-manager/controllers/tagging/tagging-controller-test.go b/cmd/aws-cloud-controller-manager/controllers/tagging/tagging-controller-test.go new file mode 100644 index 0000000000..d58e8615d4 --- /dev/null +++ b/cmd/aws-cloud-controller-manager/controllers/tagging/tagging-controller-test.go @@ -0,0 +1,14 @@ +/* +Copyright 2016 The Kubernetes Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tagging \ No newline at end of file diff --git a/cmd/aws-cloud-controller-manager/controllers/tagging/tagging-controller.go b/cmd/aws-cloud-controller-manager/controllers/tagging/tagging-controller.go index a8256834dd..8c6e48ada7 100644 --- a/cmd/aws-cloud-controller-manager/controllers/tagging/tagging-controller.go +++ b/cmd/aws-cloud-controller-manager/controllers/tagging/tagging-controller.go @@ -15,6 +15,7 @@ package tagging import ( "k8s.io/klog/v2" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" ) // eksResourceTagPrefix is the prefix for tag to group resources that are used by eks @@ -31,14 +32,13 @@ func NewTaggingController() (*TaggingController, error) { tc := &TaggingController{ } - klog.Infof("Starting the TaggingController") - return tc, nil } // Run will start the controller to tag resources attached to a cluster // and untag resources detached from a cluster. func (tc *TaggingController) Run(stopCh <-chan struct{}) { + defer utilruntime.HandleCrash() klog.Infof("Running the TaggingController, eksResourceTagPrefix is %s.", eksResourceTagPrefix) <-stopCh From d320233c2f01664846adf83331e7da2c3c4ca457 Mon Sep 17 00:00:00 2001 From: Nguyen Dinh <92940366+nguyenkndinh@users.noreply.github.com> Date: Thu, 24 Feb 2022 13:06:24 -0800 Subject: [PATCH 08/43] remove predefined tag --- .../controllers/tagging/tagging-controller.go | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/cmd/aws-cloud-controller-manager/controllers/tagging/tagging-controller.go b/cmd/aws-cloud-controller-manager/controllers/tagging/tagging-controller.go index 8c6e48ada7..6e0ef4edcf 100644 --- a/cmd/aws-cloud-controller-manager/controllers/tagging/tagging-controller.go +++ b/cmd/aws-cloud-controller-manager/controllers/tagging/tagging-controller.go @@ -18,11 +18,6 @@ import ( utilruntime "k8s.io/apimachinery/pkg/util/runtime" ) -// eksResourceTagPrefix is the prefix for tag to group resources that are used by eks -// for a particular cluster, this tag is added to the existing tags. -// Example: "Key1=Val1,aws:eks:cluster-name:my-cluster=Val2" -const eksResourceTagPrefix = "aws:eks:cluster-name:" - // TaggingController is the controller implementation for tagging cluster resources type TaggingController struct { } @@ -39,7 +34,7 @@ func NewTaggingController() (*TaggingController, error) { // and untag resources detached from a cluster. func (tc *TaggingController) Run(stopCh <-chan struct{}) { defer utilruntime.HandleCrash() - klog.Infof("Running the TaggingController, eksResourceTagPrefix is %s.", eksResourceTagPrefix) + klog.Infof("Running the TaggingController") <-stopCh } \ No newline at end of file From d12df0a41192a4046a4caa38eff4d39a79734b8b Mon Sep 17 00:00:00 2001 From: Nguyen Dinh <92940366+nguyenkndinh@users.noreply.github.com> Date: Mon, 28 Feb 2022 19:46:55 -0800 Subject: [PATCH 09/43] Refactoring based on recommendations --- cmd/aws-cloud-controller-manager/main.go | 2 +- .../controllers/aws_controllermanager.go | 11 ++++++++--- .../controllers/tagging/tagging-controller-test.go | 0 .../controllers/tagging/tagging-controller.go | 2 +- 4 files changed, 10 insertions(+), 5 deletions(-) rename {cmd/aws-cloud-controller-manager => pkg}/controllers/aws_controllermanager.go (85%) rename {cmd/aws-cloud-controller-manager => pkg}/controllers/tagging/tagging-controller-test.go (100%) rename {cmd/aws-cloud-controller-manager => pkg}/controllers/tagging/tagging-controller.go (100%) diff --git a/cmd/aws-cloud-controller-manager/main.go b/cmd/aws-cloud-controller-manager/main.go index c58ea61bba..0ebf9f61a6 100644 --- a/cmd/aws-cloud-controller-manager/main.go +++ b/cmd/aws-cloud-controller-manager/main.go @@ -45,7 +45,7 @@ import ( cloudcontrollerconfig "k8s.io/cloud-provider/app/config" - acm "k8s.io/cloud-provider-aws/cmd/aws-cloud-controller-manager/controllers" + acm "k8s.io/cloud-provider-aws/pkg/controllers" ) const ( diff --git a/cmd/aws-cloud-controller-manager/controllers/aws_controllermanager.go b/pkg/controllers/aws_controllermanager.go similarity index 85% rename from cmd/aws-cloud-controller-manager/controllers/aws_controllermanager.go rename to pkg/controllers/aws_controllermanager.go index 6e4ee514ac..951d873761 100644 --- a/cmd/aws-cloud-controller-manager/controllers/aws_controllermanager.go +++ b/pkg/controllers/aws_controllermanager.go @@ -5,13 +5,18 @@ import ( "k8s.io/klog/v2" cloudprovider "k8s.io/cloud-provider" - taggingcontroller "k8s.io/cloud-provider-aws/cmd/aws-cloud-controller-manager/controllers/tagging" + taggingcontroller "k8s.io/cloud-provider-aws/pkg/controllers/tagging" "k8s.io/cloud-provider/app" cloudcontrollerconfig "k8s.io/cloud-provider/app/config" genericcontrollermanager "k8s.io/controller-manager/app" "k8s.io/controller-manager/controller" ) +const ( + TaggingControllerClientName = "tagging-controller" + TaggingControllerKey = "tagging" +) + // BuildControllerInitializers is used to add new controllers built in this package to // the existing list of controllers from cloud-provider func BuildControllerInitializers() map[string]app.ControllerInitFuncConstructor { @@ -19,12 +24,12 @@ func BuildControllerInitializers() map[string]app.ControllerInitFuncConstructor taggingControllerInitFuncConstrustor := app.ControllerInitFuncConstructor{ InitContext: app.ControllerInitContext{ - ClientName: "tagging-controller", + ClientName: TaggingControllerClientName, }, Constructor: startTaggingControllerWrapper, } - controllerInitializers["tagging"] = taggingControllerInitFuncConstrustor + controllerInitializers[TaggingControllerKey] = taggingControllerInitFuncConstrustor return controllerInitializers } diff --git a/cmd/aws-cloud-controller-manager/controllers/tagging/tagging-controller-test.go b/pkg/controllers/tagging/tagging-controller-test.go similarity index 100% rename from cmd/aws-cloud-controller-manager/controllers/tagging/tagging-controller-test.go rename to pkg/controllers/tagging/tagging-controller-test.go diff --git a/cmd/aws-cloud-controller-manager/controllers/tagging/tagging-controller.go b/pkg/controllers/tagging/tagging-controller.go similarity index 100% rename from cmd/aws-cloud-controller-manager/controllers/tagging/tagging-controller.go rename to pkg/controllers/tagging/tagging-controller.go index 6e0ef4edcf..b090629ea4 100644 --- a/cmd/aws-cloud-controller-manager/controllers/tagging/tagging-controller.go +++ b/pkg/controllers/tagging/tagging-controller.go @@ -14,8 +14,8 @@ limitations under the License. package tagging import ( - "k8s.io/klog/v2" utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/klog/v2" ) // TaggingController is the controller implementation for tagging cluster resources From 39bd0e009cece82ef202116fbba4b30caaae488b Mon Sep 17 00:00:00 2001 From: Nguyen Dinh <92940366+nguyenkndinh@users.noreply.github.com> Date: Mon, 28 Feb 2022 19:49:11 -0800 Subject: [PATCH 10/43] Sticking to the naming convention --- .../{aws_controllermanager.go => aws_controller_manager.go} | 0 .../tagging/{tagging-controller.go => tagging_controller.go} | 0 .../{tagging-controller-test.go => tagging_controller_test.go} | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename pkg/controllers/{aws_controllermanager.go => aws_controller_manager.go} (100%) rename pkg/controllers/tagging/{tagging-controller.go => tagging_controller.go} (100%) rename pkg/controllers/tagging/{tagging-controller-test.go => tagging_controller_test.go} (100%) diff --git a/pkg/controllers/aws_controllermanager.go b/pkg/controllers/aws_controller_manager.go similarity index 100% rename from pkg/controllers/aws_controllermanager.go rename to pkg/controllers/aws_controller_manager.go diff --git a/pkg/controllers/tagging/tagging-controller.go b/pkg/controllers/tagging/tagging_controller.go similarity index 100% rename from pkg/controllers/tagging/tagging-controller.go rename to pkg/controllers/tagging/tagging_controller.go diff --git a/pkg/controllers/tagging/tagging-controller-test.go b/pkg/controllers/tagging/tagging_controller_test.go similarity index 100% rename from pkg/controllers/tagging/tagging-controller-test.go rename to pkg/controllers/tagging/tagging_controller_test.go From e38e3663d0a11dc9b5a6f2ba62072997e6b91d66 Mon Sep 17 00:00:00 2001 From: Nguyen Dinh <92940366+nguyenkndinh@users.noreply.github.com> Date: Tue, 8 Mar 2022 15:29:45 -0800 Subject: [PATCH 11/43] address more comments on naming --- cmd/aws-cloud-controller-manager/main.go | 13 +++++-------- pkg/controllers/aws_controller_manager.go | 8 +++----- 2 files changed, 8 insertions(+), 13 deletions(-) diff --git a/cmd/aws-cloud-controller-manager/main.go b/cmd/aws-cloud-controller-manager/main.go index 0ebf9f61a6..2519f43c0c 100644 --- a/cmd/aws-cloud-controller-manager/main.go +++ b/cmd/aws-cloud-controller-manager/main.go @@ -31,21 +31,19 @@ import ( "time" "k8s.io/apimachinery/pkg/util/wait" + cloudprovider "k8s.io/cloud-provider" + awsv1 "k8s.io/cloud-provider-aws/pkg/providers/v1" + awsv2 "k8s.io/cloud-provider-aws/pkg/providers/v2" "k8s.io/cloud-provider/app" "k8s.io/cloud-provider/options" cliflag "k8s.io/component-base/cli/flag" "k8s.io/component-base/logs" _ "k8s.io/component-base/metrics/prometheus/clientgo" // for client metric registration _ "k8s.io/component-base/metrics/prometheus/version" // for version metric registration - "k8s.io/klog/v2" - - cloudprovider "k8s.io/cloud-provider" - awsv1 "k8s.io/cloud-provider-aws/pkg/providers/v1" - awsv2 "k8s.io/cloud-provider-aws/pkg/providers/v2" cloudcontrollerconfig "k8s.io/cloud-provider/app/config" - acm "k8s.io/cloud-provider-aws/pkg/controllers" + awscontrollers "k8s.io/cloud-provider-aws/pkg/controllers" ) const ( @@ -65,13 +63,12 @@ func main() { klog.Fatalf("unable to initialize command options: %v", err) } - controllerInitializers := acm.BuildControllerInitializers() + controllerInitializers := awscontrollers.BuildControllerInitializers() fss := cliflag.NamedFlagSets{} command := app.NewCloudControllerManagerCommand(opts, cloudInitializer, controllerInitializers, fss, wait.NeverStop) if err := command.Execute(); err != nil { klog.Fatalf("unable to execute command: %v", err) - os.Exit(1) } } diff --git a/pkg/controllers/aws_controller_manager.go b/pkg/controllers/aws_controller_manager.go index 951d873761..8dfb5b80f3 100644 --- a/pkg/controllers/aws_controller_manager.go +++ b/pkg/controllers/aws_controller_manager.go @@ -2,8 +2,6 @@ package controllers import ( "context" - "k8s.io/klog/v2" - cloudprovider "k8s.io/cloud-provider" taggingcontroller "k8s.io/cloud-provider-aws/pkg/controllers/tagging" "k8s.io/cloud-provider/app" @@ -22,14 +20,14 @@ const ( func BuildControllerInitializers() map[string]app.ControllerInitFuncConstructor { controllerInitializers := app.DefaultInitFuncConstructors - taggingControllerInitFuncConstrustor := app.ControllerInitFuncConstructor{ + taggingControllerConstructor := app.ControllerInitFuncConstructor{ InitContext: app.ControllerInitContext{ ClientName: TaggingControllerClientName, }, Constructor: startTaggingControllerWrapper, } - controllerInitializers[TaggingControllerKey] = taggingControllerInitFuncConstrustor + controllerInitializers[TaggingControllerKey] = taggingControllerConstructor return controllerInitializers } @@ -52,4 +50,4 @@ func startTaggingController(ctx context.Context, initContext app.ControllerInitC go taggingcontroller.Run(ctx.Done()) return nil, true, nil -} \ No newline at end of file +} From 208f52eeae627257cc57e8990ba2252526300f56 Mon Sep 17 00:00:00 2001 From: Nguyen Dinh <92940366+nguyenkndinh@users.noreply.github.com> Date: Mon, 14 Mar 2022 19:38:32 -0700 Subject: [PATCH 12/43] Using ListNode to get nodes entering and leaving the cluster. * Remove route controller again * Print out clusterName for new nodes * tag new nodes when it comes online * only process a node once * check taggNodes size * add debugging * use node name as key * delete k,v from taggedNodes if node no longer exists * log if delete is done * Get a list of nodes and tag them if havent * get instance IDs of the nodes that need tagging * use MapToAWSInstanceID instead * restored v1 aws * restored from master * tag instance with a random tag * add klog * tag and untag node resources * Prepare for pr * Initialize nodeMap --- cmd/aws-cloud-controller-manager/main.go | 1 + pkg/controllers/aws_controller_manager.go | 26 +++- pkg/controllers/tagging/tagging_controller.go | 143 +++++++++++++++++- 3 files changed, 162 insertions(+), 8 deletions(-) diff --git a/cmd/aws-cloud-controller-manager/main.go b/cmd/aws-cloud-controller-manager/main.go index 2519f43c0c..177cee3afb 100644 --- a/cmd/aws-cloud-controller-manager/main.go +++ b/cmd/aws-cloud-controller-manager/main.go @@ -40,6 +40,7 @@ import ( "k8s.io/component-base/logs" _ "k8s.io/component-base/metrics/prometheus/clientgo" // for client metric registration _ "k8s.io/component-base/metrics/prometheus/version" // for version metric registration + "k8s.io/klog/v2" cloudcontrollerconfig "k8s.io/cloud-provider/app/config" diff --git a/pkg/controllers/aws_controller_manager.go b/pkg/controllers/aws_controller_manager.go index 8dfb5b80f3..7702855095 100644 --- a/pkg/controllers/aws_controller_manager.go +++ b/pkg/controllers/aws_controller_manager.go @@ -1,3 +1,16 @@ +/* +Copyright 2016 The Kubernetes Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package controllers import ( @@ -8,6 +21,7 @@ import ( cloudcontrollerconfig "k8s.io/cloud-provider/app/config" genericcontrollermanager "k8s.io/controller-manager/app" "k8s.io/controller-manager/controller" + "k8s.io/klog/v2" ) const ( @@ -29,6 +43,9 @@ func BuildControllerInitializers() map[string]app.ControllerInitFuncConstructor controllerInitializers[TaggingControllerKey] = taggingControllerConstructor + // TODO: remove the following line to enable the route controller + delete(controllerInitializers, "route") + return controllerInitializers } @@ -41,13 +58,18 @@ func startTaggingControllerWrapper(initContext app.ControllerInitContext, comple func startTaggingController(ctx context.Context, initContext app.ControllerInitContext, completedConfig *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface) (controller.Interface, bool, error) { // Start the TaggingController - taggingcontroller, err := taggingcontroller.NewTaggingController() + taggingcontroller, err := taggingcontroller.NewTaggingController( + completedConfig.SharedInformers.Core().V1().Nodes(), + // cloud node lifecycle controller uses existing cluster role from node-controller + completedConfig.ClientBuilder.ClientOrDie(initContext.ClientName), + cloud, + completedConfig.ComponentConfig.KubeCloudShared.NodeMonitorPeriod.Duration) if err != nil { klog.Warningf("failed to start tagging controller: %s", err) return nil, false, nil } - go taggingcontroller.Run(ctx.Done()) + go taggingcontroller.Run(ctx) return nil, true, nil } diff --git a/pkg/controllers/tagging/tagging_controller.go b/pkg/controllers/tagging/tagging_controller.go index b090629ea4..09aa7852b3 100644 --- a/pkg/controllers/tagging/tagging_controller.go +++ b/pkg/controllers/tagging/tagging_controller.go @@ -14,17 +14,54 @@ limitations under the License. package tagging import ( + "context" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/labels" utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/apimachinery/pkg/util/wait" + coreinformers "k8s.io/client-go/informers/core/v1" + clientset "k8s.io/client-go/kubernetes" + v1lister "k8s.io/client-go/listers/core/v1" + cloudprovider "k8s.io/cloud-provider" "k8s.io/klog/v2" + "time" ) -// TaggingController is the controller implementation for tagging cluster resources +// TaggingController is the controller implementation for tagging cluster resources. +// It periodically check for Node events (creating/deleting) to apply appropriate +// tags to resources. type TaggingController struct { + kubeClient clientset.Interface + nodeLister v1lister.NodeLister + + cloud cloudprovider.Interface + + // Value controlling TaggingController monitoring period, i.e. how often does TaggingController + // check node list. This value should be lower than nodeMonitorGracePeriod + // set in controller-manager + nodeMonitorPeriod time.Duration + + // A map presenting the node and whether it currently exists + taggedNodes map[string]bool + + // A map representing the nodes that were ever in the cluster + nodeMap map[string]*v1.Node } // NewTaggingController creates a NewTaggingController object -func NewTaggingController() (*TaggingController, error) { +func NewTaggingController( + nodeInformer coreinformers.NodeInformer, + kubeClient clientset.Interface, + cloud cloudprovider.Interface, + nodeMonitorPeriod time.Duration) (*TaggingController, error) { + tc := &TaggingController{ + kubeClient: kubeClient, + nodeLister: nodeInformer.Lister(), + cloud: cloud, + nodeMonitorPeriod: nodeMonitorPeriod, + taggedNodes: make(map[string]bool), + nodeMap: make(map[string]*v1.Node), } return tc, nil @@ -32,9 +69,103 @@ func NewTaggingController() (*TaggingController, error) { // Run will start the controller to tag resources attached to a cluster // and untag resources detached from a cluster. -func (tc *TaggingController) Run(stopCh <-chan struct{}) { +func (tc *TaggingController) Run(ctx context.Context) { defer utilruntime.HandleCrash() - klog.Infof("Running the TaggingController") - <-stopCh -} \ No newline at end of file + wait.UntilWithContext(ctx, tc.monitorNodes, tc.nodeMonitorPeriod) +} + +func (tc *TaggingController) monitorNodes(ctx context.Context) { + nodes, err := tc.nodeLister.List(labels.Everything()) + if err != nil { + klog.Errorf("error listing nodes from cache: %s", err) + return + } + + for k := range tc.taggedNodes { + tc.taggedNodes[k] = false + } + + var nodesToTag []*v1.Node + for _, node := range nodes { + if _, ok := tc.taggedNodes[node.GetName()]; !ok { + nodesToTag = append(nodesToTag, node) + } + + tc.nodeMap[node.GetName()] = node + tc.taggedNodes[node.GetName()] = true + } + tc.tagNodesResources(nodesToTag) + + var nodesToUntag []*v1.Node + for nodeName, existed := range tc.taggedNodes { + if existed == false { + nodesToUntag = append(nodesToUntag, tc.nodeMap[nodeName]) + } + } + tc.untagNodeResources(nodesToUntag) + + tc.syncDeletedNodesToTaggedNodes() +} + +// tagNodesResources tag node resources from a list of node +// If we want to tag more resources, modify this function appropriately +func (tc *TaggingController) tagNodesResources(nodes []*v1.Node) { + for _, node := range nodes { + klog.Infof("Tagging resources for node %s.", node.GetName()) + } +} + +func (tc *TaggingController) untagNodeResources(nodes []*v1.Node) { + for _, node := range nodes { + klog.Infof("Untagging resources for node %s.", node.GetName()) + } +} + +// syncDeletedNodes delete (k, v) from taggedNodes +// if it doesn't exist +func (tc *TaggingController) syncDeletedNodesToTaggedNodes() { + for k, v := range tc.taggedNodes { + if v == false { + delete(tc.taggedNodes, k) + } + } +} + +//// tagEc2Instances applies the provided tags to each EC2 instances in +//// the cluster. +//func (tc *TaggingController) tagEc2Instances(nodes []*v1.Node) { +// var instanceIds []*string +// for _, node := range nodes { +// instanceId, _ := awsv1.KubernetesInstanceID(node.Spec.ProviderID).MapToAWSInstanceID() +// instanceIds = append(instanceIds, aws.String(string(instanceId))) +// } +// +// tc.tagResources(instanceIds) +//} + +//func (tc *TaggingController) tagResources(resourceIds []*string) { +// request := &ec2.CreateTagsInput{ +// Resources: resourceIds, +// Tags: tc.getTagsFromInputs(), +// } +// +// _, error := awsv1.awsSdkEC2.CreateTags(request) +// awsv1.Cloud.TagResoures(request) +// +// if error != nil { +// klog.Errorf("Error occurred trying to tag resources, %s", error) +// } +//} +// +//// Sample function demonstrating that we'll get the tag list from user +//func (tc *TaggingController) getTagsFromInputs() []*ec2.Tag { +// var awsTags []*ec2.Tag +// tag := &ec2.Tag{ +// Key: aws.String("Sample Key"), +// Value: aws.String("Sample value"), +// } +// awsTags = append(awsTags, tag) +// +// return awsTags +//} From 9118ec0cb8a9981a29ece00f2c942c5c8f05cdbe Mon Sep 17 00:00:00 2001 From: Nguyen Dinh <92940366+nguyenkndinh@users.noreply.github.com> Date: Mon, 14 Mar 2022 19:48:58 -0700 Subject: [PATCH 13/43] refactor --- pkg/controllers/aws_controller_manager.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/pkg/controllers/aws_controller_manager.go b/pkg/controllers/aws_controller_manager.go index 7702855095..782cc6effb 100644 --- a/pkg/controllers/aws_controller_manager.go +++ b/pkg/controllers/aws_controller_manager.go @@ -43,9 +43,6 @@ func BuildControllerInitializers() map[string]app.ControllerInitFuncConstructor controllerInitializers[TaggingControllerKey] = taggingControllerConstructor - // TODO: remove the following line to enable the route controller - delete(controllerInitializers, "route") - return controllerInitializers } @@ -60,7 +57,6 @@ func startTaggingController(ctx context.Context, initContext app.ControllerInitC // Start the TaggingController taggingcontroller, err := taggingcontroller.NewTaggingController( completedConfig.SharedInformers.Core().V1().Nodes(), - // cloud node lifecycle controller uses existing cluster role from node-controller completedConfig.ClientBuilder.ClientOrDie(initContext.ClientName), cloud, completedConfig.ComponentConfig.KubeCloudShared.NodeMonitorPeriod.Duration) From af942765d9336f43b8d3429629232744407b2393 Mon Sep 17 00:00:00 2001 From: Nguyen Dinh <92940366+nguyenkndinh@users.noreply.github.com> Date: Fri, 18 Mar 2022 19:35:07 -0700 Subject: [PATCH 14/43] Add testing and controller config skeletons --- pkg/config/cloud_config.go | 6 + pkg/config/controller_config.go | 56 +++++++ pkg/config/runtime_config.go | 5 + pkg/controllers/aws_controller_manager.go | 18 +++ pkg/controllers/tagging/tagging_controller.go | 99 ++++++------ .../tagging/tagging_controller_test.go | 148 +++++++++++++++++- 6 files changed, 287 insertions(+), 45 deletions(-) create mode 100644 pkg/config/cloud_config.go create mode 100644 pkg/config/controller_config.go create mode 100644 pkg/config/runtime_config.go diff --git a/pkg/config/cloud_config.go b/pkg/config/cloud_config.go new file mode 100644 index 0000000000..8271eb8d16 --- /dev/null +++ b/pkg/config/cloud_config.go @@ -0,0 +1,6 @@ +package config + +type CloudConfig struct { +} + +var CloudCfg = &CloudConfig{} diff --git a/pkg/config/controller_config.go b/pkg/config/controller_config.go new file mode 100644 index 0000000000..c0a0d2eaa0 --- /dev/null +++ b/pkg/config/controller_config.go @@ -0,0 +1,56 @@ +package config + +import ( + "flag" + "fmt" + "github.com/spf13/pflag" + "k8s.io/cloud-provider/config" + "os" +) + +const ( + flagResourceTags = "resource-tags" + flagTaggingResources = "tagging-resources" +) + +var ControllerCFG = &ControllerConfig{} + +// ControllerConfig stores the additional flags for global usage +type ControllerConfig struct { + config.KubeCloudSharedConfiguration + ResourceTags string + TaggingResources string + + //RuntimeConfig RuntimeConfig + //CloudConfig *CloudConfig +} + +func (cfg *ControllerConfig) BindFlags(fs *pflag.FlagSet) { + fs.StringVar(&cfg.ResourceTags, flagResourceTags, "", "List of tags for the cluster.") + fs.StringVar(&cfg.TaggingResources, flagTaggingResources, "", "List of EC2 resources that need to be tagged.") +} + +// Validate the controller configuration +func (cfg *ControllerConfig) Validate() error { + if len(cfg.TaggingResources) > 0 && len(cfg.ResourceTags) == 0 { + return fmt.Errorf("--resource-tags must be set when --tagging-resources is not empty.") + } + + return nil +} + +func (cfg *ControllerConfig) LoadControllerConfig() error { + fs := pflag.NewFlagSet("", pflag.ExitOnError) + fs.AddGoFlagSet(flag.CommandLine) + cfg.BindFlags(fs) + + if err := fs.Parse(os.Args); err != nil { + return err + } + + if err := cfg.Validate(); err != nil { + return err + } + + return nil +} diff --git a/pkg/config/runtime_config.go b/pkg/config/runtime_config.go new file mode 100644 index 0000000000..b848f5a468 --- /dev/null +++ b/pkg/config/runtime_config.go @@ -0,0 +1,5 @@ +package config + +// RuntimeConfig stores the configuration for controller-runtime +type RuntimeConfig struct { +} diff --git a/pkg/controllers/aws_controller_manager.go b/pkg/controllers/aws_controller_manager.go index 782cc6effb..0876d808cc 100644 --- a/pkg/controllers/aws_controller_manager.go +++ b/pkg/controllers/aws_controller_manager.go @@ -15,6 +15,7 @@ package controllers import ( "context" + "errors" cloudprovider "k8s.io/cloud-provider" taggingcontroller "k8s.io/cloud-provider-aws/pkg/controllers/tagging" "k8s.io/cloud-provider/app" @@ -43,6 +44,9 @@ func BuildControllerInitializers() map[string]app.ControllerInitFuncConstructor controllerInitializers[TaggingControllerKey] = taggingControllerConstructor + // TODO: remove the following line to enable the route controller + delete(controllerInitializers, "route") + return controllerInitializers } @@ -54,6 +58,12 @@ func startTaggingControllerWrapper(initContext app.ControllerInitContext, comple } func startTaggingController(ctx context.Context, initContext app.ControllerInitContext, completedConfig *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface) (controller.Interface, bool, error) { + // TODO: add in validation for user input for new flags + //if ok, error := verifyTaggingControllerUserInput(completedConfig.ComponentConfig.KubeCloudShared.ClusterCIDR); ok { + // klog.Infof("Will not start the tagging controller due to invalid user input, --configure-cloud-routes: %v", error) + // return nil, false, nil + //} + // Start the TaggingController taggingcontroller, err := taggingcontroller.NewTaggingController( completedConfig.SharedInformers.Core().V1().Nodes(), @@ -69,3 +79,11 @@ func startTaggingController(ctx context.Context, initContext app.ControllerInitC return nil, true, nil } + +func verifyTaggingControllerUserInput(input string) (bool, error) { + if len(input) == 0 { + return false, errors.New("Provide inputs for --resource-tags and --tagging-resources to use the tagging controller.") + } + + return true, nil +} diff --git a/pkg/controllers/tagging/tagging_controller.go b/pkg/controllers/tagging/tagging_controller.go index 09aa7852b3..9c9c8aba21 100644 --- a/pkg/controllers/tagging/tagging_controller.go +++ b/pkg/controllers/tagging/tagging_controller.go @@ -15,6 +15,8 @@ package tagging import ( "context" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/ec2" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/labels" utilruntime "k8s.io/apimachinery/pkg/util/runtime" @@ -23,6 +25,7 @@ import ( clientset "k8s.io/client-go/kubernetes" v1lister "k8s.io/client-go/listers/core/v1" cloudprovider "k8s.io/cloud-provider" + awsv1 "k8s.io/cloud-provider-aws/pkg/providers/v1" "k8s.io/klog/v2" "time" ) @@ -44,8 +47,14 @@ type TaggingController struct { // A map presenting the node and whether it currently exists taggedNodes map[string]bool - // A map representing the nodes that were ever in the cluster + // A map representing nodes that were part of the cluster at any point in time nodeMap map[string]*v1.Node + + // Representing the user input for tags + tags string + + // Representing the resources to tag + resources string } // NewTaggingController creates a NewTaggingController object @@ -62,8 +71,10 @@ func NewTaggingController( nodeMonitorPeriod: nodeMonitorPeriod, taggedNodes: make(map[string]bool), nodeMap: make(map[string]*v1.Node), + // TODO: add controller configs including the new flags + //tags: conf.ControllerCFG.ResourceTags, + //resources: conf.ControllerCFG.TaggingResources, } - return tc, nil } @@ -72,13 +83,13 @@ func NewTaggingController( func (tc *TaggingController) Run(ctx context.Context) { defer utilruntime.HandleCrash() - wait.UntilWithContext(ctx, tc.monitorNodes, tc.nodeMonitorPeriod) + wait.UntilWithContext(ctx, tc.MonitorNodes, tc.nodeMonitorPeriod) } -func (tc *TaggingController) monitorNodes(ctx context.Context) { +func (tc *TaggingController) MonitorNodes(ctx context.Context) { nodes, err := tc.nodeLister.List(labels.Everything()) if err != nil { - klog.Errorf("error listing nodes from cache: %s", err) + klog.Errorf("error listing nodes: %s", err) return } @@ -112,13 +123,13 @@ func (tc *TaggingController) monitorNodes(ctx context.Context) { // If we want to tag more resources, modify this function appropriately func (tc *TaggingController) tagNodesResources(nodes []*v1.Node) { for _, node := range nodes { - klog.Infof("Tagging resources for node %s.", node.GetName()) + klog.Infof("Tagging resources for node %s with %s.", node.GetName(), tc.tags) } } func (tc *TaggingController) untagNodeResources(nodes []*v1.Node) { for _, node := range nodes { - klog.Infof("Untagging resources for node %s.", node.GetName()) + klog.Infof("Untagging resources for node %s with %s.", node.GetName(), tc.tags) } } @@ -132,40 +143,40 @@ func (tc *TaggingController) syncDeletedNodesToTaggedNodes() { } } -//// tagEc2Instances applies the provided tags to each EC2 instances in -//// the cluster. -//func (tc *TaggingController) tagEc2Instances(nodes []*v1.Node) { -// var instanceIds []*string -// for _, node := range nodes { -// instanceId, _ := awsv1.KubernetesInstanceID(node.Spec.ProviderID).MapToAWSInstanceID() -// instanceIds = append(instanceIds, aws.String(string(instanceId))) -// } -// -// tc.tagResources(instanceIds) -//} - -//func (tc *TaggingController) tagResources(resourceIds []*string) { -// request := &ec2.CreateTagsInput{ -// Resources: resourceIds, -// Tags: tc.getTagsFromInputs(), -// } -// -// _, error := awsv1.awsSdkEC2.CreateTags(request) -// awsv1.Cloud.TagResoures(request) -// -// if error != nil { -// klog.Errorf("Error occurred trying to tag resources, %s", error) -// } -//} -// -//// Sample function demonstrating that we'll get the tag list from user -//func (tc *TaggingController) getTagsFromInputs() []*ec2.Tag { -// var awsTags []*ec2.Tag -// tag := &ec2.Tag{ -// Key: aws.String("Sample Key"), -// Value: aws.String("Sample value"), -// } -// awsTags = append(awsTags, tag) -// -// return awsTags -//} +// tagEc2Instances applies the provided tags to each EC2 instances in +// the cluster. +func (tc *TaggingController) tagEc2Instances(nodes []*v1.Node) { + var instanceIds []*string + for _, node := range nodes { + instanceId, _ := awsv1.KubernetesInstanceID(node.Spec.ProviderID).MapToAWSInstanceID() + instanceIds = append(instanceIds, aws.String(string(instanceId))) + } + + tc.tagResources(instanceIds) +} + +// TODO: call EC2 to tag instances +func (tc *TaggingController) tagResources(resourceIds []*string) { + //request := &ec2.CreateTagsInput{ + // Resources: resourceIds, + // Tags: tc.getTagsFromInputs(), + //} + // + //_, error := awsv1..EC2.CreateTags(request) + // + //if error != nil { + // klog.Errorf("Error occurred trying to tag resources, %s", error) + //} +} + +// Sample function demonstrating that we'll get the tag list from user +func (tc *TaggingController) getTagsFromInputs() []*ec2.Tag { + var awsTags []*ec2.Tag + tag := &ec2.Tag{ + Key: aws.String("Sample Key"), + Value: aws.String("Sample value"), + } + awsTags = append(awsTags, tag) + + return awsTags +} diff --git a/pkg/controllers/tagging/tagging_controller_test.go b/pkg/controllers/tagging/tagging_controller_test.go index d58e8615d4..740b9227e5 100644 --- a/pkg/controllers/tagging/tagging_controller_test.go +++ b/pkg/controllers/tagging/tagging_controller_test.go @@ -11,4 +11,150 @@ See the License for the specific language governing permissions and limitations under the License. */ -package tagging \ No newline at end of file +package tagging + +import ( + "context" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/informers" + coreinformers "k8s.io/client-go/informers/core/v1" + "k8s.io/client-go/kubernetes/fake" + "k8s.io/client-go/tools/record" + fakecloud "k8s.io/cloud-provider/fake" + "k8s.io/klog/v2" + "testing" + "time" +) + +func Test_NodesJoining(t *testing.T) { + testcases := []struct { + name string + fakeCloud *fakecloud.Cloud + currNode *v1.Node + taggingController TaggingController + noOfNodes int + }{ + { + name: "node0 joins the cluster.", + currNode: &v1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node0", + CreationTimestamp: metav1.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC), + }, + }, + fakeCloud: &fakecloud.Cloud{ + ExistsByProviderID: false, + }, + taggingController: TaggingController{ + taggedNodes: make(map[string]bool), + nodeMap: make(map[string]*v1.Node), + }, + noOfNodes: 1, + }, + { + name: "node1 joins the cluster, node0 left.", + currNode: &v1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node1", + CreationTimestamp: metav1.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC), + }, + }, + fakeCloud: &fakecloud.Cloud{ + ExistsByProviderID: false, + }, + taggingController: TaggingController{ + taggedNodes: map[string]bool{ + "node0": true, + }, + nodeMap: map[string]*v1.Node{ + "node0": { + ObjectMeta: metav1.ObjectMeta{ + Name: "node0", + CreationTimestamp: metav1.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC), + }, + }, + }, + }, + noOfNodes: 1, + }, + { + name: "node2 joins the cluster, node0 and node1 left.", + currNode: &v1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node2", + CreationTimestamp: metav1.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC), + }, + }, + fakeCloud: &fakecloud.Cloud{ + ExistsByProviderID: false, + }, + taggingController: TaggingController{ + taggedNodes: map[string]bool{ + "node0": true, + "node1": true, + }, + nodeMap: map[string]*v1.Node{ + "node0": { + ObjectMeta: metav1.ObjectMeta{ + Name: "node0", + CreationTimestamp: metav1.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC), + }, + }, + "node1": { + ObjectMeta: metav1.ObjectMeta{ + Name: "node1", + CreationTimestamp: metav1.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC), + }, + }, + }, + }, + noOfNodes: 1, + }, + } + + for _, testcase := range testcases { + t.Run(testcase.name, func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + clientset := fake.NewSimpleClientset(testcase.currNode) + informer := informers.NewSharedInformerFactory(clientset, time.Second) + nodeInformer := informer.Core().V1().Nodes() + + if err := syncNodeStore(nodeInformer, clientset); err != nil { + t.Errorf("unexpected error: %v", err) + } + + eventBroadcaster := record.NewBroadcaster() + testcase.taggingController.nodeLister = nodeInformer.Lister() + testcase.taggingController.kubeClient = clientset + testcase.taggingController.cloud = testcase.fakeCloud + testcase.taggingController.nodeMonitorPeriod = 1 * time.Second + + w := eventBroadcaster.StartLogging(klog.Infof) + defer w.Stop() + + nodeCountBeforeTagging := len(testcase.taggingController.nodeMap) + testcase.taggingController.MonitorNodes(ctx) + + klog.Infof("testcase.taggingController.taggedNodes %s", testcase.taggingController.taggedNodes) + klog.Errorf("testcase.taggingController.nodeMap %s", testcase.taggingController.nodeMap) + + if len(testcase.taggingController.taggedNodes) != testcase.noOfNodes || len(testcase.taggingController.nodeMap) != nodeCountBeforeTagging+testcase.noOfNodes { + t.Errorf("taggedNodes must contain %d element(s), and nodeMap must contain %d element(s).", testcase.noOfNodes, nodeCountBeforeTagging+testcase.noOfNodes) + } + }) + } +} + +func syncNodeStore(nodeinformer coreinformers.NodeInformer, f *fake.Clientset) error { + nodes, err := f.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{}) + if err != nil { + return err + } + newElems := make([]interface{}, 0, len(nodes.Items)) + for i := range nodes.Items { + newElems = append(newElems, &nodes.Items[i]) + } + return nodeinformer.Informer().GetStore().Replace(newElems, "newRV") +} From 54509495c33ebfad8d5b4675d59d2b36878477a9 Mon Sep 17 00:00:00 2001 From: Nguyen Dinh <92940366+nguyenkndinh@users.noreply.github.com> Date: Mon, 21 Mar 2022 19:41:04 -0700 Subject: [PATCH 15/43] Added tagging and flags mechanisms --- .gitignore | 5 +- cmd/aws-cloud-controller-manager/main.go | 20 ++- pkg/config/cloud_config.go | 6 - pkg/config/controller_config.go | 56 -------- pkg/config/runtime_config.go | 5 - pkg/controllers/aws_controller_manager.go | 89 ------------- pkg/controllers/options/tagging_controller.go | 22 ++++ pkg/controllers/tagging/tagging_controller.go | 122 ++++++++++-------- .../tagging/tagging_controller_wrapper.go | 54 ++++++++ pkg/providers/v1/aws.go | 9 ++ pkg/providers/v1/aws_fakes.go | 5 + pkg/providers/v1/tags.go | 45 +++++++ 12 files changed, 220 insertions(+), 218 deletions(-) delete mode 100644 pkg/config/cloud_config.go delete mode 100644 pkg/config/controller_config.go delete mode 100644 pkg/config/runtime_config.go delete mode 100644 pkg/controllers/aws_controller_manager.go create mode 100644 pkg/controllers/options/tagging_controller.go create mode 100644 pkg/controllers/tagging/tagging_controller_wrapper.go diff --git a/.gitignore b/.gitignore index 53f9cfd10d..d8620b67a3 100644 --- a/.gitignore +++ b/.gitignore @@ -2,7 +2,8 @@ /ecr-credential-provider /cloudconfig -.vscode/ - docs/book/_book/ site/ +.vscode/ +e2e.test +.idea/ diff --git a/cmd/aws-cloud-controller-manager/main.go b/cmd/aws-cloud-controller-manager/main.go index 177cee3afb..5a356f4b85 100644 --- a/cmd/aws-cloud-controller-manager/main.go +++ b/cmd/aws-cloud-controller-manager/main.go @@ -26,6 +26,7 @@ limitations under the License. package main import ( + "k8s.io/cloud-provider-aws/pkg/controllers/tagging" "math/rand" "os" "time" @@ -43,8 +44,6 @@ import ( "k8s.io/klog/v2" cloudcontrollerconfig "k8s.io/cloud-provider/app/config" - - awscontrollers "k8s.io/cloud-provider-aws/pkg/controllers" ) const ( @@ -64,8 +63,23 @@ func main() { klog.Fatalf("unable to initialize command options: %v", err) } - controllerInitializers := awscontrollers.BuildControllerInitializers() + controllerInitializers := app.DefaultInitFuncConstructors + taggingControllerWrapper := tagging.TaggingControllerWrapper{} fss := cliflag.NamedFlagSets{} + taggingControllerWrapper.Options.AddFlags(fss.FlagSet("tagging controller")) + + taggingControllerConstructor := app.ControllerInitFuncConstructor{ + InitContext: app.ControllerInitContext{ + ClientName: tagging.TaggingControllerClientName, + }, + Constructor: taggingControllerWrapper.StartTaggingControllerWrapper, + } + + controllerInitializers[tagging.TaggingControllerKey] = taggingControllerConstructor + + // TODO: remove the following line to enable the route controller + delete(controllerInitializers, "route") + command := app.NewCloudControllerManagerCommand(opts, cloudInitializer, controllerInitializers, fss, wait.NeverStop) if err := command.Execute(); err != nil { diff --git a/pkg/config/cloud_config.go b/pkg/config/cloud_config.go deleted file mode 100644 index 8271eb8d16..0000000000 --- a/pkg/config/cloud_config.go +++ /dev/null @@ -1,6 +0,0 @@ -package config - -type CloudConfig struct { -} - -var CloudCfg = &CloudConfig{} diff --git a/pkg/config/controller_config.go b/pkg/config/controller_config.go deleted file mode 100644 index c0a0d2eaa0..0000000000 --- a/pkg/config/controller_config.go +++ /dev/null @@ -1,56 +0,0 @@ -package config - -import ( - "flag" - "fmt" - "github.com/spf13/pflag" - "k8s.io/cloud-provider/config" - "os" -) - -const ( - flagResourceTags = "resource-tags" - flagTaggingResources = "tagging-resources" -) - -var ControllerCFG = &ControllerConfig{} - -// ControllerConfig stores the additional flags for global usage -type ControllerConfig struct { - config.KubeCloudSharedConfiguration - ResourceTags string - TaggingResources string - - //RuntimeConfig RuntimeConfig - //CloudConfig *CloudConfig -} - -func (cfg *ControllerConfig) BindFlags(fs *pflag.FlagSet) { - fs.StringVar(&cfg.ResourceTags, flagResourceTags, "", "List of tags for the cluster.") - fs.StringVar(&cfg.TaggingResources, flagTaggingResources, "", "List of EC2 resources that need to be tagged.") -} - -// Validate the controller configuration -func (cfg *ControllerConfig) Validate() error { - if len(cfg.TaggingResources) > 0 && len(cfg.ResourceTags) == 0 { - return fmt.Errorf("--resource-tags must be set when --tagging-resources is not empty.") - } - - return nil -} - -func (cfg *ControllerConfig) LoadControllerConfig() error { - fs := pflag.NewFlagSet("", pflag.ExitOnError) - fs.AddGoFlagSet(flag.CommandLine) - cfg.BindFlags(fs) - - if err := fs.Parse(os.Args); err != nil { - return err - } - - if err := cfg.Validate(); err != nil { - return err - } - - return nil -} diff --git a/pkg/config/runtime_config.go b/pkg/config/runtime_config.go deleted file mode 100644 index b848f5a468..0000000000 --- a/pkg/config/runtime_config.go +++ /dev/null @@ -1,5 +0,0 @@ -package config - -// RuntimeConfig stores the configuration for controller-runtime -type RuntimeConfig struct { -} diff --git a/pkg/controllers/aws_controller_manager.go b/pkg/controllers/aws_controller_manager.go deleted file mode 100644 index 0876d808cc..0000000000 --- a/pkg/controllers/aws_controller_manager.go +++ /dev/null @@ -1,89 +0,0 @@ -/* -Copyright 2016 The Kubernetes Authors. -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package controllers - -import ( - "context" - "errors" - cloudprovider "k8s.io/cloud-provider" - taggingcontroller "k8s.io/cloud-provider-aws/pkg/controllers/tagging" - "k8s.io/cloud-provider/app" - cloudcontrollerconfig "k8s.io/cloud-provider/app/config" - genericcontrollermanager "k8s.io/controller-manager/app" - "k8s.io/controller-manager/controller" - "k8s.io/klog/v2" -) - -const ( - TaggingControllerClientName = "tagging-controller" - TaggingControllerKey = "tagging" -) - -// BuildControllerInitializers is used to add new controllers built in this package to -// the existing list of controllers from cloud-provider -func BuildControllerInitializers() map[string]app.ControllerInitFuncConstructor { - controllerInitializers := app.DefaultInitFuncConstructors - - taggingControllerConstructor := app.ControllerInitFuncConstructor{ - InitContext: app.ControllerInitContext{ - ClientName: TaggingControllerClientName, - }, - Constructor: startTaggingControllerWrapper, - } - - controllerInitializers[TaggingControllerKey] = taggingControllerConstructor - - // TODO: remove the following line to enable the route controller - delete(controllerInitializers, "route") - - return controllerInitializers -} - -// StartTaggingControllerWrapper is used to take cloud config as input and start the tagging controller -func startTaggingControllerWrapper(initContext app.ControllerInitContext, completedConfig *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface) app.InitFunc { - return func(ctx context.Context, controllerContext genericcontrollermanager.ControllerContext) (controller.Interface, bool, error) { - return startTaggingController(ctx, initContext, completedConfig, cloud) - } -} - -func startTaggingController(ctx context.Context, initContext app.ControllerInitContext, completedConfig *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface) (controller.Interface, bool, error) { - // TODO: add in validation for user input for new flags - //if ok, error := verifyTaggingControllerUserInput(completedConfig.ComponentConfig.KubeCloudShared.ClusterCIDR); ok { - // klog.Infof("Will not start the tagging controller due to invalid user input, --configure-cloud-routes: %v", error) - // return nil, false, nil - //} - - // Start the TaggingController - taggingcontroller, err := taggingcontroller.NewTaggingController( - completedConfig.SharedInformers.Core().V1().Nodes(), - completedConfig.ClientBuilder.ClientOrDie(initContext.ClientName), - cloud, - completedConfig.ComponentConfig.KubeCloudShared.NodeMonitorPeriod.Duration) - if err != nil { - klog.Warningf("failed to start tagging controller: %s", err) - return nil, false, nil - } - - go taggingcontroller.Run(ctx) - - return nil, true, nil -} - -func verifyTaggingControllerUserInput(input string) (bool, error) { - if len(input) == 0 { - return false, errors.New("Provide inputs for --resource-tags and --tagging-resources to use the tagging controller.") - } - - return true, nil -} diff --git a/pkg/controllers/options/tagging_controller.go b/pkg/controllers/options/tagging_controller.go new file mode 100644 index 0000000000..e921c3519a --- /dev/null +++ b/pkg/controllers/options/tagging_controller.go @@ -0,0 +1,22 @@ +package options + +import ( + "fmt" + "github.com/spf13/pflag" +) + +type TaggingControllerOptions struct { + Tags map[string]string +} + +func (o *TaggingControllerOptions) AddFlags(fs *pflag.FlagSet) { + fs.StringToStringVar(&o.Tags, "tags", o.Tags, "Tags to apply to AWS resources in the tagging controller.") +} + +func (o *TaggingControllerOptions) Validate() error { + if len(o.Tags) == 0 { + return fmt.Errorf("--tags must not be empty.") + } + + return nil +} diff --git a/pkg/controllers/tagging/tagging_controller.go b/pkg/controllers/tagging/tagging_controller.go index 9c9c8aba21..84cdd6646b 100644 --- a/pkg/controllers/tagging/tagging_controller.go +++ b/pkg/controllers/tagging/tagging_controller.go @@ -15,8 +15,7 @@ package tagging import ( "context" - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/service/ec2" + "fmt" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/labels" utilruntime "k8s.io/apimachinery/pkg/util/runtime" @@ -25,6 +24,7 @@ import ( clientset "k8s.io/client-go/kubernetes" v1lister "k8s.io/client-go/listers/core/v1" cloudprovider "k8s.io/cloud-provider" + "k8s.io/cloud-provider-aws/pkg/controllers/options" awsv1 "k8s.io/cloud-provider-aws/pkg/providers/v1" "k8s.io/klog/v2" "time" @@ -34,10 +34,10 @@ import ( // It periodically check for Node events (creating/deleting) to apply appropriate // tags to resources. type TaggingController struct { - kubeClient clientset.Interface - nodeLister v1lister.NodeLister - - cloud cloudprovider.Interface + controllerOptions options.TaggingControllerOptions + kubeClient clientset.Interface + nodeLister v1lister.NodeLister + cloud *awsv1.Cloud // Value controlling TaggingController monitoring period, i.e. how often does TaggingController // check node list. This value should be lower than nodeMonitorGracePeriod @@ -51,10 +51,10 @@ type TaggingController struct { nodeMap map[string]*v1.Node // Representing the user input for tags - tags string + tags map[string]string // Representing the resources to tag - resources string + resources []string } // NewTaggingController creates a NewTaggingController object @@ -62,18 +62,23 @@ func NewTaggingController( nodeInformer coreinformers.NodeInformer, kubeClient clientset.Interface, cloud cloudprovider.Interface, - nodeMonitorPeriod time.Duration) (*TaggingController, error) { + nodeMonitorPeriod time.Duration, + tags map[string]string) (*TaggingController, error) { + + awsCloud, ok := cloud.(*awsv1.Cloud) + if !ok { + err := fmt.Errorf("tagging controller does not support %v provider", cloud.ProviderName()) + return nil, err + } tc := &TaggingController{ kubeClient: kubeClient, nodeLister: nodeInformer.Lister(), - cloud: cloud, + cloud: awsCloud, nodeMonitorPeriod: nodeMonitorPeriod, taggedNodes: make(map[string]bool), nodeMap: make(map[string]*v1.Node), - // TODO: add controller configs including the new flags - //tags: conf.ControllerCFG.ResourceTags, - //resources: conf.ControllerCFG.TaggingResources, + tags: tags, } return tc, nil } @@ -104,7 +109,6 @@ func (tc *TaggingController) MonitorNodes(ctx context.Context) { } tc.nodeMap[node.GetName()] = node - tc.taggedNodes[node.GetName()] = true } tc.tagNodesResources(nodesToTag) @@ -115,68 +119,72 @@ func (tc *TaggingController) MonitorNodes(ctx context.Context) { } } tc.untagNodeResources(nodesToUntag) - - tc.syncDeletedNodesToTaggedNodes() } // tagNodesResources tag node resources from a list of node // If we want to tag more resources, modify this function appropriately func (tc *TaggingController) tagNodesResources(nodes []*v1.Node) { for _, node := range nodes { - klog.Infof("Tagging resources for node %s with %s.", node.GetName(), tc.tags) - } -} + nodeTagged := false + nodeTagged = tc.tagEc2Instances(node) -func (tc *TaggingController) untagNodeResources(nodes []*v1.Node) { - for _, node := range nodes { - klog.Infof("Untagging resources for node %s with %s.", node.GetName(), tc.tags) + if !nodeTagged { + // Node tagged unsuccessfully, remove from the map + // so that we can try later if it still exists + delete(tc.taggedNodes, node.GetName()) + } } } -// syncDeletedNodes delete (k, v) from taggedNodes -// if it doesn't exist -func (tc *TaggingController) syncDeletedNodesToTaggedNodes() { - for k, v := range tc.taggedNodes { - if v == false { - delete(tc.taggedNodes, k) +// tagEc2Instances applies the provided tags to each EC2 instances in +// the cluster. Return if a node is tagged or not +func (tc *TaggingController) tagEc2Instances(node *v1.Node) bool { + instanceId, err := awsv1.KubernetesInstanceID(node.Spec.ProviderID).MapToAWSInstanceID() + + if err != nil { + klog.Errorf("Error in getting instanceID for node %s, error: %v", node.GetName(), err) + return false + } else { + err := tc.cloud.TagResource(string(instanceId), tc.tags) + + if err != nil { + klog.Errorf("Error in tagging EC2 instance for node %s, error: %v", node.GetName(), err) + return false } } + + return true } -// tagEc2Instances applies the provided tags to each EC2 instances in -// the cluster. -func (tc *TaggingController) tagEc2Instances(nodes []*v1.Node) { - var instanceIds []*string +// untagNodeResources untag node resources from a list of node +// If we want to untag more resources, modify this function appropriately +func (tc *TaggingController) untagNodeResources(nodes []*v1.Node) { for _, node := range nodes { - instanceId, _ := awsv1.KubernetesInstanceID(node.Spec.ProviderID).MapToAWSInstanceID() - instanceIds = append(instanceIds, aws.String(string(instanceId))) - } + nodeUntagged := false + nodeUntagged = tc.untagEc2Instance(node) - tc.tagResources(instanceIds) + if nodeUntagged { + delete(tc.taggedNodes, node.GetName()) + } + } } -// TODO: call EC2 to tag instances -func (tc *TaggingController) tagResources(resourceIds []*string) { - //request := &ec2.CreateTagsInput{ - // Resources: resourceIds, - // Tags: tc.getTagsFromInputs(), - //} - // - //_, error := awsv1..EC2.CreateTags(request) - // - //if error != nil { - // klog.Errorf("Error occurred trying to tag resources, %s", error) - //} -} +// untagEc2Instances deletes the provided tags to each EC2 instances in +// the cluster. Return if a node is tagged or not +func (tc *TaggingController) untagEc2Instance(node *v1.Node) bool { + instanceId, err := awsv1.KubernetesInstanceID(node.Spec.ProviderID).MapToAWSInstanceID() -// Sample function demonstrating that we'll get the tag list from user -func (tc *TaggingController) getTagsFromInputs() []*ec2.Tag { - var awsTags []*ec2.Tag - tag := &ec2.Tag{ - Key: aws.String("Sample Key"), - Value: aws.String("Sample value"), + if err != nil { + klog.Errorf("Error in getting instanceID for node %s, error: %v", node.GetName(), err) + return false + } else { + err := tc.cloud.UntagResource(string(instanceId), tc.tags) + + if err != nil { + klog.Errorf("Error in untagging EC2 instance for node %s, error: %v", node.GetName(), err) + return false + } } - awsTags = append(awsTags, tag) - return awsTags + return true } diff --git a/pkg/controllers/tagging/tagging_controller_wrapper.go b/pkg/controllers/tagging/tagging_controller_wrapper.go new file mode 100644 index 0000000000..c852f333af --- /dev/null +++ b/pkg/controllers/tagging/tagging_controller_wrapper.go @@ -0,0 +1,54 @@ +package tagging + +import ( + "context" + + cloudprovider "k8s.io/cloud-provider" + "k8s.io/cloud-provider/app" + cloudcontrollerconfig "k8s.io/cloud-provider/app/config" + genericcontrollermanager "k8s.io/controller-manager/app" + "k8s.io/controller-manager/controller" + "k8s.io/klog/v2" + + "k8s.io/cloud-provider-aws/pkg/controllers/options" +) + +const ( + TaggingControllerClientName = "tagging-controller" + TaggingControllerKey = "tagging" +) + +type TaggingControllerWrapper struct { + Options options.TaggingControllerOptions +} + +// StartTaggingControllerWrapper is used to take cloud config as input and start the tagging controller +func (tc *TaggingControllerWrapper) StartTaggingControllerWrapper(initContext app.ControllerInitContext, completedConfig *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface) app.InitFunc { + return func(ctx context.Context, controllerContext genericcontrollermanager.ControllerContext) (controller.Interface, bool, error) { + return tc.startTaggingController(ctx, initContext, completedConfig, cloud) + } +} + +func (tc *TaggingControllerWrapper) startTaggingController(ctx context.Context, initContext app.ControllerInitContext, completedConfig *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface) (controller.Interface, bool, error) { + err := tc.Options.Validate() + if err != nil { + klog.Fatal("Tagging controller inputs are not properly set.") + } + + // Start the TaggingController + taggingcontroller, err := NewTaggingController( + completedConfig.SharedInformers.Core().V1().Nodes(), + completedConfig.ClientBuilder.ClientOrDie(initContext.ClientName), + cloud, + completedConfig.ComponentConfig.KubeCloudShared.NodeMonitorPeriod.Duration, + tc.Options.Tags) + + if err != nil { + klog.Warningf("failed to start tagging controller: %s", err) + return nil, false, nil + } + + go taggingcontroller.Run(ctx) + + return nil, true, nil +} diff --git a/pkg/providers/v1/aws.go b/pkg/providers/v1/aws.go index c74eef1199..985c569668 100644 --- a/pkg/providers/v1/aws.go +++ b/pkg/providers/v1/aws.go @@ -347,6 +347,7 @@ type EC2 interface { DescribeSubnets(*ec2.DescribeSubnetsInput) ([]*ec2.Subnet, error) CreateTags(*ec2.CreateTagsInput) (*ec2.CreateTagsOutput, error) + DeleteTags(input *ec2.DeleteTagsInput) (*ec2.DeleteTagsOutput, error) DescribeRouteTables(request *ec2.DescribeRouteTablesInput) ([]*ec2.RouteTable, error) CreateRoute(request *ec2.CreateRouteInput) (*ec2.CreateRouteOutput, error) @@ -1108,6 +1109,14 @@ func (s *awsSdkEC2) CreateTags(request *ec2.CreateTagsInput) (*ec2.CreateTagsOut return resp, err } +func (s *awsSdkEC2) DeleteTags(request *ec2.DeleteTagsInput) (*ec2.DeleteTagsOutput, error) { + requestTime := time.Now() + resp, err := s.ec2.DeleteTags(request) + timeTaken := time.Since(requestTime).Seconds() + recordAWSMetric("create_tags", timeTaken, err) + return resp, err +} + func (s *awsSdkEC2) DescribeRouteTables(request *ec2.DescribeRouteTablesInput) ([]*ec2.RouteTable, error) { results := []*ec2.RouteTable{} var nextToken *string diff --git a/pkg/providers/v1/aws_fakes.go b/pkg/providers/v1/aws_fakes.go index 0113c55554..5272e74c88 100644 --- a/pkg/providers/v1/aws_fakes.go +++ b/pkg/providers/v1/aws_fakes.go @@ -269,6 +269,11 @@ func (ec2i *FakeEC2Impl) CreateTags(*ec2.CreateTagsInput) (*ec2.CreateTagsOutput panic("Not implemented") } +// DeleteTags is not implemented but is required for interface conformance +func (ec2i *FakeEC2Impl) DeleteTags(input *ec2.DeleteTagsInput) (*ec2.DeleteTagsOutput, error) { + panic("Not implemented") +} + // DescribeRouteTables returns fake route table descriptions func (ec2i *FakeEC2Impl) DescribeRouteTables(request *ec2.DescribeRouteTablesInput) ([]*ec2.RouteTable, error) { ec2i.DescribeRouteTablesInput = request diff --git a/pkg/providers/v1/tags.go b/pkg/providers/v1/tags.go index 9ec0cf67e2..acced37901 100644 --- a/pkg/providers/v1/tags.go +++ b/pkg/providers/v1/tags.go @@ -303,3 +303,48 @@ func (t *awsTagging) buildTags(lifecycle ResourceLifecycle, additionalTags map[s func (t *awsTagging) clusterID() string { return t.ClusterID } + +func (c *Cloud) TagResource(resourceId string, tags map[string]string) error { + request := &ec2.CreateTagsInput{ + Resources: []*string{aws.String(resourceId)}, + Tags: buildAwsTags(tags), + } + + _, err := c.ec2.CreateTags(request) + + if err != nil { + klog.Errorf("Error occurred trying to tag resources, %v", err) + return err + } + + return nil +} + +func (c *Cloud) UntagResource(resourceId string, tags map[string]string) error { + request := &ec2.DeleteTagsInput{ + Resources: []*string{aws.String(resourceId)}, + Tags: buildAwsTags(tags), + } + + _, err := c.ec2.DeleteTags(request) + + if err != nil { + klog.Errorf("Error occurred trying to untag resources, %v", err) + return err + } + + return nil +} + +func buildAwsTags(tags map[string]string) []*ec2.Tag { + var awsTags []*ec2.Tag + for k, v := range tags { + newTag := &ec2.Tag{ + Key: aws.String(k), + Value: aws.String(v), + } + awsTags = append(awsTags, newTag) + } + + return awsTags +} From 93df23e37079d1763553016944ede095aad6c7f5 Mon Sep 17 00:00:00 2001 From: Nguyen Dinh <92940366+nguyenkndinh@users.noreply.github.com> Date: Tue, 22 Mar 2022 15:24:13 -0700 Subject: [PATCH 16/43] Disabled the tagging controller by default --- cmd/aws-cloud-controller-manager/main.go | 14 +++++--------- pkg/controllers/options/tagging_controller.go | 2 +- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/cmd/aws-cloud-controller-manager/main.go b/cmd/aws-cloud-controller-manager/main.go index 5a356f4b85..1dddfab45e 100644 --- a/cmd/aws-cloud-controller-manager/main.go +++ b/cmd/aws-cloud-controller-manager/main.go @@ -26,13 +26,9 @@ limitations under the License. package main import ( - "k8s.io/cloud-provider-aws/pkg/controllers/tagging" - "math/rand" - "os" - "time" - "k8s.io/apimachinery/pkg/util/wait" cloudprovider "k8s.io/cloud-provider" + "k8s.io/cloud-provider-aws/pkg/controllers/tagging" awsv1 "k8s.io/cloud-provider-aws/pkg/providers/v1" awsv2 "k8s.io/cloud-provider-aws/pkg/providers/v2" "k8s.io/cloud-provider/app" @@ -42,6 +38,9 @@ import ( _ "k8s.io/component-base/metrics/prometheus/clientgo" // for client metric registration _ "k8s.io/component-base/metrics/prometheus/version" // for version metric registration "k8s.io/klog/v2" + "math/rand" + "os" + "time" cloudcontrollerconfig "k8s.io/cloud-provider/app/config" ) @@ -76,10 +75,7 @@ func main() { } controllerInitializers[tagging.TaggingControllerKey] = taggingControllerConstructor - - // TODO: remove the following line to enable the route controller - delete(controllerInitializers, "route") - + app.ControllersDisabledByDefault.Insert(tagging.TaggingControllerKey) command := app.NewCloudControllerManagerCommand(opts, cloudInitializer, controllerInitializers, fss, wait.NeverStop) if err := command.Execute(); err != nil { diff --git a/pkg/controllers/options/tagging_controller.go b/pkg/controllers/options/tagging_controller.go index e921c3519a..be217696db 100644 --- a/pkg/controllers/options/tagging_controller.go +++ b/pkg/controllers/options/tagging_controller.go @@ -15,7 +15,7 @@ func (o *TaggingControllerOptions) AddFlags(fs *pflag.FlagSet) { func (o *TaggingControllerOptions) Validate() error { if len(o.Tags) == 0 { - return fmt.Errorf("--tags must not be empty.") + return fmt.Errorf("--tags must not be empty") } return nil From 4e273a7ba96c5b52cac8427bf216d53f9b290e52 Mon Sep 17 00:00:00 2001 From: Nguyen Dinh <92940366+nguyenkndinh@users.noreply.github.com> Date: Tue, 22 Mar 2022 19:28:13 -0700 Subject: [PATCH 17/43] Updated test structure --- pkg/controllers/tagging/tagging_controller.go | 1 + .../tagging/tagging_controller_test.go | 37 +++++++++++++------ pkg/providers/v1/aws.go | 4 ++ pkg/providers/v1/aws_fakes.go | 6 +-- pkg/providers/v1/tags.go | 4 +- 5 files changed, 36 insertions(+), 16 deletions(-) diff --git a/pkg/controllers/tagging/tagging_controller.go b/pkg/controllers/tagging/tagging_controller.go index 84cdd6646b..fbc8af1674 100644 --- a/pkg/controllers/tagging/tagging_controller.go +++ b/pkg/controllers/tagging/tagging_controller.go @@ -109,6 +109,7 @@ func (tc *TaggingController) MonitorNodes(ctx context.Context) { } tc.nodeMap[node.GetName()] = node + tc.taggedNodes[node.GetName()] = true } tc.tagNodesResources(nodesToTag) diff --git a/pkg/controllers/tagging/tagging_controller_test.go b/pkg/controllers/tagging/tagging_controller_test.go index 740b9227e5..807f82475c 100644 --- a/pkg/controllers/tagging/tagging_controller_test.go +++ b/pkg/controllers/tagging/tagging_controller_test.go @@ -21,16 +21,17 @@ import ( coreinformers "k8s.io/client-go/informers/core/v1" "k8s.io/client-go/kubernetes/fake" "k8s.io/client-go/tools/record" - fakecloud "k8s.io/cloud-provider/fake" + awsv1 "k8s.io/cloud-provider-aws/pkg/providers/v1" "k8s.io/klog/v2" "testing" "time" ) +const TestClusterID = "clusterid.test" + func Test_NodesJoining(t *testing.T) { testcases := []struct { name string - fakeCloud *fakecloud.Cloud currNode *v1.Node taggingController TaggingController noOfNodes int @@ -42,9 +43,9 @@ func Test_NodesJoining(t *testing.T) { Name: "node0", CreationTimestamp: metav1.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC), }, - }, - fakeCloud: &fakecloud.Cloud{ - ExistsByProviderID: false, + Spec: v1.NodeSpec{ + ProviderID: "i-00000", + }, }, taggingController: TaggingController{ taggedNodes: make(map[string]bool), @@ -59,9 +60,9 @@ func Test_NodesJoining(t *testing.T) { Name: "node1", CreationTimestamp: metav1.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC), }, - }, - fakeCloud: &fakecloud.Cloud{ - ExistsByProviderID: false, + Spec: v1.NodeSpec{ + ProviderID: "i-00001", + }, }, taggingController: TaggingController{ taggedNodes: map[string]bool{ @@ -73,6 +74,9 @@ func Test_NodesJoining(t *testing.T) { Name: "node0", CreationTimestamp: metav1.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC), }, + Spec: v1.NodeSpec{ + ProviderID: "i-00000", + }, }, }, }, @@ -85,9 +89,9 @@ func Test_NodesJoining(t *testing.T) { Name: "node2", CreationTimestamp: metav1.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC), }, - }, - fakeCloud: &fakecloud.Cloud{ - ExistsByProviderID: false, + Spec: v1.NodeSpec{ + ProviderID: "i-00002", + }, }, taggingController: TaggingController{ taggedNodes: map[string]bool{ @@ -100,12 +104,18 @@ func Test_NodesJoining(t *testing.T) { Name: "node0", CreationTimestamp: metav1.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC), }, + Spec: v1.NodeSpec{ + ProviderID: "i-00000", + }, }, "node1": { ObjectMeta: metav1.ObjectMeta{ Name: "node1", CreationTimestamp: metav1.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC), }, + Spec: v1.NodeSpec{ + ProviderID: "i-00001", + }, }, }, }, @@ -113,6 +123,9 @@ func Test_NodesJoining(t *testing.T) { }, } + awsServices := awsv1.NewFakeAWSServices(TestClusterID) + fakeAws, _ := awsv1.NewAWSCloud(awsv1.CloudConfig{}, awsServices) + for _, testcase := range testcases { t.Run(testcase.name, func(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) @@ -128,7 +141,7 @@ func Test_NodesJoining(t *testing.T) { eventBroadcaster := record.NewBroadcaster() testcase.taggingController.nodeLister = nodeInformer.Lister() testcase.taggingController.kubeClient = clientset - testcase.taggingController.cloud = testcase.fakeCloud + testcase.taggingController.cloud = fakeAws testcase.taggingController.nodeMonitorPeriod = 1 * time.Second w := eventBroadcaster.StartLogging(klog.Infof) diff --git a/pkg/providers/v1/aws.go b/pkg/providers/v1/aws.go index 985c569668..0ec6f7eab7 100644 --- a/pkg/providers/v1/aws.go +++ b/pkg/providers/v1/aws.go @@ -1363,6 +1363,10 @@ func newAWSCloud(cfg CloudConfig, awsServices Services) (*Cloud, error) { return awsCloud, nil } +func NewAWSCloud(cfg CloudConfig, awsServices Services) (*Cloud, error) { + return newAWSCloud(cfg, awsServices) +} + // isRegionValid accepts an AWS region name and returns if the region is a // valid region known to the AWS SDK. Considers the region returned from the // EC2 metadata service to be a valid region as it's only available on a host diff --git a/pkg/providers/v1/aws_fakes.go b/pkg/providers/v1/aws_fakes.go index 5272e74c88..ddc2537e8e 100644 --- a/pkg/providers/v1/aws_fakes.go +++ b/pkg/providers/v1/aws_fakes.go @@ -265,13 +265,13 @@ func (ec2i *FakeEC2Impl) RemoveSubnets() { } // CreateTags is not implemented but is required for interface conformance -func (ec2i *FakeEC2Impl) CreateTags(*ec2.CreateTagsInput) (*ec2.CreateTagsOutput, error) { - panic("Not implemented") +func (ec2i *FakeEC2Impl) CreateTags(input *ec2.CreateTagsInput) (*ec2.CreateTagsOutput, error) { + return &ec2.CreateTagsOutput{}, nil } // DeleteTags is not implemented but is required for interface conformance func (ec2i *FakeEC2Impl) DeleteTags(input *ec2.DeleteTagsInput) (*ec2.DeleteTagsOutput, error) { - panic("Not implemented") + return &ec2.DeleteTagsOutput{}, nil } // DescribeRouteTables returns fake route table descriptions diff --git a/pkg/providers/v1/tags.go b/pkg/providers/v1/tags.go index acced37901..954c379fb2 100644 --- a/pkg/providers/v1/tags.go +++ b/pkg/providers/v1/tags.go @@ -310,8 +310,10 @@ func (c *Cloud) TagResource(resourceId string, tags map[string]string) error { Tags: buildAwsTags(tags), } - _, err := c.ec2.CreateTags(request) + res, err := c.ec2.CreateTags(request) + klog.Infof("NGUYEN: %v", res) + if err != nil { klog.Errorf("Error occurred trying to tag resources, %v", err) return err From 5540042b1a213284217d868679a66d2ffef2b3c2 Mon Sep 17 00:00:00 2001 From: Nguyen Dinh <92940366+nguyenkndinh@users.noreply.github.com> Date: Tue, 22 Mar 2022 19:56:23 -0700 Subject: [PATCH 18/43] Making the tests more robust --- .../tagging/tagging_controller_test.go | 72 ++++++++++++++++--- 1 file changed, 61 insertions(+), 11 deletions(-) diff --git a/pkg/controllers/tagging/tagging_controller_test.go b/pkg/controllers/tagging/tagging_controller_test.go index 807f82475c..186ecb9c65 100644 --- a/pkg/controllers/tagging/tagging_controller_test.go +++ b/pkg/controllers/tagging/tagging_controller_test.go @@ -29,12 +29,13 @@ import ( const TestClusterID = "clusterid.test" -func Test_NodesJoining(t *testing.T) { +func Test_NodesJoiningAndLeaving(t *testing.T) { testcases := []struct { name string currNode *v1.Node taggingController TaggingController - noOfNodes int + noOfCurrNodes int + totalNodes int }{ { name: "node0 joins the cluster.", @@ -51,7 +52,8 @@ func Test_NodesJoining(t *testing.T) { taggedNodes: make(map[string]bool), nodeMap: make(map[string]*v1.Node), }, - noOfNodes: 1, + noOfCurrNodes: 1, + totalNodes: 1, }, { name: "node1 joins the cluster, node0 left.", @@ -80,7 +82,8 @@ func Test_NodesJoining(t *testing.T) { }, }, }, - noOfNodes: 1, + noOfCurrNodes: 1, + totalNodes: 2, }, { name: "node2 joins the cluster, node0 and node1 left.", @@ -119,7 +122,58 @@ func Test_NodesJoining(t *testing.T) { }, }, }, - noOfNodes: 1, + noOfCurrNodes: 1, + totalNodes: 3, + }, + { + name: "no new node joins the cluster.", + currNode: &v1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node2", + CreationTimestamp: metav1.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC), + }, + Spec: v1.NodeSpec{ + ProviderID: "i-00002", + }, + }, + taggingController: TaggingController{ + taggedNodes: map[string]bool{ + "node0": true, + "node1": true, + "node2": true, + }, + nodeMap: map[string]*v1.Node{ + "node0": { + ObjectMeta: metav1.ObjectMeta{ + Name: "node0", + CreationTimestamp: metav1.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC), + }, + Spec: v1.NodeSpec{ + ProviderID: "i-00000", + }, + }, + "node1": { + ObjectMeta: metav1.ObjectMeta{ + Name: "node1", + CreationTimestamp: metav1.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC), + }, + Spec: v1.NodeSpec{ + ProviderID: "i-00001", + }, + }, + "node2": { + ObjectMeta: metav1.ObjectMeta{ + Name: "node2", + CreationTimestamp: metav1.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC), + }, + Spec: v1.NodeSpec{ + ProviderID: "i-00002", + }, + }, + }, + }, + noOfCurrNodes: 1, + totalNodes: 3, }, } @@ -147,14 +201,10 @@ func Test_NodesJoining(t *testing.T) { w := eventBroadcaster.StartLogging(klog.Infof) defer w.Stop() - nodeCountBeforeTagging := len(testcase.taggingController.nodeMap) testcase.taggingController.MonitorNodes(ctx) - klog.Infof("testcase.taggingController.taggedNodes %s", testcase.taggingController.taggedNodes) - klog.Errorf("testcase.taggingController.nodeMap %s", testcase.taggingController.nodeMap) - - if len(testcase.taggingController.taggedNodes) != testcase.noOfNodes || len(testcase.taggingController.nodeMap) != nodeCountBeforeTagging+testcase.noOfNodes { - t.Errorf("taggedNodes must contain %d element(s), and nodeMap must contain %d element(s).", testcase.noOfNodes, nodeCountBeforeTagging+testcase.noOfNodes) + if len(testcase.taggingController.taggedNodes) != testcase.noOfCurrNodes || len(testcase.taggingController.nodeMap) != testcase.totalNodes { + t.Errorf("taggedNodes must contain %d element(s), and nodeMap must contain %d element(s).", testcase.noOfCurrNodes, testcase.totalNodes) } }) } From bd3c5b3f9671c73dbd5dccb0ca80ef42999a46de Mon Sep 17 00:00:00 2001 From: Nguyen Dinh <92940366+nguyenkndinh@users.noreply.github.com> Date: Wed, 23 Mar 2022 10:04:13 -0700 Subject: [PATCH 19/43] Renaming the maps in tagging controller --- pkg/controllers/tagging/tagging_controller.go | 28 +++++++++---------- .../tagging/tagging_controller_test.go | 20 ++++++------- 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/pkg/controllers/tagging/tagging_controller.go b/pkg/controllers/tagging/tagging_controller.go index fbc8af1674..3b98f89d45 100644 --- a/pkg/controllers/tagging/tagging_controller.go +++ b/pkg/controllers/tagging/tagging_controller.go @@ -45,10 +45,10 @@ type TaggingController struct { nodeMonitorPeriod time.Duration // A map presenting the node and whether it currently exists - taggedNodes map[string]bool + currNodes map[string]bool - // A map representing nodes that were part of the cluster at any point in time - nodeMap map[string]*v1.Node + // A map representing nodes that were ever part of the cluster + totalNodes map[string]*v1.Node // Representing the user input for tags tags map[string]string @@ -76,8 +76,8 @@ func NewTaggingController( nodeLister: nodeInformer.Lister(), cloud: awsCloud, nodeMonitorPeriod: nodeMonitorPeriod, - taggedNodes: make(map[string]bool), - nodeMap: make(map[string]*v1.Node), + currNodes: make(map[string]bool), + totalNodes: make(map[string]*v1.Node), tags: tags, } return tc, nil @@ -98,25 +98,25 @@ func (tc *TaggingController) MonitorNodes(ctx context.Context) { return } - for k := range tc.taggedNodes { - tc.taggedNodes[k] = false + for k := range tc.currNodes { + tc.currNodes[k] = false } var nodesToTag []*v1.Node for _, node := range nodes { - if _, ok := tc.taggedNodes[node.GetName()]; !ok { + if _, ok := tc.currNodes[node.GetName()]; !ok { nodesToTag = append(nodesToTag, node) } - tc.nodeMap[node.GetName()] = node - tc.taggedNodes[node.GetName()] = true + tc.totalNodes[node.GetName()] = node + tc.currNodes[node.GetName()] = true } tc.tagNodesResources(nodesToTag) var nodesToUntag []*v1.Node - for nodeName, existed := range tc.taggedNodes { + for nodeName, existed := range tc.currNodes { if existed == false { - nodesToUntag = append(nodesToUntag, tc.nodeMap[nodeName]) + nodesToUntag = append(nodesToUntag, tc.totalNodes[nodeName]) } } tc.untagNodeResources(nodesToUntag) @@ -132,7 +132,7 @@ func (tc *TaggingController) tagNodesResources(nodes []*v1.Node) { if !nodeTagged { // Node tagged unsuccessfully, remove from the map // so that we can try later if it still exists - delete(tc.taggedNodes, node.GetName()) + delete(tc.currNodes, node.GetName()) } } } @@ -165,7 +165,7 @@ func (tc *TaggingController) untagNodeResources(nodes []*v1.Node) { nodeUntagged = tc.untagEc2Instance(node) if nodeUntagged { - delete(tc.taggedNodes, node.GetName()) + delete(tc.currNodes, node.GetName()) } } } diff --git a/pkg/controllers/tagging/tagging_controller_test.go b/pkg/controllers/tagging/tagging_controller_test.go index 186ecb9c65..f5f771f0b7 100644 --- a/pkg/controllers/tagging/tagging_controller_test.go +++ b/pkg/controllers/tagging/tagging_controller_test.go @@ -49,8 +49,8 @@ func Test_NodesJoiningAndLeaving(t *testing.T) { }, }, taggingController: TaggingController{ - taggedNodes: make(map[string]bool), - nodeMap: make(map[string]*v1.Node), + currNodes: make(map[string]bool), + totalNodes: make(map[string]*v1.Node), }, noOfCurrNodes: 1, totalNodes: 1, @@ -67,10 +67,10 @@ func Test_NodesJoiningAndLeaving(t *testing.T) { }, }, taggingController: TaggingController{ - taggedNodes: map[string]bool{ + currNodes: map[string]bool{ "node0": true, }, - nodeMap: map[string]*v1.Node{ + totalNodes: map[string]*v1.Node{ "node0": { ObjectMeta: metav1.ObjectMeta{ Name: "node0", @@ -97,11 +97,11 @@ func Test_NodesJoiningAndLeaving(t *testing.T) { }, }, taggingController: TaggingController{ - taggedNodes: map[string]bool{ + currNodes: map[string]bool{ "node0": true, "node1": true, }, - nodeMap: map[string]*v1.Node{ + totalNodes: map[string]*v1.Node{ "node0": { ObjectMeta: metav1.ObjectMeta{ Name: "node0", @@ -137,12 +137,12 @@ func Test_NodesJoiningAndLeaving(t *testing.T) { }, }, taggingController: TaggingController{ - taggedNodes: map[string]bool{ + currNodes: map[string]bool{ "node0": true, "node1": true, "node2": true, }, - nodeMap: map[string]*v1.Node{ + totalNodes: map[string]*v1.Node{ "node0": { ObjectMeta: metav1.ObjectMeta{ Name: "node0", @@ -203,8 +203,8 @@ func Test_NodesJoiningAndLeaving(t *testing.T) { testcase.taggingController.MonitorNodes(ctx) - if len(testcase.taggingController.taggedNodes) != testcase.noOfCurrNodes || len(testcase.taggingController.nodeMap) != testcase.totalNodes { - t.Errorf("taggedNodes must contain %d element(s), and nodeMap must contain %d element(s).", testcase.noOfCurrNodes, testcase.totalNodes) + if len(testcase.taggingController.currNodes) != testcase.noOfCurrNodes || len(testcase.taggingController.totalNodes) != testcase.totalNodes { + t.Errorf("currNodes must contain %d element(s), and totalNodes must contain %d element(s).", testcase.noOfCurrNodes, testcase.totalNodes) } }) } From 2501748d105c6615c518295045176268d9efab59 Mon Sep 17 00:00:00 2001 From: Nguyen Dinh <92940366+nguyenkndinh@users.noreply.github.com> Date: Wed, 23 Mar 2022 10:33:04 -0700 Subject: [PATCH 20/43] Refactoring names and remove debugging logs --- pkg/controllers/tagging/tagging_controller.go | 17 ++++++++--------- pkg/providers/v1/tags.go | 4 +--- 2 files changed, 9 insertions(+), 12 deletions(-) diff --git a/pkg/controllers/tagging/tagging_controller.go b/pkg/controllers/tagging/tagging_controller.go index 3b98f89d45..0f28524ca5 100644 --- a/pkg/controllers/tagging/tagging_controller.go +++ b/pkg/controllers/tagging/tagging_controller.go @@ -83,8 +83,8 @@ func NewTaggingController( return tc, nil } -// Run will start the controller to tag resources attached to a cluster -// and untag resources detached from a cluster. +// Run will start the controller to tag resources attached to the cluster +// and untag resources detached from the cluster. func (tc *TaggingController) Run(ctx context.Context) { defer utilruntime.HandleCrash() @@ -94,8 +94,7 @@ func (tc *TaggingController) Run(ctx context.Context) { func (tc *TaggingController) MonitorNodes(ctx context.Context) { nodes, err := tc.nodeLister.List(labels.Everything()) if err != nil { - klog.Errorf("error listing nodes: %s", err) - return + klog.Fatalf("error listing nodes: %s", err) } for k := range tc.currNodes { @@ -122,7 +121,7 @@ func (tc *TaggingController) MonitorNodes(ctx context.Context) { tc.untagNodeResources(nodesToUntag) } -// tagNodesResources tag node resources from a list of node +// tagNodesResources tag node resources from a list of nodes // If we want to tag more resources, modify this function appropriately func (tc *TaggingController) tagNodesResources(nodes []*v1.Node) { for _, node := range nodes { @@ -130,15 +129,15 @@ func (tc *TaggingController) tagNodesResources(nodes []*v1.Node) { nodeTagged = tc.tagEc2Instances(node) if !nodeTagged { - // Node tagged unsuccessfully, remove from the map + // Node tagged unsuccessfully, remove from currNodes // so that we can try later if it still exists delete(tc.currNodes, node.GetName()) } } } -// tagEc2Instances applies the provided tags to each EC2 instances in -// the cluster. Return if a node is tagged or not +// tagEc2Instances applies the provided tags to each EC2 instance in +// the cluster. Return a boolean value representing if a node is tagged or not func (tc *TaggingController) tagEc2Instances(node *v1.Node) bool { instanceId, err := awsv1.KubernetesInstanceID(node.Spec.ProviderID).MapToAWSInstanceID() @@ -157,7 +156,7 @@ func (tc *TaggingController) tagEc2Instances(node *v1.Node) bool { return true } -// untagNodeResources untag node resources from a list of node +// untagNodeResources untag node resources from a list of nodes // If we want to untag more resources, modify this function appropriately func (tc *TaggingController) untagNodeResources(nodes []*v1.Node) { for _, node := range nodes { diff --git a/pkg/providers/v1/tags.go b/pkg/providers/v1/tags.go index 954c379fb2..acced37901 100644 --- a/pkg/providers/v1/tags.go +++ b/pkg/providers/v1/tags.go @@ -310,10 +310,8 @@ func (c *Cloud) TagResource(resourceId string, tags map[string]string) error { Tags: buildAwsTags(tags), } - res, err := c.ec2.CreateTags(request) + _, err := c.ec2.CreateTags(request) - klog.Infof("NGUYEN: %v", res) - if err != nil { klog.Errorf("Error occurred trying to tag resources, %v", err) return err From 0f5a59b4b1860c246d3500979a7244605e1b97cd Mon Sep 17 00:00:00 2001 From: Nguyen Dinh <92940366+nguyenkndinh@users.noreply.github.com> Date: Wed, 23 Mar 2022 11:44:44 -0700 Subject: [PATCH 21/43] Add failure test cases for when EC2 return error --- .../tagging/tagging_controller_test.go | 110 +++++++++++++++--- pkg/providers/v1/aws_fakes.go | 11 ++ 2 files changed, 106 insertions(+), 15 deletions(-) diff --git a/pkg/controllers/tagging/tagging_controller_test.go b/pkg/controllers/tagging/tagging_controller_test.go index f5f771f0b7..bf5c9a07a2 100644 --- a/pkg/controllers/tagging/tagging_controller_test.go +++ b/pkg/controllers/tagging/tagging_controller_test.go @@ -31,11 +31,11 @@ const TestClusterID = "clusterid.test" func Test_NodesJoiningAndLeaving(t *testing.T) { testcases := []struct { - name string - currNode *v1.Node - taggingController TaggingController - noOfCurrNodes int - totalNodes int + name string + currNode *v1.Node + taggingController TaggingController + noOfToBeTaggedNodes int + totalNodes int }{ { name: "node0 joins the cluster.", @@ -52,8 +52,8 @@ func Test_NodesJoiningAndLeaving(t *testing.T) { currNodes: make(map[string]bool), totalNodes: make(map[string]*v1.Node), }, - noOfCurrNodes: 1, - totalNodes: 1, + noOfToBeTaggedNodes: 1, + totalNodes: 1, }, { name: "node1 joins the cluster, node0 left.", @@ -82,8 +82,8 @@ func Test_NodesJoiningAndLeaving(t *testing.T) { }, }, }, - noOfCurrNodes: 1, - totalNodes: 2, + noOfToBeTaggedNodes: 1, + totalNodes: 2, }, { name: "node2 joins the cluster, node0 and node1 left.", @@ -122,8 +122,8 @@ func Test_NodesJoiningAndLeaving(t *testing.T) { }, }, }, - noOfCurrNodes: 1, - totalNodes: 3, + noOfToBeTaggedNodes: 1, + totalNodes: 3, }, { name: "no new node joins the cluster.", @@ -172,8 +172,88 @@ func Test_NodesJoiningAndLeaving(t *testing.T) { }, }, }, - noOfCurrNodes: 1, - totalNodes: 3, + noOfToBeTaggedNodes: 1, + totalNodes: 3, + }, + { + name: "node 3 joins the cluster but failed to be tagged.", + currNode: &v1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node3", + CreationTimestamp: metav1.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC), + }, + Spec: v1.NodeSpec{ + ProviderID: "i-error", + }, + }, + taggingController: TaggingController{ + currNodes: map[string]bool{ + "node0": true, + "node1": true, + "node2": true, + }, + totalNodes: map[string]*v1.Node{ + "node0": { + ObjectMeta: metav1.ObjectMeta{ + Name: "node0", + CreationTimestamp: metav1.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC), + }, + Spec: v1.NodeSpec{ + ProviderID: "i-00000", + }, + }, + "node1": { + ObjectMeta: metav1.ObjectMeta{ + Name: "node1", + CreationTimestamp: metav1.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC), + }, + Spec: v1.NodeSpec{ + ProviderID: "i-00001", + }, + }, + "node2": { + ObjectMeta: metav1.ObjectMeta{ + Name: "node2", + CreationTimestamp: metav1.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC), + }, + Spec: v1.NodeSpec{ + ProviderID: "i-00002", + }, + }, + }, + }, + noOfToBeTaggedNodes: 0, + totalNodes: 4, + }, + { + name: "node 1 joins the cluster, node 0 left but failed to be untagged.", + currNode: &v1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node1", + CreationTimestamp: metav1.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC), + }, + Spec: v1.NodeSpec{ + ProviderID: "i-0001", + }, + }, + taggingController: TaggingController{ + currNodes: map[string]bool{ + "node0": true, + }, + totalNodes: map[string]*v1.Node{ + "node0": { + ObjectMeta: metav1.ObjectMeta{ + Name: "node0", + CreationTimestamp: metav1.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC), + }, + Spec: v1.NodeSpec{ + ProviderID: "i-error", + }, + }, + }, + }, + noOfToBeTaggedNodes: 2, + totalNodes: 2, }, } @@ -203,8 +283,8 @@ func Test_NodesJoiningAndLeaving(t *testing.T) { testcase.taggingController.MonitorNodes(ctx) - if len(testcase.taggingController.currNodes) != testcase.noOfCurrNodes || len(testcase.taggingController.totalNodes) != testcase.totalNodes { - t.Errorf("currNodes must contain %d element(s), and totalNodes must contain %d element(s).", testcase.noOfCurrNodes, testcase.totalNodes) + if len(testcase.taggingController.currNodes) != testcase.noOfToBeTaggedNodes || len(testcase.taggingController.totalNodes) != testcase.totalNodes { + t.Errorf("currNodes must contain %d element(s), and totalNodes must contain %d element(s).", testcase.noOfToBeTaggedNodes, testcase.totalNodes) } }) } diff --git a/pkg/providers/v1/aws_fakes.go b/pkg/providers/v1/aws_fakes.go index ddc2537e8e..a4a43e9791 100644 --- a/pkg/providers/v1/aws_fakes.go +++ b/pkg/providers/v1/aws_fakes.go @@ -19,6 +19,7 @@ limitations under the License. package aws import ( + "errors" "fmt" "sort" "strings" @@ -266,11 +267,21 @@ func (ec2i *FakeEC2Impl) RemoveSubnets() { // CreateTags is not implemented but is required for interface conformance func (ec2i *FakeEC2Impl) CreateTags(input *ec2.CreateTagsInput) (*ec2.CreateTagsOutput, error) { + for _, id := range input.Resources { + if *id == "i-error" { + return nil, errors.New("Unable to tag") + } + } return &ec2.CreateTagsOutput{}, nil } // DeleteTags is not implemented but is required for interface conformance func (ec2i *FakeEC2Impl) DeleteTags(input *ec2.DeleteTagsInput) (*ec2.DeleteTagsOutput, error) { + for _, id := range input.Resources { + if *id == "i-error" { + return nil, errors.New("Unable to remove tag") + } + } return &ec2.DeleteTagsOutput{}, nil } From 1d0c7372f972b8a8daf500d0d10688142d4e0923 Mon Sep 17 00:00:00 2001 From: Nguyen Dinh <92940366+nguyenkndinh@users.noreply.github.com> Date: Thu, 24 Mar 2022 13:37:52 -0700 Subject: [PATCH 22/43] adding details for --resources * get user input for resources * Add better testing for failures to add flags * fix a small issue with --resources * finalize the --resources --- docs/tagging_controller.md | 7 ++++ pkg/controllers/options/resources.go | 9 ++++ pkg/controllers/options/tagging_controller.go | 27 +++++++++++- pkg/controllers/tagging/tagging_controller.go | 42 ++++++++++++------- .../tagging/tagging_controller_test.go | 19 +++++---- .../tagging/tagging_controller_wrapper.go | 5 ++- pkg/providers/v1/aws.go | 2 +- 7 files changed, 84 insertions(+), 27 deletions(-) create mode 100644 docs/tagging_controller.md create mode 100644 pkg/controllers/options/resources.go diff --git a/docs/tagging_controller.md b/docs/tagging_controller.md new file mode 100644 index 0000000000..86317624c2 --- /dev/null +++ b/docs/tagging_controller.md @@ -0,0 +1,7 @@ +# The Tagging Controller + +The tagging controller is responsible for tagging and untagging node resources when it joins and leaves the cluster respectively. It can add and remove tags based on user input. Unlike the existing controllers, the tagging controller is working exclusively with AWS as we want to tag the resources (EC instances for example). For functionalities used by the controller, we primarily use `CreateTags` and `DeleteTags` from `EC2`. + +| Flag | Valid Values | Default | Description | +|------| --- | --- | --- | +| tags | Comma-separated list of key=value | - | A comma-separated list of key-value pairs which will be recorded as nodes' additional tags. For example: "Key1=Val1,Key2=Val2,KeyNoVal1=,KeyNoVal2" | \ No newline at end of file diff --git a/pkg/controllers/options/resources.go b/pkg/controllers/options/resources.go new file mode 100644 index 0000000000..19cdd89ff6 --- /dev/null +++ b/pkg/controllers/options/resources.go @@ -0,0 +1,9 @@ +package options + +const ( + Instance string = "instance" +) + +var SupportedResources = map[string]string{ + "instance": Instance, +} diff --git a/pkg/controllers/options/tagging_controller.go b/pkg/controllers/options/tagging_controller.go index be217696db..21244b9bc7 100644 --- a/pkg/controllers/options/tagging_controller.go +++ b/pkg/controllers/options/tagging_controller.go @@ -1,3 +1,16 @@ +/* +Copyright 2016 The Kubernetes Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package options import ( @@ -6,11 +19,13 @@ import ( ) type TaggingControllerOptions struct { - Tags map[string]string + Tags map[string]string + Resources []string } func (o *TaggingControllerOptions) AddFlags(fs *pflag.FlagSet) { fs.StringToStringVar(&o.Tags, "tags", o.Tags, "Tags to apply to AWS resources in the tagging controller.") + fs.StringArrayVar(&o.Resources, "resources", o.Resources, "AWS resources name to add/remove tags in the tagging controller.") } func (o *TaggingControllerOptions) Validate() error { @@ -18,5 +33,15 @@ func (o *TaggingControllerOptions) Validate() error { return fmt.Errorf("--tags must not be empty") } + if len(o.Resources) == 0 { + return fmt.Errorf("--resources must not be empty") + } + + for _, r := range o.Resources { + if _, ok := SupportedResources[r]; !ok { + return fmt.Errorf("%s is not a supported resource", r) + } + } + return nil } diff --git a/pkg/controllers/tagging/tagging_controller.go b/pkg/controllers/tagging/tagging_controller.go index 0f28524ca5..4ebbe9d10d 100644 --- a/pkg/controllers/tagging/tagging_controller.go +++ b/pkg/controllers/tagging/tagging_controller.go @@ -25,6 +25,7 @@ import ( v1lister "k8s.io/client-go/listers/core/v1" cloudprovider "k8s.io/cloud-provider" "k8s.io/cloud-provider-aws/pkg/controllers/options" + opt "k8s.io/cloud-provider-aws/pkg/controllers/options" awsv1 "k8s.io/cloud-provider-aws/pkg/providers/v1" "k8s.io/klog/v2" "time" @@ -45,7 +46,7 @@ type TaggingController struct { nodeMonitorPeriod time.Duration // A map presenting the node and whether it currently exists - currNodes map[string]bool + currentNodes map[string]bool // A map representing nodes that were ever part of the cluster totalNodes map[string]*v1.Node @@ -63,7 +64,8 @@ func NewTaggingController( kubeClient clientset.Interface, cloud cloudprovider.Interface, nodeMonitorPeriod time.Duration, - tags map[string]string) (*TaggingController, error) { + tags map[string]string, + resources []string) (*TaggingController, error) { awsCloud, ok := cloud.(*awsv1.Cloud) if !ok { @@ -76,9 +78,10 @@ func NewTaggingController( nodeLister: nodeInformer.Lister(), cloud: awsCloud, nodeMonitorPeriod: nodeMonitorPeriod, - currNodes: make(map[string]bool), + currentNodes: make(map[string]bool), totalNodes: make(map[string]*v1.Node), tags: tags, + resources: resources, } return tc, nil } @@ -97,24 +100,24 @@ func (tc *TaggingController) MonitorNodes(ctx context.Context) { klog.Fatalf("error listing nodes: %s", err) } - for k := range tc.currNodes { - tc.currNodes[k] = false + for k := range tc.currentNodes { + tc.currentNodes[k] = false } var nodesToTag []*v1.Node for _, node := range nodes { - if _, ok := tc.currNodes[node.GetName()]; !ok { + if _, ok := tc.currentNodes[node.GetName()]; !ok { nodesToTag = append(nodesToTag, node) } tc.totalNodes[node.GetName()] = node - tc.currNodes[node.GetName()] = true + tc.currentNodes[node.GetName()] = true } tc.tagNodesResources(nodesToTag) var nodesToUntag []*v1.Node - for nodeName, existed := range tc.currNodes { - if existed == false { + for nodeName, existed := range tc.currentNodes { + if !existed { nodesToUntag = append(nodesToUntag, tc.totalNodes[nodeName]) } } @@ -126,12 +129,18 @@ func (tc *TaggingController) MonitorNodes(ctx context.Context) { func (tc *TaggingController) tagNodesResources(nodes []*v1.Node) { for _, node := range nodes { nodeTagged := false - nodeTagged = tc.tagEc2Instances(node) + + for _, resource := range tc.resources { + switch resource { + case opt.Instance: + nodeTagged = tc.tagEc2Instances(node) + } + } if !nodeTagged { - // Node tagged unsuccessfully, remove from currNodes + // Node tagged unsuccessfully, remove from currentNodes // so that we can try later if it still exists - delete(tc.currNodes, node.GetName()) + delete(tc.currentNodes, node.GetName()) } } } @@ -161,10 +170,15 @@ func (tc *TaggingController) tagEc2Instances(node *v1.Node) bool { func (tc *TaggingController) untagNodeResources(nodes []*v1.Node) { for _, node := range nodes { nodeUntagged := false - nodeUntagged = tc.untagEc2Instance(node) + + for _, resource := range tc.resources { + if resource == opt.Instance { + nodeUntagged = tc.untagEc2Instance(node) + } + } if nodeUntagged { - delete(tc.currNodes, node.GetName()) + delete(tc.currentNodes, node.GetName()) } } } diff --git a/pkg/controllers/tagging/tagging_controller_test.go b/pkg/controllers/tagging/tagging_controller_test.go index bf5c9a07a2..5a7fe3ef58 100644 --- a/pkg/controllers/tagging/tagging_controller_test.go +++ b/pkg/controllers/tagging/tagging_controller_test.go @@ -49,8 +49,8 @@ func Test_NodesJoiningAndLeaving(t *testing.T) { }, }, taggingController: TaggingController{ - currNodes: make(map[string]bool), - totalNodes: make(map[string]*v1.Node), + currentNodes: make(map[string]bool), + totalNodes: make(map[string]*v1.Node), }, noOfToBeTaggedNodes: 1, totalNodes: 1, @@ -67,7 +67,7 @@ func Test_NodesJoiningAndLeaving(t *testing.T) { }, }, taggingController: TaggingController{ - currNodes: map[string]bool{ + currentNodes: map[string]bool{ "node0": true, }, totalNodes: map[string]*v1.Node{ @@ -97,7 +97,7 @@ func Test_NodesJoiningAndLeaving(t *testing.T) { }, }, taggingController: TaggingController{ - currNodes: map[string]bool{ + currentNodes: map[string]bool{ "node0": true, "node1": true, }, @@ -137,7 +137,7 @@ func Test_NodesJoiningAndLeaving(t *testing.T) { }, }, taggingController: TaggingController{ - currNodes: map[string]bool{ + currentNodes: map[string]bool{ "node0": true, "node1": true, "node2": true, @@ -187,7 +187,7 @@ func Test_NodesJoiningAndLeaving(t *testing.T) { }, }, taggingController: TaggingController{ - currNodes: map[string]bool{ + currentNodes: map[string]bool{ "node0": true, "node1": true, "node2": true, @@ -237,7 +237,7 @@ func Test_NodesJoiningAndLeaving(t *testing.T) { }, }, taggingController: TaggingController{ - currNodes: map[string]bool{ + currentNodes: map[string]bool{ "node0": true, }, totalNodes: map[string]*v1.Node{ @@ -283,9 +283,10 @@ func Test_NodesJoiningAndLeaving(t *testing.T) { testcase.taggingController.MonitorNodes(ctx) - if len(testcase.taggingController.currNodes) != testcase.noOfToBeTaggedNodes || len(testcase.taggingController.totalNodes) != testcase.totalNodes { - t.Errorf("currNodes must contain %d element(s), and totalNodes must contain %d element(s).", testcase.noOfToBeTaggedNodes, testcase.totalNodes) + if len(testcase.taggingController.currentNodes) != testcase.noOfToBeTaggedNodes || len(testcase.taggingController.totalNodes) != testcase.totalNodes { + t.Errorf("currentNodes must contain %d element(s), and totalNodes must contain %d element(s).", testcase.noOfToBeTaggedNodes, testcase.totalNodes) } + }) } } diff --git a/pkg/controllers/tagging/tagging_controller_wrapper.go b/pkg/controllers/tagging/tagging_controller_wrapper.go index c852f333af..1c402d9f35 100644 --- a/pkg/controllers/tagging/tagging_controller_wrapper.go +++ b/pkg/controllers/tagging/tagging_controller_wrapper.go @@ -32,7 +32,7 @@ func (tc *TaggingControllerWrapper) StartTaggingControllerWrapper(initContext ap func (tc *TaggingControllerWrapper) startTaggingController(ctx context.Context, initContext app.ControllerInitContext, completedConfig *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface) (controller.Interface, bool, error) { err := tc.Options.Validate() if err != nil { - klog.Fatal("Tagging controller inputs are not properly set.") + klog.Fatalf("Tagging controller inputs are not properly set: %v", err) } // Start the TaggingController @@ -41,7 +41,8 @@ func (tc *TaggingControllerWrapper) startTaggingController(ctx context.Context, completedConfig.ClientBuilder.ClientOrDie(initContext.ClientName), cloud, completedConfig.ComponentConfig.KubeCloudShared.NodeMonitorPeriod.Duration, - tc.Options.Tags) + tc.Options.Tags, + tc.Options.Resources) if err != nil { klog.Warningf("failed to start tagging controller: %s", err) diff --git a/pkg/providers/v1/aws.go b/pkg/providers/v1/aws.go index 0ec6f7eab7..a4c6df35c1 100644 --- a/pkg/providers/v1/aws.go +++ b/pkg/providers/v1/aws.go @@ -1113,7 +1113,7 @@ func (s *awsSdkEC2) DeleteTags(request *ec2.DeleteTagsInput) (*ec2.DeleteTagsOut requestTime := time.Now() resp, err := s.ec2.DeleteTags(request) timeTaken := time.Since(requestTime).Seconds() - recordAWSMetric("create_tags", timeTaken, err) + recordAWSMetric("delete_tags", timeTaken, err) return resp, err } From 9be3f283257186fc1512bac8da665aa46c68f483 Mon Sep 17 00:00:00 2001 From: Nguyen Dinh <92940366+nguyenkndinh@users.noreply.github.com> Date: Thu, 24 Mar 2022 13:40:26 -0700 Subject: [PATCH 23/43] add in Copyright message --- pkg/controllers/options/resources.go | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/pkg/controllers/options/resources.go b/pkg/controllers/options/resources.go index 19cdd89ff6..3e63ae3e7d 100644 --- a/pkg/controllers/options/resources.go +++ b/pkg/controllers/options/resources.go @@ -1,3 +1,16 @@ +/* +Copyright 2016 The Kubernetes Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package options const ( From d48b90050d057e1ee6c766a3da710e0420d3c03d Mon Sep 17 00:00:00 2001 From: Nguyen Dinh <92940366+nguyenkndinh@users.noreply.github.com> Date: Thu, 24 Mar 2022 20:39:23 -0700 Subject: [PATCH 24/43] Using NodeInformer and Workqueue for tagging resources --- pkg/controllers/options/resources.go | 1 - pkg/controllers/tagging/tagging_controller.go | 156 +++++++++++------- .../tagging/tagging_controller_test.go | 1 + .../tagging/tagging_controller_wrapper.go | 2 +- 4 files changed, 98 insertions(+), 62 deletions(-) diff --git a/pkg/controllers/options/resources.go b/pkg/controllers/options/resources.go index 3e63ae3e7d..381fcde6f0 100644 --- a/pkg/controllers/options/resources.go +++ b/pkg/controllers/options/resources.go @@ -10,7 +10,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ - package options const ( diff --git a/pkg/controllers/tagging/tagging_controller.go b/pkg/controllers/tagging/tagging_controller.go index 4ebbe9d10d..7170f31862 100644 --- a/pkg/controllers/tagging/tagging_controller.go +++ b/pkg/controllers/tagging/tagging_controller.go @@ -14,15 +14,16 @@ limitations under the License. package tagging import ( - "context" "fmt" v1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/labels" + apierrors "k8s.io/apimachinery/pkg/api/errors" utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/wait" coreinformers "k8s.io/client-go/informers/core/v1" clientset "k8s.io/client-go/kubernetes" v1lister "k8s.io/client-go/listers/core/v1" + "k8s.io/client-go/tools/cache" + "k8s.io/client-go/util/workqueue" cloudprovider "k8s.io/cloud-provider" "k8s.io/cloud-provider-aws/pkg/controllers/options" opt "k8s.io/cloud-provider-aws/pkg/controllers/options" @@ -35,10 +36,12 @@ import ( // It periodically check for Node events (creating/deleting) to apply appropriate // tags to resources. type TaggingController struct { + nodeInformer coreinformers.NodeInformer controllerOptions options.TaggingControllerOptions kubeClient clientset.Interface nodeLister v1lister.NodeLister cloud *awsv1.Cloud + workqueue workqueue.RateLimitingInterface // Value controlling TaggingController monitoring period, i.e. how often does TaggingController // check node list. This value should be lower than nodeMonitorGracePeriod @@ -74,6 +77,7 @@ func NewTaggingController( } tc := &TaggingController{ + nodeInformer: nodeInformer, kubeClient: kubeClient, nodeLister: nodeInformer.Lister(), cloud: awsCloud, @@ -82,123 +86,155 @@ func NewTaggingController( totalNodes: make(map[string]*v1.Node), tags: tags, resources: resources, + workqueue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "Tagging"), } + + // Use shared informer to listen to add/update/delete of nodes. Note that any nodes + // that exist before tagging controller starts will show up in the update method + tc.nodeInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ + AddFunc: tc.enqueueNode, + UpdateFunc: func(oldObj, newObj interface{}) { tc.enqueueNode(newObj) }, + // TODO: maybe use workqueue for this to be more resilient + DeleteFunc: tc.untagNodeResources, + }) + return tc, nil } // Run will start the controller to tag resources attached to the cluster // and untag resources detached from the cluster. -func (tc *TaggingController) Run(ctx context.Context) { +func (tc *TaggingController) Run(stopCh <-chan struct{}) { defer utilruntime.HandleCrash() + defer tc.workqueue.ShutDown() + + klog.Infof("Starting the tagging controller") + go wait.Until(tc.MonitorNodes, tc.nodeMonitorPeriod, stopCh) - wait.UntilWithContext(ctx, tc.MonitorNodes, tc.nodeMonitorPeriod) + <-stopCh } -func (tc *TaggingController) MonitorNodes(ctx context.Context) { - nodes, err := tc.nodeLister.List(labels.Everything()) - if err != nil { - klog.Fatalf("error listing nodes: %s", err) +// MonitorNodes is a long-running function that continuously +// read and process a message on the work queue +func (tc *TaggingController) MonitorNodes() { + obj, shutdown := tc.workqueue.Get() + if shutdown { + return } - for k := range tc.currentNodes { - tc.currentNodes[k] = false - } + err := func(obj interface{}) error { + defer tc.workqueue.Done(obj) - var nodesToTag []*v1.Node - for _, node := range nodes { - if _, ok := tc.currentNodes[node.GetName()]; !ok { - nodesToTag = append(nodesToTag, node) + var key string + var ok bool + if key, ok = obj.(string); !ok { + tc.workqueue.Forget(obj) + utilruntime.HandleError(fmt.Errorf("expected string in workqueue but got %#v", obj)) + return nil } - tc.totalNodes[node.GetName()] = node - tc.currentNodes[node.GetName()] = true - } - tc.tagNodesResources(nodesToTag) + _, nodeName, err := cache.SplitMetaNamespaceKey(key) - var nodesToUntag []*v1.Node - for nodeName, existed := range tc.currentNodes { - if !existed { - nodesToUntag = append(nodesToUntag, tc.totalNodes[nodeName]) + if err != nil { + utilruntime.HandleError(fmt.Errorf("invalid resource key: %s", key)) + return nil + } + + if err := tc.tagNodesResources(nodeName); err != nil { + // Put the item back on the workqueue to handle any transient errors. + tc.workqueue.AddRateLimited(key) + return fmt.Errorf("error tagging '%s': %s, requeuing", key, err.Error()) } + + tc.workqueue.Forget(obj) + return nil + }(obj) + + if err != nil { + utilruntime.HandleError(err) } - tc.untagNodeResources(nodesToUntag) } // tagNodesResources tag node resources from a list of nodes // If we want to tag more resources, modify this function appropriately -func (tc *TaggingController) tagNodesResources(nodes []*v1.Node) { - for _, node := range nodes { - nodeTagged := false - - for _, resource := range tc.resources { - switch resource { - case opt.Instance: - nodeTagged = tc.tagEc2Instances(node) - } +func (tc *TaggingController) tagNodesResources(nodeName string) error { + node, err := tc.nodeInformer.Lister().Get(nodeName) + if err != nil { + if apierrors.IsNotFound(err) { + return nil } - if !nodeTagged { - // Node tagged unsuccessfully, remove from currentNodes - // so that we can try later if it still exists - delete(tc.currentNodes, node.GetName()) + return err + } + + for _, resource := range tc.resources { + switch resource { + case opt.Instance: + err = tc.tagEc2Instances(node) } } + + return err } // tagEc2Instances applies the provided tags to each EC2 instance in -// the cluster. Return a boolean value representing if a node is tagged or not -func (tc *TaggingController) tagEc2Instances(node *v1.Node) bool { +// the cluster. +func (tc *TaggingController) tagEc2Instances(node *v1.Node) error { instanceId, err := awsv1.KubernetesInstanceID(node.Spec.ProviderID).MapToAWSInstanceID() if err != nil { klog.Errorf("Error in getting instanceID for node %s, error: %v", node.GetName(), err) - return false + return err } else { err := tc.cloud.TagResource(string(instanceId), tc.tags) if err != nil { klog.Errorf("Error in tagging EC2 instance for node %s, error: %v", node.GetName(), err) - return false + return err } } - return true + return nil } // untagNodeResources untag node resources from a list of nodes // If we want to untag more resources, modify this function appropriately -func (tc *TaggingController) untagNodeResources(nodes []*v1.Node) { - for _, node := range nodes { - nodeUntagged := false - - for _, resource := range tc.resources { - if resource == opt.Instance { - nodeUntagged = tc.untagEc2Instance(node) - } - } +func (tc *TaggingController) untagNodeResources(obj interface{}) { + var node *v1.Node + var ok bool + if node, ok = obj.(*v1.Node); !ok { + utilruntime.HandleError(fmt.Errorf("unable to get Node object from %v", obj)) + } - if nodeUntagged { - delete(tc.currentNodes, node.GetName()) + for _, resource := range tc.resources { + switch resource { + case opt.Instance: + tc.untagEc2Instance(node) } } } // untagEc2Instances deletes the provided tags to each EC2 instances in -// the cluster. Return if a node is tagged or not -func (tc *TaggingController) untagEc2Instance(node *v1.Node) bool { +// the cluster. +func (tc *TaggingController) untagEc2Instance(node *v1.Node) { instanceId, err := awsv1.KubernetesInstanceID(node.Spec.ProviderID).MapToAWSInstanceID() if err != nil { - klog.Errorf("Error in getting instanceID for node %s, error: %v", node.GetName(), err) - return false + klog.Fatalf("Error in getting instanceID for node %s, error: %v", node.GetName(), err) } else { err := tc.cloud.UntagResource(string(instanceId), tc.tags) if err != nil { - klog.Errorf("Error in untagging EC2 instance for node %s, error: %v", node.GetName(), err) - return false + klog.Fatalf("Error in untagging EC2 instance for node %s, error: %v", node.GetName(), err) } } +} - return true +func (tc *TaggingController) enqueueNode(obj interface{}) { + var key string + var err error + if key, err = cache.MetaNamespaceKeyFunc(obj); err != nil { + utilruntime.HandleError(err) + return + } + tc.workqueue.Add(key) } diff --git a/pkg/controllers/tagging/tagging_controller_test.go b/pkg/controllers/tagging/tagging_controller_test.go index 5a7fe3ef58..3e684e8bae 100644 --- a/pkg/controllers/tagging/tagging_controller_test.go +++ b/pkg/controllers/tagging/tagging_controller_test.go @@ -29,6 +29,7 @@ import ( const TestClusterID = "clusterid.test" +// TODO: rework the test func Test_NodesJoiningAndLeaving(t *testing.T) { testcases := []struct { name string diff --git a/pkg/controllers/tagging/tagging_controller_wrapper.go b/pkg/controllers/tagging/tagging_controller_wrapper.go index 1c402d9f35..f250ab78f0 100644 --- a/pkg/controllers/tagging/tagging_controller_wrapper.go +++ b/pkg/controllers/tagging/tagging_controller_wrapper.go @@ -49,7 +49,7 @@ func (tc *TaggingControllerWrapper) startTaggingController(ctx context.Context, return nil, false, nil } - go taggingcontroller.Run(ctx) + go taggingcontroller.Run(ctx.Done()) return nil, true, nil } From 3b81d1248a5c50d757e4cff42a7580408b503142 Mon Sep 17 00:00:00 2001 From: Nguyen Dinh <92940366+nguyenkndinh@users.noreply.github.com> Date: Fri, 25 Mar 2022 10:44:42 -0700 Subject: [PATCH 25/43] Used workqueue for both tag and untag actions --- pkg/controllers/tagging/tagging_controller.go | 105 ++++++++++++------ 1 file changed, 71 insertions(+), 34 deletions(-) diff --git a/pkg/controllers/tagging/tagging_controller.go b/pkg/controllers/tagging/tagging_controller.go index 7170f31862..f7c31585df 100644 --- a/pkg/controllers/tagging/tagging_controller.go +++ b/pkg/controllers/tagging/tagging_controller.go @@ -29,9 +29,14 @@ import ( opt "k8s.io/cloud-provider-aws/pkg/controllers/options" awsv1 "k8s.io/cloud-provider-aws/pkg/providers/v1" "k8s.io/klog/v2" + "strings" "time" ) +const ( + tag string = "tag" +) + // TaggingController is the controller implementation for tagging cluster resources. // It periodically check for Node events (creating/deleting) to apply appropriate // tags to resources. @@ -92,10 +97,9 @@ func NewTaggingController( // Use shared informer to listen to add/update/delete of nodes. Note that any nodes // that exist before tagging controller starts will show up in the update method tc.nodeInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ - AddFunc: tc.enqueueNode, - UpdateFunc: func(oldObj, newObj interface{}) { tc.enqueueNode(newObj) }, - // TODO: maybe use workqueue for this to be more resilient - DeleteFunc: tc.untagNodeResources, + AddFunc: func(obj interface{}) { tc.enqueueNode(obj, true) }, + UpdateFunc: func(oldObj, newObj interface{}) { tc.enqueueNode(newObj, true) }, + DeleteFunc: func(obj interface{}) { tc.enqueueNode(obj, false) }, }) return tc, nil @@ -132,6 +136,9 @@ func (tc *TaggingController) MonitorNodes() { return nil } + var isTagged bool + isTagged, key = tc.getActionAndKey(key) + _, nodeName, err := cache.SplitMetaNamespaceKey(key) if err != nil { @@ -139,10 +146,26 @@ func (tc *TaggingController) MonitorNodes() { return nil } - if err := tc.tagNodesResources(nodeName); err != nil { - // Put the item back on the workqueue to handle any transient errors. - tc.workqueue.AddRateLimited(key) - return fmt.Errorf("error tagging '%s': %s, requeuing", key, err.Error()) + node, err := tc.nodeInformer.Lister().Get(nodeName) + if err != nil { + if apierrors.IsNotFound(err) { + return nil + } + + return err + } + + if isTagged { + if err := tc.tagNodesResources(node); err != nil { + // Put the item back on the workqueue to handle any transient errors. + tc.workqueue.AddRateLimited(tag + key) + return fmt.Errorf("error tagging '%s': %s, requeuing", key, err.Error()) + } + } else { + if err := tc.untagNodeResources(node); err != nil { + tc.workqueue.AddRateLimited(key) + return fmt.Errorf("error untagging '%s': %s, requeuing", key, err.Error()) + } } tc.workqueue.Forget(obj) @@ -156,29 +179,23 @@ func (tc *TaggingController) MonitorNodes() { // tagNodesResources tag node resources from a list of nodes // If we want to tag more resources, modify this function appropriately -func (tc *TaggingController) tagNodesResources(nodeName string) error { - node, err := tc.nodeInformer.Lister().Get(nodeName) - if err != nil { - if apierrors.IsNotFound(err) { - return nil - } - - return err - } - +func (tc *TaggingController) tagNodesResources(node *v1.Node) error { for _, resource := range tc.resources { switch resource { case opt.Instance: - err = tc.tagEc2Instances(node) + err := tc.tagEc2Instance(node) + if err != nil { + return err + } } } - return err + return nil } // tagEc2Instances applies the provided tags to each EC2 instance in // the cluster. -func (tc *TaggingController) tagEc2Instances(node *v1.Node) error { +func (tc *TaggingController) tagEc2Instance(node *v1.Node) error { instanceId, err := awsv1.KubernetesInstanceID(node.Spec.ProviderID).MapToAWSInstanceID() if err != nil { @@ -198,43 +215,63 @@ func (tc *TaggingController) tagEc2Instances(node *v1.Node) error { // untagNodeResources untag node resources from a list of nodes // If we want to untag more resources, modify this function appropriately -func (tc *TaggingController) untagNodeResources(obj interface{}) { - var node *v1.Node - var ok bool - if node, ok = obj.(*v1.Node); !ok { - utilruntime.HandleError(fmt.Errorf("unable to get Node object from %v", obj)) - } - +func (tc *TaggingController) untagNodeResources(node *v1.Node) error { for _, resource := range tc.resources { switch resource { case opt.Instance: - tc.untagEc2Instance(node) + err := tc.untagEc2Instance(node) + if err != nil { + return err + } } } + + return nil } // untagEc2Instances deletes the provided tags to each EC2 instances in // the cluster. -func (tc *TaggingController) untagEc2Instance(node *v1.Node) { +func (tc *TaggingController) untagEc2Instance(node *v1.Node) error { instanceId, err := awsv1.KubernetesInstanceID(node.Spec.ProviderID).MapToAWSInstanceID() if err != nil { - klog.Fatalf("Error in getting instanceID for node %s, error: %v", node.GetName(), err) + klog.Errorf("Error in getting instanceID for node %s, error: %v", node.GetName(), err) + return err } else { err := tc.cloud.UntagResource(string(instanceId), tc.tags) if err != nil { - klog.Fatalf("Error in untagging EC2 instance for node %s, error: %v", node.GetName(), err) + klog.Errorf("Error in untagging EC2 instance for node %s, error: %v", node.GetName(), err) + return err } } + + return nil } -func (tc *TaggingController) enqueueNode(obj interface{}) { +// enqueueNode takes in the object to enqueue to the workqueue and whether +// the object is to be tagged +func (tc *TaggingController) enqueueNode(obj interface{}, isTagged bool) { var key string var err error if key, err = cache.MetaNamespaceKeyFunc(obj); err != nil { utilruntime.HandleError(err) return } - tc.workqueue.Add(key) + + if isTagged { + tc.workqueue.Add(tag + key) + } else { + tc.workqueue.Add(key) + } +} + +func (tc *TaggingController) getActionAndKey(key string) (bool, string) { + isTagged := false + if strings.HasPrefix(key, tag) { + isTagged = true + key = strings.TrimPrefix(key, tag) + } + + return isTagged, key } From c009efd25e52a0f8ad04db8225c196504b22dd59 Mon Sep 17 00:00:00 2001 From: Nguyen Dinh <92940366+nguyenkndinh@users.noreply.github.com> Date: Fri, 25 Mar 2022 14:16:26 -0700 Subject: [PATCH 26/43] Update docs/tagging_controller.md Co-authored-by: Nicholas Turner <1205393+nckturner@users.noreply.github.com> --- docs/tagging_controller.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/tagging_controller.md b/docs/tagging_controller.md index 86317624c2..74161ab969 100644 --- a/docs/tagging_controller.md +++ b/docs/tagging_controller.md @@ -1,6 +1,6 @@ # The Tagging Controller -The tagging controller is responsible for tagging and untagging node resources when it joins and leaves the cluster respectively. It can add and remove tags based on user input. Unlike the existing controllers, the tagging controller is working exclusively with AWS as we want to tag the resources (EC instances for example). For functionalities used by the controller, we primarily use `CreateTags` and `DeleteTags` from `EC2`. +The tagging controller is responsible for tagging and untagging node resources when they join and leave the cluster, respectively. It can add and remove tags based on user input. Unlike the existing controllers, the tagging controller works exclusively with AWS. The AWS APIs it uses are `ec2:CreateTags` and `ec2:DeleteTags`. | Flag | Valid Values | Default | Description | |------| --- | --- | --- | From 9bb60e81cce30e1dd3c3ba20cde567caf3d4fa21 Mon Sep 17 00:00:00 2001 From: Nguyen Dinh <92940366+nguyenkndinh@users.noreply.github.com> Date: Fri, 25 Mar 2022 18:03:20 -0700 Subject: [PATCH 27/43] Renamed fields in the tagging controller to be more user friendly --- pkg/controllers/options/tagging_controller.go | 8 +- pkg/controllers/tagging/tagging_controller.go | 51 ++-- .../tagging/tagging_controller_test.go | 244 ++---------------- pkg/providers/v1/aws_fakes.go | 28 +- 4 files changed, 74 insertions(+), 257 deletions(-) diff --git a/pkg/controllers/options/tagging_controller.go b/pkg/controllers/options/tagging_controller.go index 21244b9bc7..52cc6593aa 100644 --- a/pkg/controllers/options/tagging_controller.go +++ b/pkg/controllers/options/tagging_controller.go @@ -30,7 +30,7 @@ func (o *TaggingControllerOptions) AddFlags(fs *pflag.FlagSet) { func (o *TaggingControllerOptions) Validate() error { if len(o.Tags) == 0 { - return fmt.Errorf("--tags must not be empty") + return fmt.Errorf("--tags must not be empty and must be a form of key:value") } if len(o.Resources) == 0 { @@ -39,7 +39,11 @@ func (o *TaggingControllerOptions) Validate() error { for _, r := range o.Resources { if _, ok := SupportedResources[r]; !ok { - return fmt.Errorf("%s is not a supported resource", r) + resources := []string{} + for r, _ := range SupportedResources { + resources = append(resources, r) + } + return fmt.Errorf("%s is not a supported resource. Current supported resources %v", r, resources) } } diff --git a/pkg/controllers/tagging/tagging_controller.go b/pkg/controllers/tagging/tagging_controller.go index f7c31585df..46d66fa33b 100644 --- a/pkg/controllers/tagging/tagging_controller.go +++ b/pkg/controllers/tagging/tagging_controller.go @@ -21,11 +21,9 @@ import ( "k8s.io/apimachinery/pkg/util/wait" coreinformers "k8s.io/client-go/informers/core/v1" clientset "k8s.io/client-go/kubernetes" - v1lister "k8s.io/client-go/listers/core/v1" "k8s.io/client-go/tools/cache" "k8s.io/client-go/util/workqueue" cloudprovider "k8s.io/cloud-provider" - "k8s.io/cloud-provider-aws/pkg/controllers/options" opt "k8s.io/cloud-provider-aws/pkg/controllers/options" awsv1 "k8s.io/cloud-provider-aws/pkg/providers/v1" "k8s.io/klog/v2" @@ -34,31 +32,25 @@ import ( ) const ( - tag string = "tag" + // This is a prefix used to recognized if a node in the workqueue + // is to be tagged or not + tagKeyPrefix string = "tagKeyPrefix" ) // TaggingController is the controller implementation for tagging cluster resources. // It periodically check for Node events (creating/deleting) to apply appropriate // tags to resources. type TaggingController struct { - nodeInformer coreinformers.NodeInformer - controllerOptions options.TaggingControllerOptions - kubeClient clientset.Interface - nodeLister v1lister.NodeLister - cloud *awsv1.Cloud - workqueue workqueue.RateLimitingInterface + nodeInformer coreinformers.NodeInformer + kubeClient clientset.Interface + cloud *awsv1.Cloud + workqueue workqueue.RateLimitingInterface // Value controlling TaggingController monitoring period, i.e. how often does TaggingController // check node list. This value should be lower than nodeMonitorGracePeriod // set in controller-manager nodeMonitorPeriod time.Duration - // A map presenting the node and whether it currently exists - currentNodes map[string]bool - - // A map representing nodes that were ever part of the cluster - totalNodes map[string]*v1.Node - // Representing the user input for tags tags map[string]string @@ -84,11 +76,8 @@ func NewTaggingController( tc := &TaggingController{ nodeInformer: nodeInformer, kubeClient: kubeClient, - nodeLister: nodeInformer.Lister(), cloud: awsCloud, nodeMonitorPeriod: nodeMonitorPeriod, - currentNodes: make(map[string]bool), - totalNodes: make(map[string]*v1.Node), tags: tags, resources: resources, workqueue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "Tagging"), @@ -136,8 +125,8 @@ func (tc *TaggingController) MonitorNodes() { return nil } - var isTagged bool - isTagged, key = tc.getActionAndKey(key) + var toBeTagged bool + toBeTagged, key = tc.getActionAndKey(key) _, nodeName, err := cache.SplitMetaNamespaceKey(key) @@ -155,10 +144,10 @@ func (tc *TaggingController) MonitorNodes() { return err } - if isTagged { + if toBeTagged { if err := tc.tagNodesResources(node); err != nil { // Put the item back on the workqueue to handle any transient errors. - tc.workqueue.AddRateLimited(tag + key) + tc.workqueue.AddRateLimited(tagKeyPrefix + key) return fmt.Errorf("error tagging '%s': %s, requeuing", key, err.Error()) } } else { @@ -251,7 +240,7 @@ func (tc *TaggingController) untagEc2Instance(node *v1.Node) error { // enqueueNode takes in the object to enqueue to the workqueue and whether // the object is to be tagged -func (tc *TaggingController) enqueueNode(obj interface{}, isTagged bool) { +func (tc *TaggingController) enqueueNode(obj interface{}, toBeTagged bool) { var key string var err error if key, err = cache.MetaNamespaceKeyFunc(obj); err != nil { @@ -259,19 +248,21 @@ func (tc *TaggingController) enqueueNode(obj interface{}, isTagged bool) { return } - if isTagged { - tc.workqueue.Add(tag + key) + if toBeTagged { + tc.workqueue.Add(tagKeyPrefix + key) } else { tc.workqueue.Add(key) } } +// getActionAndKey from the provided key, check if the object is to be tagged +// and extract that action together with the key func (tc *TaggingController) getActionAndKey(key string) (bool, string) { - isTagged := false - if strings.HasPrefix(key, tag) { - isTagged = true - key = strings.TrimPrefix(key, tag) + toBeTagged := false + if strings.HasPrefix(key, tagKeyPrefix) { + toBeTagged = true + key = strings.TrimPrefix(key, tagKeyPrefix) } - return isTagged, key + return toBeTagged, key } diff --git a/pkg/controllers/tagging/tagging_controller_test.go b/pkg/controllers/tagging/tagging_controller_test.go index 3e684e8bae..ceb12aa6d9 100644 --- a/pkg/controllers/tagging/tagging_controller_test.go +++ b/pkg/controllers/tagging/tagging_controller_test.go @@ -15,12 +15,14 @@ package tagging import ( "context" + "github.com/stretchr/testify/assert" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/informers" coreinformers "k8s.io/client-go/informers/core/v1" "k8s.io/client-go/kubernetes/fake" "k8s.io/client-go/tools/record" + "k8s.io/client-go/util/workqueue" awsv1 "k8s.io/cloud-provider-aws/pkg/providers/v1" "k8s.io/klog/v2" "testing" @@ -29,14 +31,11 @@ import ( const TestClusterID = "clusterid.test" -// TODO: rework the test func Test_NodesJoiningAndLeaving(t *testing.T) { testcases := []struct { - name string - currNode *v1.Node - taggingController TaggingController - noOfToBeTaggedNodes int - totalNodes int + name string + currNode *v1.Node + expectedCalls []string }{ { name: "node0 joins the cluster.", @@ -49,212 +48,7 @@ func Test_NodesJoiningAndLeaving(t *testing.T) { ProviderID: "i-00000", }, }, - taggingController: TaggingController{ - currentNodes: make(map[string]bool), - totalNodes: make(map[string]*v1.Node), - }, - noOfToBeTaggedNodes: 1, - totalNodes: 1, - }, - { - name: "node1 joins the cluster, node0 left.", - currNode: &v1.Node{ - ObjectMeta: metav1.ObjectMeta{ - Name: "node1", - CreationTimestamp: metav1.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC), - }, - Spec: v1.NodeSpec{ - ProviderID: "i-00001", - }, - }, - taggingController: TaggingController{ - currentNodes: map[string]bool{ - "node0": true, - }, - totalNodes: map[string]*v1.Node{ - "node0": { - ObjectMeta: metav1.ObjectMeta{ - Name: "node0", - CreationTimestamp: metav1.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC), - }, - Spec: v1.NodeSpec{ - ProviderID: "i-00000", - }, - }, - }, - }, - noOfToBeTaggedNodes: 1, - totalNodes: 2, - }, - { - name: "node2 joins the cluster, node0 and node1 left.", - currNode: &v1.Node{ - ObjectMeta: metav1.ObjectMeta{ - Name: "node2", - CreationTimestamp: metav1.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC), - }, - Spec: v1.NodeSpec{ - ProviderID: "i-00002", - }, - }, - taggingController: TaggingController{ - currentNodes: map[string]bool{ - "node0": true, - "node1": true, - }, - totalNodes: map[string]*v1.Node{ - "node0": { - ObjectMeta: metav1.ObjectMeta{ - Name: "node0", - CreationTimestamp: metav1.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC), - }, - Spec: v1.NodeSpec{ - ProviderID: "i-00000", - }, - }, - "node1": { - ObjectMeta: metav1.ObjectMeta{ - Name: "node1", - CreationTimestamp: metav1.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC), - }, - Spec: v1.NodeSpec{ - ProviderID: "i-00001", - }, - }, - }, - }, - noOfToBeTaggedNodes: 1, - totalNodes: 3, - }, - { - name: "no new node joins the cluster.", - currNode: &v1.Node{ - ObjectMeta: metav1.ObjectMeta{ - Name: "node2", - CreationTimestamp: metav1.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC), - }, - Spec: v1.NodeSpec{ - ProviderID: "i-00002", - }, - }, - taggingController: TaggingController{ - currentNodes: map[string]bool{ - "node0": true, - "node1": true, - "node2": true, - }, - totalNodes: map[string]*v1.Node{ - "node0": { - ObjectMeta: metav1.ObjectMeta{ - Name: "node0", - CreationTimestamp: metav1.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC), - }, - Spec: v1.NodeSpec{ - ProviderID: "i-00000", - }, - }, - "node1": { - ObjectMeta: metav1.ObjectMeta{ - Name: "node1", - CreationTimestamp: metav1.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC), - }, - Spec: v1.NodeSpec{ - ProviderID: "i-00001", - }, - }, - "node2": { - ObjectMeta: metav1.ObjectMeta{ - Name: "node2", - CreationTimestamp: metav1.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC), - }, - Spec: v1.NodeSpec{ - ProviderID: "i-00002", - }, - }, - }, - }, - noOfToBeTaggedNodes: 1, - totalNodes: 3, - }, - { - name: "node 3 joins the cluster but failed to be tagged.", - currNode: &v1.Node{ - ObjectMeta: metav1.ObjectMeta{ - Name: "node3", - CreationTimestamp: metav1.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC), - }, - Spec: v1.NodeSpec{ - ProviderID: "i-error", - }, - }, - taggingController: TaggingController{ - currentNodes: map[string]bool{ - "node0": true, - "node1": true, - "node2": true, - }, - totalNodes: map[string]*v1.Node{ - "node0": { - ObjectMeta: metav1.ObjectMeta{ - Name: "node0", - CreationTimestamp: metav1.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC), - }, - Spec: v1.NodeSpec{ - ProviderID: "i-00000", - }, - }, - "node1": { - ObjectMeta: metav1.ObjectMeta{ - Name: "node1", - CreationTimestamp: metav1.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC), - }, - Spec: v1.NodeSpec{ - ProviderID: "i-00001", - }, - }, - "node2": { - ObjectMeta: metav1.ObjectMeta{ - Name: "node2", - CreationTimestamp: metav1.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC), - }, - Spec: v1.NodeSpec{ - ProviderID: "i-00002", - }, - }, - }, - }, - noOfToBeTaggedNodes: 0, - totalNodes: 4, - }, - { - name: "node 1 joins the cluster, node 0 left but failed to be untagged.", - currNode: &v1.Node{ - ObjectMeta: metav1.ObjectMeta{ - Name: "node1", - CreationTimestamp: metav1.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC), - }, - Spec: v1.NodeSpec{ - ProviderID: "i-0001", - }, - }, - taggingController: TaggingController{ - currentNodes: map[string]bool{ - "node0": true, - }, - totalNodes: map[string]*v1.Node{ - "node0": { - ObjectMeta: metav1.ObjectMeta{ - Name: "node0", - CreationTimestamp: metav1.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC), - }, - Spec: v1.NodeSpec{ - ProviderID: "i-error", - }, - }, - }, - }, - noOfToBeTaggedNodes: 2, - totalNodes: 2, + expectedCalls: []string{"create-tags"}, }, } @@ -263,8 +57,6 @@ func Test_NodesJoiningAndLeaving(t *testing.T) { for _, testcase := range testcases { t.Run(testcase.name, func(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() clientset := fake.NewSimpleClientset(testcase.currNode) informer := informers.NewSharedInformerFactory(clientset, time.Second) nodeInformer := informer.Core().V1().Nodes() @@ -274,20 +66,26 @@ func Test_NodesJoiningAndLeaving(t *testing.T) { } eventBroadcaster := record.NewBroadcaster() - testcase.taggingController.nodeLister = nodeInformer.Lister() - testcase.taggingController.kubeClient = clientset - testcase.taggingController.cloud = fakeAws - testcase.taggingController.nodeMonitorPeriod = 1 * time.Second + tc := &TaggingController{ + nodeInformer: nodeInformer, + kubeClient: clientset, + cloud: fakeAws, + nodeMonitorPeriod: 1 * time.Second, + tags: map[string]string{"key": "value"}, + resources: []string{"instance"}, + workqueue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "Tagging"), + } w := eventBroadcaster.StartLogging(klog.Infof) defer w.Stop() - testcase.taggingController.MonitorNodes(ctx) - - if len(testcase.taggingController.currentNodes) != testcase.noOfToBeTaggedNodes || len(testcase.taggingController.totalNodes) != testcase.totalNodes { - t.Errorf("currentNodes must contain %d element(s), and totalNodes must contain %d element(s).", testcase.noOfToBeTaggedNodes, testcase.totalNodes) - } + tc.enqueueNode(testcase.currNode, true) + tc.MonitorNodes() + ec2, _ := awsServices.Compute("") + assert.EqualValues(t, testcase.expectedCalls, ec2., + "expected cloud provider methods `%v` to be called but `%v` was called ", + testcase.expectedCalls, awsServices.MadeRequest.Calls) }) } } diff --git a/pkg/providers/v1/aws_fakes.go b/pkg/providers/v1/aws_fakes.go index a4a43e9791..deaddb96e1 100644 --- a/pkg/providers/v1/aws_fakes.go +++ b/pkg/providers/v1/aws_fakes.go @@ -23,6 +23,8 @@ import ( "fmt" "sort" "strings" + "sync" + "time" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/autoscaling" @@ -50,6 +52,14 @@ type FakeAWSServices struct { kms *FakeKMS } +type FakeCloud struct { + cloud *Cloud + + Calls []string + addCallLock sync.Mutex + RequestDelay time.Duration +} + // NewFakeAWSServices creates a new FakeAWSServices func NewFakeAWSServices(clusterID string) *FakeAWSServices { s := &FakeAWSServices{} @@ -265,8 +275,8 @@ func (ec2i *FakeEC2Impl) RemoveSubnets() { ec2i.Subnets = ec2i.Subnets[:0] } -// CreateTags is not implemented but is required for interface conformance func (ec2i *FakeEC2Impl) CreateTags(input *ec2.CreateTagsInput) (*ec2.CreateTagsOutput, error) { + addCall("create-tags") for _, id := range input.Resources { if *id == "i-error" { return nil, errors.New("Unable to tag") @@ -275,8 +285,8 @@ func (ec2i *FakeEC2Impl) CreateTags(input *ec2.CreateTagsInput) (*ec2.CreateTags return &ec2.CreateTagsOutput{}, nil } -// DeleteTags is not implemented but is required for interface conformance func (ec2i *FakeEC2Impl) DeleteTags(input *ec2.DeleteTagsInput) (*ec2.DeleteTagsOutput, error) { + ec2i.addCall("delete-tags") for _, id := range input.Resources { if *id == "i-error" { return nil, errors.New("Unable to remove tag") @@ -285,6 +295,20 @@ func (ec2i *FakeEC2Impl) DeleteTags(input *ec2.DeleteTagsInput) (*ec2.DeleteTags return &ec2.DeleteTagsOutput{}, nil } +func (f *FakeCloud) addCall(desc string) { + f.addCallLock.Lock() + defer f.addCallLock.Unlock() + + time.Sleep(f.RequestDelay) + + f.Calls = append(f.Calls, desc) +} + +// ClearCalls clears internal record of method calls to this Cloud. +func (f *FakeCloud) ClearCalls() { + f.Calls = []string{} +} + // DescribeRouteTables returns fake route table descriptions func (ec2i *FakeEC2Impl) DescribeRouteTables(request *ec2.DescribeRouteTablesInput) ([]*ec2.RouteTable, error) { ec2i.DescribeRouteTablesInput = request From 9a1de579f375b228ca670ae42ac9ec65143e17ca Mon Sep 17 00:00:00 2001 From: Nguyen Dinh <92940366+nguyenkndinh@users.noreply.github.com> Date: Fri, 25 Mar 2022 18:36:58 -0700 Subject: [PATCH 28/43] Added in a loop to make sure all messages are processed before shutting down --- pkg/controllers/tagging/tagging_controller.go | 19 ++++++++++---- .../tagging/tagging_controller_test.go | 8 +----- pkg/providers/v1/aws_fakes.go | 26 ------------------- 3 files changed, 15 insertions(+), 38 deletions(-) diff --git a/pkg/controllers/tagging/tagging_controller.go b/pkg/controllers/tagging/tagging_controller.go index 46d66fa33b..a432b13787 100644 --- a/pkg/controllers/tagging/tagging_controller.go +++ b/pkg/controllers/tagging/tagging_controller.go @@ -101,17 +101,24 @@ func (tc *TaggingController) Run(stopCh <-chan struct{}) { defer tc.workqueue.ShutDown() klog.Infof("Starting the tagging controller") - go wait.Until(tc.MonitorNodes, tc.nodeMonitorPeriod, stopCh) + go wait.Until(tc.work, tc.nodeMonitorPeriod, stopCh) <-stopCh } -// MonitorNodes is a long-running function that continuously -// read and process a message on the work queue -func (tc *TaggingController) MonitorNodes() { +// work is a long-running function that continuously +// call process() for each message on the workqueue +func (tc *TaggingController) work() { + for tc.Process() { + } +} + +// Process reads each message in the queue and performs either +// tag or untag function on the Node object +func (tc *TaggingController) Process() bool { obj, shutdown := tc.workqueue.Get() if shutdown { - return + return false } err := func(obj interface{}) error { @@ -164,6 +171,8 @@ func (tc *TaggingController) MonitorNodes() { if err != nil { utilruntime.HandleError(err) } + + return true } // tagNodesResources tag node resources from a list of nodes diff --git a/pkg/controllers/tagging/tagging_controller_test.go b/pkg/controllers/tagging/tagging_controller_test.go index ceb12aa6d9..c58f9435ba 100644 --- a/pkg/controllers/tagging/tagging_controller_test.go +++ b/pkg/controllers/tagging/tagging_controller_test.go @@ -15,7 +15,6 @@ package tagging import ( "context" - "github.com/stretchr/testify/assert" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/informers" @@ -80,12 +79,7 @@ func Test_NodesJoiningAndLeaving(t *testing.T) { defer w.Stop() tc.enqueueNode(testcase.currNode, true) - tc.MonitorNodes() - ec2, _ := awsServices.Compute("") - - assert.EqualValues(t, testcase.expectedCalls, ec2., - "expected cloud provider methods `%v` to be called but `%v` was called ", - testcase.expectedCalls, awsServices.MadeRequest.Calls) + tc.Process() }) } } diff --git a/pkg/providers/v1/aws_fakes.go b/pkg/providers/v1/aws_fakes.go index deaddb96e1..5b5a3e4cd1 100644 --- a/pkg/providers/v1/aws_fakes.go +++ b/pkg/providers/v1/aws_fakes.go @@ -23,8 +23,6 @@ import ( "fmt" "sort" "strings" - "sync" - "time" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/autoscaling" @@ -52,14 +50,6 @@ type FakeAWSServices struct { kms *FakeKMS } -type FakeCloud struct { - cloud *Cloud - - Calls []string - addCallLock sync.Mutex - RequestDelay time.Duration -} - // NewFakeAWSServices creates a new FakeAWSServices func NewFakeAWSServices(clusterID string) *FakeAWSServices { s := &FakeAWSServices{} @@ -276,7 +266,6 @@ func (ec2i *FakeEC2Impl) RemoveSubnets() { } func (ec2i *FakeEC2Impl) CreateTags(input *ec2.CreateTagsInput) (*ec2.CreateTagsOutput, error) { - addCall("create-tags") for _, id := range input.Resources { if *id == "i-error" { return nil, errors.New("Unable to tag") @@ -286,7 +275,6 @@ func (ec2i *FakeEC2Impl) CreateTags(input *ec2.CreateTagsInput) (*ec2.CreateTags } func (ec2i *FakeEC2Impl) DeleteTags(input *ec2.DeleteTagsInput) (*ec2.DeleteTagsOutput, error) { - ec2i.addCall("delete-tags") for _, id := range input.Resources { if *id == "i-error" { return nil, errors.New("Unable to remove tag") @@ -295,20 +283,6 @@ func (ec2i *FakeEC2Impl) DeleteTags(input *ec2.DeleteTagsInput) (*ec2.DeleteTags return &ec2.DeleteTagsOutput{}, nil } -func (f *FakeCloud) addCall(desc string) { - f.addCallLock.Lock() - defer f.addCallLock.Unlock() - - time.Sleep(f.RequestDelay) - - f.Calls = append(f.Calls, desc) -} - -// ClearCalls clears internal record of method calls to this Cloud. -func (f *FakeCloud) ClearCalls() { - f.Calls = []string{} -} - // DescribeRouteTables returns fake route table descriptions func (ec2i *FakeEC2Impl) DescribeRouteTables(request *ec2.DescribeRouteTablesInput) ([]*ec2.RouteTable, error) { ec2i.DescribeRouteTablesInput = request From f5bb113b1a758d896dd9cd943f2a10a595832e02 Mon Sep 17 00:00:00 2001 From: Nguyen Dinh <92940366+nguyenkndinh@users.noreply.github.com> Date: Sat, 26 Mar 2022 13:04:43 -0700 Subject: [PATCH 29/43] Added more logging --- pkg/controllers/tagging/tagging_controller.go | 26 +++++--- .../tagging/tagging_controller_test.go | 60 +++++++++++++++++-- pkg/providers/v1/tags.go | 8 ++- 3 files changed, 79 insertions(+), 15 deletions(-) diff --git a/pkg/controllers/tagging/tagging_controller.go b/pkg/controllers/tagging/tagging_controller.go index a432b13787..e193aedaa0 100644 --- a/pkg/controllers/tagging/tagging_controller.go +++ b/pkg/controllers/tagging/tagging_controller.go @@ -32,9 +32,9 @@ import ( ) const ( - // This is a prefix used to recognized if a node in the workqueue - // is to be tagged or not - tagKeyPrefix string = "tagKeyPrefix" + // This is a prefix used to recognized if a node + // in the workqueue is to be tagged or not + tagKeyPrefix string = "ToBeTagged:" ) // TaggingController is the controller implementation for tagging cluster resources. @@ -121,6 +121,8 @@ func (tc *TaggingController) Process() bool { return false } + klog.Infof("Starting to process %v", obj) + err := func(obj interface{}) error { defer tc.workqueue.Done(obj) @@ -145,6 +147,7 @@ func (tc *TaggingController) Process() bool { node, err := tc.nodeInformer.Lister().Get(nodeName) if err != nil { if apierrors.IsNotFound(err) { + klog.Errorf("Unable to find a node with name %s", nodeName) return nil } @@ -152,9 +155,10 @@ func (tc *TaggingController) Process() bool { } if toBeTagged { + key = tagKeyPrefix + key if err := tc.tagNodesResources(node); err != nil { // Put the item back on the workqueue to handle any transient errors. - tc.workqueue.AddRateLimited(tagKeyPrefix + key) + tc.workqueue.AddRateLimited(key) return fmt.Errorf("error tagging '%s': %s, requeuing", key, err.Error()) } } else { @@ -165,10 +169,12 @@ func (tc *TaggingController) Process() bool { } tc.workqueue.Forget(obj) + klog.Infof("Finished processing %v", obj) return nil }(obj) if err != nil { + klog.Errorf("Error occurred while processing %v", obj) utilruntime.HandleError(err) } @@ -208,6 +214,8 @@ func (tc *TaggingController) tagEc2Instance(node *v1.Node) error { } } + klog.Infof("Successfully tagged %s with %v", instanceId, tc.tags) + return nil } @@ -244,6 +252,8 @@ func (tc *TaggingController) untagEc2Instance(node *v1.Node) error { } } + klog.Infof("Successfully tagged %s with %v", instanceId, tc.tags) + return nil } @@ -258,10 +268,12 @@ func (tc *TaggingController) enqueueNode(obj interface{}, toBeTagged bool) { } if toBeTagged { - tc.workqueue.Add(tagKeyPrefix + key) - } else { - tc.workqueue.Add(key) + key = tagKeyPrefix + key } + + tc.workqueue.Add(key) + + klog.Infof("Added %s to the workqueue", key) } // getActionAndKey from the provided key, check if the object is to be tagged diff --git a/pkg/controllers/tagging/tagging_controller_test.go b/pkg/controllers/tagging/tagging_controller_test.go index c58f9435ba..5cda9fc03b 100644 --- a/pkg/controllers/tagging/tagging_controller_test.go +++ b/pkg/controllers/tagging/tagging_controller_test.go @@ -32,10 +32,25 @@ const TestClusterID = "clusterid.test" func Test_NodesJoiningAndLeaving(t *testing.T) { testcases := []struct { - name string - currNode *v1.Node - expectedCalls []string + name string + currNode *v1.Node + noOfItemLeft int + toBeTagged bool }{ + { + name: "node0 joins the cluster, but fail to tag.", + currNode: &v1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node0", + CreationTimestamp: metav1.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC), + }, + Spec: v1.NodeSpec{ + ProviderID: "i-error", + }, + }, + noOfItemLeft: 1, + toBeTagged: true, + }, { name: "node0 joins the cluster.", currNode: &v1.Node{ @@ -44,10 +59,39 @@ func Test_NodesJoiningAndLeaving(t *testing.T) { CreationTimestamp: metav1.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC), }, Spec: v1.NodeSpec{ - ProviderID: "i-00000", + ProviderID: "i-0001", + }, + }, + noOfItemLeft: 0, + toBeTagged: true, + }, + { + name: "node0 leaves the cluster, failed to tag.", + currNode: &v1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node0", + CreationTimestamp: metav1.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC), + }, + Spec: v1.NodeSpec{ + ProviderID: "i-error", + }, + }, + noOfItemLeft: 1, + toBeTagged: false, + }, + { + name: "node0 leaves the cluster.", + currNode: &v1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node0", + CreationTimestamp: metav1.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC), + }, + Spec: v1.NodeSpec{ + ProviderID: "i-0001", }, }, - expectedCalls: []string{"create-tags"}, + noOfItemLeft: 0, + toBeTagged: false, }, } @@ -78,8 +122,12 @@ func Test_NodesJoiningAndLeaving(t *testing.T) { w := eventBroadcaster.StartLogging(klog.Infof) defer w.Stop() - tc.enqueueNode(testcase.currNode, true) + tc.enqueueNode(testcase.currNode, testcase.toBeTagged) tc.Process() + + if tc.workqueue.Len() != testcase.noOfItemLeft { + t.Fatalf("workqueue not processed properly, expected %d left, got %d.", testcase.noOfItemLeft, tc.workqueue.Len()) + } }) } } diff --git a/pkg/providers/v1/tags.go b/pkg/providers/v1/tags.go index acced37901..fa9893014f 100644 --- a/pkg/providers/v1/tags.go +++ b/pkg/providers/v1/tags.go @@ -310,13 +310,15 @@ func (c *Cloud) TagResource(resourceId string, tags map[string]string) error { Tags: buildAwsTags(tags), } - _, err := c.ec2.CreateTags(request) + output, err := c.ec2.CreateTags(request) if err != nil { klog.Errorf("Error occurred trying to tag resources, %v", err) return err } + klog.Infof("Done calling create-tags to EC2: %v", output) + return nil } @@ -326,13 +328,15 @@ func (c *Cloud) UntagResource(resourceId string, tags map[string]string) error { Tags: buildAwsTags(tags), } - _, err := c.ec2.DeleteTags(request) + output, err := c.ec2.DeleteTags(request) if err != nil { klog.Errorf("Error occurred trying to untag resources, %v", err) return err } + klog.Infof("Done calling delete-tags to EC2: %v", output) + return nil } From 54e1329bda92261c06d23a0d4825a2e186f4304f Mon Sep 17 00:00:00 2001 From: Nguyen Dinh <92940366+nguyenkndinh@users.noreply.github.com> Date: Sat, 26 Mar 2022 15:53:27 -0700 Subject: [PATCH 30/43] Added more testing --- pkg/controllers/tagging/tagging_controller.go | 4 +- .../tagging/tagging_controller_test.go | 57 ++++++++++++------- 2 files changed, 38 insertions(+), 23 deletions(-) diff --git a/pkg/controllers/tagging/tagging_controller.go b/pkg/controllers/tagging/tagging_controller.go index e193aedaa0..1d1b948bb9 100644 --- a/pkg/controllers/tagging/tagging_controller.go +++ b/pkg/controllers/tagging/tagging_controller.go @@ -26,7 +26,7 @@ import ( cloudprovider "k8s.io/cloud-provider" opt "k8s.io/cloud-provider-aws/pkg/controllers/options" awsv1 "k8s.io/cloud-provider-aws/pkg/providers/v1" - "k8s.io/klog/v2" + "k8s.io/klog" "strings" "time" ) @@ -252,7 +252,7 @@ func (tc *TaggingController) untagEc2Instance(node *v1.Node) error { } } - klog.Infof("Successfully tagged %s with %v", instanceId, tc.tags) + klog.Infof("Successfully untagged %s with %v", instanceId, tc.tags) return nil } diff --git a/pkg/controllers/tagging/tagging_controller_test.go b/pkg/controllers/tagging/tagging_controller_test.go index 5cda9fc03b..6b5268f50e 100644 --- a/pkg/controllers/tagging/tagging_controller_test.go +++ b/pkg/controllers/tagging/tagging_controller_test.go @@ -14,16 +14,19 @@ limitations under the License. package tagging import ( + "bytes" "context" + "flag" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/informers" coreinformers "k8s.io/client-go/informers/core/v1" "k8s.io/client-go/kubernetes/fake" - "k8s.io/client-go/tools/record" "k8s.io/client-go/util/workqueue" awsv1 "k8s.io/cloud-provider-aws/pkg/providers/v1" - "k8s.io/klog/v2" + "k8s.io/klog" + "os" + "strings" "testing" "time" ) @@ -31,11 +34,13 @@ import ( const TestClusterID = "clusterid.test" func Test_NodesJoiningAndLeaving(t *testing.T) { + klog.InitFlags(nil) + flag.CommandLine.Parse([]string{"--logtostderr=false"}) testcases := []struct { - name string - currNode *v1.Node - noOfItemLeft int - toBeTagged bool + name string + currNode *v1.Node + toBeTagged bool + expectedMessages []string }{ { name: "node0 joins the cluster, but fail to tag.", @@ -48,8 +53,8 @@ func Test_NodesJoiningAndLeaving(t *testing.T) { ProviderID: "i-error", }, }, - noOfItemLeft: 1, - toBeTagged: true, + toBeTagged: true, + expectedMessages: []string{"Error occurred while processing ToBeTagged:node0"}, }, { name: "node0 joins the cluster.", @@ -62,11 +67,11 @@ func Test_NodesJoiningAndLeaving(t *testing.T) { ProviderID: "i-0001", }, }, - noOfItemLeft: 0, - toBeTagged: true, + toBeTagged: true, + expectedMessages: []string{"Successfully tagged i-0001"}, }, { - name: "node0 leaves the cluster, failed to tag.", + name: "node0 leaves the cluster, failed to untag.", currNode: &v1.Node{ ObjectMeta: metav1.ObjectMeta{ Name: "node0", @@ -76,8 +81,8 @@ func Test_NodesJoiningAndLeaving(t *testing.T) { ProviderID: "i-error", }, }, - noOfItemLeft: 1, - toBeTagged: false, + toBeTagged: false, + expectedMessages: []string{"Error in untagging EC2 instance for node node0"}, }, { name: "node0 leaves the cluster.", @@ -90,8 +95,8 @@ func Test_NodesJoiningAndLeaving(t *testing.T) { ProviderID: "i-0001", }, }, - noOfItemLeft: 0, - toBeTagged: false, + toBeTagged: false, + expectedMessages: []string{"Successfully untagged i-0001"}, }, } @@ -100,6 +105,12 @@ func Test_NodesJoiningAndLeaving(t *testing.T) { for _, testcase := range testcases { t.Run(testcase.name, func(t *testing.T) { + var logBuf bytes.Buffer + klog.SetOutput(&logBuf) + defer func() { + klog.SetOutput(os.Stderr) + }() + clientset := fake.NewSimpleClientset(testcase.currNode) informer := informers.NewSharedInformerFactory(clientset, time.Second) nodeInformer := informer.Core().V1().Nodes() @@ -108,7 +119,7 @@ func Test_NodesJoiningAndLeaving(t *testing.T) { t.Errorf("unexpected error: %v", err) } - eventBroadcaster := record.NewBroadcaster() + //eventBroadcaster := record.NewBroadcaster() tc := &TaggingController{ nodeInformer: nodeInformer, kubeClient: clientset, @@ -119,14 +130,18 @@ func Test_NodesJoiningAndLeaving(t *testing.T) { workqueue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "Tagging"), } - w := eventBroadcaster.StartLogging(klog.Infof) - defer w.Stop() - tc.enqueueNode(testcase.currNode, testcase.toBeTagged) tc.Process() - if tc.workqueue.Len() != testcase.noOfItemLeft { - t.Fatalf("workqueue not processed properly, expected %d left, got %d.", testcase.noOfItemLeft, tc.workqueue.Len()) + for _, msg := range testcase.expectedMessages { + if !strings.Contains(logBuf.String(), msg) { + t.Errorf("\nMsg %q not found in log: \n%v\n", msg, logBuf.String()) + } + if strings.Contains(logBuf.String(), "error tagging ") || strings.Contains(logBuf.String(), "error untagging ") { + if !strings.Contains(logBuf.String(), ", requeuing") { + t.Errorf("\nFailed to tag or untag but logs do not contain 'requeueing': \n%v\n", logBuf.String()) + } + } } }) } From d322867b8b9318bed05689b11a31fd5eeb8314a5 Mon Sep 17 00:00:00 2001 From: Nguyen Dinh <92940366+nguyenkndinh@users.noreply.github.com> Date: Mon, 28 Mar 2022 11:12:28 -0700 Subject: [PATCH 31/43] cosmetic change --- pkg/controllers/options/tagging_controller.go | 2 +- pkg/controllers/tagging/tagging_controller.go | 2 +- pkg/controllers/tagging/tagging_controller_test.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/controllers/options/tagging_controller.go b/pkg/controllers/options/tagging_controller.go index 52cc6593aa..e3be07dfc5 100644 --- a/pkg/controllers/options/tagging_controller.go +++ b/pkg/controllers/options/tagging_controller.go @@ -30,7 +30,7 @@ func (o *TaggingControllerOptions) AddFlags(fs *pflag.FlagSet) { func (o *TaggingControllerOptions) Validate() error { if len(o.Tags) == 0 { - return fmt.Errorf("--tags must not be empty and must be a form of key:value") + return fmt.Errorf("--tags must not be empty and must be a form of key=value") } if len(o.Resources) == 0 { diff --git a/pkg/controllers/tagging/tagging_controller.go b/pkg/controllers/tagging/tagging_controller.go index 1d1b948bb9..d2cb528144 100644 --- a/pkg/controllers/tagging/tagging_controller.go +++ b/pkg/controllers/tagging/tagging_controller.go @@ -26,7 +26,7 @@ import ( cloudprovider "k8s.io/cloud-provider" opt "k8s.io/cloud-provider-aws/pkg/controllers/options" awsv1 "k8s.io/cloud-provider-aws/pkg/providers/v1" - "k8s.io/klog" + "k8s.io/klog/v2" "strings" "time" ) diff --git a/pkg/controllers/tagging/tagging_controller_test.go b/pkg/controllers/tagging/tagging_controller_test.go index 6b5268f50e..0dceefa13d 100644 --- a/pkg/controllers/tagging/tagging_controller_test.go +++ b/pkg/controllers/tagging/tagging_controller_test.go @@ -24,7 +24,7 @@ import ( "k8s.io/client-go/kubernetes/fake" "k8s.io/client-go/util/workqueue" awsv1 "k8s.io/cloud-provider-aws/pkg/providers/v1" - "k8s.io/klog" + "k8s.io/klog/v2" "os" "strings" "testing" From ea21b4661bfcebc46fa1f437467be21fda55cf99 Mon Sep 17 00:00:00 2001 From: Nguyen Dinh <92940366+nguyenkndinh@users.noreply.github.com> Date: Mon, 28 Mar 2022 12:24:50 -0700 Subject: [PATCH 32/43] use array instead of map for supported resources --- pkg/controllers/options/resources.go | 4 ++-- pkg/controllers/options/tagging_controller.go | 14 +++++++++----- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/pkg/controllers/options/resources.go b/pkg/controllers/options/resources.go index 381fcde6f0..df0ee14c83 100644 --- a/pkg/controllers/options/resources.go +++ b/pkg/controllers/options/resources.go @@ -16,6 +16,6 @@ const ( Instance string = "instance" ) -var SupportedResources = map[string]string{ - "instance": Instance, +var SupportedResources = []string{ + Instance, } diff --git a/pkg/controllers/options/tagging_controller.go b/pkg/controllers/options/tagging_controller.go index e3be07dfc5..d79a7bf265 100644 --- a/pkg/controllers/options/tagging_controller.go +++ b/pkg/controllers/options/tagging_controller.go @@ -38,12 +38,16 @@ func (o *TaggingControllerOptions) Validate() error { } for _, r := range o.Resources { - if _, ok := SupportedResources[r]; !ok { - resources := []string{} - for r, _ := range SupportedResources { - resources = append(resources, r) + found := false + + for _, resource := range SupportedResources { + if r == resource { + found = true } - return fmt.Errorf("%s is not a supported resource. Current supported resources %v", r, resources) + } + + if !found { + return fmt.Errorf("%s is not a supported resource. Current supported resources %v", r, SupportedResources) } } From 7a8f517316069e229c6bb31aede0e625cc6ebe8d Mon Sep 17 00:00:00 2001 From: Nguyen Dinh <92940366+nguyenkndinh@users.noreply.github.com> Date: Mon, 28 Mar 2022 14:28:51 -0700 Subject: [PATCH 33/43] Reworked the workqueue with workitem --- pkg/controllers/tagging/tagging_controller.go | 109 ++++++------------ .../tagging/tagging_controller_test.go | 8 +- 2 files changed, 40 insertions(+), 77 deletions(-) diff --git a/pkg/controllers/tagging/tagging_controller.go b/pkg/controllers/tagging/tagging_controller.go index d2cb528144..21c713b5e0 100644 --- a/pkg/controllers/tagging/tagging_controller.go +++ b/pkg/controllers/tagging/tagging_controller.go @@ -16,7 +16,6 @@ package tagging import ( "fmt" v1 "k8s.io/api/core/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/wait" coreinformers "k8s.io/client-go/informers/core/v1" @@ -27,15 +26,14 @@ import ( opt "k8s.io/cloud-provider-aws/pkg/controllers/options" awsv1 "k8s.io/cloud-provider-aws/pkg/providers/v1" "k8s.io/klog/v2" - "strings" "time" ) -const ( - // This is a prefix used to recognized if a node - // in the workqueue is to be tagged or not - tagKeyPrefix string = "ToBeTagged:" -) +// workItem contains the node and an action for that node +type workItem struct { + node *v1.Node + action func(node *v1.Node) error +} // TaggingController is the controller implementation for tagging cluster resources. // It periodically check for Node events (creating/deleting) to apply appropriate @@ -45,7 +43,7 @@ type TaggingController struct { kubeClient clientset.Interface cloud *awsv1.Cloud workqueue workqueue.RateLimitingInterface - + nodesSynced cache.InformerSynced // Value controlling TaggingController monitoring period, i.e. how often does TaggingController // check node list. This value should be lower than nodeMonitorGracePeriod // set in controller-manager @@ -81,14 +79,15 @@ func NewTaggingController( tags: tags, resources: resources, workqueue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "Tagging"), + nodesSynced: nodeInformer.Informer().HasSynced, } // Use shared informer to listen to add/update/delete of nodes. Note that any nodes // that exist before tagging controller starts will show up in the update method tc.nodeInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ - AddFunc: func(obj interface{}) { tc.enqueueNode(obj, true) }, - UpdateFunc: func(oldObj, newObj interface{}) { tc.enqueueNode(newObj, true) }, - DeleteFunc: func(obj interface{}) { tc.enqueueNode(obj, false) }, + AddFunc: func(obj interface{}) { tc.enqueueNode(obj, tc.tagNodesResources) }, + UpdateFunc: func(oldObj, newObj interface{}) { tc.enqueueNode(newObj, tc.tagNodesResources) }, + DeleteFunc: func(obj interface{}) { tc.enqueueNode(obj, tc.untagNodeResources) }, }) return tc, nil @@ -100,6 +99,13 @@ func (tc *TaggingController) Run(stopCh <-chan struct{}) { defer utilruntime.HandleCrash() defer tc.workqueue.ShutDown() + // Wait for the caches to be synced before starting workers + klog.Info("Waiting for informer caches to sync") + if ok := cache.WaitForCacheSync(stopCh, tc.nodesSynced); !ok { + klog.Errorf("failed to wait for caches to sync") + return + } + klog.Infof("Starting the tagging controller") go wait.Until(tc.work, tc.nodeMonitorPeriod, stopCh) @@ -126,50 +132,22 @@ func (tc *TaggingController) Process() bool { err := func(obj interface{}) error { defer tc.workqueue.Done(obj) - var key string - var ok bool - if key, ok = obj.(string); !ok { + workItem, ok := obj.(*workItem) + if !ok { tc.workqueue.Forget(obj) - utilruntime.HandleError(fmt.Errorf("expected string in workqueue but got %#v", obj)) + utilruntime.HandleError(fmt.Errorf("expected workItem in workqueue but got %#v", obj)) return nil } - var toBeTagged bool - toBeTagged, key = tc.getActionAndKey(key) - - _, nodeName, err := cache.SplitMetaNamespaceKey(key) - + err := workItem.action(workItem.node) if err != nil { - utilruntime.HandleError(fmt.Errorf("invalid resource key: %s", key)) - return nil - } - - node, err := tc.nodeInformer.Lister().Get(nodeName) - if err != nil { - if apierrors.IsNotFound(err) { - klog.Errorf("Unable to find a node with name %s", nodeName) - return nil - } - - return err - } - - if toBeTagged { - key = tagKeyPrefix + key - if err := tc.tagNodesResources(node); err != nil { - // Put the item back on the workqueue to handle any transient errors. - tc.workqueue.AddRateLimited(key) - return fmt.Errorf("error tagging '%s': %s, requeuing", key, err.Error()) - } - } else { - if err := tc.untagNodeResources(node); err != nil { - tc.workqueue.AddRateLimited(key) - return fmt.Errorf("error untagging '%s': %s, requeuing", key, err.Error()) - } + // Put the item back on the workqueue to handle any transient errors. + tc.workqueue.AddRateLimited(workItem) + return fmt.Errorf("error finishing work item '%v': %s, requeuing", workItem, err.Error()) } tc.workqueue.Forget(obj) - klog.Infof("Finished processing %v", obj) + klog.Infof("Finished processing %v", workItem) return nil }(obj) @@ -257,33 +235,14 @@ func (tc *TaggingController) untagEc2Instance(node *v1.Node) error { return nil } -// enqueueNode takes in the object to enqueue to the workqueue and whether -// the object is to be tagged -func (tc *TaggingController) enqueueNode(obj interface{}, toBeTagged bool) { - var key string - var err error - if key, err = cache.MetaNamespaceKeyFunc(obj); err != nil { - utilruntime.HandleError(err) - return - } - - if toBeTagged { - key = tagKeyPrefix + key - } - - tc.workqueue.Add(key) - - klog.Infof("Added %s to the workqueue", key) -} - -// getActionAndKey from the provided key, check if the object is to be tagged -// and extract that action together with the key -func (tc *TaggingController) getActionAndKey(key string) (bool, string) { - toBeTagged := false - if strings.HasPrefix(key, tagKeyPrefix) { - toBeTagged = true - key = strings.TrimPrefix(key, tagKeyPrefix) +// enqueueNode takes in the object and an +// action for the object for a workitem and enqueue to the workqueue +func (tc *TaggingController) enqueueNode(obj interface{}, action func(node *v1.Node) error) { + node := obj.(*v1.Node) + item := &workItem{ + node: node, + action: action, } - - return toBeTagged, key + tc.workqueue.Add(item) + klog.Infof("Added %s to the workqueue", item) } diff --git a/pkg/controllers/tagging/tagging_controller_test.go b/pkg/controllers/tagging/tagging_controller_test.go index 0dceefa13d..253f869803 100644 --- a/pkg/controllers/tagging/tagging_controller_test.go +++ b/pkg/controllers/tagging/tagging_controller_test.go @@ -54,7 +54,7 @@ func Test_NodesJoiningAndLeaving(t *testing.T) { }, }, toBeTagged: true, - expectedMessages: []string{"Error occurred while processing ToBeTagged:node0"}, + expectedMessages: []string{"Error occurred while processing"}, }, { name: "node0 joins the cluster.", @@ -130,7 +130,11 @@ func Test_NodesJoiningAndLeaving(t *testing.T) { workqueue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "Tagging"), } - tc.enqueueNode(testcase.currNode, testcase.toBeTagged) + if testcase.toBeTagged { + tc.enqueueNode(testcase.currNode, tc.tagNodesResources) + } else { + tc.enqueueNode(testcase.currNode, tc.untagNodeResources) + } tc.Process() for _, msg := range testcase.expectedMessages { From 23518086aee414617e00748aefb2408a904c0ce6 Mon Sep 17 00:00:00 2001 From: Nguyen Dinh <92940366+nguyenkndinh@users.noreply.github.com> Date: Tue, 29 Mar 2022 13:19:05 -0700 Subject: [PATCH 34/43] Addressed comments --- pkg/controllers/options/tagging_controller.go | 2 +- pkg/controllers/tagging/tagging_controller.go | 6 +++--- pkg/controllers/tagging/tagging_controller_test.go | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pkg/controllers/options/tagging_controller.go b/pkg/controllers/options/tagging_controller.go index d79a7bf265..e25c8169e0 100644 --- a/pkg/controllers/options/tagging_controller.go +++ b/pkg/controllers/options/tagging_controller.go @@ -24,7 +24,7 @@ type TaggingControllerOptions struct { } func (o *TaggingControllerOptions) AddFlags(fs *pflag.FlagSet) { - fs.StringToStringVar(&o.Tags, "tags", o.Tags, "Tags to apply to AWS resources in the tagging controller.") + fs.StringToStringVar(&o.Tags, "tags", o.Tags, "Tags to apply to AWS resources in the tagging controller, in a form of key=value.") fs.StringArrayVar(&o.Resources, "resources", o.Resources, "AWS resources name to add/remove tags in the tagging controller.") } diff --git a/pkg/controllers/tagging/tagging_controller.go b/pkg/controllers/tagging/tagging_controller.go index 21c713b5e0..282f01ac90 100644 --- a/pkg/controllers/tagging/tagging_controller.go +++ b/pkg/controllers/tagging/tagging_controller.go @@ -115,13 +115,13 @@ func (tc *TaggingController) Run(stopCh <-chan struct{}) { // work is a long-running function that continuously // call process() for each message on the workqueue func (tc *TaggingController) work() { - for tc.Process() { + for tc.process() { } } -// Process reads each message in the queue and performs either +// process reads each message in the queue and performs either // tag or untag function on the Node object -func (tc *TaggingController) Process() bool { +func (tc *TaggingController) process() bool { obj, shutdown := tc.workqueue.Get() if shutdown { return false diff --git a/pkg/controllers/tagging/tagging_controller_test.go b/pkg/controllers/tagging/tagging_controller_test.go index 253f869803..7ee4404d5d 100644 --- a/pkg/controllers/tagging/tagging_controller_test.go +++ b/pkg/controllers/tagging/tagging_controller_test.go @@ -135,7 +135,7 @@ func Test_NodesJoiningAndLeaving(t *testing.T) { } else { tc.enqueueNode(testcase.currNode, tc.untagNodeResources) } - tc.Process() + tc.process() for _, msg := range testcase.expectedMessages { if !strings.Contains(logBuf.String(), msg) { From e32c3254ce0bfe128300cb1553c4a7e7c9f71707 Mon Sep 17 00:00:00 2001 From: Nguyen Dinh <92940366+nguyenkndinh@users.noreply.github.com> Date: Tue, 29 Mar 2022 13:25:56 -0700 Subject: [PATCH 35/43] addressed verify-lint errors --- pkg/providers/v1/tags.go | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/pkg/providers/v1/tags.go b/pkg/providers/v1/tags.go index fa9893014f..4831bdc38b 100644 --- a/pkg/providers/v1/tags.go +++ b/pkg/providers/v1/tags.go @@ -304,9 +304,11 @@ func (t *awsTagging) clusterID() string { return t.ClusterID } -func (c *Cloud) TagResource(resourceId string, tags map[string]string) error { +// TagResource calls EC2 and tag the resource associated to resourceID +// with the supplied tags +func (c *Cloud) TagResource(resourceID string, tags map[string]string) error { request := &ec2.CreateTagsInput{ - Resources: []*string{aws.String(resourceId)}, + Resources: []*string{aws.String(resourceID)}, Tags: buildAwsTags(tags), } @@ -322,9 +324,11 @@ func (c *Cloud) TagResource(resourceId string, tags map[string]string) error { return nil } -func (c *Cloud) UntagResource(resourceId string, tags map[string]string) error { +// UntagResource calls EC2 and tag the resource associated to resourceID +// with the supplied tags +func (c *Cloud) UntagResource(resourceID string, tags map[string]string) error { request := &ec2.DeleteTagsInput{ - Resources: []*string{aws.String(resourceId)}, + Resources: []*string{aws.String(resourceID)}, Tags: buildAwsTags(tags), } From a9d8d75a345af62b4d13c4a9444dc5857b21633f Mon Sep 17 00:00:00 2001 From: Nguyen Dinh <92940366+nguyenkndinh@users.noreply.github.com> Date: Tue, 29 Mar 2022 13:33:12 -0700 Subject: [PATCH 36/43] addressed comments and verify-lint --- cmd/aws-cloud-controller-manager/main.go | 2 +- pkg/controllers/options/tagging_controller.go | 10 ++-------- pkg/controllers/tagging/tagging_controller_wrapper.go | 6 +++--- pkg/providers/v1/aws.go | 1 + pkg/providers/v1/aws_fakes.go | 2 ++ 5 files changed, 9 insertions(+), 12 deletions(-) diff --git a/cmd/aws-cloud-controller-manager/main.go b/cmd/aws-cloud-controller-manager/main.go index 1dddfab45e..011cc4b5aa 100644 --- a/cmd/aws-cloud-controller-manager/main.go +++ b/cmd/aws-cloud-controller-manager/main.go @@ -63,7 +63,7 @@ func main() { } controllerInitializers := app.DefaultInitFuncConstructors - taggingControllerWrapper := tagging.TaggingControllerWrapper{} + taggingControllerWrapper := tagging.ControllerWrapper{} fss := cliflag.NamedFlagSets{} taggingControllerWrapper.Options.AddFlags(fss.FlagSet("tagging controller")) diff --git a/pkg/controllers/options/tagging_controller.go b/pkg/controllers/options/tagging_controller.go index e25c8169e0..f182bd6adb 100644 --- a/pkg/controllers/options/tagging_controller.go +++ b/pkg/controllers/options/tagging_controller.go @@ -38,17 +38,11 @@ func (o *TaggingControllerOptions) Validate() error { } for _, r := range o.Resources { - found := false - for _, resource := range SupportedResources { - if r == resource { - found = true + if r != resource { + return fmt.Errorf("%s is not a supported resource. Current supported resources %v", r, SupportedResources) } } - - if !found { - return fmt.Errorf("%s is not a supported resource. Current supported resources %v", r, SupportedResources) - } } return nil diff --git a/pkg/controllers/tagging/tagging_controller_wrapper.go b/pkg/controllers/tagging/tagging_controller_wrapper.go index f250ab78f0..cef64d5bef 100644 --- a/pkg/controllers/tagging/tagging_controller_wrapper.go +++ b/pkg/controllers/tagging/tagging_controller_wrapper.go @@ -18,18 +18,18 @@ const ( TaggingControllerKey = "tagging" ) -type TaggingControllerWrapper struct { +type ControllerWrapper struct { Options options.TaggingControllerOptions } // StartTaggingControllerWrapper is used to take cloud config as input and start the tagging controller -func (tc *TaggingControllerWrapper) StartTaggingControllerWrapper(initContext app.ControllerInitContext, completedConfig *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface) app.InitFunc { +func (tc *ControllerWrapper) StartTaggingControllerWrapper(initContext app.ControllerInitContext, completedConfig *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface) app.InitFunc { return func(ctx context.Context, controllerContext genericcontrollermanager.ControllerContext) (controller.Interface, bool, error) { return tc.startTaggingController(ctx, initContext, completedConfig, cloud) } } -func (tc *TaggingControllerWrapper) startTaggingController(ctx context.Context, initContext app.ControllerInitContext, completedConfig *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface) (controller.Interface, bool, error) { +func (tc *ControllerWrapper) startTaggingController(ctx context.Context, initContext app.ControllerInitContext, completedConfig *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface) (controller.Interface, bool, error) { err := tc.Options.Validate() if err != nil { klog.Fatalf("Tagging controller inputs are not properly set: %v", err) diff --git a/pkg/providers/v1/aws.go b/pkg/providers/v1/aws.go index a4c6df35c1..75ad4513aa 100644 --- a/pkg/providers/v1/aws.go +++ b/pkg/providers/v1/aws.go @@ -1363,6 +1363,7 @@ func newAWSCloud(cfg CloudConfig, awsServices Services) (*Cloud, error) { return awsCloud, nil } +// NewAWSCloud calls and return new aws cloud from newAWSCloud with the supplied configuration func NewAWSCloud(cfg CloudConfig, awsServices Services) (*Cloud, error) { return newAWSCloud(cfg, awsServices) } diff --git a/pkg/providers/v1/aws_fakes.go b/pkg/providers/v1/aws_fakes.go index 5b5a3e4cd1..d130bc0947 100644 --- a/pkg/providers/v1/aws_fakes.go +++ b/pkg/providers/v1/aws_fakes.go @@ -265,6 +265,7 @@ func (ec2i *FakeEC2Impl) RemoveSubnets() { ec2i.Subnets = ec2i.Subnets[:0] } +// Mock CreateTags from EC2 func (ec2i *FakeEC2Impl) CreateTags(input *ec2.CreateTagsInput) (*ec2.CreateTagsOutput, error) { for _, id := range input.Resources { if *id == "i-error" { @@ -274,6 +275,7 @@ func (ec2i *FakeEC2Impl) CreateTags(input *ec2.CreateTagsInput) (*ec2.CreateTags return &ec2.CreateTagsOutput{}, nil } +// Mock DeleteTags from EC2 func (ec2i *FakeEC2Impl) DeleteTags(input *ec2.DeleteTagsInput) (*ec2.DeleteTagsOutput, error) { for _, id := range input.Resources { if *id == "i-error" { From d433e24c3d7a6f524376dd1c2bb0203ee901b9a1 Mon Sep 17 00:00:00 2001 From: Nguyen Dinh <92940366+nguyenkndinh@users.noreply.github.com> Date: Tue, 29 Mar 2022 13:41:08 -0700 Subject: [PATCH 37/43] address validate-lint error --- pkg/controllers/options/resources.go | 3 + pkg/controllers/options/tagging_controller.go | 4 ++ pkg/controllers/tagging/tagging_controller.go | 58 +++++++++---------- .../tagging/tagging_controller_test.go | 2 +- .../tagging/tagging_controller_wrapper.go | 7 ++- 5 files changed, 42 insertions(+), 32 deletions(-) diff --git a/pkg/controllers/options/resources.go b/pkg/controllers/options/resources.go index df0ee14c83..0c7201b5f5 100644 --- a/pkg/controllers/options/resources.go +++ b/pkg/controllers/options/resources.go @@ -10,12 +10,15 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ + package options const ( + // Instance presenting the string literal "instance" Instance string = "instance" ) +// SupportedResources contains the resources that can be tagged by the controller at the moment var SupportedResources = []string{ Instance, } diff --git a/pkg/controllers/options/tagging_controller.go b/pkg/controllers/options/tagging_controller.go index f182bd6adb..3c1de76724 100644 --- a/pkg/controllers/options/tagging_controller.go +++ b/pkg/controllers/options/tagging_controller.go @@ -18,16 +18,20 @@ import ( "github.com/spf13/pflag" ) +// TaggingControllerOptions contains the inputs that can +// be used in the tagging controller type TaggingControllerOptions struct { Tags map[string]string Resources []string } +// AddFlags add the additional flags for the controller func (o *TaggingControllerOptions) AddFlags(fs *pflag.FlagSet) { fs.StringToStringVar(&o.Tags, "tags", o.Tags, "Tags to apply to AWS resources in the tagging controller, in a form of key=value.") fs.StringArrayVar(&o.Resources, "resources", o.Resources, "AWS resources name to add/remove tags in the tagging controller.") } +// Validate checks for errors from user input func (o *TaggingControllerOptions) Validate() error { if len(o.Tags) == 0 { return fmt.Errorf("--tags must not be empty and must be a form of key=value") diff --git a/pkg/controllers/tagging/tagging_controller.go b/pkg/controllers/tagging/tagging_controller.go index 282f01ac90..fc84f2b5ae 100644 --- a/pkg/controllers/tagging/tagging_controller.go +++ b/pkg/controllers/tagging/tagging_controller.go @@ -35,16 +35,16 @@ type workItem struct { action func(node *v1.Node) error } -// TaggingController is the controller implementation for tagging cluster resources. +// Controller is the controller implementation for tagging cluster resources. // It periodically check for Node events (creating/deleting) to apply appropriate // tags to resources. -type TaggingController struct { +type Controller struct { nodeInformer coreinformers.NodeInformer kubeClient clientset.Interface cloud *awsv1.Cloud workqueue workqueue.RateLimitingInterface nodesSynced cache.InformerSynced - // Value controlling TaggingController monitoring period, i.e. how often does TaggingController + // Value controlling Controller monitoring period, i.e. how often does Controller // check node list. This value should be lower than nodeMonitorGracePeriod // set in controller-manager nodeMonitorPeriod time.Duration @@ -63,7 +63,7 @@ func NewTaggingController( cloud cloudprovider.Interface, nodeMonitorPeriod time.Duration, tags map[string]string, - resources []string) (*TaggingController, error) { + resources []string) (*Controller, error) { awsCloud, ok := cloud.(*awsv1.Cloud) if !ok { @@ -71,7 +71,7 @@ func NewTaggingController( return nil, err } - tc := &TaggingController{ + tc := &Controller{ nodeInformer: nodeInformer, kubeClient: kubeClient, cloud: awsCloud, @@ -95,7 +95,7 @@ func NewTaggingController( // Run will start the controller to tag resources attached to the cluster // and untag resources detached from the cluster. -func (tc *TaggingController) Run(stopCh <-chan struct{}) { +func (tc *Controller) Run(stopCh <-chan struct{}) { defer utilruntime.HandleCrash() defer tc.workqueue.ShutDown() @@ -114,14 +114,14 @@ func (tc *TaggingController) Run(stopCh <-chan struct{}) { // work is a long-running function that continuously // call process() for each message on the workqueue -func (tc *TaggingController) work() { +func (tc *Controller) work() { for tc.process() { } } // process reads each message in the queue and performs either // tag or untag function on the Node object -func (tc *TaggingController) process() bool { +func (tc *Controller) process() bool { obj, shutdown := tc.workqueue.Get() if shutdown { return false @@ -161,7 +161,7 @@ func (tc *TaggingController) process() bool { // tagNodesResources tag node resources from a list of nodes // If we want to tag more resources, modify this function appropriately -func (tc *TaggingController) tagNodesResources(node *v1.Node) error { +func (tc *Controller) tagNodesResources(node *v1.Node) error { for _, resource := range tc.resources { switch resource { case opt.Instance: @@ -177,29 +177,29 @@ func (tc *TaggingController) tagNodesResources(node *v1.Node) error { // tagEc2Instances applies the provided tags to each EC2 instance in // the cluster. -func (tc *TaggingController) tagEc2Instance(node *v1.Node) error { - instanceId, err := awsv1.KubernetesInstanceID(node.Spec.ProviderID).MapToAWSInstanceID() +func (tc *Controller) tagEc2Instance(node *v1.Node) error { + instanceID, err := awsv1.KubernetesInstanceID(node.Spec.ProviderID).MapToAWSInstanceID() if err != nil { klog.Errorf("Error in getting instanceID for node %s, error: %v", node.GetName(), err) return err - } else { - err := tc.cloud.TagResource(string(instanceId), tc.tags) + } - if err != nil { - klog.Errorf("Error in tagging EC2 instance for node %s, error: %v", node.GetName(), err) - return err - } + err = tc.cloud.TagResource(string(instanceID), tc.tags) + + if err != nil { + klog.Errorf("Error in tagging EC2 instance for node %s, error: %v", node.GetName(), err) + return err } - klog.Infof("Successfully tagged %s with %v", instanceId, tc.tags) + klog.Infof("Successfully tagged %s with %v", instanceID, tc.tags) return nil } // untagNodeResources untag node resources from a list of nodes // If we want to untag more resources, modify this function appropriately -func (tc *TaggingController) untagNodeResources(node *v1.Node) error { +func (tc *Controller) untagNodeResources(node *v1.Node) error { for _, resource := range tc.resources { switch resource { case opt.Instance: @@ -215,29 +215,29 @@ func (tc *TaggingController) untagNodeResources(node *v1.Node) error { // untagEc2Instances deletes the provided tags to each EC2 instances in // the cluster. -func (tc *TaggingController) untagEc2Instance(node *v1.Node) error { - instanceId, err := awsv1.KubernetesInstanceID(node.Spec.ProviderID).MapToAWSInstanceID() +func (tc *Controller) untagEc2Instance(node *v1.Node) error { + instanceID, err := awsv1.KubernetesInstanceID(node.Spec.ProviderID).MapToAWSInstanceID() if err != nil { klog.Errorf("Error in getting instanceID for node %s, error: %v", node.GetName(), err) return err - } else { - err := tc.cloud.UntagResource(string(instanceId), tc.tags) + } - if err != nil { - klog.Errorf("Error in untagging EC2 instance for node %s, error: %v", node.GetName(), err) - return err - } + err = tc.cloud.UntagResource(string(instanceID), tc.tags) + + if err != nil { + klog.Errorf("Error in untagging EC2 instance for node %s, error: %v", node.GetName(), err) + return err } - klog.Infof("Successfully untagged %s with %v", instanceId, tc.tags) + klog.Infof("Successfully untagged %s with %v", instanceID, tc.tags) return nil } // enqueueNode takes in the object and an // action for the object for a workitem and enqueue to the workqueue -func (tc *TaggingController) enqueueNode(obj interface{}, action func(node *v1.Node) error) { +func (tc *Controller) enqueueNode(obj interface{}, action func(node *v1.Node) error) { node := obj.(*v1.Node) item := &workItem{ node: node, diff --git a/pkg/controllers/tagging/tagging_controller_test.go b/pkg/controllers/tagging/tagging_controller_test.go index 7ee4404d5d..8f35810b30 100644 --- a/pkg/controllers/tagging/tagging_controller_test.go +++ b/pkg/controllers/tagging/tagging_controller_test.go @@ -120,7 +120,7 @@ func Test_NodesJoiningAndLeaving(t *testing.T) { } //eventBroadcaster := record.NewBroadcaster() - tc := &TaggingController{ + tc := &Controller{ nodeInformer: nodeInformer, kubeClient: clientset, cloud: fakeAws, diff --git a/pkg/controllers/tagging/tagging_controller_wrapper.go b/pkg/controllers/tagging/tagging_controller_wrapper.go index cef64d5bef..1734fb454e 100644 --- a/pkg/controllers/tagging/tagging_controller_wrapper.go +++ b/pkg/controllers/tagging/tagging_controller_wrapper.go @@ -14,8 +14,11 @@ import ( ) const ( + // TaggingControllerClientName is the name of the tagging controller TaggingControllerClientName = "tagging-controller" - TaggingControllerKey = "tagging" + + // TaggingControllerKey is the key used to register this controller + TaggingControllerKey = "tagging" ) type ControllerWrapper struct { @@ -35,7 +38,7 @@ func (tc *ControllerWrapper) startTaggingController(ctx context.Context, initCon klog.Fatalf("Tagging controller inputs are not properly set: %v", err) } - // Start the TaggingController + // Start the Controller taggingcontroller, err := NewTaggingController( completedConfig.SharedInformers.Core().V1().Nodes(), completedConfig.ClientBuilder.ClientOrDie(initContext.ClientName), From 46167e1d7bf22461bece9f52742e81a62b57fd86 Mon Sep 17 00:00:00 2001 From: Nguyen Dinh <92940366+nguyenkndinh@users.noreply.github.com> Date: Tue, 29 Mar 2022 13:45:27 -0700 Subject: [PATCH 38/43] missed a couple more lint errors --- pkg/controllers/tagging/tagging_controller_wrapper.go | 1 + pkg/providers/v1/aws_fakes.go | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/pkg/controllers/tagging/tagging_controller_wrapper.go b/pkg/controllers/tagging/tagging_controller_wrapper.go index 1734fb454e..f5d376b482 100644 --- a/pkg/controllers/tagging/tagging_controller_wrapper.go +++ b/pkg/controllers/tagging/tagging_controller_wrapper.go @@ -21,6 +21,7 @@ const ( TaggingControllerKey = "tagging" ) +// ControllerWrapper is the wrapper for the tagging controller type ControllerWrapper struct { Options options.TaggingControllerOptions } diff --git a/pkg/providers/v1/aws_fakes.go b/pkg/providers/v1/aws_fakes.go index d130bc0947..3b84a97b63 100644 --- a/pkg/providers/v1/aws_fakes.go +++ b/pkg/providers/v1/aws_fakes.go @@ -265,7 +265,7 @@ func (ec2i *FakeEC2Impl) RemoveSubnets() { ec2i.Subnets = ec2i.Subnets[:0] } -// Mock CreateTags from EC2 +// CreateTags is a mock for CreateTags from EC2 func (ec2i *FakeEC2Impl) CreateTags(input *ec2.CreateTagsInput) (*ec2.CreateTagsOutput, error) { for _, id := range input.Resources { if *id == "i-error" { @@ -275,7 +275,7 @@ func (ec2i *FakeEC2Impl) CreateTags(input *ec2.CreateTagsInput) (*ec2.CreateTags return &ec2.CreateTagsOutput{}, nil } -// Mock DeleteTags from EC2 +// DeleteTags is a mock for DeleteTags from EC2 func (ec2i *FakeEC2Impl) DeleteTags(input *ec2.DeleteTagsInput) (*ec2.DeleteTagsOutput, error) { for _, id := range input.Resources { if *id == "i-error" { From 8410726c5ad701e1e48ecc512aa728346638cd2b Mon Sep 17 00:00:00 2001 From: Nguyen Dinh <92940366+nguyenkndinh@users.noreply.github.com> Date: Tue, 29 Mar 2022 13:50:12 -0700 Subject: [PATCH 39/43] Updated doc to be clearer --- docs/tagging_controller.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/tagging_controller.md b/docs/tagging_controller.md index 74161ab969..be4bd86c2e 100644 --- a/docs/tagging_controller.md +++ b/docs/tagging_controller.md @@ -1,6 +1,6 @@ # The Tagging Controller -The tagging controller is responsible for tagging and untagging node resources when they join and leave the cluster, respectively. It can add and remove tags based on user input. Unlike the existing controllers, the tagging controller works exclusively with AWS. The AWS APIs it uses are `ec2:CreateTags` and `ec2:DeleteTags`. +The tagging controller is responsible for tagging and untagging node resources when they join and leave the cluster, respectively. It can add and remove tags based on user input. Additionally, if a tag is updated, it would leave the updated tag and reapply the user-provided tag. Unlike the existing controllers, the tagging controller works exclusively with AWS. The AWS APIs it uses are `ec2:CreateTags` and `ec2:DeleteTags`. | Flag | Valid Values | Default | Description | |------| --- | --- | --- | From 6109333bd049254d837eb5e82900cbb7fd6e2cd8 Mon Sep 17 00:00:00 2001 From: Nguyen Dinh <92940366+nguyenkndinh@users.noreply.github.com> Date: Tue, 29 Mar 2022 18:26:19 -0700 Subject: [PATCH 40/43] Add TODOs for e2e testing and non-retryable workitem --- docs/TODO.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/TODO.md b/docs/TODO.md index 2cbf0e9d56..af4b88e4f7 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -15,3 +15,8 @@ ### Kops * Add a full example (ideally with IAM roles) + +### Tagging Controller + +* Add e2e testing which enables the controller, and monitors if the resources are tagged properly +* Handle the case where potential non-retryable errors are enqeueued indefinitely. \ No newline at end of file From 2b37e3c5320493ca09f756d09bcd4d50855cac3f Mon Sep 17 00:00:00 2001 From: Nick Turner Date: Sun, 6 Mar 2022 19:55:52 -0800 Subject: [PATCH 41/43] Fix version * Fix ldflags. * Use `git describe --dirty --tags --match='v*'` for the git version. * This will produce just the tag on a commit which is tagged. * If not tagged it will be something like `v1.23.0-alpha.0-16-g67cbbd12-dirty` referencing the most recent tag on the current branch. --- Dockerfile | 10 +++++- Makefile | 45 +++++++++++++++++++----- cmd/aws-cloud-controller-manager/main.go | 2 -- 3 files changed, 45 insertions(+), 12 deletions(-) diff --git a/Dockerfile b/Dockerfile index bf678d9478..e2463ba268 100644 --- a/Dockerfile +++ b/Dockerfile @@ -32,9 +32,11 @@ ARG DISTROLESS_IMAGE=gcr.io/distroless/static@sha256:c6d5981545ce1406d33e61434c6 # libc, muscl, etc. FROM ${GOLANG_IMAGE} as builder -ARG VERSION ARG GOPROXY=https://goproxy.io,direct ARG GOOS=linux +ARG TARGETOS +ARG TARGETARCH +ARG VERSION WORKDIR /build COPY go.mod go.sum ./ @@ -42,6 +44,12 @@ COPY cmd/ cmd/ COPY pkg/ pkg/ RUN GO111MODULE=on CGO_ENABLED=0 GOOS=${GOOS} GOPROXY=${GOPROXY} go build \ -ldflags="-w -s -X 'main.version=${VERSION}'" \ +======= +RUN GO111MODULE=on CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} GOPROXY=${GOPROXY} \ + go build \ + -trimpath \ + -ldflags="-w -s -X k8s.io/component-base/version.gitVersion=${VERSION}" \ +>>>>>>> Fix version -o=aws-cloud-controller-manager \ cmd/aws-cloud-controller-manager/main.go diff --git a/Makefile b/Makefile index c1227e09db..fd9bab7616 100644 --- a/Makefile +++ b/Makefile @@ -16,27 +16,54 @@ SOURCES := $(shell find . -name '*.go') GOOS ?= $(shell go env GOOS) GOPROXY ?= $(shell go env GOPROXY) -GIT_VERSION := $(shell git describe --match=$(git rev-parse --short=8 HEAD) --always --dirty --abbrev=8) +GIT_VERSION := $(shell git describe --dirty --tags --match='v*') VERSION ?= $(GIT_VERSION) IMAGE := amazon/cloud-controller-manager:$(VERSION) +OUTPUT ?= $(shell pwd)/_output +INSTALL_PATH ?= $(OUTPUT)/bin +LDFLAGS ?= -w -s -X k8s.io/component-base/version.gitVersion=$(VERSION) aws-cloud-controller-manager: $(SOURCES) - GO111MODULE=on CGO_ENABLED=0 GOOS=$(GOOS) GOPROXY=$(GOPROXY) go build \ - -ldflags="-w -s -X 'main.version=$(VERSION)'" \ + GO111MODULE=on CGO_ENABLED=0 GOOS=$(GOOS) GOARCH=$(GOARCH) GOPROXY=$(GOPROXY) go build \ + -trimpath \ + -ldflags="$(LDFLAGS)" \ -o=aws-cloud-controller-manager \ cmd/aws-cloud-controller-manager/main.go -ecr-credential-provider: $(shell find ./cmd/ecr-credential-provider -name '*.go') - GO111MODULE=on CGO_ENABLED=0 GOOS=$(GOOS) GOPROXY=$(GOPROXY) go build \ - -ldflags="-w -s -X 'main.version=$(VERSION)'" \ +ecr-credential-provider: $(shell find ./cmd/ecr-credential-provider -name '*.go') + GO111MODULE=on CGO_ENABLED=0 GOOS=$(GOOS) GOARCH=$(GOARCH) GOPROXY=$(GOPROXY) go build \ + -trimpath \ + -ldflags="$(LDFLAGS)" \ -o=ecr-credential-provider \ cmd/ecr-credential-provider/*.go +ecr-credential-provider.exe: $(wildcard ./cmd/ecr-credential-provider/*.go) + GO111MODULE=on CGO_ENABLED=0 GOOS=windows GOPROXY=$(GOPROXY) go build \ + -trimpath \ + -ldflags="$(LDFLAGS)" \ + -o=ecr-credential-provider.exe \ + cmd/ecr-credential-provider/*.go + +.PHONY: docker-build-amd64 +docker-build-amd64: + docker buildx build --output=type=docker \ + --build-arg VERSION=$(VERSION) \ + --build-arg GOPROXY=$(GOPROXY) \ + --platform linux/amd64 \ + --tag $(IMAGE) . + +.PHONY: docker-build-arm64 +docker-build-arm64: + docker buildx build --output=type=docker \ + --build-arg VERSION=$(VERSION) \ + --build-arg GOPROXY=$(GOPROXY) \ + --platform linux/arm64 \ + --tag $(IMAGE) . + .PHONY: docker-build docker-build: - docker build \ - --build-arg VERSION=$(VERSION) \ - --build-arg GOOS=$(GOOS) \ + docker buildx build --output=type=registry \ + --build-arg LDFLAGS=$(LDFLAGS) \ --build-arg GOPROXY=$(GOPROXY) \ --tag $(IMAGE) . diff --git a/cmd/aws-cloud-controller-manager/main.go b/cmd/aws-cloud-controller-manager/main.go index 011cc4b5aa..cf1f85db9e 100644 --- a/cmd/aws-cloud-controller-manager/main.go +++ b/cmd/aws-cloud-controller-manager/main.go @@ -49,8 +49,6 @@ const ( enableAlphaV2EnvVar = "ENABLE_ALPHA_V2" ) -var version string - func main() { rand.Seed(time.Now().UTC().UnixNano()) From 72fbe7cb204cc2c2c4f048e2e664f0f38fbbc154 Mon Sep 17 00:00:00 2001 From: Nguyen Dinh <92940366+nguyenkndinh@users.noreply.github.com> Date: Tue, 5 Apr 2022 16:38:32 -0700 Subject: [PATCH 42/43] Stop retrying failed workitem after a certain amount of retries Added metrics --- docs/TODO.md | 3 +- pkg/controllers/tagging/metrics.go | 56 ++++++++++++ pkg/controllers/tagging/tagging_controller.go | 91 +++++++++++++------ .../tagging/tagging_controller_test.go | 19 +++- 4 files changed, 135 insertions(+), 34 deletions(-) create mode 100644 pkg/controllers/tagging/metrics.go diff --git a/docs/TODO.md b/docs/TODO.md index af4b88e4f7..5013e2bdcb 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -18,5 +18,4 @@ ### Tagging Controller -* Add e2e testing which enables the controller, and monitors if the resources are tagged properly -* Handle the case where potential non-retryable errors are enqeueued indefinitely. \ No newline at end of file +* Add e2e testing which enables the controller, and monitors if the resources are tagged properly \ No newline at end of file diff --git a/pkg/controllers/tagging/metrics.go b/pkg/controllers/tagging/metrics.go new file mode 100644 index 0000000000..5a263086f2 --- /dev/null +++ b/pkg/controllers/tagging/metrics.go @@ -0,0 +1,56 @@ +/* +Copyright 2020 The Kubernetes Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tagging + +import ( + "k8s.io/component-base/metrics" + "k8s.io/component-base/metrics/legacyregistry" + "sync" +) + +var register sync.Once + +var ( + workItemDuration = metrics.NewHistogramVec( + &metrics.HistogramOpts{ + Name: "cloudprovider_aws_tagging_controller_work_item_duration_seconds", + Help: "workitem latency of workitem being in the queue and time it takes to process", + StabilityLevel: metrics.ALPHA, + }, + []string{"latency_type"}) + + workItemError = metrics.NewCounterVec( + &metrics.CounterOpts{ + Name: "cloudprovider_aws_tagging_controller_work_item_errors_total", + Help: "any error in dequeueing the work queue and processing workItem", + StabilityLevel: metrics.ALPHA, + }, + []string{"error_type", "instance_id"}) +) + +// registerMetrics registers tagging-controller metrics. +func registerMetrics() { + register.Do(func() { + legacyregistry.MustRegister(workItemDuration) + legacyregistry.MustRegister(workItemError) + }) +} + +func recordWorkItemLatencyMetrics(latencyType string, timeTaken float64) { + workItemDuration.With(metrics.Labels{"latency_type": latencyType}).Observe(timeTaken) +} + +func recordWorkItemErrorMetrics(errorType string, instanceID string) { + workItemError.With(metrics.Labels{"error_type": errorType, "instance_id": instanceID}).Inc() +} diff --git a/pkg/controllers/tagging/tagging_controller.go b/pkg/controllers/tagging/tagging_controller.go index fc84f2b5ae..9e56cb6337 100644 --- a/pkg/controllers/tagging/tagging_controller.go +++ b/pkg/controllers/tagging/tagging_controller.go @@ -31,12 +31,30 @@ import ( // workItem contains the node and an action for that node type workItem struct { - node *v1.Node - action func(node *v1.Node) error + node *v1.Node + action func(node *v1.Node) error + requeuingCount int + enqueueTime time.Time } +const ( + maxRequeuingCount = 9 + + // The label for depicting total number of errors a work item encounter and succeed + totalErrorsWorkItemErrorMetric = "total_errors" + + // The label for depicting total time when work item gets queued to processed + workItemProcessingTimeWorkItemMetric = "work_item_processing_time" + + // The label for depicting total time when work item gets queued to dequeued + workItemDequeuingTimeWorkItemMetric = "work_item_dequeuing_time" + + // The label for depicting total number of errors a work item encounter and fail + errorsAfterRetriesExhaustedWorkItemErrorMetric = "errors_after_retries_exhausted" +) + // Controller is the controller implementation for tagging cluster resources. -// It periodically check for Node events (creating/deleting) to apply appropriate +// It periodically checks for Node events (creating/deleting) to apply/delete appropriate // tags to resources. type Controller struct { nodeInformer coreinformers.NodeInformer @@ -44,6 +62,7 @@ type Controller struct { cloud *awsv1.Cloud workqueue workqueue.RateLimitingInterface nodesSynced cache.InformerSynced + // Value controlling Controller monitoring period, i.e. how often does Controller // check node list. This value should be lower than nodeMonitorGracePeriod // set in controller-manager @@ -71,15 +90,16 @@ func NewTaggingController( return nil, err } + registerMetrics() tc := &Controller{ nodeInformer: nodeInformer, kubeClient: kubeClient, cloud: awsCloud, - nodeMonitorPeriod: nodeMonitorPeriod, tags: tags, resources: resources, workqueue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "Tagging"), nodesSynced: nodeInformer.Informer().HasSynced, + nodeMonitorPeriod: nodeMonitorPeriod, } // Use shared informer to listen to add/update/delete of nodes. Note that any nodes @@ -135,19 +155,42 @@ func (tc *Controller) process() bool { workItem, ok := obj.(*workItem) if !ok { tc.workqueue.Forget(obj) - utilruntime.HandleError(fmt.Errorf("expected workItem in workqueue but got %#v", obj)) + err := fmt.Errorf("expected workItem in workqueue but got %#v", obj) + utilruntime.HandleError(err) + return nil + } + + timeTaken := time.Since(workItem.enqueueTime).Seconds() + recordWorkItemLatencyMetrics(workItemDequeuingTimeWorkItemMetric, timeTaken) + + instanceID, err := awsv1.KubernetesInstanceID(workItem.node.Spec.ProviderID).MapToAWSInstanceID() + if err != nil { + err = fmt.Errorf("Error in getting instanceID for node %s, error: %v", workItem.node.GetName(), err) + utilruntime.HandleError(err) return nil } - err := workItem.action(workItem.node) + err = workItem.action(workItem.node) + if err != nil { - // Put the item back on the workqueue to handle any transient errors. - tc.workqueue.AddRateLimited(workItem) - return fmt.Errorf("error finishing work item '%v': %s, requeuing", workItem, err.Error()) + if workItem.requeuingCount < maxRequeuingCount { + // Put the item back on the workqueue to handle any transient errors. + workItem.requeuingCount++ + tc.workqueue.AddRateLimited(workItem) + + recordWorkItemErrorMetrics(totalErrorsWorkItemErrorMetric, string(instanceID)) + return fmt.Errorf("error processing work item '%v': %s, requeuing count %d", workItem, err.Error(), workItem.requeuingCount) + } + + klog.Errorf("error processing work item '%v': %s, requeuing count exceeded", workItem, err.Error()) + recordWorkItemErrorMetrics(errorsAfterRetriesExhaustedWorkItemErrorMetric, string(instanceID)) + } else { + klog.Infof("Finished processing %v", workItem) + timeTaken = time.Since(workItem.enqueueTime).Seconds() + recordWorkItemLatencyMetrics(workItemProcessingTimeWorkItemMetric, timeTaken) } tc.workqueue.Forget(obj) - klog.Infof("Finished processing %v", workItem) return nil }(obj) @@ -159,7 +202,7 @@ func (tc *Controller) process() bool { return true } -// tagNodesResources tag node resources from a list of nodes +// tagNodesResources tag node resources // If we want to tag more resources, modify this function appropriately func (tc *Controller) tagNodesResources(node *v1.Node) error { for _, resource := range tc.resources { @@ -178,14 +221,9 @@ func (tc *Controller) tagNodesResources(node *v1.Node) error { // tagEc2Instances applies the provided tags to each EC2 instance in // the cluster. func (tc *Controller) tagEc2Instance(node *v1.Node) error { - instanceID, err := awsv1.KubernetesInstanceID(node.Spec.ProviderID).MapToAWSInstanceID() + instanceID, _ := awsv1.KubernetesInstanceID(node.Spec.ProviderID).MapToAWSInstanceID() - if err != nil { - klog.Errorf("Error in getting instanceID for node %s, error: %v", node.GetName(), err) - return err - } - - err = tc.cloud.TagResource(string(instanceID), tc.tags) + err := tc.cloud.TagResource(string(instanceID), tc.tags) if err != nil { klog.Errorf("Error in tagging EC2 instance for node %s, error: %v", node.GetName(), err) @@ -197,7 +235,7 @@ func (tc *Controller) tagEc2Instance(node *v1.Node) error { return nil } -// untagNodeResources untag node resources from a list of nodes +// untagNodeResources untag node resources // If we want to untag more resources, modify this function appropriately func (tc *Controller) untagNodeResources(node *v1.Node) error { for _, resource := range tc.resources { @@ -216,14 +254,9 @@ func (tc *Controller) untagNodeResources(node *v1.Node) error { // untagEc2Instances deletes the provided tags to each EC2 instances in // the cluster. func (tc *Controller) untagEc2Instance(node *v1.Node) error { - instanceID, err := awsv1.KubernetesInstanceID(node.Spec.ProviderID).MapToAWSInstanceID() - - if err != nil { - klog.Errorf("Error in getting instanceID for node %s, error: %v", node.GetName(), err) - return err - } + instanceID, _ := awsv1.KubernetesInstanceID(node.Spec.ProviderID).MapToAWSInstanceID() - err = tc.cloud.UntagResource(string(instanceID), tc.tags) + err := tc.cloud.UntagResource(string(instanceID), tc.tags) if err != nil { klog.Errorf("Error in untagging EC2 instance for node %s, error: %v", node.GetName(), err) @@ -240,8 +273,10 @@ func (tc *Controller) untagEc2Instance(node *v1.Node) error { func (tc *Controller) enqueueNode(obj interface{}, action func(node *v1.Node) error) { node := obj.(*v1.Node) item := &workItem{ - node: node, - action: action, + node: node, + action: action, + requeuingCount: 0, + enqueueTime: time.Now(), } tc.workqueue.Add(item) klog.Infof("Added %s to the workqueue", item) diff --git a/pkg/controllers/tagging/tagging_controller_test.go b/pkg/controllers/tagging/tagging_controller_test.go index 8f35810b30..606aabdb60 100644 --- a/pkg/controllers/tagging/tagging_controller_test.go +++ b/pkg/controllers/tagging/tagging_controller_test.go @@ -135,15 +135,26 @@ func Test_NodesJoiningAndLeaving(t *testing.T) { } else { tc.enqueueNode(testcase.currNode, tc.untagNodeResources) } - tc.process() + + for tc.workqueue.Len() > 0 { + tc.process() + + // sleep briefly because of exponential backoff when requeueing failed workitem + // resulting in workqueue to be empty if checked immediately + time.Sleep(1500 * time.Millisecond) + } for _, msg := range testcase.expectedMessages { if !strings.Contains(logBuf.String(), msg) { t.Errorf("\nMsg %q not found in log: \n%v\n", msg, logBuf.String()) } - if strings.Contains(logBuf.String(), "error tagging ") || strings.Contains(logBuf.String(), "error untagging ") { - if !strings.Contains(logBuf.String(), ", requeuing") { - t.Errorf("\nFailed to tag or untag but logs do not contain 'requeueing': \n%v\n", logBuf.String()) + if strings.Contains(logBuf.String(), "Unable to tag") || strings.Contains(logBuf.String(), "Unable to untag") { + if !strings.Contains(logBuf.String(), ", requeuing count ") { + t.Errorf("\nFailed to tag or untag but logs did not requeue: \n%v\n", logBuf.String()) + } + + if !strings.Contains(logBuf.String(), "requeuing count exceeded") { + t.Errorf("\nExceeded requeue count but did not stop: \n%v\n", logBuf.String()) } } } From 1ad79e9d428a42d4bedfff64953666fef2084286 Mon Sep 17 00:00:00 2001 From: Saurav Agarwalla Date: Wed, 11 May 2022 16:16:33 -0400 Subject: [PATCH 43/43] Sync Docker and Make files from master --- Dockerfile | 20 +++---- Makefile | 41 +++++++++++++- kops-example/cluster.yaml | 111 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 157 insertions(+), 15 deletions(-) create mode 100644 kops-example/cluster.yaml diff --git a/Dockerfile b/Dockerfile index e2463ba268..8b79f61d92 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,26 +14,24 @@ ## BUILD ARGS ## ################################################################################ # This build arg allows the specification of a custom Golang image. -ARG GOLANG_IMAGE=golang:1.15.3 +ARG GOLANG_IMAGE=golang:1.18.1 # The distroless image on which the CPI manager image is built. # # Please do not use "latest". Explicit tags should be used to provide -# deterministic builds. This image doesn't have semantic version tags, but -# the fully-qualified image can be obtained by entering -# "gcr.io/distroless/static:latest" in a browser and then copying the -# fully-qualified image from the web page. -ARG DISTROLESS_IMAGE=gcr.io/distroless/static@sha256:c6d5981545ce1406d33e61434c61e9452dad93ecd8397c41e89036ef977a88f4 +# deterministic builds. Follow what kubernetes uses to build +# kube-controller-manager, for example for 1.23.x: +# https://github.com/kubernetes/kubernetes/blob/release-1.24/build/common.sh#L94 +ARG DISTROLESS_IMAGE=k8s.gcr.io/build-image/go-runner:v2.3.1-go1.18.1-bullseye.0 ################################################################################ ## BUILD STAGE ## ################################################################################ # Build the manager as a statically compiled binary so it has no dependencies # libc, muscl, etc. -FROM ${GOLANG_IMAGE} as builder +FROM --platform=linux/amd64 ${GOLANG_IMAGE} as builder ARG GOPROXY=https://goproxy.io,direct -ARG GOOS=linux ARG TARGETOS ARG TARGETARCH ARG VERSION @@ -42,14 +40,10 @@ WORKDIR /build COPY go.mod go.sum ./ COPY cmd/ cmd/ COPY pkg/ pkg/ -RUN GO111MODULE=on CGO_ENABLED=0 GOOS=${GOOS} GOPROXY=${GOPROXY} go build \ - -ldflags="-w -s -X 'main.version=${VERSION}'" \ -======= RUN GO111MODULE=on CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} GOPROXY=${GOPROXY} \ go build \ -trimpath \ -ldflags="-w -s -X k8s.io/component-base/version.gitVersion=${VERSION}" \ ->>>>>>> Fix version -o=aws-cloud-controller-manager \ cmd/aws-cloud-controller-manager/main.go @@ -57,6 +51,6 @@ RUN GO111MODULE=on CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} GOPROXY=$ ## MAIN STAGE ## ################################################################################ # Copy the manager into the distroless image. -FROM ${DISTROLESS_IMAGE} +FROM --platform=${TARGETPLATFORM} ${DISTROLESS_IMAGE} COPY --from=builder /build/aws-cloud-controller-manager /bin/aws-cloud-controller-manager ENTRYPOINT [ "/bin/aws-cloud-controller-manager" ] diff --git a/Makefile b/Makefile index fd9bab7616..09b85d23af 100644 --- a/Makefile +++ b/Makefile @@ -13,12 +13,16 @@ # limitations under the License. # +.EXPORT_ALL_VARIABLES: + +SHELL := /bin/bash SOURCES := $(shell find . -name '*.go') GOOS ?= $(shell go env GOOS) +GOARCH ?= $(shell go env GOARCH) GOPROXY ?= $(shell go env GOPROXY) GIT_VERSION := $(shell git describe --dirty --tags --match='v*') VERSION ?= $(GIT_VERSION) -IMAGE := amazon/cloud-controller-manager:$(VERSION) +IMAGE ?= amazon/cloud-controller-manager:$(VERSION) OUTPUT ?= $(shell pwd)/_output INSTALL_PATH ?= $(OUTPUT)/bin LDFLAGS ?= -w -s -X k8s.io/component-base/version.gitVersion=$(VERSION) @@ -65,8 +69,14 @@ docker-build: docker buildx build --output=type=registry \ --build-arg LDFLAGS=$(LDFLAGS) \ --build-arg GOPROXY=$(GOPROXY) \ + --platform linux/amd64,linux/arm64 \ --tag $(IMAGE) . +e2e.test: + pushd tests/e2e > /dev/null && \ + go test -c && popd + mv tests/e2e/e2e.test e2e.test + .PHONY: check check: verify-fmt verify-lint vet @@ -80,7 +90,7 @@ verify-fmt: .PHONY: verify-lint verify-lint: - which golint 2>&1 >/dev/null || go get golang.org/x/lint/golint + which golint 2>&1 >/dev/null || go install golang.org/x/lint/golint@latest golint -set_exit_status $(shell go list ./...) .PHONY: verify-codegen @@ -102,3 +112,30 @@ docs: .PHONY: publish-docs publish-docs: ./hack/publish-docs.sh + +.PHONY: kops-example +kops-example: + ./hack/kops-example.sh + +.PHONY: test-e2e +test-e2e: e2e.test docker-build-amd64 install-e2e-tools + AWS_REGION=us-west-2 \ + TEST_PATH=./tests/e2e/... \ + BUILD_IMAGE=$(IMAGE) \ + BUILD_VERSION=$(VERSION) \ + INSTALL_PATH=$(INSTALL_PATH) \ + GINKGO_FOCUS="\[cloud-provider-aws-e2e\]" \ + ./hack/e2e/run.sh + +# Use `make install-e2e-tools KOPS_ROOT=` +# to skip the kops download, test local changes to the kubetest2-kops +# deployer, etc. +.PHONY: install-e2e-tools +install-e2e-tools: + mkdir -p $(INSTALL_PATH) + INSTALL_PATH=$(INSTALL_PATH) \ + ./hack/install-e2e-tools.sh + +.PHONY: print-image-tag +print-image-tag: + @echo $(IMAGE) diff --git a/kops-example/cluster.yaml b/kops-example/cluster.yaml new file mode 100644 index 0000000000..6a6769b15a --- /dev/null +++ b/kops-example/cluster.yaml @@ -0,0 +1,111 @@ +apiVersion: kops.k8s.io/v1alpha2 +kind: Cluster +metadata: + name: aws-external-cloud-provider-example.k8s.local +spec: + api: + loadBalancer: + type: Public + authorization: + rbac: {} + channel: stable + cloudControllerManager: + cloudProvider: aws + configureCloudRoutes: false + useServiceAccountCredentials: true + cloudProvider: aws + containerRuntime: docker + etcdClusters: + - cpuRequest: 200m + etcdMembers: + - instanceGroup: master-us-west-2a + name: a + memoryRequest: 100Mi + name: main + - cpuRequest: 100m + etcdMembers: + - instanceGroup: master-us-west-2a + name: a + memoryRequest: 100Mi + name: events + fileAssets: + - content: | + apiVersion: audit.k8s.io/v1 + kind: Policy + rules: + - level: Metadata + name: audit-policy-config + path: /srv/kubernetes/audit/policy-config.yaml + roles: + - Master + iam: + allowContainerRegistry: true + legacy: false + kubeAPIServer: + auditLogMaxAge: 10 + auditLogMaxBackups: 1 + auditLogMaxSize: 100 + auditLogPath: /var/log/kube-apiserver-audit.log + auditPolicyFile: /srv/kubernetes/audit/policy-config.yaml + cloudProvider: external + kubeControllerManager: + externalCloudVolumePlugin: aws + kubelet: + anonymousAuth: false + cloudProvider: aws + kubernetesApiAccess: + - 0.0.0.0/0 + kubernetesVersion: 1.21.0 + masterInternalName: api.internal.aws-external-cloud-provider-example.k8s.local + masterPublicName: api.aws-external-cloud-provider-example.k8s.local + networkCIDR: 172.20.0.0/16 + networking: + amazonvpc: {} + nonMasqueradeCIDR: 100.64.0.0/10 + sshAccess: + - 0.0.0.0/0 + sshKeyName: test2 + subnets: + - cidr: 172.20.32.0/19 + name: us-west-2a + type: Public + zone: us-west-2a + topology: + dns: + type: Public + masters: public + nodes: public +--- +apiVersion: kops.k8s.io/v1alpha2 +kind: InstanceGroup +metadata: + labels: + kops.k8s.io/cluster: aws-external-cloud-provider-example.k8s.local + name: master-us-west-2a +spec: + image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20200716 + machineType: t3.medium + maxSize: 1 + minSize: 1 + nodeLabels: + kops.k8s.io/instancegroup: master-us-west-2a + role: Master + subnets: + - us-west-2a +--- +apiVersion: kops.k8s.io/v1alpha2 +kind: InstanceGroup +metadata: + labels: + kops.k8s.io/cluster: aws-external-cloud-provider-example.k8s.local + name: nodes-us-west-2a +spec: + image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20200716 + machineType: t3.medium + maxSize: 1 + minSize: 1 + nodeLabels: + kops.k8s.io/instancegroup: nodes-us-west-2a + role: Node + subnets: + - us-west-2a