[AWS EKS - Scale-to-0] Add EKS service and DescribeNodegroup API call

This change adds the AWS EKS service to the vendor service directory and adds the DescribeNodegroup API call to the AWSWrapper. This will be used for the AWS scale-to-0 project to get more information about empty managed nodegroups

Related proposal: https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/proposals/circumvent-tag-limit-aws.md

cluster-autoscaler/cloudprovider/aws/aws_cloud_provider_test.go

@@ -38,8 +38,8 @@ var testAwsManager = &AwsManager{
 	awsService: testAwsService,
 }
 
-func newTestAwsManagerWithMockServices(mockAutoScaling autoScalingI, mockEC2 ec2I, autoDiscoverySpecs []asgAutoDiscoveryConfig) *AwsManager {
-	awsService := awsWrapper{mockAutoScaling, mockEC2}
+func newTestAwsManagerWithMockServices(mockAutoScaling autoScalingI, mockEC2 ec2I, mockEKS eksI, autoDiscoverySpecs []asgAutoDiscoveryConfig) *AwsManager {
+	awsService := awsWrapper{mockAutoScaling, mockEC2, mockEKS}
 	return &AwsManager{
 		awsService: awsService,
 		asgCache: &asgCache{
@@ -56,13 +56,13 @@ func newTestAwsManagerWithMockServices(mockAutoScaling autoScalingI, mockEC2 ec2
 }
 
 func newTestAwsManagerWithAsgs(t *testing.T, mockAutoScaling autoScalingI, mockEC2 ec2I, specs []string) *AwsManager {
-	m := newTestAwsManagerWithMockServices(mockAutoScaling, mockEC2, nil)
+	m := newTestAwsManagerWithMockServices(mockAutoScaling, mockEC2, nil, nil)
 	m.asgCache.parseExplicitAsgs(specs)
 	return m
 }
 
 func newTestAwsManagerWithAutoAsgs(t *testing.T, mockAutoScaling autoScalingI, mockEC2 ec2I, specs []string, autoDiscoverySpecs []asgAutoDiscoveryConfig) *AwsManager {
-	m := newTestAwsManagerWithMockServices(mockAutoScaling, mockEC2, autoDiscoverySpecs)
+	m := newTestAwsManagerWithMockServices(mockAutoScaling, mockEC2, nil, autoDiscoverySpecs)
 	m.asgCache.parseExplicitAsgs(specs)
 	return m
 }
@@ -524,7 +524,7 @@ func TestDeleteNodesAfterMultipleRefreshes(t *testing.T) {
 func TestGetResourceLimiter(t *testing.T) {
 	mockAutoScaling := &autoScalingMock{}
 	mockEC2 := &ec2Mock{}
-	m := newTestAwsManagerWithMockServices(mockAutoScaling, mockEC2, nil)
+	m := newTestAwsManagerWithMockServices(mockAutoScaling, mockEC2, nil, nil)
 
 	provider := testProvider(t, m)
 	_, err := provider.GetResourceLimiter()

cluster-autoscaler/cloudprovider/aws/aws_manager.go

@@ -34,6 +34,7 @@ import (
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/autoscaling"
 	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/aws/aws-sdk-go/service/eks"
 	"gopkg.in/gcfg.v1"
 	apiv1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
@@ -194,7 +195,7 @@ func createAWSManagerInternal(
 			return nil, err
 		}
 
-		awsService = &awsWrapper{autoscaling.New(sess), ec2.New(sess)}
+		awsService = &awsWrapper{autoscaling.New(sess), ec2.New(sess), eks.New(sess)}
 	}
 
 	specs, err := parseASGAutoDiscoverySpecs(discoveryOpts)

cluster-autoscaler/cloudprovider/aws/aws_manager_test.go

@@ -330,7 +330,7 @@ func TestFetchExplicitAsgs(t *testing.T) {
 	defer resetAWSRegion(os.LookupEnv("AWS_REGION"))
 	os.Setenv("AWS_REGION", "fanghorn")
 	instanceTypes, _ := GetStaticEC2InstanceTypes()
-	m, err := createAWSManagerInternal(nil, do, &awsWrapper{a, nil}, instanceTypes)
+	m, err := createAWSManagerInternal(nil, do, &awsWrapper{a, nil, nil}, instanceTypes)
 	assert.NoError(t, err)
 
 	asgs := m.asgCache.Get()
@@ -395,7 +395,7 @@ func TestGetASGTemplate(t *testing.T) {
 			instanceTypes, _ := GetStaticEC2InstanceTypes()
 			do := cloudprovider.NodeGroupDiscoveryOptions{}
 
-			m, err := createAWSManagerInternal(nil, do, &awsWrapper{nil, e}, instanceTypes)
+			m, err := createAWSManagerInternal(nil, do, &awsWrapper{nil, e, nil}, instanceTypes)
 			origGetInstanceTypeFunc := getInstanceTypeForAsg
 			defer func() { getInstanceTypeForAsg = origGetInstanceTypeFunc }()
 			getInstanceTypeForAsg = func(m *asgCache, asg *asg) (string, error) {
@@ -483,7 +483,7 @@ func TestFetchAutoAsgs(t *testing.T) {
 	os.Setenv("AWS_REGION", "fanghorn")
 	// fetchAutoASGs is called at manager creation time, via forceRefresh
 	instanceTypes, _ := GetStaticEC2InstanceTypes()
-	m, err := createAWSManagerInternal(nil, do, &awsWrapper{a, nil}, instanceTypes)
+	m, err := createAWSManagerInternal(nil, do, &awsWrapper{a, nil, nil}, instanceTypes)
 	assert.NoError(t, err)
 
 	asgs := m.asgCache.Get()

cluster-autoscaler/cloudprovider/aws/aws_wrapper.go

@@ -18,11 +18,14 @@ package aws
 
 import (
 	"fmt"
+	"strconv"
 	"time"
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/autoscaling"
 	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/aws/aws-sdk-go/service/eks"
+	apiv1 "k8s.io/api/core/v1"
 	klog "k8s.io/klog/v2"
 )
 
@@ -40,10 +43,73 @@ type ec2I interface {
 	DescribeLaunchTemplateVersions(input *ec2.DescribeLaunchTemplateVersionsInput) (*ec2.DescribeLaunchTemplateVersionsOutput, error)
 }
 
+// eksI is the interface that represents a specific aspect of EKS (Elastic Kubernetes Service) which is provided by AWS SDK for use in CA
+type eksI interface {
+	DescribeNodegroup(input *eks.DescribeNodegroupInput) (*eks.DescribeNodegroupOutput, error)
+}
+
 // awsWrapper provides several utility methods over the services provided by the AWS SDK
 type awsWrapper struct {
 	autoScalingI
 	ec2I
+	eksI
+}
+
+func (m *awsWrapper) getManagedNodegroupInfo(nodegroupName string, clusterName string) ([]apiv1.Taint, map[string]string, error) {
+	params := &eks.DescribeNodegroupInput{
+		ClusterName:   &clusterName,
+		NodegroupName: &nodegroupName,
+	}
+	start := time.Now()
+	r, err := m.DescribeNodegroup(params)
+	observeAWSRequest("DescribeNodegroup", err, start)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	taints := make([]apiv1.Taint, 0)
+	labels := make(map[string]string)
+
+	// Labels will include diskSize, amiType, capacityType, version
+	if r.Nodegroup.DiskSize != nil {
+		labels["diskSize"] = strconv.FormatInt(*r.Nodegroup.DiskSize, 10)
+	}
+
+	if r.Nodegroup.AmiType != nil && len(*r.Nodegroup.AmiType) > 0 {
+		labels["amiType"] = *r.Nodegroup.AmiType
+	}
+
+	if r.Nodegroup.CapacityType != nil && len(*r.Nodegroup.CapacityType) > 0 {
+		labels["capacityType"] = *r.Nodegroup.CapacityType
+	}
+
+	if r.Nodegroup.Version != nil && len(*r.Nodegroup.Version) > 0 {
+		labels["k8sVersion"] = *r.Nodegroup.Version
+	}
+
+	if r.Nodegroup.Labels != nil && len(r.Nodegroup.Labels) > 0 {
+		labelsMap := r.Nodegroup.Labels
+		for k, v := range labelsMap {
+			if v != nil {
+				labels[k] = *v
+			}
+		}
+	}
+
+	if r.Nodegroup.Taints != nil && len(r.Nodegroup.Taints) > 0 {
+		taintList := r.Nodegroup.Taints
+		for _, taint := range taintList {
+			if taint != nil && taint.Effect != nil && taint.Key != nil && taint.Value != nil {
+				taints = append(taints, apiv1.Taint{
+					Key:    *taint.Key,
+					Value:  *taint.Value,
+					Effect: apiv1.TaintEffect(*taint.Effect),
+				})
+			}
+		}
+	}
+
+	return taints, labels, nil
 }
 
 func (m *awsWrapper) getInstanceTypeByLaunchConfigNames(launchConfigToQuery []*string) (map[string]string, error) {

cluster-autoscaler/cloudprovider/aws/aws_wrapper_test.go

@@ -19,13 +19,16 @@ package aws
 import (
 	"fmt"
 	"os"
+	"strconv"
 	"testing"
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/autoscaling"
 	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/aws/aws-sdk-go/service/eks"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/mock"
+	apiv1 "k8s.io/api/core/v1"
 )
 
 type autoScalingMock struct {
@@ -66,13 +69,178 @@ func (e *ec2Mock) DescribeLaunchTemplateVersions(i *ec2.DescribeLaunchTemplateVe
 	return args.Get(0).(*ec2.DescribeLaunchTemplateVersionsOutput), nil
 }
 
-var testAwsService = awsWrapper{&autoScalingMock{}, &ec2Mock{}}
+type eksMock struct {
+	mock.Mock
+}
+
+func (k *eksMock) DescribeNodegroup(i *eks.DescribeNodegroupInput) (*eks.DescribeNodegroupOutput, error) {
+	args := k.Called(i)
+	return args.Get(0).(*eks.DescribeNodegroupOutput), nil
+}
+
+var testAwsService = awsWrapper{&autoScalingMock{}, &ec2Mock{}, &eksMock{}}
+
+func TestGetManagedNodegroup(t *testing.T) {
+	k := &eksMock{}
+	awsWrapper := &awsWrapper{
+		autoScalingI: nil,
+		ec2I:         nil,
+		eksI:         k,
+	}
+
+	labelKey1 := "labelKey 1"
+	labelKey2 := "labelKey 2"
+	labelValue1 := "testValue 1"
+	labelValue2 := "testValue 2"
+	nodegroupName := "testNodegroup"
+	clusterName := "testCluster"
+
+	taintEffect1 := "effect 1"
+	taintKey1 := "key 1"
+	taintValue1 := "value 1"
+	taint1 := eks.Taint{
+		Effect: &taintEffect1,
+		Key:    &taintKey1,
+		Value:  &taintValue1,
+	}
+
+	taintEffect2 := "effect 2"
+	taintKey2 := "key 2"
+	taintValue2 := "value 2"
+	taint2 := eks.Taint{
+		Effect: &taintEffect2,
+		Key:    &taintKey2,
+		Value:  &taintValue2,
+	}
+
+	amiType := "testAmiType"
+	diskSize := int64(100)
+	capacityType := "testCapacityType"
+	k8sVersion := "1.19"
+
+	// Create test nodegroup
+	testNodegroup := eks.Nodegroup{
+		AmiType:       &amiType,
+		ClusterName:   &clusterName,
+		DiskSize:      &diskSize,
+		Labels:        map[string]*string{labelKey1: &labelValue1, labelKey2: &labelValue2},
+		NodegroupName: &nodegroupName,
+		CapacityType:  &capacityType,
+		Version:       &k8sVersion,
+		Taints:        []*eks.Taint{&taint1, &taint2},
+	}
+
+	k.On("DescribeNodegroup", &eks.DescribeNodegroupInput{
+		ClusterName:   &clusterName,
+		NodegroupName: &nodegroupName,
+	}).Return(&eks.DescribeNodegroupOutput{Nodegroup: &testNodegroup})
+
+	taintList, labelMap, err := awsWrapper.getManagedNodegroupInfo(nodegroupName, clusterName)
+	assert.Nil(t, err)
+	assert.Equal(t, len(taintList), 2)
+	assert.Equal(t, taintList[0].Effect, apiv1.TaintEffect(taintEffect1))
+	assert.Equal(t, taintList[0].Key, taintKey1)
+	assert.Equal(t, taintList[0].Value, taintValue1)
+	assert.Equal(t, taintList[1].Effect, apiv1.TaintEffect(taintEffect2))
+	assert.Equal(t, taintList[1].Key, taintKey2)
+	assert.Equal(t, taintList[1].Value, taintValue2)
+	assert.Equal(t, len(labelMap), 6)
+	assert.Equal(t, labelMap[labelKey1], labelValue1)
+	assert.Equal(t, labelMap[labelKey2], labelValue2)
+	assert.Equal(t, labelMap["diskSize"], strconv.FormatInt(diskSize, 10))
+	assert.Equal(t, labelMap["amiType"], amiType)
+	assert.Equal(t, labelMap["capacityType"], capacityType)
+	assert.Equal(t, labelMap["k8sVersion"], k8sVersion)
+}
+
+func TestGetManagedNodegroupWithNilValues(t *testing.T) {
+	k := &eksMock{}
+	awsWrapper := &awsWrapper{
+		autoScalingI: nil,
+		ec2I:         nil,
+		eksI:         k,
+	}
+
+	nodegroupName := "testNodegroup"
+	clusterName := "testCluster"
+
+	amiType := "testAmiType"
+	capacityType := "testCapacityType"
+	k8sVersion := "1.19"
+
+	// Create test nodegroup
+	testNodegroup := eks.Nodegroup{
+		AmiType:       &amiType,
+		ClusterName:   &clusterName,
+		DiskSize:      nil,
+		Labels:        nil,
+		NodegroupName: &nodegroupName,
+		CapacityType:  &capacityType,
+		Version:       &k8sVersion,
+		Taints:        nil,
+	}
+
+	k.On("DescribeNodegroup", &eks.DescribeNodegroupInput{
+		ClusterName:   &clusterName,
+		NodegroupName: &nodegroupName,
+	}).Return(&eks.DescribeNodegroupOutput{Nodegroup: &testNodegroup})
+
+	taintList, labelMap, err := awsWrapper.getManagedNodegroupInfo(nodegroupName, clusterName)
+	assert.Nil(t, err)
+	assert.Equal(t, len(taintList), 0)
+	assert.Equal(t, len(labelMap), 3)
+	assert.Equal(t, labelMap["amiType"], amiType)
+	assert.Equal(t, labelMap["capacityType"], capacityType)
+	assert.Equal(t, labelMap["k8sVersion"], k8sVersion)
+}
+
+func TestGetManagedNodegroupWithEmptyValues(t *testing.T) {
+	k := &eksMock{}
+	awsWrapper := &awsWrapper{
+		autoScalingI: nil,
+		ec2I:         nil,
+		eksI:         k,
+	}
+
+	nodegroupName := "testNodegroup"
+	clusterName := "testCluster"
+
+	amiType := "testAmiType"
+	capacityType := "testCapacityType"
+	k8sVersion := "1.19"
+
+	// Create test nodegroup
+	testNodegroup := eks.Nodegroup{
+		AmiType:       &amiType,
+		ClusterName:   &clusterName,
+		DiskSize:      nil,
+		Labels:        make(map[string]*string),
+		NodegroupName: &nodegroupName,
+		CapacityType:  &capacityType,
+		Version:       &k8sVersion,
+		Taints:        make([]*eks.Taint, 0),
+	}
+
+	k.On("DescribeNodegroup", &eks.DescribeNodegroupInput{
+		ClusterName:   &clusterName,
+		NodegroupName: &nodegroupName,
+	}).Return(&eks.DescribeNodegroupOutput{Nodegroup: &testNodegroup})
+
+	taintList, labelMap, err := awsWrapper.getManagedNodegroupInfo(nodegroupName, clusterName)
+	assert.Nil(t, err)
+	assert.Equal(t, len(taintList), 0)
+	assert.Equal(t, len(labelMap), 3)
+	assert.Equal(t, labelMap["amiType"], amiType)
+	assert.Equal(t, labelMap["capacityType"], capacityType)
+	assert.Equal(t, labelMap["k8sVersion"], k8sVersion)
+}
 
 func TestMoreThen100Groups(t *testing.T) {
 	a := &autoScalingMock{}
 	awsWrapper := &awsWrapper{
 		autoScalingI: a,
 		ec2I:         nil,
+		eksI:         nil,
 	}
 
 	// Generate 101 ASG names
@@ -152,6 +320,7 @@ func TestGetInstanceTypesForAsgs(t *testing.T) {
 	awsWrapper := &awsWrapper{
 		autoScalingI: a,
 		ec2I:         e,
+		eksI:         nil,
 	}
 
 	cases := []struct {

cluster-autoscaler/cloudprovider/aws/instance_type_cache_test.go

@@ -76,7 +76,7 @@ func TestLTVersionChange(t *testing.T) {
 		},
 		fakeClock,
 	)
-	m := newAsgInstanceTypeCacheWithClock(&awsWrapper{a, e}, fakeClock, fakeStore)
+	m := newAsgInstanceTypeCacheWithClock(&awsWrapper{a, e, nil}, fakeClock, fakeStore)
 
 	for i := 0; i < 2; i++ {
 		err := m.populate([]*asg{

cluster-autoscaler/go.mod

@@ -41,6 +41,8 @@ require (
 	k8s.io/utils v0.0.0-20210802155522-efc7438f0176
 )
 
+replace github.com/aws/aws-sdk-go/service/eks => github.com/aws/aws-sdk-go/service/eks v1.38.49
+
 replace github.com/digitalocean/godo => github.com/digitalocean/godo v1.27.0
 
 replace github.com/rancher/go-rancher => github.com/rancher/go-rancher v0.1.0