diff --git a/.landscaper/blueprint/blueprint.yaml b/.landscaper/blueprint/blueprint.yaml index 1564d86c..e3a3a0a9 100644 --- a/.landscaper/blueprint/blueprint.yaml +++ b/.landscaper/blueprint/blueprint.yaml @@ -73,8 +73,6 @@ imports: handleETCDPersistentVolumes: description: "If true, the persistent volumes for the etcds are also deleted." type: boolean - hvpaEnabled: - type: boolean kubeAPIServer: type: object properties: @@ -150,46 +148,6 @@ imports: type: boolean certificateAuthorityData: type: string - hvpaEnabled: - type: boolean - hvpa: - type: object - properties: - maxReplicas: - type: integer - minReplicas: - type: integer - targetAverageUtilizationCpu: - type: integer - targetAverageUtilizationMemory: - type: integer - vpaScaleUpMode: - type: string - vpaScaleDownMode: - type: string - vpaScaleUpStabilization: - type: object - properties: - stabilizationDuration: - type: string - minChange: - "$ref": "#/definitions/scaleParams" - vpaScaleDownStabilization: - type: object - properties: - stabilizationDuration: - type: string - minChange: - "$ref": "#/definitions/scaleParams" - limitsRequestsGapScaleParams: - "$ref": "#/definitions/scaleParams" - maintenanceWindow: - type: object - properties: - begin: - type: string - end: - type: string eventTTL: type: string oidcIssuerURL: diff --git a/cmd/virtual-garden/app/virtual_garden.go b/cmd/virtual-garden/app/virtual_garden.go index d5d7c226..9cfd19d2 100644 --- a/cmd/virtual-garden/app/virtual_garden.go +++ b/cmd/virtual-garden/app/virtual_garden.go @@ -27,15 +27,13 @@ import ( "github.com/gardener/virtual-garden/pkg/api/validation" "github.com/gardener/virtual-garden/pkg/virtualgarden" - hvpav1alpha1 "github.com/gardener/hvpa-controller/api/v1alpha1" lsv1alpha1 "github.com/gardener/landscaper/apis/core/v1alpha1" "github.com/sirupsen/logrus" "github.com/spf13/cobra" "github.com/spf13/pflag" - apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" "k8s.io/apimachinery/pkg/runtime" utilruntime "k8s.io/apimachinery/pkg/util/runtime" - autoscalingv1beta2 "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1beta2" kubernetesscheme "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/tools/clientcmd" "k8s.io/component-base/version/verflag" @@ -178,9 +176,7 @@ func NewClientFromKubeconfig(kubeconfig []byte) (client.Client, error) { scheme := runtime.NewScheme() utilruntime.Must(kubernetesscheme.AddToScheme(scheme)) - utilruntime.Must(hvpav1alpha1.AddToScheme(scheme)) - utilruntime.Must(autoscalingv1beta2.AddToScheme(scheme)) - utilruntime.Must(apiextensionsv1beta1.AddToScheme(scheme)) + utilruntime.Must(apiextensionsv1.AddToScheme(scheme)) return client.New(restConfig, client.Options{Scheme: scheme}) } diff --git a/go.mod b/go.mod index d1d65138..26ea56ab 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,6 @@ require ( github.com/gardener/component-cli v0.36.0 github.com/gardener/component-spec/bindings-go v0.0.53 github.com/gardener/gardener v1.26.0 - github.com/gardener/hvpa-controller v0.3.1 github.com/gardener/landscaper/apis v0.20.0 github.com/ghodss/yaml v1.0.0 github.com/golang/mock v1.6.0 @@ -25,7 +24,6 @@ require ( k8s.io/apiextensions-apiserver v0.21.2 k8s.io/apimachinery v0.21.2 k8s.io/apiserver v0.21.2 - k8s.io/autoscaler v0.0.0-20190805135949-100e91ba756e k8s.io/client-go v11.0.1-0.20190409021438-1a26190bd76a+incompatible k8s.io/component-base v0.21.2 k8s.io/utils v0.0.0-20210527160623-6fdb442a123b diff --git a/hack/install-requirements.sh b/hack/install-requirements.sh index ad15f031..97ae5aa2 100755 --- a/hack/install-requirements.sh +++ b/hack/install-requirements.sh @@ -19,7 +19,7 @@ set -e CURRENT_DIR=$(dirname $0) PROJECT_ROOT="${CURRENT_DIR}"/.. -curl -sfL "https://install.goreleaser.com/github.com/golangci/golangci-lint.sh" | sh -s -- -b $(go env GOPATH)/bin v1.32.2 +curl -sfL "https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh" | sh -s -- -b $(go env GOPATH)/bin v1.32.2 GO111MODULE=off go get golang.org/x/tools/cmd/goimports diff --git a/hack/setup-testenv.sh b/hack/setup-testenv.sh index 463b4a53..e443d2b9 100755 --- a/hack/setup-testenv.sh +++ b/hack/setup-testenv.sh @@ -22,7 +22,14 @@ echo "> Setup Test Environment for K8s Version ${K8S_VERSION}" CURRENT_DIR=$(dirname $0) PROJECT_ROOT="${CURRENT_DIR}"/.. -export KUBEBUILDER_ASSETS=$(setup-envtest use -p path ${K8S_VERSION}) + +ARCH_ARG="" +if [[ $(go env GOOS) == "darwin" && $(go env GOARCH) == "arm64" ]]; then + ARCH_ARG="--arch amd64" +fi + +export KUBEBUILDER_ASSETS=$(setup-envtest use -p path ${K8S_VERSION} ${ARCH_ARG}) + mkdir -p ${PROJECT_ROOT}/tmp/test rm -f ${PROJECT_ROOT}/tmp/test/bin ln -s "${KUBEBUILDER_ASSETS}" ${PROJECT_ROOT}/tmp/test/bin diff --git a/pkg/api/helper/helper.go b/pkg/api/helper/helper.go index a288f466..8438e567 100644 --- a/pkg/api/helper/helper.go +++ b/pkg/api/helper/helper.go @@ -27,8 +27,3 @@ func KubeAPIServerSNIEnabled(kubeAPIServer *api.KubeAPIServer) bool { func ETCDBackupEnabled(etcd *api.ETCD) bool { return etcd != nil && etcd.Backup != nil } - -// ETCDHVPAEnabled returns true if the etcd HVPA settings are configured. -func ETCDHVPAEnabled(etcd *api.ETCD) bool { - return etcd != nil && etcd.HVPAEnabled -} diff --git a/pkg/api/helper/helper_test.go b/pkg/api/helper/helper_test.go index e6c0c3ff..c8ace53e 100644 --- a/pkg/api/helper/helper_test.go +++ b/pkg/api/helper/helper_test.go @@ -44,13 +44,4 @@ var _ = Describe("Helper", func() { Entry("backup is nil", &api.ETCD{}, BeFalse()), Entry("backup is not nil", &api.ETCD{Backup: &api.ETCDBackup{}}, BeTrue()), ) - - DescribeTable("#ETCDHVPAEnabled", - func(etcd *api.ETCD, matcher types.GomegaMatcher) { - Expect(ETCDHVPAEnabled(etcd)).To(matcher) - }, - - Entry("etcd is nil", nil, BeFalse()), - Entry("hvpa is nil", &api.ETCD{}, BeFalse()), - ) }) diff --git a/pkg/api/hvpa_types.go b/pkg/api/hvpa_types.go deleted file mode 100644 index 94152d49..00000000 --- a/pkg/api/hvpa_types.go +++ /dev/null @@ -1,152 +0,0 @@ -// Copyright (c) 2021 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package api - -import ( - "github.com/gardener/hvpa-controller/api/v1alpha1" - "k8s.io/utils/pointer" -) - -// HvpaConfig defines the structure of HVPA configuration data. -type HvpaConfig struct { - MaxReplicas *int32 `json:"maxReplicas,omitempty" yaml:"maxReplicas,omitempty"` - MinReplicas *int32 `json:"minReplicas,omitempty" yaml:"minReplicas,omitempty"` - TargetAverageUtilizationCpu *int32 `json:"targetAverageUtilizationCpu,omitempty" yaml:"targetAverageUtilizationCpu,omitempty"` - TargetAverageUtilizationMemory *int32 `json:"targetAverageUtilizationMemory,omitempty" yaml:"targetAverageUtilizationMemory,omitempty"` - VpaScaleUpMode *string `json:"vpaScaleUpMode,omitempty" yaml:"vpaScaleUpMode,omitempty"` - VpaScaleDownMode *string `json:"vpaScaleDownMode,omitempty" yaml:"vpaScaleDownMode,omitempty"` - VpaScaleUpStabilization *ScaleType `json:"vpaScaleUpStabilization,omitempty" yaml:"vpaScaleUpStabilization,omitempty"` - VpaScaleDownStabilization *ScaleType `json:"vpaScaleDownStabilization,omitempty" yaml:"vpaScaleDownStabilization,omitempty"` - LimitsRequestsGapScaleParams *v1alpha1.ScaleParams `json:"limitsRequestsGapScaleParams,omitempty" yaml:"limitsRequestsGapScaleParams,omitempty"` - MaintenanceWindow *v1alpha1.MaintenanceTimeWindow `json:"maintenanceWindow,omitempty" yaml:"maintenanceWindow,omitempty"` -} - -// ScaleType defines the scaling parameters for the limits. -type ScaleType struct { - StabilizationDuration *string `json:"stabilizationDuration,omitempty" yaml:"stabilizationDuration,omitempty"` - MinChange *v1alpha1.ScaleParams `json:"minChange,omitempty" yaml:"minChange,omitempty"` -} - -func (h *HvpaConfig) GetMaxReplicas(def int32) int32 { - if h == nil || h.MaxReplicas == nil { - return def - } - return *h.MaxReplicas -} - -func (h *HvpaConfig) GetMinReplicas() *int32 { - var def int32 = 1 - if h == nil || h.MinReplicas == nil { - return pointer.Int32Ptr(def) - } - return h.MinReplicas -} - -func (h *HvpaConfig) GetTargetAverageUtilizationMemory(def int32) *int32 { - if h == nil || h.TargetAverageUtilizationMemory == nil { - return pointer.Int32Ptr(def) - } - return h.TargetAverageUtilizationMemory -} - -func (h *HvpaConfig) GetTargetAverageUtilizationCpu(def int32) *int32 { - if h == nil || h.TargetAverageUtilizationCpu == nil { - return pointer.Int32Ptr(def) - } - return h.TargetAverageUtilizationCpu -} - -func (h *HvpaConfig) GetVpaScaleUpMode(def string) *string { - if h == nil || h.VpaScaleUpMode == nil { - return pointer.StringPtr(def) - } - return h.VpaScaleUpMode -} - -func (h *HvpaConfig) GetVpaScaleUpStabilisationDuration(def string) *string { - if h == nil || h.VpaScaleUpStabilization == nil { - return pointer.StringPtr(def) - } - return h.VpaScaleUpStabilization.StabilizationDuration -} - -func (h *HvpaConfig) GetVpaScaleUpMinChange(def v1alpha1.ScaleParams) v1alpha1.ScaleParams { - if h == nil || h.VpaScaleUpStabilization == nil || h.VpaScaleUpStabilization.MinChange == nil { - return def - } - - return mergeScaleParams(h.VpaScaleUpStabilization.MinChange, def) -} - -func (h *HvpaConfig) GetVpaScaleDownMinChange(def v1alpha1.ScaleParams) v1alpha1.ScaleParams { - if h == nil || h.VpaScaleDownStabilization == nil || h.VpaScaleDownStabilization.MinChange == nil { - return def - } - - return mergeScaleParams(h.VpaScaleDownStabilization.MinChange, def) -} - -func (h *HvpaConfig) GetLimitsRequestsGapScaleParams(def v1alpha1.ScaleParams) v1alpha1.ScaleParams { - if h == nil || h.LimitsRequestsGapScaleParams == nil { - return def - } - - return mergeScaleParams(h.LimitsRequestsGapScaleParams, def) -} - -func mergeScaleParams(externalValues *v1alpha1.ScaleParams, defaultValues v1alpha1.ScaleParams) v1alpha1.ScaleParams { - scaleParams := defaultValues - - if externalValues.CPU.Value != nil { - scaleParams.CPU.Value = externalValues.CPU.Value - } - - if externalValues.CPU.Percentage != nil { - scaleParams.CPU.Percentage = externalValues.CPU.Percentage - } - - if externalValues.Memory.Value != nil { - scaleParams.Memory.Value = externalValues.Memory.Value - } - - if externalValues.Memory.Percentage != nil { - scaleParams.Memory.Percentage = externalValues.Memory.Percentage - } - - if externalValues.Replicas.Value != nil { - scaleParams.Replicas.Value = externalValues.Replicas.Value - } - - if externalValues.Replicas.Percentage != nil { - scaleParams.Replicas.Percentage = externalValues.Replicas.Percentage - } - - // return *h.VpaScaleUpStabilization.MinChange - return scaleParams -} - -func (h *HvpaConfig) GetVpaScaleDownMode(def string) *string { - if h == nil || h.VpaScaleDownMode == nil { - return pointer.StringPtr(def) - } - return h.VpaScaleDownMode -} - -func (h *HvpaConfig) GetVpaScaleDownStabilisationDuration(def string) *string { - if h == nil || h.VpaScaleDownStabilization == nil { - return pointer.StringPtr(def) - } - return h.VpaScaleDownStabilization.StabilizationDuration -} diff --git a/pkg/api/imports.go b/pkg/api/imports.go index 636d8bfe..a9fd784d 100644 --- a/pkg/api/imports.go +++ b/pkg/api/imports.go @@ -72,8 +72,6 @@ type ETCD struct { // +optional Backup *ETCDBackup `json:"backup,omitempty" yaml:"backup,omitempty"` - HVPAEnabled bool `json:"hvpaEnabled,omitempty" yaml:"hvpaEnabled,omitempty"` - // HandleETCDPersistentVolumes defines whether the PV(C)s that are getting automatically created by the etcd // statefulset shall be handled or not (false by default). If true then they will be deleted when the virtual // garden is deleted. Otherwise, they will remain in the system for manual cleanup (to prevent data loss). @@ -118,17 +116,12 @@ type KubeAPIServer struct { OidcWebhookAuthenticator OidcWebhookAuthenticator `json:"oidcWebhookAuthenticator,omitempty" yaml:"oidcWebhookAuthenticator,omitempty"` - HVPAEnabled bool `json:"hvpaEnabled,omitempty" yaml:"hvpaEnabled,omitempty"` - HVPA *HvpaConfig `json:"hvpa,omitempty" yaml:"hvpa,omitempty"` - EventTTL *string `json:"eventTTL,omitempty" yaml:"eventTTL,omitempty"` OidcIssuerURL *string `json:"oidcIssuerURL,omitempty" yaml:"oidcIssuerURL,omitempty"` AdditionalVolumeMounts []corev1.VolumeMount `json:"additionalVolumeMounts,omitempty" yaml:"additionalVolumeMounts,omitempty"` AdditionalVolumes []corev1.Volume `json:"additionalVolumes,omitempty" yaml:"additionalVolumes,omitempty"` - HorizontalPodAutoscaler *HorizontalPodAutoscaler `json:"horizontalPodAutoscaler,omitempty" yaml:"horizontalPodAutoscaler,omitempty"` - MaxRequestsInflight *int `json:"maxRequestsInflight,omitempty" yaml:"maxRequestsInflight,omitempty"` MaxMutatingRequestsInflight *int `json:"maxMutatingRequestsInflight,omitempty" yaml:"maxMutatingRequestsInflight,omitempty"` @@ -149,15 +142,6 @@ func (r *KubeAPIServer) GetMaxMutatingRequestsInflight(defaultValue int) int { return *r.MaxMutatingRequestsInflight } -// HorizontalPodAutoscaler contains configuration for the horizontal pod auto scaler. -type HorizontalPodAutoscaler struct { - DownscaleStabilization string `json:"downscaleStabilization,omitempty" yaml:"downscaleStabilization,omitempty"` - ReadinessDelay string `json:"readinessDelay,omitempty" yaml:"readinessDelay,omitempty"` - CpuInitializationPeriod string `json:"cpuInitializationPeriod,omitempty" yaml:"cpuInitializationPeriod,omitempty"` - SyncPeriod string `json:"syncPeriod,omitempty" yaml:"syncPeriod,omitempty"` - Tolerance string `json:"tolerance,omitempty" yaml:"tolerance,omitempty"` -} - // GardenerControlplane contains the activation info for webhooks type GardenerControlplane struct { ValidatingWebhook AdmissionWebhookConfig `json:"validatingWebhook,omitempty" yaml:"validatingWebhook,omitempty"` diff --git a/pkg/virtualgarden/crd_util.go b/pkg/virtualgarden/crd_util.go deleted file mode 100644 index 4be7b134..00000000 --- a/pkg/virtualgarden/crd_util.go +++ /dev/null @@ -1,116 +0,0 @@ -package virtualgarden - -import ( - "bytes" - "context" - _ "embed" - "fmt" - "time" - - "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/apimachinery/pkg/util/yaml" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" -) - -//go:embed resources/hvpa.yaml -var hvpaCrd []byte - -//go:embed resources/vpa.yaml -var vpaCrd []byte - -// DeployHVPACRD deploys the HVPA CRD. -func DeployHVPACRD(ctx context.Context, c client.Client) error { - newCrd, err := loadCRD(hvpaCrd) - if err != nil { - return err - } - - crd := emptyHVPACRD() - _, err = controllerutil.CreateOrUpdate(ctx, c, crd, func() error { - crd.Spec = newCrd.Spec - return nil - }) - - if err != nil { - return err - } - - err = waitToBeEstablished(ctx, c, crd) - - return err -} - -// DeployVPACRD deploys the VPA CRD. -func DeployVPACRD(ctx context.Context, c client.Client) error { - newCrd, err := loadCRD(vpaCrd) - if err != nil { - return err - } - - crd := emptyVPACRD() - _, err = controllerutil.CreateOrUpdate(ctx, c, crd, func() error { - crd.Spec = newCrd.Spec - return nil - }) - - if err != nil { - return err - } - - err = waitToBeEstablished(ctx, c, crd) - - return err -} - -func waitToBeEstablished(ctx context.Context, c client.Client, crd *v1beta1.CustomResourceDefinition) error { - timeoutCtx, cancel := context.WithTimeout(ctx, 2*time.Minute) - defer cancel() - - err := wait.PollImmediateUntil(2*time.Second, func() (done bool, err error) { - if err := c.Get(ctx, client.ObjectKeyFromObject(crd), crd); err != nil { - if client.IgnoreNotFound(err) != nil { - return false, err - } - return false, nil - } - - conditions := crd.Status.Conditions - for _, condition := range conditions { - if condition.Type == v1beta1.Established && condition.Status == v1beta1.ConditionTrue { - return true, nil - } - } - - return false, nil - }, timeoutCtx.Done()) - - return err -} - -// DeleteHPVACRD deletes the HPVA CRD. -func DeleteHPVACRD(ctx context.Context, c client.Client) error { - return client.IgnoreNotFound(c.Delete(ctx, emptyHVPACRD())) -} - -// loadCRD loads the CRD. -func loadCRD(crdBytes []byte) (*v1beta1.CustomResourceDefinition, error) { - crd := &v1beta1.CustomResourceDefinition{} - decoder := yaml.NewYAMLOrJSONDecoder(bytes.NewReader(crdBytes), 32) - err := decoder.Decode(crd) - if err != nil { - return nil, fmt.Errorf("failed to decode CRD: %w", err) - } - - return crd, nil -} - -func emptyVPACRD() *v1beta1.CustomResourceDefinition { - return &v1beta1.CustomResourceDefinition{ - ObjectMeta: metav1.ObjectMeta{ - Name: "verticalpodautoscalers.autoscaling.k8s.io", - }, - } -} diff --git a/pkg/virtualgarden/etcd.go b/pkg/virtualgarden/etcd.go index 2b862a35..93c943a6 100644 --- a/pkg/virtualgarden/etcd.go +++ b/pkg/virtualgarden/etcd.go @@ -21,7 +21,6 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/meta" "k8s.io/utils/pointer" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -123,13 +122,6 @@ func (o *operation) deployETCDResources(ctx context.Context) error { if err := o.deployETCDStatefulSet(ctx, role, checksums, storageCapacity, storageClassName, storageProviderName, environment); err != nil { return err } - - if helper.ETCDHVPAEnabled(o.imports.VirtualGarden.ETCD) { - o.log.Infof("Deploying etcd HVPA for role %q if CRD is installed", role) - if err := o.deployETCDHVPA(ctx, role); err != nil && !meta.IsNoMatchError(err) { - return err - } - } } return nil @@ -138,12 +130,6 @@ func (o *operation) deployETCDResources(ctx context.Context) error { // DeleteETCD deletes etcd and all related resources. func (o *operation) DeleteETCD(ctx context.Context) error { for _, role := range []string{ETCDRoleMain, ETCDRoleEvents} { - if helper.ETCDHVPAEnabled(o.imports.VirtualGarden.ETCD) { - o.log.Infof("Deleting etcd HVPA for role %q if CRD is installed", role) - if err := o.deleteETCDHVPA(ctx, role); err != nil && !meta.IsNoMatchError(err) { - return err - } - } o.log.Infof("Deleting etcd statefulset for role %q", role) if err := o.deleteETCDStatefulSet(ctx, role); err != nil { diff --git a/pkg/virtualgarden/etcd_hvpa.go b/pkg/virtualgarden/etcd_hvpa.go deleted file mode 100644 index b7e4ba0a..00000000 --- a/pkg/virtualgarden/etcd_hvpa.go +++ /dev/null @@ -1,178 +0,0 @@ -// Copyright (c) 2021 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package virtualgarden - -import ( - "context" - - hvpav1alpha1 "github.com/gardener/hvpa-controller/api/v1alpha1" - autoscalingv2beta1 "k8s.io/api/autoscaling/v2beta1" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - autoscalingv1beta2 "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1beta2" - "k8s.io/utils/pointer" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" -) - -// ETCDHVPAName returns the name of the HVPA object for the given role. -func ETCDHVPAName(role string) string { - return Prefix + "-etcd-" + role -} - -func (o *operation) deployETCDHVPA(ctx context.Context, role string) error { - var ( - containerScalingOff = autoscalingv1beta2.ContainerScalingModeOff - hvpa = emptyETCDHVPA(o.namespace, role) - ) - - _, err := controllerutil.CreateOrUpdate(ctx, o.client, hvpa, func() error { - hvpa.Spec.Replicas = pointer.Int32Ptr(1) - hvpa.Spec.TargetRef = &autoscalingv2beta1.CrossVersionObjectReference{ - APIVersion: "apps/v1", - Kind: "StatefulSet", - Name: ETCDStatefulSetName(role), - } - hvpa.Spec.Hpa = hvpav1alpha1.HpaSpec{ - Selector: &metav1.LabelSelector{ - MatchLabels: etcdHPALabels(role), - }, - Deploy: false, - Template: hvpav1alpha1.HpaTemplate{ - ObjectMeta: metav1.ObjectMeta{ - Labels: etcdHPALabels(role), - }, - Spec: hvpav1alpha1.HpaTemplateSpec{ - MinReplicas: pointer.Int32Ptr(1), - MaxReplicas: 1, - Metrics: []autoscalingv2beta1.MetricSpec{ - { - Resource: &autoscalingv2beta1.ResourceMetricSource{ - Name: corev1.ResourceMemory, - TargetAverageUtilization: pointer.Int32Ptr(80), - }, - }, - { - Resource: &autoscalingv2beta1.ResourceMetricSource{ - Name: corev1.ResourceCPU, - TargetAverageUtilization: pointer.Int32Ptr(80), - }, - }, - }, - }, - }, - } - hvpa.Spec.Vpa = hvpav1alpha1.VpaSpec{ - Selector: &metav1.LabelSelector{ - MatchLabels: etcdVPALabels(role), - }, - Deploy: true, - Template: hvpav1alpha1.VpaTemplate{ - ObjectMeta: metav1.ObjectMeta{ - Labels: etcdVPALabels(role), - }, - Spec: hvpav1alpha1.VpaTemplateSpec{ - ResourcePolicy: &autoscalingv1beta2.PodResourcePolicy{ - ContainerPolicies: []autoscalingv1beta2.ContainerResourcePolicy{ - { - ContainerName: etcdContainerName, - MaxAllowed: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("4"), - corev1.ResourceMemory: resource.MustParse("30G"), - }, - MinAllowed: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("200m"), - corev1.ResourceMemory: resource.MustParse("700M"), - }, - }, - { - ContainerName: backupRestoreSidecarContainerName, - Mode: &containerScalingOff, - }, - }, - }, - }, - }, - ScaleUp: hvpav1alpha1.ScaleType{ - UpdatePolicy: hvpav1alpha1.UpdatePolicy{ - UpdateMode: pointer.StringPtr(hvpav1alpha1.UpdateModeAuto), - }, - StabilizationDuration: pointer.StringPtr("5m"), - MinChange: hvpav1alpha1.ScaleParams{ - CPU: hvpav1alpha1.ChangeParams{ - Value: pointer.StringPtr("1"), - Percentage: pointer.Int32Ptr(80), - }, - Memory: hvpav1alpha1.ChangeParams{ - Value: pointer.StringPtr("2G"), - Percentage: pointer.Int32Ptr(80), - }, - }, - }, - ScaleDown: hvpav1alpha1.ScaleType{ - UpdatePolicy: hvpav1alpha1.UpdatePolicy{ - UpdateMode: pointer.StringPtr(hvpav1alpha1.UpdateModeOff), - }, - StabilizationDuration: pointer.StringPtr("15m"), - MinChange: hvpav1alpha1.ScaleParams{ - CPU: hvpav1alpha1.ChangeParams{ - Value: pointer.StringPtr("1"), - Percentage: pointer.Int32Ptr(80), - }, - Memory: hvpav1alpha1.ChangeParams{ - Value: pointer.StringPtr("2G"), - Percentage: pointer.Int32Ptr(80), - }, - }, - }, - LimitsRequestsGapScaleParams: hvpav1alpha1.ScaleParams{ - CPU: hvpav1alpha1.ChangeParams{ - Value: pointer.StringPtr("2"), - Percentage: pointer.Int32Ptr(40), - }, - Memory: hvpav1alpha1.ChangeParams{ - Value: pointer.StringPtr("3G"), - Percentage: pointer.Int32Ptr(40), - }, - }, - } - hvpa.Spec.WeightBasedScalingIntervals = []hvpav1alpha1.WeightBasedScalingInterval{ - { - VpaWeight: hvpav1alpha1.VpaOnly, - StartReplicaCount: 1, - LastReplicaCount: 1, - }, - } - return nil - }) - return err -} - -func (o *operation) deleteETCDHVPA(ctx context.Context, role string) error { - return client.IgnoreNotFound(o.client.Delete(ctx, emptyETCDHVPA(o.namespace, role))) -} - -func emptyETCDHVPA(namespace, role string) *hvpav1alpha1.Hvpa { - return &hvpav1alpha1.Hvpa{ObjectMeta: metav1.ObjectMeta{Name: ETCDHVPAName(role), Namespace: namespace}} -} - -func etcdVPALabels(role string) map[string]string { - return map[string]string{"role": "etcd-vpa-" + role} -} - -func etcdHPALabels(role string) map[string]string { - return map[string]string{"role": "etcd-hpa-" + role} -} diff --git a/pkg/virtualgarden/etcd_hvpa_test.go b/pkg/virtualgarden/etcd_hvpa_test.go deleted file mode 100644 index 32d04237..00000000 --- a/pkg/virtualgarden/etcd_hvpa_test.go +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright (c) 2021 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package virtualgarden_test - -import ( - . "github.com/gardener/virtual-garden/pkg/virtualgarden" - - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" -) - -var _ = Describe("EtcdHVPA", func() { - role := "foobar" - - Describe("#ETCDHVPAName", func() { - It("should return the correct hvpa name", func() { - Expect(ETCDHVPAName(role)).To(Equal("virtual-garden-etcd-" + role)) - }) - }) -}) diff --git a/pkg/virtualgarden/etcd_test.go b/pkg/virtualgarden/etcd_test.go index b64b96cb..3c48f44c 100644 --- a/pkg/virtualgarden/etcd_test.go +++ b/pkg/virtualgarden/etcd_test.go @@ -26,13 +26,11 @@ import ( "github.com/gardener/gardener/pkg/utils" secretutils "github.com/gardener/gardener/pkg/utils/secrets" - hvpav1alpha1 "github.com/gardener/hvpa-controller/api/v1alpha1" "github.com/golang/mock/gomock" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" "github.com/sirupsen/logrus" appsv1 "k8s.io/api/apps/v1" - autoscalingv2beta1 "k8s.io/api/autoscaling/v2beta1" corev1 "k8s.io/api/core/v1" storagev1 "k8s.io/api/storage/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -41,7 +39,6 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/intstr" - autoscalingv1beta2 "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1beta2" "k8s.io/utils/pointer" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -90,7 +87,6 @@ var _ = Describe("Etcd", func() { BucketName: bucketName, }, StorageClassName: &storageClassName, - HVPAEnabled: true, HandleETCDPersistentVolumes: true, }, PriorityClassName: "garden-controlplane", @@ -118,13 +114,11 @@ var _ = Describe("Etcd", func() { configMapMainObjectMeta metav1.ObjectMeta secretServerMainObjectMeta metav1.ObjectMeta statefulSetMainObjectMeta metav1.ObjectMeta - hvpaMainObjectMeta metav1.ObjectMeta serviceEventsObjectMeta metav1.ObjectMeta configMapEventsObjectMeta metav1.ObjectMeta secretServerEventsObjectMeta metav1.ObjectMeta statefulSetEventsObjectMeta metav1.ObjectMeta - hvpaEventsObjectMeta metav1.ObjectMeta secretChecksumCA string secretChecksumClient string @@ -527,146 +521,6 @@ var _ = Describe("Etcd", func() { }, } } - hvpaFor = func(role string) *hvpav1alpha1.Hvpa { - var ( - hvpaContainerScalingOff = autoscalingv1beta2.ContainerScalingModeOff - objectMeta metav1.ObjectMeta - statefulSetName string - ) - - if role == ETCDRoleMain { - objectMeta = hvpaMainObjectMeta - statefulSetName = statefulSetMainObjectMeta.Name - } else if role == ETCDRoleEvents { - objectMeta = hvpaEventsObjectMeta - statefulSetName = statefulSetEventsObjectMeta.Name - } - - return &hvpav1alpha1.Hvpa{ - ObjectMeta: metav1.ObjectMeta{ - Name: objectMeta.Name, - Namespace: objectMeta.Namespace, - }, - Spec: hvpav1alpha1.HvpaSpec{ - Replicas: pointer.Int32Ptr(1), - TargetRef: &autoscalingv2beta1.CrossVersionObjectReference{ - APIVersion: "apps/v1", - Kind: "StatefulSet", - Name: statefulSetName, - }, - Hpa: hvpav1alpha1.HpaSpec{ - Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{"role": "etcd-hpa-" + role}, - }, - Deploy: false, - Template: hvpav1alpha1.HpaTemplate{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{"role": "etcd-hpa-" + role}, - }, - Spec: hvpav1alpha1.HpaTemplateSpec{ - MinReplicas: pointer.Int32Ptr(1), - MaxReplicas: 1, - Metrics: []autoscalingv2beta1.MetricSpec{ - { - Resource: &autoscalingv2beta1.ResourceMetricSource{ - Name: corev1.ResourceMemory, - TargetAverageUtilization: pointer.Int32Ptr(80), - }, - }, - { - Resource: &autoscalingv2beta1.ResourceMetricSource{ - Name: corev1.ResourceCPU, - TargetAverageUtilization: pointer.Int32Ptr(80), - }, - }, - }, - }, - }, - }, - Vpa: hvpav1alpha1.VpaSpec{ - Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{"role": "etcd-vpa-" + role}, - }, - Deploy: true, - Template: hvpav1alpha1.VpaTemplate{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{"role": "etcd-vpa-" + role}, - }, - Spec: hvpav1alpha1.VpaTemplateSpec{ - ResourcePolicy: &autoscalingv1beta2.PodResourcePolicy{ - ContainerPolicies: []autoscalingv1beta2.ContainerResourcePolicy{ - { - ContainerName: etcdContainerName, - MaxAllowed: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("4"), - corev1.ResourceMemory: resource.MustParse("30G"), - }, - MinAllowed: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("200m"), - corev1.ResourceMemory: resource.MustParse("700M"), - }, - }, - { - ContainerName: backupRestoreSidecarContainerName, - Mode: &hvpaContainerScalingOff, - }, - }, - }, - }, - }, - ScaleUp: hvpav1alpha1.ScaleType{ - UpdatePolicy: hvpav1alpha1.UpdatePolicy{ - UpdateMode: pointer.StringPtr(hvpav1alpha1.UpdateModeAuto), - }, - StabilizationDuration: pointer.StringPtr("5m"), - MinChange: hvpav1alpha1.ScaleParams{ - CPU: hvpav1alpha1.ChangeParams{ - Value: pointer.StringPtr("1"), - Percentage: pointer.Int32Ptr(80), - }, - Memory: hvpav1alpha1.ChangeParams{ - Value: pointer.StringPtr("2G"), - Percentage: pointer.Int32Ptr(80), - }, - }, - }, - ScaleDown: hvpav1alpha1.ScaleType{ - UpdatePolicy: hvpav1alpha1.UpdatePolicy{ - UpdateMode: pointer.StringPtr(hvpav1alpha1.UpdateModeOff), - }, - StabilizationDuration: pointer.StringPtr("15m"), - MinChange: hvpav1alpha1.ScaleParams{ - CPU: hvpav1alpha1.ChangeParams{ - Value: pointer.StringPtr("1"), - Percentage: pointer.Int32Ptr(80), - }, - Memory: hvpav1alpha1.ChangeParams{ - Value: pointer.StringPtr("2G"), - Percentage: pointer.Int32Ptr(80), - }, - }, - }, - LimitsRequestsGapScaleParams: hvpav1alpha1.ScaleParams{ - CPU: hvpav1alpha1.ChangeParams{ - Value: pointer.StringPtr("2"), - Percentage: pointer.Int32Ptr(40), - }, - Memory: hvpav1alpha1.ChangeParams{ - Value: pointer.StringPtr("3G"), - Percentage: pointer.Int32Ptr(40), - }, - }, - }, - WeightBasedScalingIntervals: []hvpav1alpha1.WeightBasedScalingInterval{ - { - VpaWeight: hvpav1alpha1.VpaOnly, - StartReplicaCount: 1, - LastReplicaCount: 1, - }, - }, - }, - } - } ) BeforeEach(func() { @@ -679,16 +533,14 @@ var _ = Describe("Etcd", func() { configMapMainObjectMeta = objectMeta(ETCDConfigMapName(ETCDRoleMain), namespace) secretServerMainObjectMeta = objectMeta(ETCDSecretNameServerCertificate(ETCDRoleMain), namespace) statefulSetMainObjectMeta = objectMeta(ETCDStatefulSetName(ETCDRoleMain), namespace) - hvpaMainObjectMeta = objectMeta(ETCDHVPAName(ETCDRoleMain), namespace) serviceEventsObjectMeta = objectMeta(ETCDServiceName(ETCDRoleEvents), namespace) configMapEventsObjectMeta = objectMeta(ETCDConfigMapName(ETCDRoleEvents), namespace) secretServerEventsObjectMeta = objectMeta(ETCDSecretNameServerCertificate(ETCDRoleEvents), namespace) statefulSetEventsObjectMeta = objectMeta(ETCDStatefulSetName(ETCDRoleEvents), namespace) - hvpaEventsObjectMeta = objectMeta(ETCDHVPAName(ETCDRoleEvents), namespace) }) - It("should correctly deploy all etcd resources (w/ backup, w/ hvpa)", func() { + It("should correctly deploy all etcd resources (w/ backup)", func() { gomock.InOrder( c.EXPECT().Get(ctx, client.ObjectKey{Name: storageClassObjectMeta.Name}, gomock.AssignableToTypeOf(&storagev1.StorageClass{})).Return(apierrors.NewNotFound(schema.GroupResource{}, "")), c.EXPECT().Create(ctx, gomock.AssignableToTypeOf(&storagev1.StorageClass{})).Do(func(ctx context.Context, obj runtime.Object, opts ...client.CreateOption) { @@ -751,10 +603,6 @@ var _ = Describe("Etcd", func() { c.EXPECT().Create(ctx, gomock.AssignableToTypeOf(&appsv1.StatefulSet{})).Do(func(ctx context.Context, obj runtime.Object, opts ...client.CreateOption) { Expect(obj).To(Equal(statefulSetFor(ETCDRoleMain, op.imports.VirtualGarden.ETCD.Backup))) }), - c.EXPECT().Get(ctx, client.ObjectKey{Name: hvpaMainObjectMeta.Name, Namespace: hvpaMainObjectMeta.Namespace}, gomock.AssignableToTypeOf(&hvpav1alpha1.Hvpa{})).Return(apierrors.NewNotFound(schema.GroupResource{}, "")), - c.EXPECT().Create(ctx, gomock.AssignableToTypeOf(&hvpav1alpha1.Hvpa{})).Do(func(ctx context.Context, obj runtime.Object, opts ...client.CreateOption) { - Expect(obj).To(Equal(hvpaFor(ETCDRoleMain))) - }), c.EXPECT().Get(ctx, client.ObjectKey{Name: serviceEventsObjectMeta.Name, Namespace: serviceEventsObjectMeta.Namespace}, gomock.AssignableToTypeOf(&corev1.Service{})).Return(apierrors.NewNotFound(schema.GroupResource{}, "")), c.EXPECT().Create(ctx, gomock.AssignableToTypeOf(&corev1.Service{})).Do(func(ctx context.Context, obj runtime.Object, opts ...client.CreateOption) { @@ -777,18 +625,13 @@ var _ = Describe("Etcd", func() { c.EXPECT().Create(ctx, gomock.AssignableToTypeOf(&appsv1.StatefulSet{})).Do(func(ctx context.Context, obj runtime.Object, opts ...client.CreateOption) { Expect(obj).To(Equal(statefulSetFor(ETCDRoleEvents, nil))) }), - c.EXPECT().Get(ctx, client.ObjectKey{Name: hvpaEventsObjectMeta.Name, Namespace: hvpaEventsObjectMeta.Namespace}, gomock.AssignableToTypeOf(&hvpav1alpha1.Hvpa{})).Return(apierrors.NewNotFound(schema.GroupResource{}, "")), - c.EXPECT().Create(ctx, gomock.AssignableToTypeOf(&hvpav1alpha1.Hvpa{})).Do(func(ctx context.Context, obj runtime.Object, opts ...client.CreateOption) { - Expect(obj).To(Equal(hvpaFor(ETCDRoleEvents))) - }), ) Expect(op.deployETCDResources(ctx)).To(Succeed()) }) - It("should correctly deploy all etcd resources (w/o backup, w/o hvpa)", func() { + It("should correctly deploy all etcd resources (w/o backup)", func() { op.imports.VirtualGarden.ETCD.Backup = nil - op.imports.VirtualGarden.ETCD.HVPAEnabled = false gomock.InOrder( c.EXPECT().Get(ctx, client.ObjectKey{Name: storageClassObjectMeta.Name}, gomock.AssignableToTypeOf(&storagev1.StorageClass{})).Return(apierrors.NewNotFound(schema.GroupResource{}, "")), @@ -872,16 +715,14 @@ var _ = Describe("Etcd", func() { }) Describe("#DeleteETCD", func() { - It("should correctly delete all etcd resources (w/ backup, w/ pvc handling, w/ hvpa)", func() { + It("should correctly delete all etcd resources (w/ backup, w/ pvc handling)", func() { gomock.InOrder( - c.EXPECT().Delete(ctx, &hvpav1alpha1.Hvpa{ObjectMeta: objectMeta(ETCDHVPAName(ETCDRoleMain), namespace)}), c.EXPECT().Delete(ctx, &appsv1.StatefulSet{ObjectMeta: objectMeta(ETCDStatefulSetName(ETCDRoleMain), namespace)}), c.EXPECT().Delete(ctx, &corev1.PersistentVolumeClaim{ObjectMeta: objectMeta(ETCDPersistentVolumeClaimName(ETCDRoleMain), namespace)}), c.EXPECT().Delete(ctx, &corev1.Secret{ObjectMeta: objectMeta(ETCDSecretNameBackup, namespace)}), c.EXPECT().Delete(ctx, &corev1.ConfigMap{ObjectMeta: objectMeta(ETCDConfigMapName(ETCDRoleMain), namespace)}), c.EXPECT().Delete(ctx, &corev1.Service{ObjectMeta: objectMeta(ETCDServiceName(ETCDRoleMain), namespace)}), - c.EXPECT().Delete(ctx, &hvpav1alpha1.Hvpa{ObjectMeta: objectMeta(ETCDHVPAName(ETCDRoleEvents), namespace)}), c.EXPECT().Delete(ctx, &appsv1.StatefulSet{ObjectMeta: objectMeta(ETCDStatefulSetName(ETCDRoleEvents), namespace)}), c.EXPECT().Delete(ctx, &corev1.PersistentVolumeClaim{ObjectMeta: objectMeta(ETCDPersistentVolumeClaimName(ETCDRoleEvents), namespace)}), c.EXPECT().Delete(ctx, &corev1.ConfigMap{ObjectMeta: objectMeta(ETCDConfigMapName(ETCDRoleEvents), namespace)}), @@ -898,9 +739,8 @@ var _ = Describe("Etcd", func() { Expect(op.DeleteETCD(ctx)).To(Succeed()) }) - It("should correctly delete all etcd resources (w/o backup, w/o pvc handling, w/o hvpa)", func() { + It("should correctly delete all etcd resources (w/o backup, w/o pvc handling)", func() { op.imports.VirtualGarden.ETCD.Backup = nil - op.imports.VirtualGarden.ETCD.HVPAEnabled = false op.imports.VirtualGarden.ETCD.HandleETCDPersistentVolumes = false gomock.InOrder( diff --git a/pkg/virtualgarden/hvpa_crd.go b/pkg/virtualgarden/hvpa_crd.go deleted file mode 100644 index 7dd8aec6..00000000 --- a/pkg/virtualgarden/hvpa_crd.go +++ /dev/null @@ -1,58 +0,0 @@ -// Copyright (c) 2021 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package virtualgarden - -import ( - "context" - _ "embed" - - "github.com/gardener/virtual-garden/pkg/api/helper" - - "sigs.k8s.io/controller-runtime/pkg/client" - - "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -// checkHVPACRD checks if the HVPA CRD is deployed if HVPA is enabled in the etcd or api server -func (o *operation) checkHVPACRD(ctx context.Context) error { - o.log.Infof("Check if hvpa crd exists if hvpa is enabled") - - if helper.ETCDHVPAEnabled(o.imports.VirtualGarden.ETCD) || o.imports.VirtualGarden.KubeAPIServer.HVPAEnabled { - _, err := o.getHVPACRD(ctx) - return err - } - - return nil -} - -// getHVPACRD return the HVPA CRD. -func (o *operation) getHVPACRD(ctx context.Context) (*v1beta1.CustomResourceDefinition, error) { - hvpaCrd := emptyHVPACRD() - err := o.client.Get(ctx, client.ObjectKeyFromObject(hvpaCrd), hvpaCrd) - if err != nil { - return nil, err - } - - return hvpaCrd, nil -} - -func emptyHVPACRD() *v1beta1.CustomResourceDefinition { - return &v1beta1.CustomResourceDefinition{ - ObjectMeta: metav1.ObjectMeta{ - Name: "hvpas.autoscaling.k8s.io", - }, - } -} diff --git a/pkg/virtualgarden/kube_api_server.go b/pkg/virtualgarden/kube_api_server.go index 459ba918..1df9693f 100644 --- a/pkg/virtualgarden/kube_api_server.go +++ b/pkg/virtualgarden/kube_api_server.go @@ -49,11 +49,6 @@ func (o *operation) DeployKubeAPIServer(ctx context.Context) error { return err } - err = o.deployKubeAPIServerPodAutoscaling(ctx) - if err != nil { - return err - } - err = o.deployMisc(ctx) if err != nil { return err @@ -88,10 +83,6 @@ func (o *operation) DeleteKubeAPIServer(ctx context.Context) error { return err } - if err := o.deleteKubeAPIServerPodAutoscaling(ctx); err != nil { - return err - } - if err := o.deleteKubeAPIServerConfigMaps(ctx); err != nil { return err } diff --git a/pkg/virtualgarden/kube_api_server_certificates_test.go b/pkg/virtualgarden/kube_api_server_certificates_test.go index f130eb95..a4956fa0 100644 --- a/pkg/virtualgarden/kube_api_server_certificates_test.go +++ b/pkg/virtualgarden/kube_api_server_certificates_test.go @@ -123,13 +123,10 @@ func getImportsApiServerCertificatesTest() api.Imports { Enabled: true, CertificateAuthorityData: "test-ca-data", }, - HVPAEnabled: false, - HVPA: nil, - EventTTL: nil, - OidcIssuerURL: nil, - AdditionalVolumeMounts: nil, - AdditionalVolumes: nil, - HorizontalPodAutoscaler: nil, + EventTTL: nil, + OidcIssuerURL: nil, + AdditionalVolumeMounts: nil, + AdditionalVolumes: nil, }, DeleteNamespace: false, PriorityClassName: "", diff --git a/pkg/virtualgarden/kube_api_server_configmaps_test.go b/pkg/virtualgarden/kube_api_server_configmaps_test.go index a073ae1d..872d8503 100644 --- a/pkg/virtualgarden/kube_api_server_configmaps_test.go +++ b/pkg/virtualgarden/kube_api_server_configmaps_test.go @@ -118,13 +118,10 @@ func getImportsForApiServerConfigMapsTest() api.Imports { Enabled: true, CertificateAuthorityData: "test-ca-data", }, - HVPAEnabled: false, - HVPA: nil, - EventTTL: nil, - OidcIssuerURL: nil, - AdditionalVolumeMounts: nil, - AdditionalVolumes: nil, - HorizontalPodAutoscaler: nil, + EventTTL: nil, + OidcIssuerURL: nil, + AdditionalVolumeMounts: nil, + AdditionalVolumes: nil, }, DeleteNamespace: false, PriorityClassName: "", diff --git a/pkg/virtualgarden/kube_api_server_deployment_controller_manager.go b/pkg/virtualgarden/kube_api_server_deployment_controller_manager.go index 459060e8..63bab59d 100644 --- a/pkg/virtualgarden/kube_api_server_deployment_controller_manager.go +++ b/pkg/virtualgarden/kube_api_server_deployment_controller_manager.go @@ -16,9 +16,7 @@ package virtualgarden import ( "context" - "fmt" - "github.com/gardener/virtual-garden/pkg/api" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" @@ -126,8 +124,6 @@ func (o *operation) getKubeControllerManagerContainers() []corev1.Container { } func (o *operation) getKubeControllerManagerCommand() []string { - hpaConfig := o.getKubeControllerManagerHPAConfig() - return []string{ "/usr/local/bin/kube-controller-manager", "--authentication-kubeconfig=/srv/kubernetes/controller-manager/kubeconfig", @@ -140,11 +136,6 @@ func (o *operation) getKubeControllerManagerCommand() []string { "--concurrent-namespace-syncs=100", "--concurrent-resource-quota-syncs=100", "--concurrent-serviceaccount-token-syncs=100", - fmt.Sprintf("--horizontal-pod-autoscaler-downscale-stabilization=%s", hpaConfig.DownscaleStabilization), - fmt.Sprintf("--horizontal-pod-autoscaler-cpu-initialization-period=%s", hpaConfig.CpuInitializationPeriod), - fmt.Sprintf("--horizontal-pod-autoscaler-initial-readiness-delay=%s", hpaConfig.ReadinessDelay), - fmt.Sprintf("--horizontal-pod-autoscaler-sync-period=%s", hpaConfig.SyncPeriod), - fmt.Sprintf("--horizontal-pod-autoscaler-tolerance=%s", hpaConfig.Tolerance), "--kubeconfig=/srv/kubernetes/controller-manager/kubeconfig", "--root-ca-file=/srv/kubernetes/ca/ca.crt", "--service-account-private-key-file=/srv/kubernetes/service-account-key/service_account.key", @@ -177,36 +168,3 @@ func (o *operation) getKubeControllerManagerVolumes() []corev1.Volume { volumeWithSecretSource(volumeNameServiceAccountKey, KubeApiServerSecretNameServiceAccountKey), } } - -func (o *operation) getKubeControllerManagerHPAConfig() *api.HorizontalPodAutoscaler { - // Start with the default values - config := api.HorizontalPodAutoscaler{ - DownscaleStabilization: "30m0s", - ReadinessDelay: "30s", - CpuInitializationPeriod: "5m0s", - SyncPeriod: "30s", - Tolerance: "0.1", - } - - // Overwrite with imported values - importedConfig := o.imports.VirtualGarden.KubeAPIServer.HorizontalPodAutoscaler - if importedConfig != nil { - if len(importedConfig.DownscaleStabilization) != 0 { - config.DownscaleStabilization = importedConfig.DownscaleStabilization - } - if len(importedConfig.ReadinessDelay) != 0 { - config.ReadinessDelay = importedConfig.ReadinessDelay - } - if len(importedConfig.CpuInitializationPeriod) != 0 { - config.CpuInitializationPeriod = importedConfig.CpuInitializationPeriod - } - if len(importedConfig.SyncPeriod) != 0 { - config.SyncPeriod = importedConfig.SyncPeriod - } - if len(importedConfig.Tolerance) != 0 { - config.Tolerance = importedConfig.Tolerance - } - } - - return &config -} diff --git a/pkg/virtualgarden/kube_api_server_deployment_controller_manager_test.go b/pkg/virtualgarden/kube_api_server_deployment_controller_manager_test.go index b8c0a556..61de502b 100644 --- a/pkg/virtualgarden/kube_api_server_deployment_controller_manager_test.go +++ b/pkg/virtualgarden/kube_api_server_deployment_controller_manager_test.go @@ -111,13 +111,10 @@ func getImportsForControllerManagerDeploymentTest() *api.Imports { Enabled: true, CertificateAuthorityData: "test-ca-data", }, - HVPAEnabled: false, - HVPA: nil, - EventTTL: nil, - OidcIssuerURL: nil, - AdditionalVolumeMounts: nil, - AdditionalVolumes: nil, - HorizontalPodAutoscaler: nil, + EventTTL: nil, + OidcIssuerURL: nil, + AdditionalVolumeMounts: nil, + AdditionalVolumes: nil, }, DeleteNamespace: false, PriorityClassName: "", diff --git a/pkg/virtualgarden/kube_api_server_deployments.go b/pkg/virtualgarden/kube_api_server_deployments.go index 1ceff0f3..76307f10 100644 --- a/pkg/virtualgarden/kube_api_server_deployments.go +++ b/pkg/virtualgarden/kube_api_server_deployments.go @@ -58,9 +58,6 @@ func (o *operation) deployKubeAPIServerDeployment(ctx context.Context, checksums apiServerImports := o.imports.VirtualGarden.KubeAPIServer replicas := pointer.Int32Ptr(int32(apiServerImports.Replicas)) - if apiServerImports.HVPAEnabled { - replicas = apiServerImports.HVPA.GetMinReplicas() - } annotations := o.computeApiServerAnnotations(checksums) diff --git a/pkg/virtualgarden/kube_api_server_deployments_test.go b/pkg/virtualgarden/kube_api_server_deployments_test.go index 3defc637..eb6244d1 100644 --- a/pkg/virtualgarden/kube_api_server_deployments_test.go +++ b/pkg/virtualgarden/kube_api_server_deployments_test.go @@ -127,13 +127,10 @@ func getImportsForDeploymentTest() *api.Imports { Enabled: true, CertificateAuthorityData: "test-ca-data", }, - HVPAEnabled: false, - HVPA: nil, - EventTTL: nil, - OidcIssuerURL: nil, - AdditionalVolumeMounts: nil, - AdditionalVolumes: nil, - HorizontalPodAutoscaler: nil, + EventTTL: nil, + OidcIssuerURL: nil, + AdditionalVolumeMounts: nil, + AdditionalVolumes: nil, }, DeleteNamespace: false, PriorityClassName: "", diff --git a/pkg/virtualgarden/kube_api_server_misc_test.go b/pkg/virtualgarden/kube_api_server_misc_test.go index 1865f10b..12e2e171 100644 --- a/pkg/virtualgarden/kube_api_server_misc_test.go +++ b/pkg/virtualgarden/kube_api_server_misc_test.go @@ -153,13 +153,10 @@ func getImportsForMiscTest() *api.Imports { Enabled: true, CertificateAuthorityData: "test-ca-data", }, - HVPAEnabled: false, - HVPA: nil, - EventTTL: nil, - OidcIssuerURL: nil, - AdditionalVolumeMounts: nil, - AdditionalVolumes: nil, - HorizontalPodAutoscaler: nil, + EventTTL: nil, + OidcIssuerURL: nil, + AdditionalVolumeMounts: nil, + AdditionalVolumes: nil, }, DeleteNamespace: false, PriorityClassName: "", diff --git a/pkg/virtualgarden/kube_api_server_podautoscaling.go b/pkg/virtualgarden/kube_api_server_podautoscaling.go deleted file mode 100644 index 1980a2fd..00000000 --- a/pkg/virtualgarden/kube_api_server_podautoscaling.go +++ /dev/null @@ -1,375 +0,0 @@ -// Copyright (c) 2021 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package virtualgarden - -import ( - "context" - _ "embed" - - autoscalingv1 "k8s.io/api/autoscaling/v1" - autoscalingv2beta1 "k8s.io/api/autoscaling/v2beta1" - corev1 "k8s.io/api/core/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/api/resource" - autoscalingv1beta2 "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1beta2" - "k8s.io/utils/pointer" - "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" - - hvpav1alpha1 "github.com/gardener/hvpa-controller/api/v1alpha1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "sigs.k8s.io/controller-runtime/pkg/client" -) - -const ( - VirtualGardenKubeControllerManagerName = "virtual-garden-kube-controller-manager" -) - -func (o *operation) deployKubeAPIServerPodAutoscaling(ctx context.Context) error { - o.log.Infof("Deploying manifests for pod autoscaling for the kube-apiserver") - - if err := o.deployKubeApiServerHvpa(ctx); err != nil { - return err - } - - if err := o.deployKubeApiServerVpa(ctx); err != nil { - return err - } - - if err := o.deployKubeApiServerVpaForControllerManager(ctx); err != nil { - return err - } - - return nil -} - -func (o *operation) deleteKubeAPIServerPodAutoscaling(ctx context.Context) error { - o.log.Infof("Deleting manifests for pod autoscaling for the kube-apiserver") - - if err := o.deleteKubeApiServerHvpa(ctx); err != nil { - return err - } - - if err := o.deleteKubeApiServerVpa(ctx); err != nil { - return err - } - - if err := o.deleteKubeApiServerVpaForControllerManager(ctx); err != nil { - return err - } - - return nil -} - -func (o *operation) deployKubeApiServerVpaForControllerManager(ctx context.Context) error { - o.log.Infof("Deploying vpa for controller manager for the kube-apiserver") - if o.imports.VirtualGarden.KubeAPIServer.HVPAEnabled { - return nil - } - - vpa := o.emptyKubeAPIServerVpa(VirtualGardenKubeControllerManagerName) - - auto := autoscalingv1beta2.UpdateModeAuto - - _, err := controllerutil.CreateOrUpdate(ctx, o.client, vpa, func() error { - vpa.Spec = autoscalingv1beta2.VerticalPodAutoscalerSpec{ - ResourcePolicy: &autoscalingv1beta2.PodResourcePolicy{ - ContainerPolicies: []autoscalingv1beta2.ContainerResourcePolicy{ - { - ContainerName: "kube-controller-manager", - MaxAllowed: corev1.ResourceList{ - corev1.ResourceMemory: resource.MustParse("10G"), - corev1.ResourceCPU: resource.MustParse("5"), - }, - MinAllowed: corev1.ResourceList{ - corev1.ResourceMemory: resource.MustParse("400M"), - corev1.ResourceCPU: resource.MustParse("400m"), - }, - }, - }, - }, - TargetRef: &autoscalingv1.CrossVersionObjectReference{ - Kind: "Deployment", - Name: VirtualGardenKubeControllerManagerName, - APIVersion: "apps/v1", - }, - UpdatePolicy: &autoscalingv1beta2.PodUpdatePolicy{ - UpdateMode: &auto, - }, - } - - return nil - }) - - return err -} - -func (o *operation) deleteKubeApiServerVpaForControllerManager(ctx context.Context) error { - o.log.Infof("Delete vpa for controller manager for the kube-apiserver") - - hvpa := o.emptyKubeAPIServerVpa(VirtualGardenKubeControllerManagerName) - - if err := o.client.Delete(ctx, hvpa); client.IgnoreNotFound(err) != nil { - return err - } - - return nil -} - -func (o *operation) deployKubeApiServerVpa(ctx context.Context) error { - o.log.Infof("Deploying vpa for the kube-apiserver") - if o.imports.VirtualGarden.KubeAPIServer.HVPAEnabled { - return nil - } - - vpa := o.emptyKubeAPIServerVpa(KubeAPIServerServiceName) - - auto := autoscalingv1beta2.UpdateModeAuto - - _, err := controllerutil.CreateOrUpdate(ctx, o.client, vpa, func() error { - vpa.Spec = autoscalingv1beta2.VerticalPodAutoscalerSpec{ - ResourcePolicy: &autoscalingv1beta2.PodResourcePolicy{ - ContainerPolicies: []autoscalingv1beta2.ContainerResourcePolicy{ - { - ContainerName: "kube-apiserver", - MaxAllowed: corev1.ResourceList{ - corev1.ResourceMemory: resource.MustParse("25G"), - corev1.ResourceCPU: resource.MustParse("8"), - }, - MinAllowed: corev1.ResourceList{ - corev1.ResourceMemory: resource.MustParse("400M"), - corev1.ResourceCPU: resource.MustParse("400m"), - }, - }, - }, - }, - TargetRef: &autoscalingv1.CrossVersionObjectReference{ - Kind: "Deployment", - Name: KubeAPIServerServiceName, - APIVersion: "apps/v1", - }, - UpdatePolicy: &autoscalingv1beta2.PodUpdatePolicy{ - UpdateMode: &auto, - }, - } - - return nil - }) - - return err -} - -func (o *operation) deleteKubeApiServerVpa(ctx context.Context) error { - o.log.Infof("Delete vpa for the kube-apiserver") - - hvpa := o.emptyKubeAPIServerVpa(KubeAPIServerServiceName) - - if err := o.client.Delete(ctx, hvpa); client.IgnoreNotFound(err) != nil { - return err - } - - return nil -} - -func (o *operation) deployKubeApiServerHvpa(ctx context.Context) error { - o.log.Infof("Deploying hvpa for the kube-apiserver") - if !o.imports.VirtualGarden.KubeAPIServer.HVPAEnabled || o.imports.VirtualGarden.KubeAPIServer.Replicas == 0 { - return nil - } - - hvpaConfig := o.imports.VirtualGarden.KubeAPIServer.HVPA - - maxReplicas := hvpaConfig.GetMaxReplicas(6) - minReplicas := hvpaConfig.GetMinReplicas() - - hvpa := o.emptyKubeAPIServerHvpa() - _, err := controllerutil.CreateOrUpdate(ctx, o.client, hvpa, func() error { - hvpa.Spec.Replicas = pointer.Int32Ptr(1) - - hvpa.Spec.Hpa = hvpav1alpha1.HpaSpec{ - Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{ - "role": "virtual-gardener-apiserver-hpa", - }, - }, - Deploy: true, - ScaleUp: hvpav1alpha1.ScaleType{ - UpdatePolicy: hvpav1alpha1.UpdatePolicy{ - UpdateMode: pointer.StringPtr(hvpav1alpha1.UpdateModeAuto), - }, - }, - ScaleDown: hvpav1alpha1.ScaleType{ - UpdatePolicy: hvpav1alpha1.UpdatePolicy{ - UpdateMode: pointer.StringPtr(hvpav1alpha1.UpdateModeAuto), - }, - }, - Template: hvpav1alpha1.HpaTemplate{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - "role": "virtual-gardener-apiserver-hpa", - }, - }, - Spec: hvpav1alpha1.HpaTemplateSpec{ - MinReplicas: minReplicas, - MaxReplicas: maxReplicas, - Metrics: []autoscalingv2beta1.MetricSpec{ - { - Resource: &autoscalingv2beta1.ResourceMetricSource{ - Name: corev1.ResourceMemory, - TargetAverageUtilization: hvpaConfig.GetTargetAverageUtilizationMemory(80), - }, - }, - { - Resource: &autoscalingv2beta1.ResourceMetricSource{ - Name: corev1.ResourceCPU, - TargetAverageUtilization: hvpaConfig.GetTargetAverageUtilizationCpu(80), - }, - }, - }, - }, - }, - } - - hvpa.Spec.Vpa = hvpav1alpha1.VpaSpec{ - Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{ - "role": "virtual-gardener-apiserver-vpa", - }, - }, - Deploy: true, - ScaleUp: hvpav1alpha1.ScaleType{ - UpdatePolicy: hvpav1alpha1.UpdatePolicy{ - UpdateMode: hvpaConfig.GetVpaScaleUpMode(hvpav1alpha1.UpdateModeAuto), - }, - MinChange: hvpaConfig.GetVpaScaleUpMinChange(hvpav1alpha1.ScaleParams{ - CPU: hvpav1alpha1.ChangeParams{ - Value: pointer.StringPtr("300m"), - Percentage: pointer.Int32Ptr(80), - }, - Memory: hvpav1alpha1.ChangeParams{ - Value: pointer.StringPtr("200M"), - Percentage: pointer.Int32Ptr(80), - }, - }), - StabilizationDuration: hvpaConfig.GetVpaScaleUpStabilisationDuration("3m"), - }, - ScaleDown: hvpav1alpha1.ScaleType{ - UpdatePolicy: hvpav1alpha1.UpdatePolicy{ - UpdateMode: hvpaConfig.GetVpaScaleDownMode(hvpav1alpha1.UpdateModeAuto), - }, - MinChange: hvpaConfig.GetVpaScaleDownMinChange(hvpav1alpha1.ScaleParams{ - CPU: hvpav1alpha1.ChangeParams{ - Value: pointer.StringPtr("600m"), - Percentage: pointer.Int32Ptr(80), - }, - Memory: hvpav1alpha1.ChangeParams{ - Value: pointer.StringPtr("600M"), - Percentage: pointer.Int32Ptr(80), - }, - }), - StabilizationDuration: hvpaConfig.GetVpaScaleDownStabilisationDuration("15m"), - }, - LimitsRequestsGapScaleParams: hvpaConfig.GetLimitsRequestsGapScaleParams(hvpav1alpha1.ScaleParams{ - CPU: hvpav1alpha1.ChangeParams{ - Value: pointer.StringPtr("1"), - Percentage: pointer.Int32Ptr(40), - }, - Memory: hvpav1alpha1.ChangeParams{ - Value: pointer.StringPtr("1G"), - Percentage: pointer.Int32Ptr(40), - }, - }), - Template: hvpav1alpha1.VpaTemplate{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - "role": "virtual-gardener-apiserver-vpa", - }, - }, - Spec: hvpav1alpha1.VpaTemplateSpec{ - ResourcePolicy: &autoscalingv1beta2.PodResourcePolicy{ - ContainerPolicies: []autoscalingv1beta2.ContainerResourcePolicy{ - { - ContainerName: kubeAPIServerContainerName, - MaxAllowed: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("8"), - corev1.ResourceMemory: resource.MustParse("25G"), - }, - MinAllowed: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("400m"), - corev1.ResourceMemory: resource.MustParse("400M"), - }, - }, - }, - }, - }, - }, - } - - hvpa.Spec.WeightBasedScalingIntervals = []hvpav1alpha1.WeightBasedScalingInterval{} - if maxReplicas > *minReplicas { - hvpa.Spec.WeightBasedScalingIntervals = append(hvpa.Spec.WeightBasedScalingIntervals, hvpav1alpha1.WeightBasedScalingInterval{ - VpaWeight: hvpav1alpha1.HpaOnly, - StartReplicaCount: *minReplicas, - LastReplicaCount: maxReplicas - 1, - }) - } - hvpa.Spec.WeightBasedScalingIntervals = append(hvpa.Spec.WeightBasedScalingIntervals, hvpav1alpha1.WeightBasedScalingInterval{ - VpaWeight: hvpav1alpha1.VpaOnly, - StartReplicaCount: maxReplicas, - LastReplicaCount: maxReplicas, - }) - - hvpa.Spec.TargetRef = &autoscalingv2beta1.CrossVersionObjectReference{ - APIVersion: "apps/v1", - Kind: "Deployment", - Name: KubeAPIServerServiceName, - } - - hvpa.Spec.MaintenanceTimeWindow = nil - if hvpaConfig != nil { - hvpa.Spec.MaintenanceTimeWindow = hvpaConfig.MaintenanceWindow - } - - return nil - }) - - return err -} - -func (o *operation) deleteKubeApiServerHvpa(ctx context.Context) error { - o.log.Infof("Delete hvpa for the kube-apiserver") - - if _, err := o.getHVPACRD(ctx); err != nil { - if apierrors.IsNotFound(err) { - return nil - } - - return err - } - - hvpa := o.emptyKubeAPIServerHvpa() - if err := o.client.Delete(ctx, hvpa); client.IgnoreNotFound(err) != nil { - return err - } - - return nil -} - -func (o *operation) emptyKubeAPIServerHvpa() *hvpav1alpha1.Hvpa { - return &hvpav1alpha1.Hvpa{ObjectMeta: metav1.ObjectMeta{Name: KubeAPIServerServiceName, Namespace: o.namespace}} -} - -func (o *operation) emptyKubeAPIServerVpa(name string) *autoscalingv1beta2.VerticalPodAutoscaler { - return &autoscalingv1beta2.VerticalPodAutoscaler{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: o.namespace}} -} diff --git a/pkg/virtualgarden/kube_api_server_podautoscaling_test.go b/pkg/virtualgarden/kube_api_server_podautoscaling_test.go deleted file mode 100644 index 053ab319..00000000 --- a/pkg/virtualgarden/kube_api_server_podautoscaling_test.go +++ /dev/null @@ -1,189 +0,0 @@ -// Copyright (c) 2021 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package virtualgarden - -import ( - "context" - "reflect" - - hvpav1alpha1 "github.com/gardener/hvpa-controller/api/v1alpha1" - "sigs.k8s.io/controller-runtime/pkg/client" - - lsv1alpha1 "github.com/gardener/landscaper/apis/core/v1alpha1" - "github.com/gardener/virtual-garden/pkg/api" - "github.com/gardener/virtual-garden/pkg/provider" - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/utils/pointer" -) - -const ( - maxReplicas = int32(6) - minReplicas = int32(3) - vpaScaleUpMode = hvpav1alpha1.UpdateModeOff - vpaScaleDownMode = hvpav1alpha1.UpdateModeOff -) - -var vpaScaleUpStabilization = api.ScaleType{ - MinChange: &hvpav1alpha1.ScaleParams{ - CPU: hvpav1alpha1.ChangeParams{ - Value: pointer.StringPtr("350m"), - Percentage: pointer.Int32Ptr(40), - }, - Memory: hvpav1alpha1.ChangeParams{ - Value: pointer.StringPtr("250M"), - Percentage: pointer.Int32Ptr(87), - }, - }, - StabilizationDuration: pointer.StringPtr("4m"), -} - -var vpaScaleDownStabilization = api.ScaleType{ - MinChange: &hvpav1alpha1.ScaleParams{ - CPU: hvpav1alpha1.ChangeParams{ - Value: pointer.StringPtr("360m"), - Percentage: pointer.Int32Ptr(42), - }, - Memory: hvpav1alpha1.ChangeParams{ - Value: pointer.StringPtr("260M"), - Percentage: pointer.Int32Ptr(89), - }, - }, - StabilizationDuration: pointer.StringPtr("9m"), -} - -var limitsRequestsGapScaleParams = hvpav1alpha1.ScaleParams{ - CPU: hvpav1alpha1.ChangeParams{ - Value: pointer.StringPtr("320m"), - Percentage: pointer.Int32Ptr(45), - }, - Memory: hvpav1alpha1.ChangeParams{ - Value: pointer.StringPtr("220M"), - Percentage: pointer.Int32Ptr(81), - }, -} - -var _ = Describe("Api Server pod auto scaling test", func() { - - It("Should create api auto scaling settings", func() { - namespaceName := "apiserverhvpa" - - ctx := context.Background() - defer ctx.Done() - - namespace := v1.Namespace{ - ObjectMeta: metav1.ObjectMeta{Name: namespaceName}, - } - err := testenv.Client.Create(ctx, &namespace) - Expect(err).To(BeNil()) - - imports := getImportsForHvpa() - - infrastructureProvider, err := provider.NewInfrastructureProvider(api.InfrastructureProviderGCP) - Expect(err).To(BeNil()) - - operation := &operation{ - client: testenv.Client, - log: testenv.Logger, - infrastructureProvider: infrastructureProvider, - backupProvider: nil, - namespace: namespaceName, - imports: &imports, - exports: api.Exports{}, - imageRefs: api.ImageRefs{}, - } - - err = DeployHVPACRD(ctx, testenv.Client) - Expect(err).To(BeNil()) - - err = DeployVPACRD(ctx, testenv.Client) - Expect(err).To(BeNil()) - - err = operation.deployKubeAPIServerPodAutoscaling(ctx) - Expect(err).To(BeNil()) - - hvpa := operation.emptyKubeAPIServerHvpa() - - err = testenv.Client.Get(ctx, client.ObjectKeyFromObject(hvpa), hvpa) - Expect(err).To(BeNil()) - - Expect(*hvpa.Spec.Replicas == 1).To(BeTrue()) - Expect(hvpa.Spec.Hpa.Template.Spec.MaxReplicas == maxReplicas).To(BeTrue()) - Expect(*hvpa.Spec.Hpa.Template.Spec.MinReplicas == minReplicas).To(BeTrue()) - Expect(*hvpa.Spec.Vpa.ScaleUp.UpdatePolicy.UpdateMode == vpaScaleUpMode).To(BeTrue()) - Expect(*hvpa.Spec.Vpa.ScaleDown.UpdatePolicy.UpdateMode == vpaScaleDownMode).To(BeTrue()) - Expect(reflect.DeepEqual(hvpa.Spec.Vpa.ScaleUp.MinChange, *vpaScaleUpStabilization.MinChange)).To(BeTrue()) - Expect(reflect.DeepEqual(*hvpa.Spec.Vpa.ScaleUp.StabilizationDuration, *vpaScaleUpStabilization.StabilizationDuration)).To(BeTrue()) - Expect(reflect.DeepEqual(hvpa.Spec.Vpa.ScaleDown.MinChange, *vpaScaleDownStabilization.MinChange)).To(BeTrue()) - Expect(reflect.DeepEqual(*hvpa.Spec.Vpa.ScaleDown.StabilizationDuration, *vpaScaleDownStabilization.StabilizationDuration)).To(BeTrue()) - Expect(reflect.DeepEqual(hvpa.Spec.Vpa.LimitsRequestsGapScaleParams, limitsRequestsGapScaleParams)).To(BeTrue()) - - Expect(operation.deleteKubeAPIServerPodAutoscaling(ctx)).To(Succeed()) - }) -}) - -func getHVPASettings() *api.HvpaConfig { - return &api.HvpaConfig{ - MaxReplicas: pointer.Int32Ptr(maxReplicas), - MinReplicas: pointer.Int32Ptr(minReplicas), - VpaScaleUpMode: pointer.StringPtr(vpaScaleUpMode), - VpaScaleDownMode: pointer.StringPtr(vpaScaleDownMode), - VpaScaleUpStabilization: &vpaScaleUpStabilization, - VpaScaleDownStabilization: &vpaScaleDownStabilization, - LimitsRequestsGapScaleParams: &limitsRequestsGapScaleParams, - MaintenanceWindow: nil, - } -} - -func getImportsForHvpa() api.Imports { - return api.Imports{ - RuntimeCluster: lsv1alpha1.Target{}, - RuntimeClusterSettings: api.ClusterSettings{}, - VirtualGarden: api.VirtualGarden{ - ETCD: nil, - KubeAPIServer: &api.KubeAPIServer{ - Replicas: 1, - SNI: nil, - DnsAccessDomain: "com.our.test", - GardenerControlplane: api.GardenerControlplane{ - ValidatingWebhook: api.AdmissionWebhookConfig{ - Token: api.AdmissionWebhookTokenConfig{Enabled: true}, - }, - MutatingWebhook: api.AdmissionWebhookConfig{ - Token: api.AdmissionWebhookTokenConfig{Enabled: true}, - }, - }, - ServiceAccountKeyPem: pointer.String("test-service-account-key"), - AuditWebhookConfig: api.AuditWebhookConfig{Config: "testconfig"}, - AuditWebhookBatchMaxSize: "", - SeedAuthorizer: api.SeedAuthorizer{ - Enabled: true, - CertificateAuthorityData: "test-ca-data", - }, - HVPAEnabled: true, - HVPA: getHVPASettings(), - EventTTL: nil, - OidcIssuerURL: nil, - AdditionalVolumeMounts: nil, - AdditionalVolumes: nil, - HorizontalPodAutoscaler: nil, - }, - DeleteNamespace: false, - PriorityClassName: "", - }, - } -} diff --git a/pkg/virtualgarden/kube_api_server_secrets_test.go b/pkg/virtualgarden/kube_api_server_secrets_test.go index 34b96002..ff944223 100644 --- a/pkg/virtualgarden/kube_api_server_secrets_test.go +++ b/pkg/virtualgarden/kube_api_server_secrets_test.go @@ -127,13 +127,10 @@ func getImportsForApiServerSecretsTest() api.Imports { Enabled: true, CertificateAuthorityData: "test-ca-data", }, - HVPAEnabled: false, - HVPA: nil, - EventTTL: nil, - OidcIssuerURL: nil, - AdditionalVolumeMounts: nil, - AdditionalVolumes: nil, - HorizontalPodAutoscaler: nil, + EventTTL: nil, + OidcIssuerURL: nil, + AdditionalVolumeMounts: nil, + AdditionalVolumes: nil, }, DeleteNamespace: false, PriorityClassName: "", diff --git a/pkg/virtualgarden/operation_reconcile.go b/pkg/virtualgarden/operation_reconcile.go index 15336b1d..c72c5036 100644 --- a/pkg/virtualgarden/operation_reconcile.go +++ b/pkg/virtualgarden/operation_reconcile.go @@ -29,11 +29,6 @@ func (o *operation) Reconcile(ctx context.Context) (*api.Exports, error) { var ( graph = flow.NewGraph("Virtual Garden Reconciliation") - checkHVPACRD = graph.Add(flow.Task{ - Name: "Check if HVPA CRD in hosting cluster exists if it should be used", - Fn: o.checkHVPACRD, - }) - createNamespace = graph.Add(flow.Task{ Name: "Creating namespace for virtual-garden deployment in hosting cluster", Fn: o.CreateNamespace, @@ -47,7 +42,7 @@ func (o *operation) Reconcile(ctx context.Context) (*api.Exports, error) { createETCD = graph.Add(flow.Task{ Name: "Deploying the main and events etcds", Fn: o.DeployETCD, - Dependencies: flow.NewTaskIDs(checkHVPACRD, createNamespace, deployBackupBucket), + Dependencies: flow.NewTaskIDs(createNamespace, deployBackupBucket), }) createKubeAPIServerService = graph.Add(flow.Task{ diff --git a/pkg/virtualgarden/resources/hvpa.yaml b/pkg/virtualgarden/resources/hvpa.yaml deleted file mode 100644 index 38c1f73b..00000000 --- a/pkg/virtualgarden/resources/hvpa.yaml +++ /dev/null @@ -1,1204 +0,0 @@ -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: hvpas.autoscaling.k8s.io -spec: - group: autoscaling.k8s.io - names: - kind: Hvpa - plural: hvpas - scope: "" - preserveUnknownFields: false - subresources: - scale: - labelSelectorPath: .status.targetSelector - specReplicasPath: .spec.replicas - statusReplicasPath: .status.replicas - status: {} - validation: - openAPIV3Schema: - description: Hvpa is the Schema for the hvpas API - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: HvpaSpec defines the desired state of Hvpa - properties: - hpa: - description: Hpa defines the spec of HPA - properties: - deploy: - description: Deploy defines whether the HPA is deployed or not - type: boolean - scaleDown: - description: ScaleDown defines the parameters for scale down - properties: - minChange: - description: MinChange is the minimum change in the resource - on which HVPA acts HVPA uses minimum of the Value and Percentage - value - properties: - cpu: - description: Scale parameters for CPU - properties: - percentage: - description: Percentage is the percentage of currently - allocated value to be used for scaling - format: int32 - type: integer - value: - description: Value is the absolute value of the scaling - type: string - type: object - memory: - description: Scale parameters for memory - properties: - percentage: - description: Percentage is the percentage of currently - allocated value to be used for scaling - format: int32 - type: integer - value: - description: Value is the absolute value of the scaling - type: string - type: object - replicas: - description: Scale patameters for replicas - properties: - percentage: - description: Percentage is the percentage of currently - allocated value to be used for scaling - format: int32 - type: integer - value: - description: Value is the absolute value of the scaling - type: string - type: object - type: object - stabilizationDuration: - description: StabilizationDuration defines the minimum delay - in minutes between 2 consecutive scale operations Valid time - units are "ns", "us" (or "µs"), "ms", "s", "m", "h" - type: string - updatePolicy: - description: Describes the rules on when changes are applied. - If not specified, all fields in the `UpdatePolicy` are set - to their default values. - properties: - updateMode: - description: Controls when autoscaler applies changes to - the resources. The default is 'Auto'. - type: string - type: object - type: object - scaleUp: - description: ScaleUp defines the parameters for scale up - properties: - minChange: - description: MinChange is the minimum change in the resource - on which HVPA acts HVPA uses minimum of the Value and Percentage - value - properties: - cpu: - description: Scale parameters for CPU - properties: - percentage: - description: Percentage is the percentage of currently - allocated value to be used for scaling - format: int32 - type: integer - value: - description: Value is the absolute value of the scaling - type: string - type: object - memory: - description: Scale parameters for memory - properties: - percentage: - description: Percentage is the percentage of currently - allocated value to be used for scaling - format: int32 - type: integer - value: - description: Value is the absolute value of the scaling - type: string - type: object - replicas: - description: Scale patameters for replicas - properties: - percentage: - description: Percentage is the percentage of currently - allocated value to be used for scaling - format: int32 - type: integer - value: - description: Value is the absolute value of the scaling - type: string - type: object - type: object - stabilizationDuration: - description: StabilizationDuration defines the minimum delay - in minutes between 2 consecutive scale operations Valid time - units are "ns", "us" (or "µs"), "ms", "s", "m", "h" - type: string - updatePolicy: - description: Describes the rules on when changes are applied. - If not specified, all fields in the `UpdatePolicy` are set - to their default values. - properties: - updateMode: - description: Controls when autoscaler applies changes to - the resources. The default is 'Auto'. - type: string - type: object - type: object - selector: - description: 'Selector is a label query that should match HPA. Must - match in order to be controlled. If empty, defaulted to labels - on HPA template. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#label-selectors' - properties: - matchExpressions: - description: matchExpressions is a list of label selector requirements. - The requirements are ANDed. - items: - description: A label selector requirement is a selector that - contains values, a key, and an operator that relates the - key and values. - properties: - key: - description: key is the label key that the selector applies - to. - type: string - operator: - description: operator represents a key's relationship - to a set of values. Valid operators are In, NotIn, Exists - and DoesNotExist. - type: string - values: - description: values is an array of string values. If the - operator is In or NotIn, the values array must be non-empty. - If the operator is Exists or DoesNotExist, the values - array must be empty. This array is replaced during a - strategic merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: matchLabels is a map of {key,value} pairs. A single - {key,value} in the matchLabels map is equivalent to an element - of matchExpressions, whose key field is "key", the operator - is "In", and the values array contains only "value". The requirements - are ANDed. - type: object - type: object - template: - description: Template is the object that describes the HPA that - will be created. - properties: - metadata: - description: Metadata of the pods created from this template. - type: object - x-kubernetes-preserve-unknown-fields: true # To be replaced by proper OpenAPI spec - spec: - description: Spec defines the behavior of a HPA. - properties: - maxReplicas: - description: maxReplicas is the upper limit for the number - of replicas to which the autoscaler can scale up. It cannot - be less that minReplicas. - format: int32 - type: integer - metrics: - description: metrics contains the specifications for which - to use to calculate the desired replica count (the maximum - replica count across all metrics will be used). The desired - replica count is calculated multiplying the ratio between - the target value and the current value by the current - number of pods. Ergo, metrics used must decrease as the - pod count is increased, and vice-versa. See the individual - metric source types for more information about how each - type of metric must respond. If not set, the default metric - will be set to 80% average CPU utilization. - items: - description: MetricSpec specifies how to scale based on - a single metric (only `type` and one other matching - field should be set at once). - properties: - external: - description: external refers to a global metric that - is not associated with any Kubernetes object. It - allows autoscaling based on information coming from - components running outside of cluster (for example - length of queue in cloud messaging service, or QPS - from loadbalancer running outside of cluster). - properties: - metricName: - description: metricName is the name of the metric - in question. - type: string - metricSelector: - description: metricSelector is used to identify - a specific time series within a given metric. - properties: - matchExpressions: - description: matchExpressions is a list of - label selector requirements. The requirements - are ANDed. - items: - description: A label selector requirement - is a selector that contains values, a - key, and an operator that relates the - key and values. - properties: - key: - description: key is the label key that - the selector applies to. - type: string - operator: - description: operator represents a key's - relationship to a set of values. Valid - operators are In, NotIn, Exists and - DoesNotExist. - type: string - values: - description: values is an array of string - values. If the operator is In or NotIn, - the values array must be non-empty. - If the operator is Exists or DoesNotExist, - the values array must be empty. This - array is replaced during a strategic - merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: matchLabels is a map of {key,value} - pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, - whose key field is "key", the operator is - "In", and the values array contains only - "value". The requirements are ANDed. - type: object - type: object - targetAverageValue: - description: targetAverageValue is the target - per-pod value of global metric (as a quantity). - Mutually exclusive with TargetValue. - type: string - targetValue: - description: targetValue is the target value of - the metric (as a quantity). Mutually exclusive - with TargetAverageValue. - type: string - required: - - metricName - type: object - object: - description: object refers to a metric describing - a single kubernetes object (for example, hits-per-second - on an Ingress object). - properties: - averageValue: - description: averageValue is the target value - of the average of the metric across all relevant - pods (as a quantity) - type: string - metricName: - description: metricName is the name of the metric - in question. - type: string - selector: - description: selector is the string-encoded form - of a standard kubernetes label selector for - the given metric When set, it is passed as an - additional parameter to the metrics server for - more specific metrics scoping When unset, just - the metricName will be used to gather metrics. - properties: - matchExpressions: - description: matchExpressions is a list of - label selector requirements. The requirements - are ANDed. - items: - description: A label selector requirement - is a selector that contains values, a - key, and an operator that relates the - key and values. - properties: - key: - description: key is the label key that - the selector applies to. - type: string - operator: - description: operator represents a key's - relationship to a set of values. Valid - operators are In, NotIn, Exists and - DoesNotExist. - type: string - values: - description: values is an array of string - values. If the operator is In or NotIn, - the values array must be non-empty. - If the operator is Exists or DoesNotExist, - the values array must be empty. This - array is replaced during a strategic - merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: matchLabels is a map of {key,value} - pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, - whose key field is "key", the operator is - "In", and the values array contains only - "value". The requirements are ANDed. - type: object - type: object - target: - description: target is the described Kubernetes - object. - properties: - apiVersion: - description: API version of the referent - type: string - kind: - description: 'Kind of the referent; More info: - https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds"' - type: string - name: - description: 'Name of the referent; More info: - http://kubernetes.io/docs/user-guide/identifiers#names' - type: string - required: - - kind - - name - type: object - targetValue: - description: targetValue is the target value of - the metric (as a quantity). - type: string - required: - - metricName - - target - - targetValue - type: object - pods: - description: pods refers to a metric describing each - pod in the current scale target (for example, transactions-processed-per-second). The - values will be averaged together before being compared - to the target value. - properties: - metricName: - description: metricName is the name of the metric - in question - type: string - selector: - description: selector is the string-encoded form - of a standard kubernetes label selector for - the given metric When set, it is passed as an - additional parameter to the metrics server for - more specific metrics scoping When unset, just - the metricName will be used to gather metrics. - properties: - matchExpressions: - description: matchExpressions is a list of - label selector requirements. The requirements - are ANDed. - items: - description: A label selector requirement - is a selector that contains values, a - key, and an operator that relates the - key and values. - properties: - key: - description: key is the label key that - the selector applies to. - type: string - operator: - description: operator represents a key's - relationship to a set of values. Valid - operators are In, NotIn, Exists and - DoesNotExist. - type: string - values: - description: values is an array of string - values. If the operator is In or NotIn, - the values array must be non-empty. - If the operator is Exists or DoesNotExist, - the values array must be empty. This - array is replaced during a strategic - merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: matchLabels is a map of {key,value} - pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, - whose key field is "key", the operator is - "In", and the values array contains only - "value". The requirements are ANDed. - type: object - type: object - targetAverageValue: - description: targetAverageValue is the target - value of the average of the metric across all - relevant pods (as a quantity) - type: string - required: - - metricName - - targetAverageValue - type: object - resource: - description: resource refers to a resource metric - (such as those specified in requests and limits) - known to Kubernetes describing each pod in the current - scale target (e.g. CPU or memory). Such metrics - are built in to Kubernetes, and have special scaling - options on top of those available to normal per-pod - metrics using the "pods" source. - properties: - name: - description: name is the name of the resource - in question. - type: string - targetAverageUtilization: - description: targetAverageUtilization is the target - value of the average of the resource metric - across all relevant pods, represented as a percentage - of the requested value of the resource for the - pods. - format: int32 - type: integer - targetAverageValue: - description: targetAverageValue is the target - value of the average of the resource metric - across all relevant pods, as a raw value (instead - of as a percentage of the request), similar - to the "pods" metric source type. - type: string - required: - - name - type: object - type: - description: type is the type of metric source. It - should be one of "Object", "Pods" or "Resource", - each mapping to a matching field in the object. - type: string - required: - - type - type: object - type: array - minReplicas: - description: minReplicas is the lower limit for the number - of replicas to which the autoscaler can scale down. It - defaults to 1 pod. - format: int32 - type: integer - required: - - maxReplicas - type: object - type: object - type: object - maintenanceTimeWindow: - description: MaintenanceTimeWindow contains information about the time - window for maintenance operations. - properties: - begin: - description: Begin is the beginning of the time window in the format - HHMMSS+ZONE, e.g. "220000+0100". - type: string - end: - description: End is the end of the time window in the format HHMMSS+ZONE, - e.g. "220000+0100". - type: string - required: - - begin - - end - type: object - replicas: - description: Replicas is the number of replicas of target resource - format: int32 - type: integer - targetRef: - description: TargetRef points to the controller managing the set of - pods for the autoscaler to control - properties: - apiVersion: - description: API version of the referent - type: string - kind: - description: 'Kind of the referent; More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds"' - type: string - name: - description: 'Name of the referent; More info: http://kubernetes.io/docs/user-guide/identifiers#names' - type: string - required: - - kind - - name - type: object - vpa: - description: Vpa defines the spec of VPA - properties: - deploy: - description: Deploy defines whether the VPA is deployed or not - type: boolean - limitsRequestsGapScaleParams: - description: LimitsRequestsGapScaleParams is the scaling thresholds - for limits - properties: - cpu: - description: Scale parameters for CPU - properties: - percentage: - description: Percentage is the percentage of currently allocated - value to be used for scaling - format: int32 - type: integer - value: - description: Value is the absolute value of the scaling - type: string - type: object - memory: - description: Scale parameters for memory - properties: - percentage: - description: Percentage is the percentage of currently allocated - value to be used for scaling - format: int32 - type: integer - value: - description: Value is the absolute value of the scaling - type: string - type: object - replicas: - description: Scale patameters for replicas - properties: - percentage: - description: Percentage is the percentage of currently allocated - value to be used for scaling - format: int32 - type: integer - value: - description: Value is the absolute value of the scaling - type: string - type: object - type: object - scaleDown: - description: ScaleDown defines the parameters for scale down - properties: - minChange: - description: MinChange is the minimum change in the resource - on which HVPA acts HVPA uses minimum of the Value and Percentage - value - properties: - cpu: - description: Scale parameters for CPU - properties: - percentage: - description: Percentage is the percentage of currently - allocated value to be used for scaling - format: int32 - type: integer - value: - description: Value is the absolute value of the scaling - type: string - type: object - memory: - description: Scale parameters for memory - properties: - percentage: - description: Percentage is the percentage of currently - allocated value to be used for scaling - format: int32 - type: integer - value: - description: Value is the absolute value of the scaling - type: string - type: object - replicas: - description: Scale patameters for replicas - properties: - percentage: - description: Percentage is the percentage of currently - allocated value to be used for scaling - format: int32 - type: integer - value: - description: Value is the absolute value of the scaling - type: string - type: object - type: object - stabilizationDuration: - description: StabilizationDuration defines the minimum delay - in minutes between 2 consecutive scale operations Valid time - units are "ns", "us" (or "µs"), "ms", "s", "m", "h" - type: string - updatePolicy: - description: Describes the rules on when changes are applied. - If not specified, all fields in the `UpdatePolicy` are set - to their default values. - properties: - updateMode: - description: Controls when autoscaler applies changes to - the resources. The default is 'Auto'. - type: string - type: object - type: object - scaleUp: - description: ScaleUp defines the parameters for scale up - properties: - minChange: - description: MinChange is the minimum change in the resource - on which HVPA acts HVPA uses minimum of the Value and Percentage - value - properties: - cpu: - description: Scale parameters for CPU - properties: - percentage: - description: Percentage is the percentage of currently - allocated value to be used for scaling - format: int32 - type: integer - value: - description: Value is the absolute value of the scaling - type: string - type: object - memory: - description: Scale parameters for memory - properties: - percentage: - description: Percentage is the percentage of currently - allocated value to be used for scaling - format: int32 - type: integer - value: - description: Value is the absolute value of the scaling - type: string - type: object - replicas: - description: Scale patameters for replicas - properties: - percentage: - description: Percentage is the percentage of currently - allocated value to be used for scaling - format: int32 - type: integer - value: - description: Value is the absolute value of the scaling - type: string - type: object - type: object - stabilizationDuration: - description: StabilizationDuration defines the minimum delay - in minutes between 2 consecutive scale operations Valid time - units are "ns", "us" (or "µs"), "ms", "s", "m", "h" - type: string - updatePolicy: - description: Describes the rules on when changes are applied. - If not specified, all fields in the `UpdatePolicy` are set - to their default values. - properties: - updateMode: - description: Controls when autoscaler applies changes to - the resources. The default is 'Auto'. - type: string - type: object - type: object - selector: - description: 'Selector is a label query that should match VPA. Must - match in order to be controlled. If empty, defaulted to labels - on VPA template. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#label-selectors' - properties: - matchExpressions: - description: matchExpressions is a list of label selector requirements. - The requirements are ANDed. - items: - description: A label selector requirement is a selector that - contains values, a key, and an operator that relates the - key and values. - properties: - key: - description: key is the label key that the selector applies - to. - type: string - operator: - description: operator represents a key's relationship - to a set of values. Valid operators are In, NotIn, Exists - and DoesNotExist. - type: string - values: - description: values is an array of string values. If the - operator is In or NotIn, the values array must be non-empty. - If the operator is Exists or DoesNotExist, the values - array must be empty. This array is replaced during a - strategic merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: matchLabels is a map of {key,value} pairs. A single - {key,value} in the matchLabels map is equivalent to an element - of matchExpressions, whose key field is "key", the operator - is "In", and the values array contains only "value". The requirements - are ANDed. - type: object - type: object - template: - description: Template is the object that describes the VPA that - will be created. - properties: - metadata: - description: Metadata of the pods created from this template. - type: object - x-kubernetes-preserve-unknown-fields: true # To be replaced by proper OpenAPI spec - spec: - description: Spec defines the behavior of a VPA. - properties: - resourcePolicy: - description: Controls how the autoscaler computes recommended - resources. The resource policy may be used to set constraints - on the recommendations for individual containers. If not - specified, the autoscaler computes recommended resources - for all containers in the pod, without additional constraints. - properties: - containerPolicies: - description: Per-container resource policies. - items: - description: ContainerResourcePolicy controls how - autoscaler computes the recommended resources for - a specific container. - properties: - containerName: - description: Name of the container or DefaultContainerResourcePolicy, - in which case the policy is used by the containers - that don't have their own policy specified. - type: string - maxAllowed: - additionalProperties: - type: string - description: Specifies the maximum amount of resources - that will be recommended for the container. - The default is no maximum. - type: object - minAllowed: - additionalProperties: - type: string - description: Specifies the minimal amount of resources - that will be recommended for the container. - The default is no minimum. - type: object - mode: - description: Whether autoscaler is enabled for - the container. The default is "Auto". - type: string - type: object - type: array - type: object - type: object - type: object - type: object - weightBasedScalingIntervals: - description: WeightBasedScalingIntervals defines the intervals of replica - counts, and the weights for scaling a deployment vertically If there - are overlapping intervals, then the vpaWeight will be taken from the - first matching interval - items: - description: WeightBasedScalingInterval defines the interval of replica - counts in which VpaWeight is applied to VPA scaling - properties: - lastReplicaCount: - description: LastReplicaCount is the number of replicas till which - VpaWeight is applied to VPA scaling If this field is not provided, - it will default to maxReplicas of HPA - format: int32 - type: integer - startReplicaCount: - description: StartReplicaCount is the number of replicas from - which VpaWeight is applied to VPA scaling If this field is not - provided, it will default to minReplicas of HPA - format: int32 - type: integer - vpaWeight: - description: VpaWeight defines the weight (in percentage) to be - given to VPA's recommendationd for the interval of number of - replicas provided - format: int32 - type: integer - type: object - type: array - required: - - targetRef - type: object - status: - description: HvpaStatus defines the observed state of Hvpa - properties: - hpaScaleDownUpdatePolicy: - description: Current HPA UpdatePolicy set in the spec - properties: - updateMode: - description: Controls when autoscaler applies changes to the resources. - The default is 'Auto'. - type: string - type: object - hpaScaleUpUpdatePolicy: - description: Current HPA UpdatePolicy set in the spec - properties: - updateMode: - description: Controls when autoscaler applies changes to the resources. - The default is 'Auto'. - type: string - type: object - hpaWeight: - description: VpaWeight - weight to provide to VPA scaling - format: int32 - type: integer - lastBlockedScaling: - items: - description: BlockedScaling defines the details for blocked scaling - properties: - reason: - description: BlockingReason defines the reason for blocking. - type: string - scalingStatus: - description: ScalingStatus defines the status of scaling - properties: - hpaStatus: - description: HpaStatus defines the status of HPA - properties: - currentReplicas: - format: int32 - type: integer - desiredReplicas: - format: int32 - type: integer - type: object - lastScaleTime: - format: date-time - type: string - vpaStatus: - description: VerticalPodAutoscalerStatus describes the runtime - state of the autoscaler. - properties: - conditions: - description: Conditions is the set of conditions required - for this autoscaler to scale its target, and indicates - whether or not those conditions are met. - items: - description: VerticalPodAutoscalerCondition describes - the state of a VerticalPodAutoscaler at a certain - point. - properties: - lastTransitionTime: - description: lastTransitionTime is the last time - the condition transitioned from one status to - another - format: date-time - type: string - message: - description: message is a human-readable explanation - containing details about the transition - type: string - reason: - description: reason is the reason for the condition's - last transition. - type: string - status: - description: status is the status of the condition - (True, False, Unknown) - type: string - type: - description: type describes the current condition - type: string - required: - - status - - type - type: object - type: array - recommendation: - description: The most recently computed amount of resources - recommended by the autoscaler for the controlled pods. - properties: - containerRecommendations: - description: Resources recommended by the autoscaler - for each container. - items: - description: RecommendedContainerResources is the - recommendation of resources computed by autoscaler - for a specific container. Respects the container - resource policy if present in the spec. In particular - the recommendation is not produced for containers - with `ContainerScalingMode` set to 'Off'. - properties: - containerName: - description: Name of the container. - type: string - lowerBound: - additionalProperties: - type: string - description: Minimum recommended amount of resources. - Observes ContainerResourcePolicy. This amount - is not guaranteed to be sufficient for the - application to operate in a stable way, however - running with less resources is likely to have - significant impact on performance/availability. - type: object - target: - additionalProperties: - type: string - description: Recommended amount of resources. - Observes ContainerResourcePolicy. - type: object - uncappedTarget: - additionalProperties: - type: string - description: The most recent recommended resources - target computed by the autoscaler for the - controlled pods, based only on actual resource - usage, not taking into account the ContainerResourcePolicy. - May differ from the Recommendation if the - actual resource usage causes the target to - violate the ContainerResourcePolicy (lower - than MinAllowed or higher that MaxAllowed). - Used only as status indication, will not affect - actual resource assignment. - type: object - upperBound: - additionalProperties: - type: string - description: Maximum recommended amount of resources. - Observes ContainerResourcePolicy. Any resources - allocated beyond this value are likely wasted. - This value may be larger than the maximum - amount of application is actually capable - of consuming. - type: object - required: - - target - type: object - type: array - type: object - type: object - type: object - type: object - type: array - lastError: - description: LastError has details of any errors that occured - properties: - description: - description: Description of the error - type: string - lastOperation: - description: LastOperation is the type of operation for which error - occurred - type: string - lastUpdateTime: - description: Time at which the error occurred - format: date-time - type: string - type: object - lastScaling: - description: ScalingStatus defines the status of scaling - properties: - hpaStatus: - description: HpaStatus defines the status of HPA - properties: - currentReplicas: - format: int32 - type: integer - desiredReplicas: - format: int32 - type: integer - type: object - lastScaleTime: - format: date-time - type: string - vpaStatus: - description: VerticalPodAutoscalerStatus describes the runtime state - of the autoscaler. - properties: - conditions: - description: Conditions is the set of conditions required for - this autoscaler to scale its target, and indicates whether - or not those conditions are met. - items: - description: VerticalPodAutoscalerCondition describes the - state of a VerticalPodAutoscaler at a certain point. - properties: - lastTransitionTime: - description: lastTransitionTime is the last time the condition - transitioned from one status to another - format: date-time - type: string - message: - description: message is a human-readable explanation containing - details about the transition - type: string - reason: - description: reason is the reason for the condition's - last transition. - type: string - status: - description: status is the status of the condition (True, - False, Unknown) - type: string - type: - description: type describes the current condition - type: string - required: - - status - - type - type: object - type: array - recommendation: - description: The most recently computed amount of resources - recommended by the autoscaler for the controlled pods. - properties: - containerRecommendations: - description: Resources recommended by the autoscaler for - each container. - items: - description: RecommendedContainerResources is the recommendation - of resources computed by autoscaler for a specific container. - Respects the container resource policy if present in - the spec. In particular the recommendation is not produced - for containers with `ContainerScalingMode` set to 'Off'. - properties: - containerName: - description: Name of the container. - type: string - lowerBound: - additionalProperties: - type: string - description: Minimum recommended amount of resources. - Observes ContainerResourcePolicy. This amount is - not guaranteed to be sufficient for the application - to operate in a stable way, however running with - less resources is likely to have significant impact - on performance/availability. - type: object - target: - additionalProperties: - type: string - description: Recommended amount of resources. Observes - ContainerResourcePolicy. - type: object - uncappedTarget: - additionalProperties: - type: string - description: The most recent recommended resources - target computed by the autoscaler for the controlled - pods, based only on actual resource usage, not taking - into account the ContainerResourcePolicy. May differ - from the Recommendation if the actual resource usage - causes the target to violate the ContainerResourcePolicy - (lower than MinAllowed or higher that MaxAllowed). - Used only as status indication, will not affect - actual resource assignment. - type: object - upperBound: - additionalProperties: - type: string - description: Maximum recommended amount of resources. - Observes ContainerResourcePolicy. Any resources - allocated beyond this value are likely wasted. This - value may be larger than the maximum amount of application - is actually capable of consuming. - type: object - required: - - target - type: object - type: array - type: object - type: object - type: object - overrideScaleUpStabilization: - description: Override scale up stabilization window - type: boolean - replicas: - description: Replicas is the number of replicas of the target resource. - format: int32 - type: integer - targetSelector: - description: TargetSelector is the string form of the label selector - of HPA. This is required for HPA to work with scale subresource. - type: string - vpaScaleDownUpdatePolicy: - description: Current VPA UpdatePolicy set in the spec - properties: - updateMode: - description: Controls when autoscaler applies changes to the resources. - The default is 'Auto'. - type: string - type: object - vpaScaleUpUpdatePolicy: - description: Current VPA UpdatePolicy set in the spec - properties: - updateMode: - description: Controls when autoscaler applies changes to the resources. - The default is 'Auto'. - type: string - type: object - vpaWeight: - description: VpaWeight - weight to provide to VPA scaling - format: int32 - type: integer - type: object - type: object - versions: - - name: v1alpha1 - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] diff --git a/pkg/virtualgarden/resources/vpa.yaml b/pkg/virtualgarden/resources/vpa.yaml deleted file mode 100644 index b010f747..00000000 --- a/pkg/virtualgarden/resources/vpa.yaml +++ /dev/null @@ -1,527 +0,0 @@ -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: verticalpodautoscalers.autoscaling.k8s.io -spec: - conversion: - strategy: None - group: autoscaling.k8s.io - names: - kind: VerticalPodAutoscaler - listKind: VerticalPodAutoscalerList - plural: verticalpodautoscalers - shortNames: - - vpa - singular: verticalpodautoscaler - scope: Namespaced - versions: - - additionalPrinterColumns: - - jsonPath: .spec.updatePolicy.updateMode - name: Mode - type: string - - jsonPath: .status.recommendation.containerRecommendations[0].target.cpu - name: CPU - type: string - - jsonPath: .status.recommendation.containerRecommendations[0].target.memory - name: Mem - type: string - - jsonPath: .status.conditions[?(@.type=='RecommendationProvided')].status - name: Provided - type: string - - jsonPath: .metadata.creationTimestamp - name: Age - type: date - name: v1 - schema: - openAPIV3Schema: - description: VerticalPodAutoscaler is the configuration for a vertical pod - autoscaler, which automatically manages pod resources based on historical - and real time resource utilization. - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: 'Specification of the behavior of the autoscaler. More info: - https://git.k8s.io/community/contributors/devel/api-conventions.md#spec-and-status.' - properties: - resourcePolicy: - description: Controls how the autoscaler computes recommended resources. - The resource policy may be used to set constraints on the recommendations - for individual containers. If not specified, the autoscaler computes - recommended resources for all containers in the pod, without additional - constraints. - properties: - containerPolicies: - description: Per-container resource policies. - items: - description: ContainerResourcePolicy controls how autoscaler - computes the recommended resources for a specific container. - properties: - containerName: - description: Name of the container or DefaultContainerResourcePolicy, - in which case the policy is used by the containers that - don't have their own policy specified. - type: string - controlledResources: - description: Specifies the type of recommendations that - will be computed (and possibly applied) by VPA. If not - specified, the default of [ResourceCPU, ResourceMemory] - will be used. - items: - description: ResourceName is the name identifying various - resources in a ResourceList. - type: string - type: array - controlledValues: - description: Specifies which resource values should be controlled. - The default is "RequestsAndLimits". - enum: - - RequestsAndLimits - - RequestsOnly - type: string - maxAllowed: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: Specifies the maximum amount of resources that - will be recommended for the container. The default is - no maximum. - type: object - minAllowed: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: Specifies the minimal amount of resources that - will be recommended for the container. The default is - no minimum. - type: object - mode: - description: Whether autoscaler is enabled for the container. - The default is "Auto". - enum: - - Auto - - "Off" - type: string - type: object - type: array - type: object - targetRef: - description: TargetRef points to the controller managing the set of - pods for the autoscaler to control - e.g. Deployment, StatefulSet. - VerticalPodAutoscaler can be targeted at controller implementing - scale subresource (the pod set is retrieved from the controller's - ScaleStatus) or some well known controllers (e.g. for DaemonSet - the pod set is read from the controller's spec). If VerticalPodAutoscaler - cannot use specified target it will report ConfigUnsupported condition. - Note that VerticalPodAutoscaler does not require full implementation - of scale subresource - it will not use it to modify the replica - count. The only thing retrieved is a label selector matching pods - grouped by the target resource. - properties: - apiVersion: - description: API version of the referent - type: string - kind: - description: 'Kind of the referent; More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds"' - type: string - name: - description: 'Name of the referent; More info: http://kubernetes.io/docs/user-guide/identifiers#names' - type: string - required: - - kind - - name - type: object - updatePolicy: - description: Describes the rules on how changes are applied to the - pods. If not specified, all fields in the `PodUpdatePolicy` are - set to their default values. - properties: - updateMode: - description: Controls when autoscaler applies changes to the pod - resources. The default is 'Auto'. - enum: - - "Off" - - Initial - - Recreate - - Auto - type: string - type: object - required: - - targetRef - type: object - status: - description: Current information about the autoscaler. - nullable: true - properties: - conditions: - description: Conditions is the set of conditions required for this - autoscaler to scale its target, and indicates whether or not those - conditions are met. - items: - description: VerticalPodAutoscalerCondition describes the state - of a VerticalPodAutoscaler at a certain point. - properties: - lastTransitionTime: - description: lastTransitionTime is the last time the condition - transitioned from one status to another - format: date-time - type: string - message: - description: message is a human-readable explanation containing - details about the transition - type: string - reason: - description: reason is the reason for the condition's last transition. - type: string - status: - description: status is the status of the condition (True, False, - Unknown) - type: string - type: - description: type describes the current condition - type: string - required: - - status - - type - type: object - type: array - recommendation: - description: The most recently computed amount of resources recommended - by the autoscaler for the controlled pods. - properties: - containerRecommendations: - description: Resources recommended by the autoscaler for each - container. - items: - description: RecommendedContainerResources is the recommendation - of resources computed by autoscaler for a specific container. - Respects the container resource policy if present in the spec. - In particular the recommendation is not produced for containers - with `ContainerScalingMode` set to 'Off'. - properties: - containerName: - description: Name of the container. - type: string - lowerBound: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: Minimum recommended amount of resources. Observes - ContainerResourcePolicy. This amount is not guaranteed - to be sufficient for the application to operate in a stable - way, however running with less resources is likely to - have significant impact on performance/availability. - type: object - target: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: Recommended amount of resources. Observes ContainerResourcePolicy. - type: object - uncappedTarget: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: The most recent recommended resources target - computed by the autoscaler for the controlled pods, based - only on actual resource usage, not taking into account - the ContainerResourcePolicy. May differ from the Recommendation - if the actual resource usage causes the target to violate - the ContainerResourcePolicy (lower than MinAllowed or - higher that MaxAllowed). Used only as status indication, - will not affect actual resource assignment. - type: object - upperBound: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: Maximum recommended amount of resources. Observes - ContainerResourcePolicy. Any resources allocated beyond - this value are likely wasted. This value may be larger - than the maximum amount of application is actually capable - of consuming. - type: object - required: - - target - type: object - type: array - type: object - type: object - required: - - spec - type: object - served: true - storage: true - - name: v1beta2 - schema: - openAPIV3Schema: - description: VerticalPodAutoscaler is the configuration for a vertical pod - autoscaler, which automatically manages pod resources based on historical - and real time resource utilization. - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: 'Specification of the behavior of the autoscaler. More info: - https://git.k8s.io/community/contributors/devel/api-conventions.md#spec-and-status.' - properties: - resourcePolicy: - description: Controls how the autoscaler computes recommended resources. - The resource policy may be used to set constraints on the recommendations - for individual containers. If not specified, the autoscaler computes - recommended resources for all containers in the pod, without additional - constraints. - properties: - containerPolicies: - description: Per-container resource policies. - items: - description: ContainerResourcePolicy controls how autoscaler - computes the recommended resources for a specific container. - properties: - containerName: - description: Name of the container or DefaultContainerResourcePolicy, - in which case the policy is used by the containers that - don't have their own policy specified. - type: string - controlledResources: - description: Specifies the type of recommendations that - will be computed (and possibly applied) by VPA. If not - specified, the default of [ResourceCPU, ResourceMemory] - will be used. - items: - description: ResourceName is the name identifying various - resources in a ResourceList. - enum: - - cpu - - memory - type: string - type: array - maxAllowed: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: Specifies the maximum amount of resources that - will be recommended for the container. The default is - no maximum. - type: object - minAllowed: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: Specifies the minimal amount of resources that - will be recommended for the container. The default is - no minimum. - type: object - mode: - description: Whether autoscaler is enabled for the container. - The default is "Auto". - enum: - - Auto - - "Off" - type: string - type: object - type: array - type: object - targetRef: - description: TargetRef points to the controller managing the set of - pods for the autoscaler to control - e.g. Deployment, StatefulSet. - VerticalPodAutoscaler can be targeted at controller implementing - scale subresource (the pod set is retrieved from the controller's - ScaleStatus) or some well known controllers (e.g. for DaemonSet - the pod set is read from the controller's spec). If VerticalPodAutoscaler - cannot use specified target it will report ConfigUnsupported condition. - Note that VerticalPodAutoscaler does not require full implementation - of scale subresource - it will not use it to modify the replica - count. The only thing retrieved is a label selector matching pods - grouped by the target resource. - properties: - apiVersion: - description: API version of the referent - type: string - kind: - description: 'Kind of the referent; More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds"' - type: string - name: - description: 'Name of the referent; More info: http://kubernetes.io/docs/user-guide/identifiers#names' - type: string - required: - - kind - - name - type: object - updatePolicy: - description: Describes the rules on how changes are applied to the - pods. If not specified, all fields in the `PodUpdatePolicy` are - set to their default values. - properties: - updateMode: - description: Controls when autoscaler applies changes to the pod - resources. The default is 'Auto'. - enum: - - "Off" - - Initial - - Recreate - - Auto - type: string - type: object - required: - - targetRef - type: object - status: - description: Current information about the autoscaler. - nullable: true - properties: - conditions: - description: Conditions is the set of conditions required for this - autoscaler to scale its target, and indicates whether or not those - conditions are met. - items: - description: VerticalPodAutoscalerCondition describes the state - of a VerticalPodAutoscaler at a certain point. - properties: - lastTransitionTime: - description: lastTransitionTime is the last time the condition - transitioned from one status to another - format: date-time - type: string - message: - description: message is a human-readable explanation containing - details about the transition - type: string - reason: - description: reason is the reason for the condition's last transition. - type: string - status: - description: status is the status of the condition (True, False, - Unknown) - type: string - type: - description: type describes the current condition - type: string - required: - - status - - type - type: object - type: array - recommendation: - description: The most recently computed amount of resources recommended - by the autoscaler for the controlled pods. - properties: - containerRecommendations: - description: Resources recommended by the autoscaler for each - container. - items: - description: RecommendedContainerResources is the recommendation - of resources computed by autoscaler for a specific container. - Respects the container resource policy if present in the spec. - In particular the recommendation is not produced for containers - with `ContainerScalingMode` set to 'Off'. - properties: - containerName: - description: Name of the container. - type: string - lowerBound: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: Minimum recommended amount of resources. Observes - ContainerResourcePolicy. This amount is not guaranteed - to be sufficient for the application to operate in a stable - way, however running with less resources is likely to - have significant impact on performance/availability. - type: object - target: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: Recommended amount of resources. Observes ContainerResourcePolicy. - type: object - uncappedTarget: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: The most recent recommended resources target - computed by the autoscaler for the controlled pods, based - only on actual resource usage, not taking into account - the ContainerResourcePolicy. May differ from the Recommendation - if the actual resource usage causes the target to violate - the ContainerResourcePolicy (lower than MinAllowed or - higher that MaxAllowed). Used only as status indication, - will not affect actual resource assignment. - type: object - upperBound: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: Maximum recommended amount of resources. Observes - ContainerResourcePolicy. Any resources allocated beyond - this value are likely wasted. This value may be larger - than the maximum amount of application is actually capable - of consuming. - type: object - required: - - target - type: object - type: array - type: object - type: object - required: - - spec - type: object - served: true - storage: false diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index bf883b26..095defbf 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -40,7 +40,6 @@ import ( "github.com/gardener/virtual-garden/pkg/virtualgarden" secretsutil "github.com/gardener/gardener/pkg/utils/secrets" - hvpav1alpha1 "github.com/gardener/hvpa-controller/api/v1alpha1" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" . "github.com/onsi/gomega/gstruct" @@ -92,7 +91,6 @@ var _ = Describe("VirtualGarden E2E tests", func() { c, err = app.NewClientFromTarget(imports.RuntimeCluster) Expect(err).NotTo(HaveOccurred()) - err = virtualgarden.DeployHVPACRD(ctx, c) Expect(err).NotTo(HaveOccurred()) }) @@ -101,9 +99,6 @@ var _ = Describe("VirtualGarden E2E tests", func() { Expect(os.Setenv("OPERATION", "DELETE")).To(Succeed()) Expect(app.NewCommandVirtualGarden().ExecuteContext(ctx)).To(Succeed()) - err := virtualgarden.DeleteHPVACRD(ctx, c) - Expect(err).NotTo(HaveOccurred()) - verifyDeletion(ctx, c, imports) }) @@ -217,8 +212,6 @@ func verifyReconciliation(ctx context.Context, c client.Client, imports *api.Imp verifyReconciliationOfETCDServerCertificate(ctx, c, imports, role, etcdCACertificate) verifyReconciliationOfETCDStatefulSet(ctx, c, imports, role, backupProvider) - - verifyReconciliationOfETCDHVPA(ctx, c, imports, role) } verifyReconciliationOfKubeAPIServerCertificates(ctx, c, imports) @@ -464,15 +457,6 @@ func verifyReconciliationOfETCDStatefulSet(ctx context.Context, c client.Client, }, timeoutCtx.Done())).To(Succeed()) } -func verifyReconciliationOfETCDHVPA(ctx context.Context, c client.Client, imports *api.Imports, role string) { - if helper.ETCDHVPAEnabled(imports.VirtualGarden.ETCD) { - By("Checking that the etcd HVPA was created as expected (" + role + ")") - etcdHVPA := &hvpav1alpha1.Hvpa{} - err := c.Get(ctx, client.ObjectKey{Name: virtualgarden.ETCDHVPAName(role), Namespace: imports.RuntimeClusterSettings.Namespace}, etcdHVPA) - Expect(err).To(Succeed()) - } -} - func verifyReconciliationOfKubeAPIServerCertificates(ctx context.Context, c client.Client, imports *api.Imports) { By("Checking that the certificates for the kube-apiserver were created as expected") @@ -581,8 +565,6 @@ func verifyDeletion(ctx context.Context, c client.Client, imports *api.Imports) verifyDeletionOfKubeAPIServerService(ctx, c, imports) for _, role := range []string{virtualgarden.ETCDRoleMain, virtualgarden.ETCDRoleEvents} { - verifyDeletionOfETCDHVPA(ctx, c, imports, role) - verifyDeletionOfETCDStatefulSet(ctx, c, imports, role) By("Checking that the etcd bootstrap configmap was deleted successfully (" + role + ")") @@ -623,14 +605,6 @@ func verifyDeletionOfKubeAPIServerService(ctx context.Context, c client.Client, }, timeoutCtx.Done())).To(Succeed()) } -func verifyDeletionOfETCDHVPA(ctx context.Context, c client.Client, imports *api.Imports, role string) { - if helper.ETCDHVPAEnabled(imports.VirtualGarden.ETCD) { - By("Checking that the etcd HVPA was deleted successfully (" + role + ")") - err := c.Get(ctx, client.ObjectKey{Name: virtualgarden.ETCDHVPAName(role), Namespace: imports.RuntimeClusterSettings.Namespace}, &hvpav1alpha1.Hvpa{}) - Expect(apierrors.IsNotFound(err)).To(BeTrue()) - } -} - func verifyDeletionOfETCDStatefulSet(ctx context.Context, c client.Client, imports *api.Imports, role string) { By("Checking that the etcd statefulset was deleted successfully (" + role + ")") diff --git a/test/utils/localenvtest/environment.go b/test/utils/localenvtest/environment.go index c9db7988..4e6289c8 100644 --- a/test/utils/localenvtest/environment.go +++ b/test/utils/localenvtest/environment.go @@ -11,15 +11,12 @@ import ( appsv1 "k8s.io/api/apps/v1" policyv1 "k8s.io/api/policy/v1beta1" - hvpav1alpha1 "github.com/gardener/hvpa-controller/api/v1alpha1" + "github.com/sirupsen/logrus" corev1 "k8s.io/api/core/v1" "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions" - apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" "k8s.io/apimachinery/pkg/runtime" utilruntime "k8s.io/apimachinery/pkg/util/runtime" - autoscalingv1beta2 "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1beta2" - - "github.com/sirupsen/logrus" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/envtest" ) @@ -76,10 +73,8 @@ func (e *Environment) Start() (client.Client, error) { scheme := runtime.NewScheme() utilruntime.Must(corev1.AddToScheme(scheme)) utilruntime.Must(appsv1.AddToScheme(scheme)) - utilruntime.Must(hvpav1alpha1.AddToScheme(scheme)) - utilruntime.Must(autoscalingv1beta2.AddToScheme(scheme)) utilruntime.Must(apiextensions.AddToScheme(scheme)) - utilruntime.Must(apiextensionsv1beta1.AddToScheme(scheme)) + utilruntime.Must(apiextensionsv1.AddToScheme(scheme)) utilruntime.Must(policyv1.AddToScheme(scheme)) fakeClient, err := client.New(restConfig, client.Options{Scheme: scheme}) diff --git a/vendor/modules.txt b/vendor/modules.txt index 20ac667b..c3b184bb 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -182,7 +182,6 @@ github.com/gardener/gardener/pkg/utils/version github.com/gardener/gardener-resource-manager/pkg/apis/resources github.com/gardener/gardener-resource-manager/pkg/apis/resources/v1alpha1 # github.com/gardener/hvpa-controller v0.3.1 -## explicit github.com/gardener/hvpa-controller/api/v1alpha1 # github.com/gardener/landscaper/apis v0.20.0 => github.com/gardener/landscaper/apis v0.20.0 ## explicit @@ -801,7 +800,6 @@ k8s.io/apiserver/pkg/apis/config k8s.io/apiserver/pkg/apis/config/v1 k8s.io/apiserver/pkg/authentication/user # k8s.io/autoscaler v0.0.0-20190805135949-100e91ba756e -## explicit k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1beta2 # k8s.io/client-go v11.0.1-0.20190409021438-1a26190bd76a+incompatible => k8s.io/client-go v0.21.2 ## explicit