FlowLogs contains the flow log configuration for the subnet.
+privateServiceConnect
+
+
+PrivateServiceConnect
+
+
+PrivateServiceConnect
++(Appears on: +NetworkConfig) +
++
+| Field | +Description | +
|---|---|
+enabled
+
+bool
+
+ |
++ | +
+endpointIP
+
+string
+
+ |
++ | +
diff --git a/pkg/admission/validator/shoot.go b/pkg/admission/validator/shoot.go index 68cc5f82f..97a31b124 100644 --- a/pkg/admission/validator/shoot.go +++ b/pkg/admission/validator/shoot.go @@ -19,6 +19,9 @@ import ( "fmt" "reflect" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/utils/pointer" + "github.com/gardener/gardener-extension-provider-gcp/pkg/admission" apisgcp "github.com/gardener/gardener-extension-provider-gcp/pkg/apis/gcp" gcpvalidation "github.com/gardener/gardener-extension-provider-gcp/pkg/apis/gcp/validation" @@ -89,6 +92,7 @@ var ( type validationContext struct { shoot *core.Shoot + seed *gardencorev1beta1.Seed infrastructureConfig *apisgcp.InfrastructureConfig controlPlaneConfig *apisgcp.ControlPlaneConfig cloudProfile *gardencorev1beta1.CloudProfile @@ -120,12 +124,13 @@ func getAllowedRegionZonesFromCloudProfile(shoot *core.Shoot, cloudProfile *gard func (s *shoot) validateContext(valContext *validationContext) field.ErrorList { var ( - allErrors = field.ErrorList{} - allowedZones = getAllowedRegionZonesFromCloudProfile(valContext.shoot, valContext.cloudProfile) + allErrors = field.ErrorList{} + allowedZones = getAllowedRegionZonesFromCloudProfile(valContext.shoot, valContext.cloudProfile) + seedServiceCIDR *string ) allErrors = append(allErrors, gcpvalidation.ValidateNetworking(valContext.shoot.Spec.Networking, networkPath)...) - allErrors = append(allErrors, gcpvalidation.ValidateInfrastructureConfig(valContext.infrastructureConfig, valContext.shoot.Spec.Networking.Nodes, valContext.shoot.Spec.Networking.Pods, valContext.shoot.Spec.Networking.Services, infrastructureConfigPath)...) + allErrors = append(allErrors, gcpvalidation.ValidateInfrastructureConfig(valContext.infrastructureConfig, valContext.shoot.Spec.Networking.Nodes, valContext.shoot.Spec.Networking.Pods, valContext.shoot.Spec.Networking.Services, seedServiceCIDR, infrastructureConfigPath)...) allErrors = append(allErrors, gcpvalidation.ValidateWorkers(valContext.shoot.Spec.Provider.Workers, workersPath)...) allErrors = append(allErrors, gcpvalidation.ValidateControlPlaneConfig(valContext.controlPlaneConfig, allowedZones, workersZones(valContext.shoot.Spec.Provider.Workers), valContext.shoot.Spec.Kubernetes.Version, controlPlaneConfigPath)...) @@ -169,9 +174,11 @@ func (s *shoot) validateUpdate(ctx context.Context, oldShoot, currentShoot *core allErrors = field.ErrorList{} ) - if !reflect.DeepEqual(oldInfrastructureConfig, currentInfrastructureConfig) { - allErrors = append(allErrors, gcpvalidation.ValidateInfrastructureConfigUpdate(oldInfrastructureConfig, currentInfrastructureConfig, infrastructureConfigPath)...) + var seedServices *string + if currentValContext.seed != nil { + seedServices = pointer.String(currentValContext.seed.Spec.Networks.Services) } + allErrors = append(allErrors, gcpvalidation.ValidateInfrastructureConfigUpdate(oldInfrastructureConfig, currentInfrastructureConfig, seedServices, infrastructureConfigPath)...) if !reflect.DeepEqual(oldControlPlaneConfig, currentControlPlaneConfig) { allErrors = append(allErrors, gcpvalidation.ValidateControlPlaneConfigUpdate(oldControlPlaneConfig, currentControlPlaneConfig, controlPlaneConfigPath)...) @@ -209,13 +216,24 @@ func newValidationContext(ctx context.Context, decoder runtime.Decoder, c client if cloudProfile.Spec.ProviderConfig == nil { return nil, fmt.Errorf("providerConfig is not given for cloud profile %q", cloudProfile.Name) } + cloudProfileConfig, err := admission.DecodeCloudProfileConfig(decoder, cloudProfile.Spec.ProviderConfig) if err != nil { return nil, fmt.Errorf("an error occurred while reading the cloud profile %q: %v", cloudProfile.Name, err) } + var seed *gardencorev1beta1.Seed + if shoot.Spec.SeedName != nil && len(*shoot.Spec.SeedName) > 0 { + seed = &gardencorev1beta1.Seed{} + err = c.Get(ctx, kutil.Key(*shoot.Spec.SeedName), seed) + if err != nil && !errors.IsNotFound(err) { + return nil, fmt.Errorf("an error occured while reading seed information: %v", err) + } + } + return &validationContext{ shoot: shoot, + seed: seed, infrastructureConfig: infrastructureConfig, controlPlaneConfig: controlPlaneConfig, cloudProfile: cloudProfile, diff --git a/pkg/apis/gcp/types_infrastructure.go b/pkg/apis/gcp/types_infrastructure.go index 8768ec0c4..bd235c1f4 100644 --- a/pkg/apis/gcp/types_infrastructure.go +++ b/pkg/apis/gcp/types_infrastructure.go @@ -44,6 +44,14 @@ type NetworkConfig struct { Workers string // FlowLogs contains the flow log configuration for the subnet. FlowLogs *FlowLogs + // PrivateServiceConnect configures access to Google services via Private Service Connect. + PrivateServiceConnect *PrivateServiceConnect +} + +// PrivateServiceConnect holds the configuration for Private Service Connect endpoints. +type PrivateServiceConnect struct { + Enabled bool + EndpointIP string } // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object diff --git a/pkg/apis/gcp/v1alpha1/types_infrastructure.go b/pkg/apis/gcp/v1alpha1/types_infrastructure.go index 220cfe0f1..de2b22449 100644 --- a/pkg/apis/gcp/v1alpha1/types_infrastructure.go +++ b/pkg/apis/gcp/v1alpha1/types_infrastructure.go @@ -48,6 +48,15 @@ type NetworkConfig struct { // FlowLogs contains the flow log configuration for the subnet. // +optional FlowLogs *FlowLogs `json:"flowLogs,omitempty"` + // PrivateServiceConnect configures access to Google services via Private Service Connect. + // +optional + PrivateServiceConnect *PrivateServiceConnect `json:"privateServiceConnect,omitempty"` +} + +// PrivateServiceConnect holds the configuration for Private Service Connect endpoints. +type PrivateServiceConnect struct { + Enabled bool `json:"enabled"` + EndpointIP string `json:"endpointIP"` } // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object diff --git a/pkg/apis/gcp/v1alpha1/zz_generated.conversion.go b/pkg/apis/gcp/v1alpha1/zz_generated.conversion.go index 205e9e7b9..0f43ee3f0 100644 --- a/pkg/apis/gcp/v1alpha1/zz_generated.conversion.go +++ b/pkg/apis/gcp/v1alpha1/zz_generated.conversion.go @@ -196,6 +196,16 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } + if err := s.AddGeneratedConversionFunc((*PrivateServiceConnect)(nil), (*gcp.PrivateServiceConnect)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1alpha1_PrivateServiceConnect_To_gcp_PrivateServiceConnect(a.(*PrivateServiceConnect), b.(*gcp.PrivateServiceConnect), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*gcp.PrivateServiceConnect)(nil), (*PrivateServiceConnect)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_gcp_PrivateServiceConnect_To_v1alpha1_PrivateServiceConnect(a.(*gcp.PrivateServiceConnect), b.(*PrivateServiceConnect), scope) + }); err != nil { + return err + } if err := s.AddGeneratedConversionFunc((*ServiceAccount)(nil), (*gcp.ServiceAccount)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1alpha1_ServiceAccount_To_gcp_ServiceAccount(a.(*ServiceAccount), b.(*gcp.ServiceAccount), scope) }); err != nil { @@ -574,6 +584,7 @@ func autoConvert_v1alpha1_NetworkConfig_To_gcp_NetworkConfig(in *NetworkConfig, out.Worker = in.Worker out.Workers = in.Workers out.FlowLogs = (*gcp.FlowLogs)(unsafe.Pointer(in.FlowLogs)) + out.PrivateServiceConnect = (*gcp.PrivateServiceConnect)(unsafe.Pointer(in.PrivateServiceConnect)) return nil } @@ -589,6 +600,7 @@ func autoConvert_gcp_NetworkConfig_To_v1alpha1_NetworkConfig(in *gcp.NetworkConf out.Worker = in.Worker out.Workers = in.Workers out.FlowLogs = (*FlowLogs)(unsafe.Pointer(in.FlowLogs)) + out.PrivateServiceConnect = (*PrivateServiceConnect)(unsafe.Pointer(in.PrivateServiceConnect)) return nil } @@ -625,6 +637,28 @@ func Convert_gcp_NetworkStatus_To_v1alpha1_NetworkStatus(in *gcp.NetworkStatus, return autoConvert_gcp_NetworkStatus_To_v1alpha1_NetworkStatus(in, out, s) } +func autoConvert_v1alpha1_PrivateServiceConnect_To_gcp_PrivateServiceConnect(in *PrivateServiceConnect, out *gcp.PrivateServiceConnect, s conversion.Scope) error { + out.Enabled = in.Enabled + out.EndpointIP = in.EndpointIP + return nil +} + +// Convert_v1alpha1_PrivateServiceConnect_To_gcp_PrivateServiceConnect is an autogenerated conversion function. +func Convert_v1alpha1_PrivateServiceConnect_To_gcp_PrivateServiceConnect(in *PrivateServiceConnect, out *gcp.PrivateServiceConnect, s conversion.Scope) error { + return autoConvert_v1alpha1_PrivateServiceConnect_To_gcp_PrivateServiceConnect(in, out, s) +} + +func autoConvert_gcp_PrivateServiceConnect_To_v1alpha1_PrivateServiceConnect(in *gcp.PrivateServiceConnect, out *PrivateServiceConnect, s conversion.Scope) error { + out.Enabled = in.Enabled + out.EndpointIP = in.EndpointIP + return nil +} + +// Convert_gcp_PrivateServiceConnect_To_v1alpha1_PrivateServiceConnect is an autogenerated conversion function. +func Convert_gcp_PrivateServiceConnect_To_v1alpha1_PrivateServiceConnect(in *gcp.PrivateServiceConnect, out *PrivateServiceConnect, s conversion.Scope) error { + return autoConvert_gcp_PrivateServiceConnect_To_v1alpha1_PrivateServiceConnect(in, out, s) +} + func autoConvert_v1alpha1_ServiceAccount_To_gcp_ServiceAccount(in *ServiceAccount, out *gcp.ServiceAccount, s conversion.Scope) error { out.Email = in.Email out.Scopes = *(*[]string)(unsafe.Pointer(&in.Scopes)) diff --git a/pkg/apis/gcp/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/gcp/v1alpha1/zz_generated.deepcopy.go index baf9e4c91..ca7fd773d 100644 --- a/pkg/apis/gcp/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/gcp/v1alpha1/zz_generated.deepcopy.go @@ -359,6 +359,11 @@ func (in *NetworkConfig) DeepCopyInto(out *NetworkConfig) { *out = new(FlowLogs) (*in).DeepCopyInto(*out) } + if in.PrivateServiceConnect != nil { + in, out := &in.PrivateServiceConnect, &out.PrivateServiceConnect + *out = new(PrivateServiceConnect) + **out = **in + } return } @@ -399,6 +404,22 @@ func (in *NetworkStatus) DeepCopy() *NetworkStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PrivateServiceConnect) DeepCopyInto(out *PrivateServiceConnect) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PrivateServiceConnect. +func (in *PrivateServiceConnect) DeepCopy() *PrivateServiceConnect { + if in == nil { + return nil + } + out := new(PrivateServiceConnect) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ServiceAccount) DeepCopyInto(out *ServiceAccount) { *out = *in diff --git a/pkg/apis/gcp/validation/infrastructure.go b/pkg/apis/gcp/validation/infrastructure.go index 516cafbe6..a1cace8d4 100644 --- a/pkg/apis/gcp/validation/infrastructure.go +++ b/pkg/apis/gcp/validation/infrastructure.go @@ -15,17 +15,19 @@ package validation import ( + "fmt" "reflect" apivalidation "k8s.io/apimachinery/pkg/api/validation" - apisgcp "github.com/gardener/gardener-extension-provider-gcp/pkg/apis/gcp" cidrvalidation "github.com/gardener/gardener/pkg/utils/validation/cidr" "k8s.io/apimachinery/pkg/util/validation/field" + + apisgcp "github.com/gardener/gardener-extension-provider-gcp/pkg/apis/gcp" ) // ValidateInfrastructureConfig validates a InfrastructureConfig object. -func ValidateInfrastructureConfig(infra *apisgcp.InfrastructureConfig, nodesCIDR, podsCIDR, servicesCIDR *string, fldPath *field.Path) field.ErrorList { +func ValidateInfrastructureConfig(infra *apisgcp.InfrastructureConfig, nodesCIDR, podsCIDR, servicesCIDR, seedServiceCIDR *string, fldPath *field.Path) field.ErrorList { allErrs := field.ErrorList{} var ( @@ -51,7 +53,7 @@ func ValidateInfrastructureConfig(infra *apisgcp.InfrastructureConfig, nodesCIDR allErrs = append(allErrs, field.Required(networksPath.Child("workers"), "must specify the network range for the worker network")) } - var workerCIDR cidrvalidation.CIDR + var workerCIDR, internalCIDR cidrvalidation.CIDR if infra.Networks.Worker != "" { workerCIDR = cidrvalidation.NewCIDR(infra.Networks.Worker, networksPath.Child("worker")) allErrs = append(allErrs, cidrvalidation.ValidateCIDRParse(workerCIDR)...) @@ -64,7 +66,7 @@ func ValidateInfrastructureConfig(infra *apisgcp.InfrastructureConfig, nodesCIDR } if infra.Networks.Internal != nil { - internalCIDR := cidrvalidation.NewCIDR(*infra.Networks.Internal, networksPath.Child("internal")) + internalCIDR = cidrvalidation.NewCIDR(*infra.Networks.Internal, networksPath.Child("internal")) allErrs = append(allErrs, cidrvalidation.ValidateCIDRParse(internalCIDR)...) allErrs = append(allErrs, cidrvalidation.ValidateCIDRIsCanonical(networksPath.Child("internal"), *infra.Networks.Internal)...) if pods != nil { @@ -124,11 +126,25 @@ func ValidateInfrastructureConfig(infra *apisgcp.InfrastructureConfig, nodesCIDR } } + if infra.Networks.PrivateServiceConnect != nil && infra.Networks.PrivateServiceConnect.Enabled { + ranges := []cidrvalidation.CIDR{ + pods, services, nodes, workerCIDR, + } + if infra.Networks.Internal != nil { + ranges = append(ranges, internalCIDR) + } + if seedServiceCIDR != nil { + ranges = append(ranges, cidrvalidation.NewCIDR(*seedServiceCIDR, field.NewPath("seed service range"))) + } + + allErrs = append(allErrs, ValidatePrivateServiceConnect(networksPath.Child("privateServiceConnect"), infra, ranges)...) + } + return allErrs } // ValidateInfrastructureConfigUpdate validates a InfrastructureConfig object. -func ValidateInfrastructureConfigUpdate(oldConfig, newConfig *apisgcp.InfrastructureConfig, fldPath *field.Path) field.ErrorList { +func ValidateInfrastructureConfigUpdate(oldConfig, newConfig *apisgcp.InfrastructureConfig, seedServices *string, fldPath *field.Path) field.ErrorList { var ( allErrs = field.ErrorList{} networksPath = fldPath.Child("networks") @@ -150,6 +166,37 @@ func ValidateInfrastructureConfigUpdate(oldConfig, newConfig *apisgcp.Infrastruc allErrs = append(allErrs, apivalidation.ValidateImmutableField(newConfig.Networks.Worker, oldConfig.Networks.Worker, networksPath.Child("worker"))...) } + if newConfig.Networks.PrivateServiceConnect != nil && newConfig.Networks.PrivateServiceConnect.Enabled { + var ranges []cidrvalidation.CIDR + if seedServices != nil { + ranges = append(ranges, cidrvalidation.NewCIDR(*seedServices, field.NewPath("seed service range"))) + } + + allErrs = append(allErrs, ValidatePrivateServiceConnect(networksPath.Child("privateServiceConnect"), newConfig, ranges)...) + } + return allErrs +} + +// ValidatePrivateServiceConnect validates the Private Service Connect configuration. +// The endpointCIDR must not overlap with the following ranges: +// - Shoot's functional CIDRs (pod, service, node) +// - Seed's service CIDRs +// - The VPC CIDR (the sub of the subnet CIDRs used by workers) +// - The static ranges used by reverse VPN (192.168.123.0/24) +func ValidatePrivateServiceConnect(fldPath *field.Path, infra *apisgcp.InfrastructureConfig, ranges []cidrvalidation.CIDR) field.ErrorList { + allErrs := field.ErrorList{} + + if infra.Networks.PrivateServiceConnect != nil && infra.Networks.PrivateServiceConnect.Enabled { + endpointCIDR := cidrvalidation.NewCIDR(fmt.Sprintf("%s/32", infra.Networks.PrivateServiceConnect.EndpointIP), fldPath) + allErrs = append(allErrs, cidrvalidation.ValidateCIDRParse(endpointCIDR)...) + + var rangesToVerify = []cidrvalidation.CIDR{ + cidrvalidation.NewCIDR("192.168.123.0/24", nil), + } + rangesToVerify = append(rangesToVerify, ranges...) + + allErrs = append(allErrs, endpointCIDR.ValidateNotOverlap(rangesToVerify...)...) + } return allErrs } diff --git a/pkg/apis/gcp/validation/infrastructure_test.go b/pkg/apis/gcp/validation/infrastructure_test.go index 466093d34..60b50ede0 100644 --- a/pkg/apis/gcp/validation/infrastructure_test.go +++ b/pkg/apis/gcp/validation/infrastructure_test.go @@ -15,9 +15,10 @@ package validation_test import ( + "k8s.io/utils/pointer" + apisgcp "github.com/gardener/gardener-extension-provider-gcp/pkg/apis/gcp" . "github.com/gardener/gardener-extension-provider-gcp/pkg/apis/gcp/validation" - "k8s.io/utils/pointer" . "github.com/gardener/gardener/pkg/utils/test/matchers" . "github.com/onsi/ginkgo/v2" @@ -71,7 +72,7 @@ var _ = Describe("InfrastructureConfig validation", func() { It("should forbid invalid worker CIDRs", func() { infrastructureConfig.Networks.Workers = invalidCIDR - errorList := ValidateInfrastructureConfig(infrastructureConfig, &nodes, &pods, &services, fldPath) + errorList := ValidateInfrastructureConfig(infrastructureConfig, &nodes, &pods, &services, nil, nil, fldPath) Expect(errorList).To(ConsistOfFields(Fields{ "Type": Equal(field.ErrorTypeInvalid), @@ -84,7 +85,7 @@ var _ = Describe("InfrastructureConfig validation", func() { invalidCIDR = "invalid-cidr" infrastructureConfig.Networks.Internal = &invalidCIDR - errorList := ValidateInfrastructureConfig(infrastructureConfig, &nodes, &pods, &services, fldPath) + errorList := ValidateInfrastructureConfig(infrastructureConfig, &nodes, &pods, &services, nil, nil, fldPath) Expect(errorList).To(ConsistOfFields(Fields{ "Type": Equal(field.ErrorTypeInvalid), @@ -96,7 +97,7 @@ var _ = Describe("InfrastructureConfig validation", func() { It("should forbid workers CIDR which are not in Nodes CIDR", func() { infrastructureConfig.Networks.Workers = "1.1.1.1/32" - errorList := ValidateInfrastructureConfig(infrastructureConfig, &nodes, &pods, &services, fldPath) + errorList := ValidateInfrastructureConfig(infrastructureConfig, &nodes, &pods, &services, nil, nil, fldPath) Expect(errorList).To(ConsistOfFields(Fields{ "Type": Equal(field.ErrorTypeInvalid), @@ -110,7 +111,7 @@ var _ = Describe("InfrastructureConfig validation", func() { infrastructureConfig.Networks.Internal = &overlappingCIDR infrastructureConfig.Networks.Workers = overlappingCIDR - errorList := ValidateInfrastructureConfig(infrastructureConfig, &overlappingCIDR, &pods, &services, fldPath) + errorList := ValidateInfrastructureConfig(infrastructureConfig, &overlappingCIDR, &pods, &services, nil, nil, fldPath) Expect(errorList).To(ConsistOfFields(Fields{ "Type": Equal(field.ErrorTypeInvalid), @@ -131,7 +132,7 @@ var _ = Describe("InfrastructureConfig validation", func() { infrastructureConfig.Networks.Internal = &internal infrastructureConfig.Networks.Workers = "10.250.3.8/24" - errorList := ValidateInfrastructureConfig(infrastructureConfig, &nodeCIDR, &podCIDR, &serviceCIDR, fldPath) + errorList := ValidateInfrastructureConfig(infrastructureConfig, &nodeCIDR, &podCIDR, &serviceCIDR, nil, nil, fldPath) Expect(errorList).To(HaveLen(2)) Expect(errorList).To(ConsistOfFields(Fields{ @@ -146,12 +147,12 @@ var _ = Describe("InfrastructureConfig validation", func() { }) It("should allow specifying valid config", func() { - errorList := ValidateInfrastructureConfig(infrastructureConfig, &nodes, &pods, &services, fldPath) + errorList := ValidateInfrastructureConfig(infrastructureConfig, &nodes, &pods, &services, nil, nil, fldPath) Expect(errorList).To(BeEmpty()) }) It("should allow specifying valid config with podsCIDR=nil and servicesCIDR=nil", func() { - errorList := ValidateInfrastructureConfig(infrastructureConfig, &nodes, nil, nil, fldPath) + errorList := ValidateInfrastructureConfig(infrastructureConfig, &nodes, nil, nil, nil, nil, fldPath) Expect(errorList).To(BeEmpty()) }) }) @@ -166,7 +167,7 @@ var _ = Describe("InfrastructureConfig validation", func() { testInfrastructureConfig.Networks.VPC = &apisgcp.VPC{} testInfrastructureConfig.Networks.VPC.CloudRouter = &apisgcp.CloudRouter{} - errorList := ValidateInfrastructureConfig(testInfrastructureConfig, &nodes, &pods, &services, fldPath) + errorList := ValidateInfrastructureConfig(testInfrastructureConfig, &nodes, &pods, &services, nil, nil, fldPath) Expect(errorList).To(ConsistOfFields(Fields{ "Type": Equal(field.ErrorTypeInvalid), "Field": Equal("networks.vpc.cloudRouter"), @@ -180,7 +181,7 @@ var _ = Describe("InfrastructureConfig validation", func() { It("should forbid empty VPC flow log config", func() { infrastructureConfig.Networks.FlowLogs = &apisgcp.FlowLogs{} - errorList := ValidateInfrastructureConfig(infrastructureConfig, &nodes, &pods, &services, fldPath) + errorList := ValidateInfrastructureConfig(infrastructureConfig, &nodes, &pods, &services, nil, nil, fldPath) Expect(errorList).To(ConsistOfFields(Fields{ "Type": Equal(field.ErrorTypeRequired), "Field": Equal("networks.flowLogs"), @@ -193,7 +194,7 @@ var _ = Describe("InfrastructureConfig validation", func() { metadata := "foo" infrastructureConfig.Networks.FlowLogs = &apisgcp.FlowLogs{AggregationInterval: &aggregationInterval, FlowSampling: &flowSampling, Metadata: &metadata} - errorList := ValidateInfrastructureConfig(infrastructureConfig, &nodes, &pods, &services, fldPath) + errorList := ValidateInfrastructureConfig(infrastructureConfig, &nodes, &pods, &services, nil, nil, fldPath) Expect(errorList).To(ConsistOfFields(Fields{ "Type": Equal(field.ErrorTypeNotSupported), "Field": Equal("networks.flowLogs.aggregationInterval"), @@ -213,7 +214,7 @@ var _ = Describe("InfrastructureConfig validation", func() { Name: "test-vpc", } - errorList := ValidateInfrastructureConfig(testInfrastructureConfig, &nodes, &pods, &services, fldPath) + errorList := ValidateInfrastructureConfig(testInfrastructureConfig, &nodes, &pods, &services, nil, nil, fldPath) Expect(errorList).To(ConsistOfFields(Fields{ "Type": Equal(field.ErrorTypeInvalid), "Field": Equal("networks.vpc.cloudRouter"), @@ -226,7 +227,7 @@ var _ = Describe("InfrastructureConfig validation", func() { CloudRouter: &apisgcp.CloudRouter{}, } - errorList := ValidateInfrastructureConfig(testInfrastructureConfig, &nodes, &pods, &services, fldPath) + errorList := ValidateInfrastructureConfig(testInfrastructureConfig, &nodes, &pods, &services, nil, nil, fldPath) Expect(errorList).To(ConsistOfFields(Fields{ "Type": Equal(field.ErrorTypeInvalid), "Field": Equal("networks.vpc.cloudRouter.name"), @@ -238,7 +239,7 @@ var _ = Describe("InfrastructureConfig validation", func() { Name: "test-vpc", } - errorList := ValidateInfrastructureConfig(testInfrastructureConfig, &nodes, &pods, &services, fldPath) + errorList := ValidateInfrastructureConfig(testInfrastructureConfig, &nodes, &pods, &services, nil, nil, fldPath) Expect(errorList).To(ConsistOfFields(Fields{ "Type": Equal(field.ErrorTypeInvalid), "Field": Equal("networks.vpc.cloudRouter"), @@ -251,7 +252,7 @@ var _ = Describe("InfrastructureConfig validation", func() { CloudRouter: &apisgcp.CloudRouter{}, } - errorList := ValidateInfrastructureConfig(testInfrastructureConfig, &nodes, &pods, &services, fldPath) + errorList := ValidateInfrastructureConfig(testInfrastructureConfig, &nodes, &pods, &services, nil, nil, fldPath) Expect(errorList).To(ConsistOfFields(Fields{ "Type": Equal(field.ErrorTypeInvalid), "Field": Equal("networks.vpc.cloudRouter.name"), @@ -262,7 +263,7 @@ var _ = Describe("InfrastructureConfig validation", func() { It("should forbid empty VPC flow log config", func() { infrastructureConfig.Networks.FlowLogs = &apisgcp.FlowLogs{} - errorList := ValidateInfrastructureConfig(infrastructureConfig, &nodes, &pods, &services, fldPath) + errorList := ValidateInfrastructureConfig(infrastructureConfig, &nodes, &pods, &services, nil, nil, fldPath) Expect(errorList).To(ConsistOfFields(Fields{ "Type": Equal(field.ErrorTypeRequired), "Field": Equal("networks.flowLogs"), @@ -275,7 +276,7 @@ var _ = Describe("InfrastructureConfig validation", func() { metadata := "foo" infrastructureConfig.Networks.FlowLogs = &apisgcp.FlowLogs{AggregationInterval: &aggregationInterval, FlowSampling: &flowSampling, Metadata: &metadata} - errorList := ValidateInfrastructureConfig(infrastructureConfig, &nodes, &pods, &services, fldPath) + errorList := ValidateInfrastructureConfig(infrastructureConfig, &nodes, &pods, &services, nil, nil, fldPath) Expect(errorList).To(ConsistOfFields(Fields{ "Type": Equal(field.ErrorTypeNotSupported), "Field": Equal("networks.flowLogs.aggregationInterval"), @@ -296,7 +297,7 @@ var _ = Describe("InfrastructureConfig validation", func() { metadata := "INCLUDE_ALL_METADATA" infrastructureConfig.Networks.FlowLogs = &apisgcp.FlowLogs{AggregationInterval: &aggregationInterval, FlowSampling: &flowSampling, Metadata: &metadata} - errorList := ValidateInfrastructureConfig(infrastructureConfig, &nodes, &pods, &services, fldPath) + errorList := ValidateInfrastructureConfig(infrastructureConfig, &nodes, &pods, &services, nil, nil, fldPath) Expect(errorList).To(BeEmpty()) }) }) @@ -309,7 +310,7 @@ var _ = Describe("InfrastructureConfig validation", func() { FlowSampling: pointer.Float32Ptr(0.5), } - errorList := ValidateInfrastructureConfig(newInfrastructureConfig, &nodes, &pods, &services, fldPath) + errorList := ValidateInfrastructureConfig(newInfrastructureConfig, &nodes, &pods, &services, nil, nil, fldPath) Expect(errorList).To(BeEmpty()) }) }) @@ -317,7 +318,7 @@ var _ = Describe("InfrastructureConfig validation", func() { Describe("#ValidateInfrastructureConfigUpdate", func() { It("should return no errors for an unchanged config", func() { - Expect(ValidateInfrastructureConfigUpdate(infrastructureConfig, infrastructureConfig, fldPath)).To(BeEmpty()) + Expect(ValidateInfrastructureConfigUpdate(infrastructureConfig, infrastructureConfig, nil, nil, fldPath)).To(BeEmpty()) }) It("should allow changing cloudNAT AND Flow-logs details", func() { @@ -332,7 +333,7 @@ var _ = Describe("InfrastructureConfig validation", func() { AggregationInterval: pointer.StringPtr("INTERVAL_30_SEC"), } - errorList := ValidateInfrastructureConfigUpdate(infrastructureConfig, newInfrastructureConfig, fldPath) + errorList := ValidateInfrastructureConfigUpdate(infrastructureConfig, newInfrastructureConfig, nil, nil, fldPath) Expect(errorList).To(BeEmpty()) }) @@ -348,7 +349,7 @@ var _ = Describe("InfrastructureConfig validation", func() { newInfrastructureConfig.Networks.Worker = "10.96.0.0/16" newInfrastructureConfig.Networks.Internal = pointer.StringPtr("10.96.0.0/16") - errorList := ValidateInfrastructureConfigUpdate(infrastructureConfig, newInfrastructureConfig, fldPath) + errorList := ValidateInfrastructureConfigUpdate(infrastructureConfig, newInfrastructureConfig, nil, nil, fldPath) Expect(errorList).To(ConsistOfFields(Fields{ "Type": Equal(field.ErrorTypeInvalid), "Field": Equal("networks.vpc.name"), @@ -371,7 +372,7 @@ var _ = Describe("InfrastructureConfig validation", func() { newInfrastructureConfig := infrastructureConfig.DeepCopy() newInfrastructureConfig.Networks.VPC = nil - errorList := ValidateInfrastructureConfigUpdate(infrastructureConfig, newInfrastructureConfig, fldPath) + errorList := ValidateInfrastructureConfigUpdate(infrastructureConfig, newInfrastructureConfig, nil, nil, fldPath) Expect(errorList).To(ConsistOfFields(Fields{ "Type": Equal(field.ErrorTypeInvalid), "Field": Equal("networks.vpc"), diff --git a/pkg/apis/gcp/zz_generated.deepcopy.go b/pkg/apis/gcp/zz_generated.deepcopy.go index 4ad1b05d1..c0ed17b4d 100644 --- a/pkg/apis/gcp/zz_generated.deepcopy.go +++ b/pkg/apis/gcp/zz_generated.deepcopy.go @@ -359,6 +359,11 @@ func (in *NetworkConfig) DeepCopyInto(out *NetworkConfig) { *out = new(FlowLogs) (*in).DeepCopyInto(*out) } + if in.PrivateServiceConnect != nil { + in, out := &in.PrivateServiceConnect, &out.PrivateServiceConnect + *out = new(PrivateServiceConnect) + **out = **in + } return } @@ -399,6 +404,22 @@ func (in *NetworkStatus) DeepCopy() *NetworkStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PrivateServiceConnect) DeepCopyInto(out *PrivateServiceConnect) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PrivateServiceConnect. +func (in *PrivateServiceConnect) DeepCopy() *PrivateServiceConnect { + if in == nil { + return nil + } + out := new(PrivateServiceConnect) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ServiceAccount) DeepCopyInto(out *ServiceAccount) { *out = *in diff --git a/pkg/internal/infrastructure/templates/main.tpl.tf b/pkg/internal/infrastructure/templates/main.tpl.tf index 9c49a711b..f22c5977f 100644 --- a/pkg/internal/infrastructure/templates/main.tpl.tf +++ b/pkg/internal/infrastructure/templates/main.tpl.tf @@ -4,6 +4,14 @@ provider "google" { region = "{{ .google.region }}" } +{{ if .google.enableBeta -}} +provider "google-beta" { + credentials = var.SERVICEACCOUNT + project = "{{ .google.project }}" + region = "{{ .google.region }}" +} +{{- end }} + //===================================================================== //= Service Account //===================================================================== @@ -44,6 +52,9 @@ resource "google_compute_subnetwork" "subnetwork-nodes" { {{ if .networks.flowLogs.metadata }}metadata = "{{ .networks.flowLogs.metadata }}"{{ end }} } {{- end }} +{{- if .networks.privateServiceConnect }} + private_ip_google_access = true +{{- end }} timeouts { create = "5m" @@ -98,14 +109,14 @@ resource "google_compute_router_nat" "nat" { delete = "5m" } } -{{- end}} +{{- end }} {{ if .networks.cloudNAT.natIPNames -}} {{range $index, $natIP := .networks.cloudNAT.natIPNames}} data "google_compute_address" "{{ $natIP }}" { name = "{{ $natIP }}" } -{{end}} +{{ end }} {{- end }} {{ if .networks.internal -}} @@ -121,7 +132,33 @@ resource "google_compute_subnetwork" "subnetwork-internal" { delete = "5m" } } -{{- end}} +{{- end }} + +{{ if .networks.privateServiceConnect }} + resource "google_compute_global_address" "default" { + provider = google-beta + name = "{{ .clusterName }}" + address_type = "INTERNAL" + purpose = "PRIVATE_SERVICE_CONNECT" + network = {{ .vpc.name }} + address = "{{ .networks.privateServiceConnect.address }}" +} + +// There is an issue currently that TF will always delete and recreate the address during infrastructure reconciliation +// because it cannot handle the labels properly. +resource "google_compute_global_forwarding_rule" "default" { + provider = google-beta + name = "{{ .networks.privateServiceConnect.ruleName }}" + target = "all-apis" + network = {{ .vpc.name }} + ip_address = google_compute_global_address.default.id + load_balancing_scheme = "" + description = "{{ .clusterName }}" + +// labels = {} +} + +{{- end }} //===================================================================== //= Firewall @@ -260,4 +297,5 @@ output "{{ .outputKeys.subnetNodes }}" { output "{{ .outputKeys.subnetInternal }}" { value = google_compute_subnetwork.subnetwork-internal.name } -{{- end}} + +{{- end }} diff --git a/pkg/internal/infrastructure/terraform.go b/pkg/internal/infrastructure/terraform.go index db5ef10b0..dc51b8364 100644 --- a/pkg/internal/infrastructure/terraform.go +++ b/pkg/internal/infrastructure/terraform.go @@ -17,6 +17,8 @@ package infrastructure import ( "bytes" "context" + "crypto/sha1" + "encoding/base32" "fmt" "strconv" "strings" @@ -127,8 +129,9 @@ func ComputeTerraformerTemplateValues( values := map[string]interface{}{ "google": map[string]interface{}{ - "region": infra.Spec.Region, - "project": account.ProjectID, + "region": infra.Spec.Region, + "project": account.ProjectID, + "enableBeta": false, }, "create": map[string]interface{}{ "vpc": createVPC, @@ -145,6 +148,19 @@ func ComputeTerraformerTemplateValues( "outputKeys": outputKeys, } + if config.Networks.PrivateServiceConnect != nil && config.Networks.PrivateServiceConnect.Enabled { + values["google"].(map[string]interface{})["enableBeta"] = true + + psc := make(map[string]interface{}) + psc["address"] = config.Networks.PrivateServiceConnect.EndpointIP + name, err := privateServiceConnectName(infra) + if err != nil { + return nil, err + } + psc["ruleName"] = name + values["networks"].(map[string]interface{})["privateServiceConnect"] = psc + } + if config.Networks.FlowLogs != nil { fl := make(map[string]interface{}) @@ -333,3 +349,15 @@ func ComputeStatus(ctx context.Context, tf terraformer.Terraformer, config *api. func manualNatIPsSet(config *api.InfrastructureConfig) bool { return config.Networks.CloudNAT != nil && config.Networks.CloudNAT.NatIPNames != nil } + +func privateServiceConnectName(infra *extensionsv1alpha1.Infrastructure) (string, error) { + sha := sha1.New() + _, err := sha.Write([]byte(infra.Namespace)) + if err != nil { + return "", fmt.Errorf("failed to create name for PrivateServiceConnect: %v", err) + } + + var buf []byte + name := fmt.Sprintf("g%19s", strings.ToLower(base32.StdEncoding.EncodeToString(sha.Sum(buf))))[:20] + return name, nil +} diff --git a/pkg/internal/infrastructure/terraform_test.go b/pkg/internal/infrastructure/terraform_test.go index 8aa2946b6..b4023412f 100644 --- a/pkg/internal/infrastructure/terraform_test.go +++ b/pkg/internal/infrastructure/terraform_test.go @@ -19,10 +19,11 @@ import ( "fmt" "strconv" + mockterraformer "github.com/gardener/gardener/extensions/pkg/terraformer/mock" + api "github.com/gardener/gardener-extension-provider-gcp/pkg/apis/gcp" apiv1alpha1 "github.com/gardener/gardener-extension-provider-gcp/pkg/apis/gcp/v1alpha1" "github.com/gardener/gardener-extension-provider-gcp/pkg/gcp" - mockterraformer "github.com/gardener/gardener/extensions/pkg/terraformer/mock" extensionsv1alpha1 "github.com/gardener/gardener/pkg/apis/extensions/v1alpha1" "github.com/golang/mock/gomock" @@ -204,8 +205,9 @@ var _ = Describe("Terraform", func() { Expect(err).To(BeNil()) Expect(values).To(Equal(map[string]interface{}{ "google": map[string]interface{}{ - "region": infra.Spec.Region, - "project": projectID, + "region": infra.Spec.Region, + "project": projectID, + "enableBeta": false, }, "create": map[string]interface{}{ "vpc": false, @@ -242,8 +244,9 @@ var _ = Describe("Terraform", func() { Expect(err).To(BeNil()) Expect(values).To(Equal(map[string]interface{}{ "google": map[string]interface{}{ - "region": infra.Spec.Region, - "project": projectID, + "region": infra.Spec.Region, + "project": projectID, + "enableBeta": false, }, "create": map[string]interface{}{ "vpc": false, @@ -305,8 +308,9 @@ var _ = Describe("Terraform", func() { Expect(err).To(BeNil()) Expect(values).To(Equal(map[string]interface{}{ "google": map[string]interface{}{ - "region": infra.Spec.Region, - "project": projectID, + "region": infra.Spec.Region, + "project": projectID, + "enableBeta": false, }, "create": map[string]interface{}{ "vpc": false, @@ -367,8 +371,9 @@ var _ = Describe("Terraform", func() { Expect(err).To(BeNil()) Expect(values).To(Equal(map[string]interface{}{ "google": map[string]interface{}{ - "region": infra.Spec.Region, - "project": projectID, + "region": infra.Spec.Region, + "project": projectID, + "enableBeta": false, }, "create": map[string]interface{}{ "vpc": false, @@ -411,8 +416,9 @@ var _ = Describe("Terraform", func() { Expect(err).To(BeNil()) Expect(values).To(Equal(map[string]interface{}{ "google": map[string]interface{}{ - "region": infra.Spec.Region, - "project": projectID, + "region": infra.Spec.Region, + "project": projectID, + "enableBeta": false, }, "create": map[string]interface{}{ "vpc": true,