Configurable HAP Shared Secrets Plans and Regions

cmd/broker/main.go

@@ -18,6 +18,7 @@ import (
 	imv1 "github.com/kyma-project/infrastructure-manager/api/v1"
 
 	"github.com/kyma-project/kyma-environment-broker/internal/expiration"
+	"github.com/kyma-project/kyma-environment-broker/internal/hap"
 	"github.com/kyma-project/kyma-environment-broker/internal/metricsv2"
 	"github.com/kyma-project/kyma-environment-broker/internal/whitelist"
 
@@ -142,6 +143,7 @@ type Config struct {
 	Events events.Config
 
 	MetricsV2 metricsv2.Config
+	Hap       hap.Config
 
 	Provisioning    process.StagedManagerConfiguration
 	Deprovisioning  process.StagedManagerConfiguration
@@ -408,6 +410,7 @@ func logConfiguration(logs *slog.Logger, cfg Config) {
 		cfg.Broker.KimConfig.KimOnlyPlans))
 	logs.Info(fmt.Sprintf("Is SubaccountMovementEnabled: %t", cfg.Broker.SubaccountMovementEnabled))
 	logs.Info(fmt.Sprintf("Is UpdateCustomResourcesLabelsOnAccountMove enabled: %t", cfg.Broker.UpdateCustomResourcesLabelsOnAccountMove))
+	logs.Info(fmt.Sprintf("HAP's shared secerts enabled for plans: %s", cfg.Hap.SharedSecretPlans))
 }
 
 func createAPI(router *mux.Router, servicesConfig broker.ServicesConfig, planValidator broker.PlanValidator, cfg *Config, db storage.BrokerStorage,

cmd/broker/provisioning.go

@@ -68,7 +68,7 @@ func NewProvisioningProcessingQueue(ctx context.Context, provisionManager *proce
 		},
 		{
 			stage:     createRuntimeStageName,
-			step:      provisioning.NewResolveCredentialsStep(db.Operations(), accountProvider),
+			step:      provisioning.NewResolveCredentialsStep(db.Operations(), accountProvider, cfg.Hap),
 			condition: provisioning.SkipForOwnClusterPlan,
 		},
 		{

docs/contributor/03-10-hyperscaler-account-pool.md

@@ -2,15 +2,16 @@
 
 To provision clusters through Gardener using Runtime Provisioner, Kyma Environment Broker (KEB) requires a hyperscaler (GCP, Azure, AWS, etc.) account/subscription. Managing the available hyperscaler accounts is not in the scope of KEB. Instead, the available accounts are handled by Hyperscaler Account Pool (HAP).
 
-HAP stores credentials for the hyperscaler accounts that have been set up in advance in Kubernetes Secrets. The credentials are stored separately for each provider and tenant. The content of the credentials Secrets may vary for different use cases. The Secrets are labeled with the **hyperscaler-type** and **tenant-name** labels to manage pools of credentials for use by the provisioning process. This way, the in-use credentials and unassigned credentials available for use are tracked. Only the **hyperscaler-type** label is added during Secret creation, and the **tenant-name** label is added when the account respective for a given Secret is claimed. The content of the Secrets is opaque to HAP.
+HAP stores credentials for the hyperscaler accounts that have been set up in advance in Kubernetes Secrets. The credentials are stored separately for each provider and tenant. The content of the credentials Secrets may vary for different use cases. The Secrets are labeled with the **hyperscaler-type**, **euAccess**, **shared** and **tenant-name** labels to manage pools of credentials for use by the provisioning process. The **hyperscaler-type** contains hyperscaler name and region information in the format `hyperscaler_type: <HYPERSCALER_NAME>[_<PLATFORM_REGION>][_<HYPERSCALER_REGION>]`, where both `_<PLATFORM_REGION>` and `_<HYPERSCALER_REGION>` are optional. The **euAccess** and **shared** labels contain boolean values and they used to divide existing pools to secrets used by EU restricted regions and secrets shared by multiple Global Accounts. The **tenant-name** label is added when the account respective for a given Secret is claimed and it is the only one not added during Secret creation. This way, the in-use credentials and unassigned credentials available for use are tracked. The content of the Secrets is opaque to HAP.
 
 The Secrets are stored in a Gardener seed cluster pointed to by HAP. They are available within a given Gardener project specified in the KEB and Runtime Provisioner configuration. This configuration uses a `kubeconfig` that gives KEB and Runtime Provisioner access to a specific Gardener seed cluster, which, in turn, enables access to those Secrets.
 
 This diagram shows the HAP workflow:
 
 ![hap-workflow](../assets/hap-flow.drawio.svg)
 
-Before a new cluster is provisioned, KEB queries for a Secret based on the **tenant-name** and **hyperscaler-type** labels.
+Before a new cluster is provisioned, KEB queries for a Secret based on the mandatory **hyperscaler-type** and optional **tenant-name**, **euAccess** and **shared** labels.
+
 If a Secret is found, KEB uses the credentials stored in this Secret. If a matching Secret is not found, KEB queries again for an unassigned Secret for a given hyperscaler and adds the **tenant-name** label to claim the account and use the credentials for provisioning.
 
 One tenant can use only one account per given hyperscaler type.
@@ -47,6 +48,7 @@ metadata:
     shared: "true"
 ```
 
+
 ### Shared Credentials for `sap-converged-cloud` Plan
 
 For the `sap-converged-cloud` plan, each region is treated as a separate hyperscaler. Hence, Secrets are labeled with **openstack_{region name}**, for example, **openstack_eu-de-1**.
@@ -82,3 +84,34 @@ metadata:
     tenant-name: {TENANT_NAME}
     hyperscaler-type: "gcp_cf-sa30"
 ```
+
+## Secret Bindings Selection Rules
+
+<!-- rules Overview -->
+HAP evaluates a set of rules to determine what labels to use when querying secret bindings. Input to the rules consists of an SKR's plan, hyperscaler and region. If evaluated to true the rules modify or add labels used in the secret bindings resource query. There are four possible rules to configure:
+* `hap.platformRegionRule` - if evaluated to true the `_<PLATFOR_REGION>` is appended to the `hyperscaler-type` label when searching, refered to as platform region based search,
+* `hap.clusterRegionRule` - if evaluated to true the `_<HYPERSCALER_REGION>` is appended to the `hyperscaler-type` label when searching, refered to as cluster region based search,
+* `hap.sharedRule` - if evaluated to true the `shared: true` label is used when searching, refered to as shared based search,
+* `hap.euAccessRule` - if evaluated to true the `shared: true` label is used when searching, refered to as euAccess based search.
+The configuration is done by specifying above rules in KEB helm values. 
+
+<!-- rules format -->
+Each rule consists of a semicolon separated list of plans that the rule applies to. Additionally, a plan can be extended with `:region` suffix that makes the rule evaluate to true only if a cluster is provisioned in the specified region. List entries comply with the format `<PLAN_ID_1>:<REGION_ID_1>;<PLAN_ID_2>:<REGION_ID_2>`. Either plan or region (but never both) can be specified as wildcard `*` meaning all plans or regions should apply for specific second value. The following example lists valid and invalid configuration values:
+* `trial` - valid, rule will be evaluated to true for the `trial` plan in all regions,
+* `trial:*` - valid, rule will be evaluated to true for the `trial` plan in all regions,
+* `trial:eu` - valid, rule will be evaluated to true for the `trial` plan in the `eu` region,
+
+* `*:eu` - valid, rule will be evaluated to true for all plans in the `eu` region,
+* `eu:*` - invalid, plan must be specified in the first part of `<PLAN_ID>:<REGION_ID>` pair,
+* `*:*` - invalid, at least one of the values must be specified in the `<PLAN_ID>:<REGION_ID>` pair.
+
+* `*:eu;trial:eu` - valid, rule will be evaluated to true for all plans in the `eu` region and for the `trial` plan in the `eu` region, configuration can be duplicated
+* `trial:eu;trial:gcp` - valid, rule will be evaluated to true for trials plans but only in `eu` and `gcp` regions,
+
+<!-- search construction -->
+![alt text](image.png)
+
+<!-- TODO: rules validation -->
+Rules validation is done during the KEB startup. If the configuration is invalid, KEB will not start and an error message will be displayed in the logs. The constraints used for validation include:
+* Rules format check - all the rules must comply with the format specified above.
+* Plan ?and region? existence check.

internal/hap/config.go

@@ -0,0 +1,9 @@
+package hap
+
+import (
+	"github.com/kyma-project/kyma-environment-broker/internal/utils"
+)
+
+type Config struct {
+	SharedSecretPlans utils.Whitelist `envconfig:"default=trial:*;sap-converged-cloud:*"`
+}

internal/hap/config_test.go

@@ -0,0 +1,37 @@
+package hap
+
+import (
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	"github.com/vrischmann/envconfig"
+)
+
+func TestHyperscalerConfigs(t *testing.T) {
+
+	t.Run("should read default values from env variables", func(t *testing.T) {
+		// given
+		var cfg Config
+		err := envconfig.InitWithPrefix(&cfg, "APP_HAP")
+		require.NoError(t, err)
+
+		require.True(t, cfg.SharedSecretPlans.Contains("trial:*"))
+		require.True(t, cfg.SharedSecretPlans.Contains("sap-converged-cloud:*"))
+	})
+
+	t.Run("should read single values from env variables", func(t *testing.T) {
+		err := os.Setenv("APP_HAP_SHARED_SECRET_PLANS", "aws:*;azure:*;gcp:eu1")
+		require.NoError(t, err)
+
+		// given
+		var cfg Config
+		err = envconfig.InitWithPrefix(&cfg, "APP_HAP")
+		require.NoError(t, err)
+
+		require.True(t, cfg.SharedSecretPlans.Contains("aws:*"))
+		require.False(t, cfg.SharedSecretPlans.Contains("azure"))
+		require.True(t, cfg.SharedSecretPlans.Contains("azure:*"))
+		require.True(t, cfg.SharedSecretPlans.Contains("gcp:eu1"))
+	})
+}

internal/process/provisioning/resolve_creds.go

@@ -6,6 +6,7 @@ import (
 	"time"
 
 	kebError "github.com/kyma-project/kyma-environment-broker/internal/error"
+	"github.com/kyma-project/kyma-environment-broker/internal/hap"
 
 	"github.com/kyma-project/kyma-environment-broker/internal/euaccess"
 
@@ -23,12 +24,14 @@ type ResolveCredentialsStep struct {
 	accountProvider  hyperscaler.AccountProvider
 	opStorage        storage.Operations
 	tenant           string
+	hapConfig        *hap.Config
 }
 
-func NewResolveCredentialsStep(os storage.Operations, accountProvider hyperscaler.AccountProvider) *ResolveCredentialsStep {
+func NewResolveCredentialsStep(os storage.Operations, accountProvider hyperscaler.AccountProvider, hapConfig hap.Config) *ResolveCredentialsStep {
 	step := &ResolveCredentialsStep{
 		opStorage:       os,
 		accountProvider: accountProvider,
+		hapConfig:       &hapConfig,
 	}
 	step.operationManager = process.NewOperationManager(os, step.Name(), kebError.AccountPoolDependency)
 	return step
@@ -90,7 +93,9 @@ func (s *ResolveCredentialsStep) retryOrFailOperation(operation internal.Operati
 func (s *ResolveCredentialsStep) getTargetSecretFromGardener(operation internal.Operation, log *slog.Logger, hypType hyperscaler.Type, euAccess bool) (string, error) {
 	var secretName string
 	var err error
-	if broker.IsTrialPlan(operation.ProvisioningParameters.PlanID) || broker.IsSapConvergedCloudPlan(operation.ProvisioningParameters.PlanID) {
+
+	if broker.IsTrialPlan(operation.ProvisioningParameters.PlanID) || broker.IsSapConvergedCloudPlan(operation.ProvisioningParameters.PlanID) ||
+		s.isSharedSecret(operation.ProvisioningParameters) {
 		log.Info("HAP lookup for shared secret binding")
 		secretName, err = s.accountProvider.GardenerSharedSecretName(hypType, euAccess)
 	} else {
@@ -100,6 +105,15 @@ func (s *ResolveCredentialsStep) getTargetSecretFromGardener(operation internal.
 	return secretName, err
 }
 
+func (s *ResolveCredentialsStep) isSharedSecret(provisioningParameters internal.ProvisioningParameters) bool {
+
+	var planName = broker.PlanNamesMapping[provisioningParameters.PlanID]
+	var platformRegion = provisioningParameters.PlatformRegion
+
+	return s.hapConfig.SharedSecretPlans.Contains(planName+":"+platformRegion) ||
+		s.hapConfig.SharedSecretPlans.Contains(planName+":*") || s.hapConfig.SharedSecretPlans.Contains("*:"+platformRegion)
+}
+
 // TODO: Calculate the region parameter using default SapConvergedCloud region. This is to be removed when region is mandatory (Jan 2024).
 func getEffectiveRegionForSapConvergedCloud(provisioningParametersRegion *string) string {
 	if provisioningParametersRegion != nil && *provisioningParametersRegion != "" {