From 685bba1613d45cfa778123818b90726a8e04d357 Mon Sep 17 00:00:00 2001 From: Arkadiusz Galwas Date: Tue, 7 Jan 2025 07:20:32 +0100 Subject: [PATCH 1/7] Removed setting fields that are not needed on Runtime CR --- .../internal/migration/migrator.go | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/hack/runtime-migrator/internal/migration/migrator.go b/hack/runtime-migrator/internal/migration/migrator.go index c457c76b..08d2317a 100644 --- a/hack/runtime-migrator/internal/migration/migrator.go +++ b/hack/runtime-migrator/internal/migration/migrator.go @@ -60,16 +60,10 @@ func (m Migrator) Do(ctx context.Context, shoot v1beta1.Shoot) (v1.Runtime, erro APIVersion: "infrastructuremanager.kyma-project.io/v1", }, ObjectMeta: metav1.ObjectMeta{ - Name: labels["kyma-project.io/runtime-id"], - GenerateName: shoot.GenerateName, - Namespace: "kcp-system", - DeletionTimestamp: shoot.DeletionTimestamp, - DeletionGracePeriodSeconds: shoot.DeletionGracePeriodSeconds, - Labels: labels, - Annotations: shoot.Annotations, - OwnerReferences: nil, // deliberately left empty, as without that we will not be able to delete Runtime CRs - Finalizers: nil, // deliberately left empty, as without that we will not be able to delete Runtime CRs - ManagedFields: nil, // deliberately left empty "This is mostly for migrator housekeeping, and users typically shouldn't need to set or understand this field." + Name: labels["kyma-project.io/runtime-id"], + Namespace: "kcp-system", + Labels: labels, + Annotations: shoot.Annotations, }, Spec: v1.RuntimeSpec{ Shoot: v1.RuntimeShoot{ @@ -112,10 +106,6 @@ func (m Migrator) Do(ctx context.Context, shoot v1beta1.Shoot) (v1.Runtime, erro }, }, }, - Status: v1.RuntimeStatus{ - State: "", // deliberately left empty by our migrator to show that controller has not picked it yet - Conditions: nil, // deliberately left nil by our migrator to show that controller has not picked it yet - }, } controlPlane := getControlPlane(shoot) From 5c26811bd168e64778b506d546be16d77997fe1e Mon Sep 17 00:00:00 2001 From: Arkadiusz Galwas Date: Tue, 7 Jan 2025 07:49:27 +0100 Subject: [PATCH 2/7] Added functions for labeling crbs to backup --- hack/runtime-migrator/cmd/backup/backup.go | 30 ++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/hack/runtime-migrator/cmd/backup/backup.go b/hack/runtime-migrator/cmd/backup/backup.go index a881a13f..e22b5613 100644 --- a/hack/runtime-migrator/cmd/backup/backup.go +++ b/hack/runtime-migrator/cmd/backup/backup.go @@ -8,8 +8,12 @@ import ( "github.com/kyma-project/infrastructure-manager/hack/runtime-migrator-app/internal/config" "github.com/kyma-project/infrastructure-manager/hack/runtime-migrator-app/internal/shoot" "github.com/kyma-project/infrastructure-manager/pkg/gardener/kubeconfig" + "github.com/pkg/errors" + rbacv1 "k8s.io/api/rbac/v1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" "log/slog" + "slices" "time" ) @@ -106,3 +110,29 @@ func (b Backup) Do(ctx context.Context, runtimeIDs []string) error { return nil } + +func filterOnlySupportedTypesOfCRBs(bindings []rbacv1.ClusterRoleBinding) []rbacv1.ClusterRoleBinding { + return slices.DeleteFunc(bindings, func(clusterRoleBinding rbacv1.ClusterRoleBinding) bool { + if clusterRoleBinding.RoleRef.Kind != "ClusterRole" || clusterRoleBinding.RoleRef.Name != "cluster-admin" { + return true + } + // leave only cluster-admin CRBs where at least one subject is of a user type + if slices.ContainsFunc(clusterRoleBinding.Subjects, func(subject rbacv1.Subject) bool { return subject.Kind == rbacv1.UserKind }) { + return false + } + return true + }) +} + +func labelsCRBsAsDeprecated(ctx context.Context, clientset *kubernetes.Clientset, deprecatedCRBs []rbacv1.ClusterRoleBinding) error { + for _, clusterRoleBinding := range deprecatedCRBs { + clusterRoleBinding.ObjectMeta.Labels["kyma-project.io/deprecation"] = "to-be-removed-soon" + _, err := clientset.RbacV1().ClusterRoleBindings().Update(ctx, &clusterRoleBinding, v1.UpdateOptions{}) + + if err != nil { + return errors.Wrap(err, fmt.Sprintf("Failed to update ClusterRoleBinding with deprecation label %s", clusterRoleBinding.Name)) + } + slog.Info(fmt.Sprintf("ClusterRoleBinding %s has been labeled as deprecated", clusterRoleBinding.Name)) + } + return nil +} From dadb5fb37e239100f1e67b5cfde4fead661f1bc4 Mon Sep 17 00:00:00 2001 From: Arkadiusz Galwas Date: Tue, 7 Jan 2025 11:20:25 +0100 Subject: [PATCH 3/7] Code for labeling CRBs moved to internal/backup.go --- hack/runtime-migrator/cmd/backup/backup.go | 30 ----------------- hack/runtime-migrator/cmd/restore/restore.go | 2 +- .../internal/backup/backup.go | 33 +++++++++++++++++-- 3 files changed, 32 insertions(+), 33 deletions(-) diff --git a/hack/runtime-migrator/cmd/backup/backup.go b/hack/runtime-migrator/cmd/backup/backup.go index a71995c9..0f6a0659 100644 --- a/hack/runtime-migrator/cmd/backup/backup.go +++ b/hack/runtime-migrator/cmd/backup/backup.go @@ -8,13 +8,9 @@ import ( "github.com/kyma-project/infrastructure-manager/hack/runtime-migrator-app/internal/initialisation" "github.com/kyma-project/infrastructure-manager/hack/runtime-migrator-app/internal/shoot" "github.com/kyma-project/infrastructure-manager/pkg/gardener/kubeconfig" - "github.com/pkg/errors" - rbacv1 "k8s.io/api/rbac/v1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/kubernetes" "log/slog" "sigs.k8s.io/controller-runtime/pkg/client" - "slices" "time" ) @@ -112,29 +108,3 @@ func (b Backup) Do(ctx context.Context, runtimeIDs []string) error { return nil } - -func filterOnlySupportedTypesOfCRBs(bindings []rbacv1.ClusterRoleBinding) []rbacv1.ClusterRoleBinding { - return slices.DeleteFunc(bindings, func(clusterRoleBinding rbacv1.ClusterRoleBinding) bool { - if clusterRoleBinding.RoleRef.Kind != "ClusterRole" || clusterRoleBinding.RoleRef.Name != "cluster-admin" { - return true - } - // leave only cluster-admin CRBs where at least one subject is of a user type - if slices.ContainsFunc(clusterRoleBinding.Subjects, func(subject rbacv1.Subject) bool { return subject.Kind == rbacv1.UserKind }) { - return false - } - return true - }) -} - -func labelsCRBsAsDeprecated(ctx context.Context, clientset *kubernetes.Clientset, deprecatedCRBs []rbacv1.ClusterRoleBinding) error { - for _, clusterRoleBinding := range deprecatedCRBs { - clusterRoleBinding.ObjectMeta.Labels["kyma-project.io/deprecation"] = "to-be-removed-soon" - _, err := clientset.RbacV1().ClusterRoleBindings().Update(ctx, &clusterRoleBinding, v1.UpdateOptions{}) - - if err != nil { - return errors.Wrap(err, fmt.Sprintf("Failed to update ClusterRoleBinding with deprecation label %s", clusterRoleBinding.Name)) - } - slog.Info(fmt.Sprintf("ClusterRoleBinding %s has been labeled as deprecated", clusterRoleBinding.Name)) - } - return nil -} diff --git a/hack/runtime-migrator/cmd/restore/restore.go b/hack/runtime-migrator/cmd/restore/restore.go index ea685820..bd298550 100644 --- a/hack/runtime-migrator/cmd/restore/restore.go +++ b/hack/runtime-migrator/cmd/restore/restore.go @@ -60,7 +60,7 @@ func (r Restore) Do(ctx context.Context, runtimeIDs []string) error { return err } - restorer := restore.NewRestorer(r.cfg.BackupDir) + restorer := restore.NewRestorer(r.cfg.BackupDir, r.cfg.RestoreCRB, r.cfg.RestoreOIDC) for _, runtimeID := range runtimeIDs { currentShoot, err := shoot.Fetch(ctx, shootList, r.shootClient, runtimeID) diff --git a/hack/runtime-migrator/internal/backup/backup.go b/hack/runtime-migrator/internal/backup/backup.go index df368edb..f4c2e034 100644 --- a/hack/runtime-migrator/internal/backup/backup.go +++ b/hack/runtime-migrator/internal/backup/backup.go @@ -2,14 +2,17 @@ package backup import ( "context" + "fmt" "github.com/gardener/gardener/pkg/apis/core/v1beta1" authenticationv1alpha1 "github.com/gardener/oidc-webhook-authenticator/apis/authentication/v1alpha1" "github.com/kyma-project/infrastructure-manager/hack/runtime-migrator-app/internal/initialisation" "github.com/kyma-project/infrastructure-manager/pkg/gardener/kubeconfig" "github.com/pkg/errors" + "golang.org/x/exp/slices" rbacv1 "k8s.io/api/rbac/v1" crdv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "log/slog" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -40,7 +43,7 @@ func (b Backuper) Do(ctx context.Context, shoot v1beta1.Shoot, runtimeID string) return RuntimeBackup{}, err } - crbs, err := b.getCRBs(runtimeClient) + crbs, err := b.getAllCRBs(runtimeClient) if err != nil { return RuntimeBackup{}, errors.Wrap(err, "failed to get Cluster Role Bindings") } @@ -104,7 +107,7 @@ func (b Backuper) getShootToRestore(shootFromGardener v1beta1.Shoot) v1beta1.Sho } } -func (b Backuper) getCRBs(runtimeClient client.Client) ([]rbacv1.ClusterRoleBinding, error) { +func (b Backuper) getAllCRBs(runtimeClient client.Client) ([]rbacv1.ClusterRoleBinding, error) { var crbList rbacv1.ClusterRoleBindingList err := runtimeClient.List(context.Background(), &crbList) @@ -159,3 +162,29 @@ func oidcCRDExists(runtimeClient client.Client) (bool, error) { return false, nil } + +func filterOnlySupportedTypesOfCRBs(bindings []rbacv1.ClusterRoleBinding) []rbacv1.ClusterRoleBinding { + return slices.DeleteFunc(bindings, func(clusterRoleBinding rbacv1.ClusterRoleBinding) bool { + if clusterRoleBinding.RoleRef.Kind != "ClusterRole" || clusterRoleBinding.RoleRef.Name != "cluster-admin" { + return true + } + // leave only cluster-admin CRBs where at least one subject is of a user type + if slices.ContainsFunc(clusterRoleBinding.Subjects, func(subject rbacv1.Subject) bool { return subject.Kind == rbacv1.UserKind }) { + return false + } + return true + }) +} + +func labelsCRBsAsDeprecated(ctx context.Context, runtimeClient client.Client, deprecatedCRBs []rbacv1.ClusterRoleBinding) error { + for _, clusterRoleBinding := range deprecatedCRBs { + clusterRoleBinding.ObjectMeta.Labels["kyma-project.io/deprecation"] = "to-be-removed-soon" + err := runtimeClient.Patch(ctx, &clusterRoleBinding, client.Apply, &client.PatchOptions{}) + + if err != nil { + return errors.Wrap(err, fmt.Sprintf("Failed to update ClusterRoleBinding with deprecation label %s", clusterRoleBinding.Name)) + } + slog.Info(fmt.Sprintf("ClusterRoleBinding %s has been labeled as deprecated", clusterRoleBinding.Name)) + } + return nil +} From b0c57a07761315063ff58be75fbb7f6e9eb7cb9f Mon Sep 17 00:00:00 2001 From: Arkadiusz Galwas Date: Wed, 8 Jan 2025 15:39:03 +0100 Subject: [PATCH 4/7] go.mod update --- hack/runtime-migrator/go.mod | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hack/runtime-migrator/go.mod b/hack/runtime-migrator/go.mod index a22b47bf..159d67dc 100644 --- a/hack/runtime-migrator/go.mod +++ b/hack/runtime-migrator/go.mod @@ -10,7 +10,9 @@ require ( github.com/kyma-project/infrastructure-manager/hack/shoot-comparator v0.0.0-20241023155010-55a6abeb1690 github.com/pkg/errors v0.9.1 github.com/stretchr/testify v1.10.0 + golang.org/x/exp v0.0.0-20241204233417-43b7b7cde48d k8s.io/api v0.31.3 + k8s.io/apiextensions-apiserver v0.31.3 k8s.io/apimachinery v0.31.3 k8s.io/client-go v0.31.3 k8s.io/utils v0.0.0-20241210054802-24370beab758 @@ -78,7 +80,6 @@ require ( gopkg.in/square/go-jose.v2 v2.6.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/apiextensions-apiserver v0.31.3 // indirect k8s.io/klog/v2 v2.130.1 // indirect k8s.io/kube-openapi v0.0.0-20240903163716-9e1beecbcb38 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect From b7e5481497f4a10ae8371ef810dc6960804d4205 Mon Sep 17 00:00:00 2001 From: Arkadiusz Galwas Date: Wed, 8 Jan 2025 17:56:33 +0100 Subject: [PATCH 5/7] Moved deprecated crb labeling from migrator --- hack/runtime-migrator/cmd/backup/backup.go | 80 ++++++++++++++++++- .../cmd/migration/migration.go | 9 ++- .../internal/backup/backup.go | 40 +--------- .../internal/backup/results.go | 29 ++++--- .../internal/restore/results.go | 8 +- 5 files changed, 110 insertions(+), 56 deletions(-) diff --git a/hack/runtime-migrator/cmd/backup/backup.go b/hack/runtime-migrator/cmd/backup/backup.go index 3532b6d3..ca7b9ba2 100644 --- a/hack/runtime-migrator/cmd/backup/backup.go +++ b/hack/runtime-migrator/cmd/backup/backup.go @@ -8,14 +8,19 @@ import ( "github.com/kyma-project/infrastructure-manager/hack/runtime-migrator-app/internal/initialisation" "github.com/kyma-project/infrastructure-manager/hack/runtime-migrator-app/internal/shoot" "github.com/kyma-project/infrastructure-manager/pkg/gardener/kubeconfig" + "github.com/pkg/errors" + rbacv1 "k8s.io/api/rbac/v1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" "log/slog" "sigs.k8s.io/controller-runtime/pkg/client" + "slices" "time" ) const ( timeoutK8sOperation = 20 * time.Second + fieldManagerName = "kim-backup" ) type Backup struct { @@ -70,7 +75,11 @@ func (b Backup) Do(ctx context.Context, runtimeIDs []string) error { continue } - runtimeBackup, err := backuper.Do(ctx, *shootToBackup, runtimeID) + runtimeClient, err := initialisation.GetRuntimeClient(ctx, b.kcpClient, runtimeID) + if err != nil { + continue + } + runtimeBackup, err := backuper.Do(ctx, runtimeClient, *shootToBackup) if err != nil { errMsg := fmt.Sprintf("Failed to backup runtime: %v", err) b.results.ErrorOccurred(runtimeID, shootToBackup.Name, errMsg) @@ -81,7 +90,7 @@ func (b Backup) Do(ctx context.Context, runtimeIDs []string) error { if b.cfg.IsDryRun { slog.Info("Runtime processed successfully (dry-run)", "runtimeID", runtimeID) - b.results.OperationSucceeded(runtimeID, shootToBackup.Name) + b.results.OperationSucceeded(runtimeID, shootToBackup.Name, nil) continue } @@ -94,8 +103,17 @@ func (b Backup) Do(ctx context.Context, runtimeIDs []string) error { continue } + deprecatedCRBs, err := labelDeprecatedCRBs(ctx, runtimeClient) + if err != nil { + errMsg := fmt.Sprintf("Failed to deprecate Cluster Role Bindings: %v", err) + b.results.ErrorOccurred(runtimeID, shootToBackup.Name, errMsg) + slog.Error(errMsg, "runtimeID", runtimeID) + + continue + } + slog.Info("Runtime backup created successfully", "runtimeID", runtimeID) - b.results.OperationSucceeded(runtimeID, shootToBackup.Name) + b.results.OperationSucceeded(runtimeID, shootToBackup.Name, deprecatedCRBs) } resultsFile, err := b.outputWriter.SaveBackupResults(b.results) @@ -108,3 +126,59 @@ func (b Backup) Do(ctx context.Context, runtimeIDs []string) error { return nil } + +func labelDeprecatedCRBs(ctx context.Context, runtimeClient client.Client) ([]rbacv1.ClusterRoleBinding, error) { + var crbList rbacv1.ClusterRoleBindingList + + listCtx, cancel := context.WithTimeout(ctx, timeoutK8sOperation) + defer cancel() + + selector, err := labels.Parse("reconciler.kyma-project.io/managed-by=reconciler,app=kyma") + if err != nil { + return nil, err + } + + err = runtimeClient.List(listCtx, &crbList, &client.ListOptions{ + LabelSelector: selector, + }) + + if err != nil { + return nil, err + } + + deprecatedCRBs := slices.DeleteFunc(crbList.Items, func(clusterRoleBinding rbacv1.ClusterRoleBinding) bool { + if clusterRoleBinding.RoleRef.Kind != "ClusterRole" || clusterRoleBinding.RoleRef.Name != "cluster-admin" { + return true + } + // leave only cluster-admin CRBs where at least one subject is of a user type + if slices.ContainsFunc(clusterRoleBinding.Subjects, func(subject rbacv1.Subject) bool { return subject.Kind == rbacv1.UserKind }) { + return false + } + return true + }) + + patchCRB := func(clusterRoleBinding rbacv1.ClusterRoleBinding) error { + patchCtx, cancelPatch := context.WithTimeout(ctx, timeoutK8sOperation) + defer cancelPatch() + + clusterRoleBinding.Kind = "ClusterRoleBinding" + clusterRoleBinding.APIVersion = "rbac.authorization.k8s.io/v1" + clusterRoleBinding.ManagedFields = nil + + return runtimeClient.Patch(patchCtx, &clusterRoleBinding, client.Apply, &client.PatchOptions{ + FieldManager: fieldManagerName, + }) + } + + for _, clusterRoleBinding := range deprecatedCRBs { + clusterRoleBinding.ObjectMeta.Labels["kyma-project.io/deprecation"] = "to-be-removed-soon" + err := patchCRB(clusterRoleBinding) + if err != nil { + if err != nil { + return nil, errors.Wrap(err, fmt.Sprintf("failed to update ClusterRoleBinding with deprecation label %s", clusterRoleBinding.Name)) + } + } + } + + return deprecatedCRBs, nil +} diff --git a/hack/runtime-migrator/cmd/migration/migration.go b/hack/runtime-migrator/cmd/migration/migration.go index 9f6914e7..c7c8bf96 100644 --- a/hack/runtime-migrator/cmd/migration/migration.go +++ b/hack/runtime-migrator/cmd/migration/migration.go @@ -5,6 +5,7 @@ import ( "fmt" "github.com/kyma-project/infrastructure-manager/hack/runtime-migrator-app/internal/shoot" "log/slog" + "time" gardener_types "github.com/gardener/gardener/pkg/client/core/clientset/versioned/typed/core/v1beta1" runtimev1 "github.com/kyma-project/infrastructure-manager/api/v1" @@ -26,6 +27,10 @@ type Migration struct { isDryRun bool } +const ( + timeoutK8sOperation = 20 * time.Second +) + func NewMigration(migratorConfig initialisation.Config, converterConfig config.ConverterConfig, auditLogConfig auditlogs.Configuration, kubeconfigProvider kubeconfig.Provider, kcpClient client.Client, shootClient gardener_types.ShootInterface) (Migration, error) { outputWriter, err := migration.NewOutputWriter(migratorConfig.OutputPath) @@ -44,7 +49,7 @@ func NewMigration(migratorConfig initialisation.Config, converterConfig config.C } func (m Migration) Do(ctx context.Context, runtimeIDs []string) error { - listCtx, cancel := context.WithTimeout(ctx, initialisation.TimeoutK8sOperation) + listCtx, cancel := context.WithTimeout(ctx, timeoutK8sOperation) defer cancel() shootList, err := m.shootClient.List(listCtx, v1.ListOptions{}) @@ -92,7 +97,7 @@ func (m Migration) Do(ctx context.Context, runtimeIDs []string) error { return } - migrationCtx, cancel := context.WithTimeout(ctx, initialisation.TimeoutK8sOperation) + migrationCtx, cancel := context.WithTimeout(ctx, timeoutK8sOperation) defer cancel() runtime, err := m.runtimeMigrator.Do(migrationCtx, *shootToMigrate) diff --git a/hack/runtime-migrator/internal/backup/backup.go b/hack/runtime-migrator/internal/backup/backup.go index bdd9762a..f0909f5c 100644 --- a/hack/runtime-migrator/internal/backup/backup.go +++ b/hack/runtime-migrator/internal/backup/backup.go @@ -2,17 +2,14 @@ package backup import ( "context" - "fmt" "github.com/gardener/gardener/pkg/apis/core/v1beta1" authenticationv1alpha1 "github.com/gardener/oidc-webhook-authenticator/apis/authentication/v1alpha1" "github.com/kyma-project/infrastructure-manager/hack/runtime-migrator-app/internal/initialisation" "github.com/kyma-project/infrastructure-manager/pkg/gardener/kubeconfig" "github.com/pkg/errors" - "golang.org/x/exp/slices" rbacv1 "k8s.io/api/rbac/v1" crdv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "log/slog" "sigs.k8s.io/controller-runtime/pkg/client" "time" ) @@ -40,13 +37,8 @@ type RuntimeBackup struct { OIDCConfig []authenticationv1alpha1.OpenIDConnect } -func (b Backuper) Do(ctx context.Context, shoot v1beta1.Shoot, runtimeID string) (RuntimeBackup, error) { - runtimeClient, err := initialisation.GetRuntimeClient(ctx, b.kcpClient, runtimeID) - if err != nil { - return RuntimeBackup{}, err - } - - crbs, err := b.getCRBs(ctx, runtimeClient) +func (b Backuper) Do(ctx context.Context, runtimeClient client.Client, shoot v1beta1.Shoot) (RuntimeBackup, error) { + crbs, err := b.getAllCRBs(ctx, runtimeClient) if err != nil { return RuntimeBackup{}, errors.Wrap(err, "failed to get Cluster Role Bindings") } @@ -111,7 +103,7 @@ func (b Backuper) getShootForPatch(shootFromGardener v1beta1.Shoot) v1beta1.Shoo } } -func (b Backuper) getCRBs(ctx context.Context, runtimeClient client.Client) ([]rbacv1.ClusterRoleBinding, error) { +func (b Backuper) getAllCRBs(ctx context.Context, runtimeClient client.Client) ([]rbacv1.ClusterRoleBinding, error) { var crbList rbacv1.ClusterRoleBindingList listCtx, cancel := context.WithTimeout(ctx, b.timeoutK8sOperation) @@ -176,29 +168,3 @@ func (b Backuper) oidcCRDExists(ctx context.Context, runtimeClient client.Client return false, nil } - -func filterOnlySupportedTypesOfCRBs(bindings []rbacv1.ClusterRoleBinding) []rbacv1.ClusterRoleBinding { - return slices.DeleteFunc(bindings, func(clusterRoleBinding rbacv1.ClusterRoleBinding) bool { - if clusterRoleBinding.RoleRef.Kind != "ClusterRole" || clusterRoleBinding.RoleRef.Name != "cluster-admin" { - return true - } - // leave only cluster-admin CRBs where at least one subject is of a user type - if slices.ContainsFunc(clusterRoleBinding.Subjects, func(subject rbacv1.Subject) bool { return subject.Kind == rbacv1.UserKind }) { - return false - } - return true - }) -} - -func labelsCRBsAsDeprecated(ctx context.Context, runtimeClient client.Client, deprecatedCRBs []rbacv1.ClusterRoleBinding) error { - for _, clusterRoleBinding := range deprecatedCRBs { - clusterRoleBinding.ObjectMeta.Labels["kyma-project.io/deprecation"] = "to-be-removed-soon" - err := runtimeClient.Patch(ctx, &clusterRoleBinding, client.Apply, &client.PatchOptions{}) - - if err != nil { - return errors.Wrap(err, fmt.Sprintf("Failed to update ClusterRoleBinding with deprecation label %s", clusterRoleBinding.Name)) - } - slog.Info(fmt.Sprintf("ClusterRoleBinding %s has been labeled as deprecated", clusterRoleBinding.Name)) - } - return nil -} diff --git a/hack/runtime-migrator/internal/backup/results.go b/hack/runtime-migrator/internal/backup/results.go index 46489c27..2897d14e 100644 --- a/hack/runtime-migrator/internal/backup/results.go +++ b/hack/runtime-migrator/internal/backup/results.go @@ -2,6 +2,7 @@ package backup import ( "fmt" + v12 "k8s.io/api/rbac/v1" ) type StatusType string @@ -12,11 +13,12 @@ const ( ) type RuntimeResult struct { - RuntimeID string `json:"runtimeId"` - ShootName string `json:"shootName"` - Status StatusType `json:"status"` - ErrorMessage string `json:"errorMessage,omitempty"` - BackupDirPath string `json:"backupDirPath,omitempty"` + RuntimeID string `json:"runtimeId"` + ShootName string `json:"shootName"` + Status StatusType `json:"status"` + ErrorMessage string `json:"errorMessage,omitempty"` + BackupDirPath string `json:"backupDirPath,omitempty"` + DeprecatedCRBs []string `json:"deprecatedCRBs,omitempty"` } type Results struct { @@ -45,12 +47,19 @@ func (br *Results) ErrorOccurred(runtimeID, shootName string, errorMsg string) { br.Results = append(br.Results, result) } -func (br *Results) OperationSucceeded(runtimeID string, shootName string) { +func (br *Results) OperationSucceeded(runtimeID string, shootName string, deprecatedCRBs []v12.ClusterRoleBinding) { + + var deprecatedCRBsString []string + for _, crb := range deprecatedCRBs { + deprecatedCRBsString = append(deprecatedCRBsString, crb.Name) + } + result := RuntimeResult{ - RuntimeID: runtimeID, - ShootName: shootName, - Status: StatusSuccess, - BackupDirPath: br.getBackupDirPath(runtimeID), + RuntimeID: runtimeID, + ShootName: shootName, + Status: StatusSuccess, + BackupDirPath: br.getBackupDirPath(runtimeID), + DeprecatedCRBs: deprecatedCRBsString, } br.Succeeded++ diff --git a/hack/runtime-migrator/internal/restore/results.go b/hack/runtime-migrator/internal/restore/results.go index 30479331..8a54264d 100644 --- a/hack/runtime-migrator/internal/restore/results.go +++ b/hack/runtime-migrator/internal/restore/results.go @@ -18,8 +18,8 @@ type RuntimeResult struct { ShootName string `json:"shootName"` Status StatusType `json:"status"` ErrorMessage string `json:"errorMessage,omitempty"` - RestoredCRBs []string `json:"restoredCRBs"` - RestoredOIDCs []string `json:"restoredOIDCs"` + RestoredCRBs []string `json:"restoredCRBs,omitempty"` + RestoredOIDCs []string `json:"restoredOIDCs,omitempty"` } type Results struct { @@ -51,12 +51,12 @@ func (rr *Results) ErrorOccurred(runtimeID, shootName string, errorMsg string) { func (rr *Results) OperationSucceeded(runtimeID string, shootName string, appliedCRBs []v12.ClusterRoleBinding, appliedOIDCs []authenticationv1alpha1.OpenIDConnect) { - appliedCRBsString := make([]string, 0) + var appliedCRBsString []string for _, crb := range appliedCRBs { appliedCRBsString = append(appliedCRBsString, crb.Name) } - appliedOIDCsString := make([]string, 0) + var appliedOIDCsString []string for _, oidc := range appliedOIDCs { appliedOIDCsString = append(appliedOIDCsString, oidc.Name) } From c3ec246cbfa6976afe52e004dc999a7bd7188dbd Mon Sep 17 00:00:00 2001 From: Arkadiusz Galwas Date: Thu, 9 Jan 2025 08:51:05 +0100 Subject: [PATCH 6/7] Runtime is switched to KIM after backup --- hack/runtime-migrator/cmd/backup/backup.go | 48 +++++++++++++++++-- hack/runtime-migrator/cmd/backup/main.go | 8 ++-- .../internal/backup/results.go | 26 +++++----- .../internal/initialisation/params.go | 27 +++++++++++ 4 files changed, 89 insertions(+), 20 deletions(-) diff --git a/hack/runtime-migrator/cmd/backup/backup.go b/hack/runtime-migrator/cmd/backup/backup.go index ca7b9ba2..d04a89d6 100644 --- a/hack/runtime-migrator/cmd/backup/backup.go +++ b/hack/runtime-migrator/cmd/backup/backup.go @@ -4,6 +4,7 @@ import ( "context" "fmt" gardener_types "github.com/gardener/gardener/pkg/client/core/clientset/versioned/typed/core/v1beta1" + runtimev1 "github.com/kyma-project/infrastructure-manager/api/v1" "github.com/kyma-project/infrastructure-manager/hack/runtime-migrator-app/internal/backup" "github.com/kyma-project/infrastructure-manager/hack/runtime-migrator-app/internal/initialisation" "github.com/kyma-project/infrastructure-manager/hack/runtime-migrator-app/internal/shoot" @@ -12,6 +13,7 @@ import ( rbacv1 "k8s.io/api/rbac/v1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/types" "log/slog" "sigs.k8s.io/controller-runtime/pkg/client" "slices" @@ -29,10 +31,10 @@ type Backup struct { kcpClient client.Client outputWriter backup.OutputWriter results backup.Results - cfg initialisation.Config + cfg initialisation.BackupConfig } -func NewBackup(cfg initialisation.Config, kcpClient client.Client, shootClient gardener_types.ShootInterface) (Backup, error) { +func NewBackup(cfg initialisation.BackupConfig, kcpClient client.Client, shootClient gardener_types.ShootInterface) (Backup, error) { outputWriter, err := backup.NewOutputWriter(cfg.OutputPath) if err != nil { return Backup{}, err @@ -77,6 +79,10 @@ func (b Backup) Do(ctx context.Context, runtimeIDs []string) error { runtimeClient, err := initialisation.GetRuntimeClient(ctx, b.kcpClient, runtimeID) if err != nil { + errMsg := fmt.Sprintf("Failed to get kubernetes client for runtime: %v", err) + b.results.ErrorOccurred(runtimeID, shootToBackup.Name, errMsg) + slog.Error(errMsg, "runtimeID", runtimeID) + continue } runtimeBackup, err := backuper.Do(ctx, runtimeClient, *shootToBackup) @@ -90,7 +96,7 @@ func (b Backup) Do(ctx context.Context, runtimeIDs []string) error { if b.cfg.IsDryRun { slog.Info("Runtime processed successfully (dry-run)", "runtimeID", runtimeID) - b.results.OperationSucceeded(runtimeID, shootToBackup.Name, nil) + b.results.OperationSucceeded(runtimeID, shootToBackup.Name, nil, false) continue } @@ -112,8 +118,17 @@ func (b Backup) Do(ctx context.Context, runtimeIDs []string) error { continue } + if b.cfg.SetControlledByKim { + err := setControlledByKim(ctx, b.kcpClient, runtimeID) + if err != nil { + errMsg := fmt.Sprintf("Failed to set the rutnime to be controlled by KIM: %v", err) + b.results.ErrorOccurred(runtimeID, shootToBackup.Name, errMsg) + slog.Error(errMsg, "runtimeID", runtimeID) + } + } + slog.Info("Runtime backup created successfully", "runtimeID", runtimeID) - b.results.OperationSucceeded(runtimeID, shootToBackup.Name, deprecatedCRBs) + b.results.OperationSucceeded(runtimeID, shootToBackup.Name, deprecatedCRBs, b.cfg.SetControlledByKim) } resultsFile, err := b.outputWriter.SaveBackupResults(b.results) @@ -182,3 +197,28 @@ func labelDeprecatedCRBs(ctx context.Context, runtimeClient client.Client) ([]rb return deprecatedCRBs, nil } + +func setControlledByKim(ctx context.Context, kcpClient client.Client, runtimeID string) error { + getCtx, cancelGet := context.WithTimeout(ctx, timeoutK8sOperation) + defer cancelGet() + + key := types.NamespacedName{ + Name: runtimeID, + Namespace: "kcp-system", + } + var runtime runtimev1.Runtime + + err := kcpClient.Get(getCtx, key, &runtime, &client.GetOptions{}) + if err != nil { + return err + } + + runtime.Labels["kyma-project.io/controlled-by-provisioner"] = "false" + + patchCtx, cancelPatch := context.WithTimeout(ctx, timeoutK8sOperation) + defer cancelPatch() + + return kcpClient.Patch(patchCtx, &runtime, client.Apply, &client.PatchOptions{ + FieldManager: fieldManagerName, + }) +} diff --git a/hack/runtime-migrator/cmd/backup/main.go b/hack/runtime-migrator/cmd/backup/main.go index e7af47b8..d9d191c1 100644 --- a/hack/runtime-migrator/cmd/backup/main.go +++ b/hack/runtime-migrator/cmd/backup/main.go @@ -12,9 +12,9 @@ import ( func main() { slog.Info("Starting runtime-backuper") - cfg := initialisation.NewConfig() + cfg := initialisation.NewBackupConfig() - initialisation.PrintConfig(cfg) + initialisation.PrintBackupConfig(cfg) opts := zap.Options{ Development: true, @@ -30,7 +30,7 @@ func main() { os.Exit(1) } - kcpClient, err := initialisation.CreateKcpClient(&cfg) + kcpClient, err := initialisation.CreateKcpClient(&cfg.Config) if err != nil { slog.Error("Failed to create kcp client", slog.Any("error", err)) os.Exit(1) @@ -43,7 +43,7 @@ func main() { } slog.Info("Reading runtimeIds from input file") - runtimeIds, err := initialisation.GetRuntimeIDsFromInputFile(cfg) + runtimeIds, err := initialisation.GetRuntimeIDsFromInputFile(cfg.Config) if err != nil { slog.Error("Failed to read runtime Ids from input", slog.Any("error", err)) os.Exit(1) diff --git a/hack/runtime-migrator/internal/backup/results.go b/hack/runtime-migrator/internal/backup/results.go index 2897d14e..101b5f44 100644 --- a/hack/runtime-migrator/internal/backup/results.go +++ b/hack/runtime-migrator/internal/backup/results.go @@ -13,12 +13,13 @@ const ( ) type RuntimeResult struct { - RuntimeID string `json:"runtimeId"` - ShootName string `json:"shootName"` - Status StatusType `json:"status"` - ErrorMessage string `json:"errorMessage,omitempty"` - BackupDirPath string `json:"backupDirPath,omitempty"` - DeprecatedCRBs []string `json:"deprecatedCRBs,omitempty"` + RuntimeID string `json:"runtimeId"` + ShootName string `json:"shootName"` + Status StatusType `json:"status"` + ErrorMessage string `json:"errorMessage,omitempty"` + BackupDirPath string `json:"backupDirPath,omitempty"` + DeprecatedCRBs []string `json:"deprecatedCRBs,omitempty"` + SetControlledByKIM bool `json:"setControlledByKIM"` } type Results struct { @@ -47,7 +48,7 @@ func (br *Results) ErrorOccurred(runtimeID, shootName string, errorMsg string) { br.Results = append(br.Results, result) } -func (br *Results) OperationSucceeded(runtimeID string, shootName string, deprecatedCRBs []v12.ClusterRoleBinding) { +func (br *Results) OperationSucceeded(runtimeID string, shootName string, deprecatedCRBs []v12.ClusterRoleBinding, setControlledByKIM bool) { var deprecatedCRBsString []string for _, crb := range deprecatedCRBs { @@ -55,11 +56,12 @@ func (br *Results) OperationSucceeded(runtimeID string, shootName string, deprec } result := RuntimeResult{ - RuntimeID: runtimeID, - ShootName: shootName, - Status: StatusSuccess, - BackupDirPath: br.getBackupDirPath(runtimeID), - DeprecatedCRBs: deprecatedCRBsString, + RuntimeID: runtimeID, + ShootName: shootName, + Status: StatusSuccess, + BackupDirPath: br.getBackupDirPath(runtimeID), + DeprecatedCRBs: deprecatedCRBsString, + SetControlledByKIM: setControlledByKIM, } br.Succeeded++ diff --git a/hack/runtime-migrator/internal/initialisation/params.go b/hack/runtime-migrator/internal/initialisation/params.go index 1c503c87..f7401683 100644 --- a/hack/runtime-migrator/internal/initialisation/params.go +++ b/hack/runtime-migrator/internal/initialisation/params.go @@ -80,6 +80,33 @@ func GetRuntimeIDsFromInputFile(cfg Config) ([]string, error) { return runtimeIDs, err } +type BackupConfig struct { + Config + SetControlledByKim bool +} + +func NewBackupConfig() BackupConfig { + backupConfig := BackupConfig{} + + flag.StringVar(&backupConfig.KcpKubeconfigPath, "kcp-kubeconfig-path", "/path/to/kcp/kubeconfig", "Path to the Kubeconfig file of KCP cluster.") + flag.StringVar(&backupConfig.GardenerKubeconfigPath, "gardener-kubeconfig-path", "/path/to/gardener/kubeconfig", "Kubeconfig file for Gardener cluster.") + flag.StringVar(&backupConfig.GardenerProjectName, "gardener-project-name", "gardener-project-name", "Name of the Gardener project.") + flag.StringVar(&backupConfig.OutputPath, "output-path", "/tmp/", "Path where generated yamls will be saved. Directory has to exist.") + flag.BoolVar(&backupConfig.IsDryRun, "dry-run", true, "Dry-run flag. Has to be set to 'false' otherwise it will not apply the Custom Resources on the KCP cluster.") + flag.StringVar(&backupConfig.InputType, "input-type", InputTypeJSON, "Type of input to be used. Possible values: **txt** (see the example hack/runtime-migrator/input/runtimeids_sample.txt), and **json** (see the example hack/runtime-migrator/input/runtimeids_sample.json).") + flag.StringVar(&backupConfig.InputFilePath, "input-file-path", "/path/to/input/file", "Path to the input file containing RuntimeCRs to be migrated.") + flag.BoolVar(&backupConfig.SetControlledByKim, "set-controlled-by-kim", false, "Flag determining whether Runtime CR should be modified to be controlled by KIM.") + + flag.Parse() + + return backupConfig +} + +func PrintBackupConfig(cfg BackupConfig) { + PrintConfig(cfg.Config) + log.Println("set-controlled-by-kim:", cfg.SetControlledByKim) +} + type RestoreConfig struct { Config BackupDir string From 7941c0cafc4c1b2cf9c2a8ce37a16f79b88b51c8 Mon Sep 17 00:00:00 2001 From: Arkadiusz Galwas Date: Thu, 9 Jan 2025 12:04:18 +0100 Subject: [PATCH 7/7] Changes to the migrator: setting only fields set by KEB --- .../internal/backup/backup.go | 9 ++++++- .../internal/migration/migrator.go | 10 ++++---- .../internal/shoot/workers.go | 25 +++++++++++++++++++ 3 files changed, 38 insertions(+), 6 deletions(-) create mode 100644 hack/runtime-migrator/internal/shoot/workers.go diff --git a/hack/runtime-migrator/internal/backup/backup.go b/hack/runtime-migrator/internal/backup/backup.go index f0909f5c..464f4925 100644 --- a/hack/runtime-migrator/internal/backup/backup.go +++ b/hack/runtime-migrator/internal/backup/backup.go @@ -5,6 +5,7 @@ import ( "github.com/gardener/gardener/pkg/apis/core/v1beta1" authenticationv1alpha1 "github.com/gardener/oidc-webhook-authenticator/apis/authentication/v1alpha1" "github.com/kyma-project/infrastructure-manager/hack/runtime-migrator-app/internal/initialisation" + "github.com/kyma-project/infrastructure-manager/hack/runtime-migrator-app/internal/shoot" "github.com/kyma-project/infrastructure-manager/pkg/gardener/kubeconfig" "github.com/pkg/errors" rbacv1 "k8s.io/api/rbac/v1" @@ -93,7 +94,13 @@ func (b Backuper) getShootForPatch(shootFromGardener v1beta1.Shoot) v1beta1.Shoo Services: shootFromGardener.Spec.Networking.Services, }, // TODO: consider if we need to do the backup selectively (workers) - Provider: shootFromGardener.Spec.Provider, + Provider: v1beta1.Provider{ + Type: shootFromGardener.Spec.Provider.Type, + ControlPlaneConfig: shootFromGardener.Spec.Provider.ControlPlaneConfig, + InfrastructureConfig: shootFromGardener.Spec.Provider.InfrastructureConfig, + Workers: shoot.FilterOutFields(shootFromGardener.Spec.Provider.Workers), + WorkersSettings: shootFromGardener.Spec.Provider.WorkersSettings, + }, Purpose: shootFromGardener.Spec.Purpose, Region: shootFromGardener.Spec.Region, Resources: shootFromGardener.Spec.Resources, diff --git a/hack/runtime-migrator/internal/migration/migrator.go b/hack/runtime-migrator/internal/migration/migrator.go index 95c17857..c70c3b0b 100644 --- a/hack/runtime-migrator/internal/migration/migrator.go +++ b/hack/runtime-migrator/internal/migration/migrator.go @@ -6,6 +6,7 @@ import ( "github.com/gardener/gardener/pkg/apis/core/v1beta1" v1 "github.com/kyma-project/infrastructure-manager/api/v1" "github.com/kyma-project/infrastructure-manager/hack/runtime-migrator-app/internal/initialisation" + shootutil "github.com/kyma-project/infrastructure-manager/hack/runtime-migrator-app/internal/shoot" "github.com/kyma-project/infrastructure-manager/pkg/config" "github.com/kyma-project/infrastructure-manager/pkg/gardener/kubeconfig" "github.com/pkg/errors" @@ -60,10 +61,9 @@ func (m Migrator) Do(ctx context.Context, shoot v1beta1.Shoot) (v1.Runtime, erro APIVersion: "infrastructuremanager.kyma-project.io/v1", }, ObjectMeta: metav1.ObjectMeta{ - Name: labels["kyma-project.io/runtime-id"], - Namespace: "kcp-system", - Labels: labels, - Annotations: shoot.Annotations, + Name: labels["kyma-project.io/runtime-id"], + Namespace: "kcp-system", + Labels: labels, }, Spec: v1.RuntimeSpec{ Shoot: v1.RuntimeShoot{ @@ -81,7 +81,7 @@ func (m Migrator) Do(ctx context.Context, shoot v1beta1.Shoot) (v1.Runtime, erro }, Provider: v1.Provider{ Type: shoot.Spec.Provider.Type, - Workers: shoot.Spec.Provider.Workers, + Workers: shootutil.FilterOutFields(shoot.Spec.Provider.Workers), ControlPlaneConfig: shoot.Spec.Provider.ControlPlaneConfig, InfrastructureConfig: shoot.Spec.Provider.InfrastructureConfig, }, diff --git a/hack/runtime-migrator/internal/shoot/workers.go b/hack/runtime-migrator/internal/shoot/workers.go new file mode 100644 index 00000000..ddf4ea72 --- /dev/null +++ b/hack/runtime-migrator/internal/shoot/workers.go @@ -0,0 +1,25 @@ +package shoot + +import "github.com/gardener/gardener/pkg/apis/core/v1beta1" + +// FilterOutFields creates a new slice with workers containing only the fields KEB sets +func FilterOutFields(workers []v1beta1.Worker) []v1beta1.Worker { + newWorkers := make([]v1beta1.Worker, 0) + + for _, worker := range workers { + newWorker := v1beta1.Worker{ + Machine: worker.Machine, + Maximum: worker.Maximum, + Minimum: worker.Minimum, + MaxSurge: worker.MaxSurge, + MaxUnavailable: worker.MaxUnavailable, + Name: worker.Name, + Volume: worker.Volume, + Zones: worker.Zones, + } + + newWorkers = append(newWorkers, newWorker) + } + + return newWorkers +}