KCP CLI - List Deprovisioned Instances (#920)

* Add storage layer

* List instances archived by filter

* Remove instances from the operations table

cmd/broker/deprovisioning_test.go

@@ -165,9 +165,9 @@ func TestRuntimesEndpointForDeprovisionedInstance(t *testing.T) {
 	cfg.EDP.Disabled = true
 	suite := NewBrokerSuiteTestWithConfig(t, cfg)
 	defer suite.TearDown()
-	iid := uuid.New().String()
+	iid1 := uuid.New().String()
 
-	resp := suite.CallAPI("PUT", fmt.Sprintf("oauth/cf-eu10/v2/service_instances/%s?accepts_incomplete=true&plan_id=7d55d31d-35ae-4438-bf13-6ffdfa107d9f&service_id=47c9dcbf-ff30-448e-ab36-d3bad66ba281", iid),
+	resp := suite.CallAPI("PUT", fmt.Sprintf("oauth/cf-eu10/v2/service_instances/%s?accepts_incomplete=true&plan_id=7d55d31d-35ae-4438-bf13-6ffdfa107d9f&service_id=47c9dcbf-ff30-448e-ab36-d3bad66ba281", iid1),
 		`{
 				   "service_id": "47c9dcbf-ff30-448e-ab36-d3bad66ba281",
 				   "plan_id": "7d55d31d-35ae-4438-bf13-6ffdfa107d9f",
@@ -190,14 +190,47 @@ func TestRuntimesEndpointForDeprovisionedInstance(t *testing.T) {
 	suite.processProvisioningByOperationID(opID)
 
 	// deprovision
-	resp = suite.CallAPI("DELETE", fmt.Sprintf("oauth/cf-eu10/v2/service_instances/%s?accepts_incomplete=true&plan_id=7d55d31d-35ae-4438-bf13-6ffdfa107d9f&service_id=47c9dcbf-ff30-448e-ab36-d3bad66ba281", iid),
+	resp = suite.CallAPI("DELETE", fmt.Sprintf("oauth/cf-eu10/v2/service_instances/%s?accepts_incomplete=true&plan_id=7d55d31d-35ae-4438-bf13-6ffdfa107d9f&service_id=47c9dcbf-ff30-448e-ab36-d3bad66ba281", iid1),
 		``)
 	depOpID := suite.DecodeOperationID(resp)
 
 	suite.FinishDeprovisioningOperationByProvisioner(depOpID)
-	suite.WaitForOperationsNotExists(iid) // deprovisioning completed, no operations in the DB
+	suite.WaitForOperationsNotExists(iid1) // deprovisioning completed, no operations in the DB
+
+	iid2 := uuid.New().String()
+	resp = suite.CallAPI("PUT", fmt.Sprintf("oauth/cf-eu10/v2/service_instances/%s?accepts_incomplete=true&plan_id=b1a5764e-2ea1-4f95-94c0-2b4538b37b55&service_id=47c9dcbf-ff30-448e-ab36-d3bad66ba281", iid2),
+		`{
+				   "service_id": "47c9dcbf-ff30-448e-ab36-d3bad66ba281",
+				   "plan_id": "b1a5764e-2ea1-4f95-94c0-2b4538b37b55",
+				   "context": {
+					   "sm_operator_credentials": {
+						   "clientid": "cid",
+						   "clientsecret": "cs",
+						   "url": "url",
+						   "sm_url": "sm_url"
+					   },
+					   "globalaccount_id": "g-account-id",
+					   "subaccount_id": "sub-id",
+					   "user_id": "[email protected]"
+				   },
+					"parameters": {
+						"name": "testing-cluster",
+						"region": "eu-central-1"
+				}
+   }`)
+	opID = suite.DecodeOperationID(resp)
+	suite.processProvisioningByOperationID(opID)
+
+	// deprovision
+	resp = suite.CallAPI("DELETE", fmt.Sprintf("oauth/cf-eu10/v2/service_instances/%s?accepts_incomplete=true&plan_id=b1a5764e-2ea1-4f95-94c0-2b4538b37b55&service_id=47c9dcbf-ff30-448e-ab36-d3bad66ba281", iid2),
+		``)
+	depOpID = suite.DecodeOperationID(resp)
+
+	suite.FinishDeprovisioningOperationByProvisioner(depOpID)
+	suite.WaitForOperationsNotExists(iid2) // deprovisioning completed, no operations in the DB
+
 	// when
-	resp = suite.CallAPI("GET", fmt.Sprintf("runtimes?instance_id=%s&state=deprovisioned", iid), "")
+	resp = suite.CallAPI("GET", fmt.Sprintf("runtimes?instance_id=%s&state=deprovisioned", iid1), "")
 
 	// then
 	assert.Equal(t, http.StatusOK, resp.StatusCode)
@@ -208,5 +241,45 @@ func TestRuntimesEndpointForDeprovisionedInstance(t *testing.T) {
 	require.NoError(t, err)
 
 	assert.Len(t, runtimes.Data, 1)
-	assert.Equal(t, iid, runtimes.Data[0].InstanceID)
+	assert.Equal(t, iid1, runtimes.Data[0].InstanceID)
+
+	// when
+	resp = suite.CallAPI("GET", fmt.Sprintf("runtimes?account=%s&subaccount=%s&state=deprovisioned", "g-account-id", "sub-id"), "")
+
+	// then
+	assert.Equal(t, http.StatusOK, resp.StatusCode)
+	response, err = io.ReadAll(resp.Body)
+	assert.NoError(t, err)
+	err = json.Unmarshal(response, &runtimes)
+	require.NoError(t, err)
+
+	assert.Len(t, runtimes.Data, 2)
+	assert.Equal(t, iid1, runtimes.Data[0].InstanceID)
+	assert.Equal(t, iid2, runtimes.Data[1].InstanceID)
+
+	// when
+	resp = suite.CallAPI("GET", fmt.Sprintf("runtimes?plan=%s&state=deprovisioned", "trial"), "")
+
+	// then
+	assert.Equal(t, http.StatusOK, resp.StatusCode)
+	response, err = io.ReadAll(resp.Body)
+	assert.NoError(t, err)
+	err = json.Unmarshal(response, &runtimes)
+	require.NoError(t, err)
+
+	assert.Len(t, runtimes.Data, 1)
+	assert.Equal(t, iid1, runtimes.Data[0].InstanceID)
+
+	// when
+	resp = suite.CallAPI("GET", fmt.Sprintf("runtimes?region=%s&state=deprovisioned", "eu-central-1"), "")
+
+	// then
+	assert.Equal(t, http.StatusOK, resp.StatusCode)
+	response, err = io.ReadAll(resp.Body)
+	assert.NoError(t, err)
+	err = json.Unmarshal(response, &runtimes)
+	require.NoError(t, err)
+
+	assert.Len(t, runtimes.Data, 1)
+	assert.Equal(t, iid2, runtimes.Data[0].InstanceID)
 }

internal/runtime/handler.go

@@ -10,7 +10,6 @@ import (
 
 	"github.com/kyma-project/kyma-environment-broker/internal/provisioner"
 	"github.com/kyma-project/kyma-environment-broker/internal/ptr"
-	"github.com/pivotal-cf/brokerapi/v8/domain"
 	"golang.org/x/exp/slices"
 
 	"github.com/gorilla/mux"
@@ -55,44 +54,6 @@ func (h *Handler) AttachRoutes(router *mux.Router) {
 	router.HandleFunc("/runtimes", h.getRuntimes)
 }
 
-func findLastDeprovisioning(operations []internal.Operation) internal.Operation {
-	for i := len(operations) - 1; i > 0; i-- {
-		o := operations[i]
-		if o.Type != internal.OperationTypeDeprovision {
-			continue
-		}
-		if o.State != domain.Succeeded {
-			continue
-		}
-		return o
-	}
-	return operations[len(operations)-1]
-}
-
-func recreateInstances(operations []internal.Operation) []internal.Instance {
-	byInstance := make(map[string][]internal.Operation)
-	for _, o := range operations {
-		byInstance[o.InstanceID] = append(byInstance[o.InstanceID], o)
-	}
-	var instances []internal.Instance
-	for id, op := range byInstance {
-		o := op[0]
-		last := findLastDeprovisioning(op)
-		instances = append(instances, internal.Instance{
-			InstanceID:      id,
-			GlobalAccountID: o.GlobalAccountID,
-			SubAccountID:    o.SubAccountID,
-			RuntimeID:       o.RuntimeID,
-			CreatedAt:       o.CreatedAt,
-			ServicePlanID:   o.ProvisioningParameters.PlanID,
-			DeletedAt:       last.UpdatedAt,
-			InstanceDetails: last.InstanceDetails,
-			Parameters:      last.ProvisioningParameters,
-		})
-	}
-	return instances
-}
-
 func unionInstances(sets ...[]pkg.RuntimeDTO) (union []pkg.RuntimeDTO) {
 	m := make(map[string]pkg.RuntimeDTO)
 	for _, s := range sets {
@@ -109,47 +70,16 @@ func unionInstances(sets ...[]pkg.RuntimeDTO) (union []pkg.RuntimeDTO) {
 }
 
 func (h *Handler) listInstances(filter dbmodel.InstanceFilter) ([]pkg.RuntimeDTO, int, int, error) {
-	archived := []pkg.RuntimeDTO{}
 	if slices.Contains(filter.States, dbmodel.InstanceDeprovisioned) {
 		// try to list instances where deletion didn't finish successfully
 		// entry in the Instances table still exists but has deletion timestamp and contains list of incomplete steps
 		deletionAttempted := true
 		filter.DeletionAttempted = &deletionAttempted
 		instances, instancesCount, instancesTotalCount, _ := h.instancesDb.List(filter)
 
-		// try to recreate instances from the operations table where entry in the instances table is gone
-		opFilter := dbmodel.OperationFilter{}
-		opFilter.InstanceFilter = &filter
-		opFilter.Page = filter.Page
-		opFilter.PageSize = filter.PageSize
-		operations, _, _, err := h.operationsDb.ListOperations(opFilter)
+		instancesArchived, instancesArchivedCount, instancesArchivedTotalCount, err := h.instancesArchivedDb.List(filter)
 		if err != nil {
-			return []pkg.RuntimeDTO{}, instancesCount, instancesTotalCount, err
-		}
-		instancesFromOperations := recreateInstances(operations)
-
-		if len(instancesFromOperations) == 0 && len(filter.InstanceIDs) == 1 {
-			instanceArchived, err := h.instancesArchivedDb.GetByInstanceID(filter.InstanceIDs[0])
-			if err != nil && !dberr.IsNotFound(err) {
-				return archived, instancesCount, instancesTotalCount, err
-			}
-			instance := h.InstanceFromInstanceArchived(instanceArchived)
-			dto, err := h.converter.NewDTO(instance)
-			dto.Status = pkg.RuntimeStatus{
-				Provisioning: &pkg.Operation{
-					CreatedAt: instanceArchived.ProvisioningStartedAt,
-					UpdatedAt: instanceArchived.ProvisioningFinishedAt,
-					State:     string(instanceArchived.ProvisioningState),
-				},
-				Deprovisioning: &pkg.Operation{
-					UpdatedAt: instanceArchived.LastDeprovisioningFinishedAt,
-				},
-			}
-
-			if err != nil {
-				return archived, instancesCount, instancesTotalCount, err
-			}
-			archived = append(archived, dto)
+			return []pkg.RuntimeDTO{}, instancesArchivedCount, instancesArchivedTotalCount, err
 		}
 
 		// return union of all sets of instances
@@ -161,17 +91,27 @@ func (h *Handler) listInstances(filter dbmodel.InstanceFilter) ([]pkg.RuntimeDTO
 			}
 			instanceDTOs = append(instanceDTOs, dto)
 		}
-		instanceDTOsFromOperations := []pkg.RuntimeDTO{}
-		for _, i := range instancesFromOperations {
-			dto, err := h.converter.NewDTO(i)
+		archived := []pkg.RuntimeDTO{}
+		for _, i := range instancesArchived {
+			instance := h.InstanceFromInstanceArchived(i)
+			dto, err := h.converter.NewDTO(instance)
 			if err != nil {
-				return []pkg.RuntimeDTO{}, instancesCount, instancesTotalCount, err
+				return archived, instancesArchivedCount, instancesArchivedTotalCount, err
+			}
+			dto.Status = pkg.RuntimeStatus{
+				Provisioning: &pkg.Operation{
+					CreatedAt: i.ProvisioningStartedAt,
+					UpdatedAt: i.ProvisioningFinishedAt,
+					State:     string(i.ProvisioningState),
+				},
+				Deprovisioning: &pkg.Operation{
+					UpdatedAt: i.LastDeprovisioningFinishedAt,
+				},
 			}
-			instanceDTOsFromOperations = append(instanceDTOsFromOperations, dto)
+			archived = append(archived, dto)
 		}
-		instancesUnion := unionInstances(instanceDTOs, instanceDTOsFromOperations, archived)
-		count := len(instancesFromOperations)
-		return instancesUnion, count + instancesCount, count + instancesTotalCount, nil
+		instancesUnion := unionInstances(instanceDTOs, archived)
+		return instancesUnion, instancesCount + instancesArchivedCount, instancesTotalCount + instancesArchivedTotalCount, nil
 	}
 
 	var result []pkg.RuntimeDTO

internal/storage/driver/memory/instance_archived.go

@@ -1,8 +1,13 @@
 package memory
 
 import (
+	"sort"
 	"sync"
 
+	"github.com/kyma-project/kyma-environment-broker/common/pagination"
+
+	"github.com/kyma-project/kyma-environment-broker/internal/storage/dbmodel"
+
 	"github.com/pivotal-cf/brokerapi/v8/domain"
 
 	"github.com/kyma-project/kyma-environment-broker/internal"
@@ -60,3 +65,61 @@ func (s *InstanceArchivedInMemoryStorage) TotalNumberOfInstancesArchivedForGloba
 
 	return numberOfInstances, nil
 }
+
+func (s *InstanceArchivedInMemoryStorage) List(filter dbmodel.InstanceFilter) ([]internal.InstanceArchived, int, int, error) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	var toReturn []internal.InstanceArchived
+
+	instancesArchived := s.filterInstancesArchived(filter)
+	sortInstancesArchivedByLastDeprovisioningFinishedAt(instancesArchived)
+
+	offset := pagination.ConvertPageAndPageSizeToOffset(filter.PageSize, filter.Page)
+	for i := offset; (filter.PageSize < 1 || i < offset+filter.PageSize) && i < len(instancesArchived); i++ {
+		toReturn = append(toReturn, s.data[instancesArchived[i].InstanceID])
+	}
+
+	return toReturn, len(toReturn), len(instancesArchived), nil
+}
+
+func (s *InstanceArchivedInMemoryStorage) filterInstancesArchived(filter dbmodel.InstanceFilter) []internal.InstanceArchived {
+	instancesArchived := make([]internal.InstanceArchived, 0, len(s.data))
+	var ok bool
+	equal := func(a, b string) bool {
+		return a == b
+	}
+
+	for _, i := range s.data {
+		if ok = matchFilter(i.InstanceID, filter.InstanceIDs, equal); !ok {
+			continue
+		}
+		if ok = matchFilter(i.GlobalAccountID, filter.GlobalAccountIDs, equal); !ok {
+			continue
+		}
+		if ok = matchFilter(i.SubaccountID, filter.SubAccountIDs, equal); !ok {
+			continue
+		}
+		if ok = matchFilter(i.PlanName, filter.Plans, equal); !ok {
+			continue
+		}
+		if ok = matchFilter(i.Region, filter.Regions, equal); !ok {
+			continue
+		}
+		if ok = matchFilter(i.LastRuntimeID, filter.RuntimeIDs, equal); !ok {
+			continue
+		}
+		if ok = matchFilter(i.ShootName, filter.Shoots, equal); !ok {
+			continue
+		}
+
+		instancesArchived = append(instancesArchived, i)
+	}
+
+	return instancesArchived
+}
+
+func sortInstancesArchivedByLastDeprovisioningFinishedAt(instances []internal.InstanceArchived) {
+	sort.Slice(instances, func(i, j int) bool {
+		return instances[i].LastDeprovisioningFinishedAt.Before(instances[j].LastDeprovisioningFinishedAt)
+	})
+}

internal/storage/driver/postsql/instance_archived.go

@@ -32,3 +32,16 @@ func (s *instanceArchived) TotalNumberOfInstancesArchived() (int, error) {
 func (s *instanceArchived) TotalNumberOfInstancesArchivedForGlobalAccountID(globalAccountID string, planID string) (int, error) {
 	return s.factory.NewReadSession().TotalNumberOfInstancesArchivedForGlobalAccountID(globalAccountID, planID)
 }
+
+func (s *instanceArchived) List(filter dbmodel.InstanceFilter) ([]internal.InstanceArchived, int, int, error) {
+	dtos, count, totalCount, err := s.factory.NewReadSession().ListInstancesArchived(filter)
+	if err != nil {
+		return []internal.InstanceArchived{}, 0, 0, err
+	}
+	var instancesArchived []internal.InstanceArchived
+	for _, dto := range dtos {
+		instanceArchived := dbmodel.NewInstanceArchivedFromDTO(dto)
+		instancesArchived = append(instancesArchived, instanceArchived)
+	}
+	return instancesArchived, count, totalCount, err
+}