Refactor provisioner tests env (#3424)

* Bump Go in github action, remove redundant caching * Bump buildpack * Change test environment Now tests are run without buildpack by default. This changes DB code to remove special handling of test environment. * Update e2e provision request * Improve Makefile aesthetically * Fix test command in docs
kyma-project · Apr 22, 2024 · a798597 · a798597
1 parent 438ebf8
commit a798597
Show file tree

Hide file tree

Showing 8 changed files with 92 additions and 106 deletions.
diff --git a/.github/workflows/provisioner-e2e-test.yaml b/.github/workflows/provisioner-e2e-test.yaml
@@ -14,15 +14,12 @@ jobs:
     steps:
       - name: Checkout code
         uses: actions/checkout@v3
-      - name: Set up cache
-        uses: actions/cache@v3
+      - name: Set up go environment
+        uses: actions/setup-go@v5
         with:
-          path: |
-            ~/.cache/go-build
-            ~/go/pkg/mod
-            /home/runner/work/common/bin
-          key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
-          restore-keys: |
-            ${{ runner.os }}-go-
+          go-version-file: 'components/provisioner/go.mod'
+          cache-dependency-path: 'components/provisioner/go.mod'
       - name: Run unit tests
-        run: make -C components/provisioner verify
+        run: |
+          mkdir -p /home/runner/go/pkg/mod
+          make -C components/provisioner verify
diff --git a/components/provisioner/Makefile b/components/provisioner/Makefile
@@ -1,7 +1,7 @@
 APP_NAME = provisioner
 APP_PATH = components/provisioner
 ENTRYPOINT = cmd/main.go
-BUILDPACK = eu.gcr.io/kyma-project/test-infra/buildpack-golang:v20220809-002bc8cf3
+BUILDPACK = europe-docker.pkg.dev/kyma-project/prod/testimages/buildpack-go:v20240404-de49794e
 SCRIPTS_DIR = $(realpath $(shell pwd)/../..)/scripts
 DOCKER_SOCKET = /var/run/docker.sock
 TESTING_DB_NETWORK = test_network
@@ -10,9 +10,9 @@ export PIPELINE_BUILD
 
 include $(SCRIPTS_DIR)/generic_make_go.mk
 
-.PHONY: gqlgen check-gqlgen testing-with-database-network clean-up mod-verify go-mod-check
+.PHONY: gqlgen check-gqlgen test-really-local mod-verify go-mod-check
 
-verify:: gqlgen check-gqlgen testing-with-database-network mod-verify go-mod-check
+verify:: gqlgen check-gqlgen test-really-local mod-verify go-mod-check
 
 resolve-local:
 	GO111MODULE=on go mod vendor -v
@@ -38,10 +38,12 @@ go-mod-check-local:
 		exit 1; \
 	fi;
 
+# Check imports doesn't work with current buildpack,
+# and the build won't pass anyway, if imports aren't correct
+check-imports-local: ;
 
 # We have to override test-local and errcheck, because we need to run provisioner with database
-#as docker container connected with custom network and the buildpack container itsefl has to be connected to the network
-
+# as docker container connected with custom network and the buildpack container itsefl has to be connected to the network
 test-local: ;
 
 errcheck-local: ;
@@ -50,21 +52,15 @@ errcheck-local: ;
 # 		-v $(COMPONENT_DIR):$(WORKSPACE_COMPONENT_DIR):delegated \
 # 		$(DOCKER_CREATE_OPTS) errcheck -blank -asserts -ignorepkg '$$($(DIRS_TO_CHECK) | tr '\n' ',')' -ignoregenerated ./...
 
+LOCALBIN ?= $(shell pwd)/bin
+ENVTEST ?= $(LOCALBIN)/setup-envtest
+$(LOCALBIN):
+	mkdir -p $(LOCALBIN)
 
-testing-with-database-network:
-	@echo testing-with-database-network
-	@docker network inspect $(TESTING_DB_NETWORK) >/dev/null 2>&1 || \
-	docker network create --driver bridge $(TESTING_DB_NETWORK)
-	@docker run $(DOCKER_INTERACTIVE) \
-		-v $(DOCKER_SOCKET):$(DOCKER_SOCKET) \
-		-v $(COMPONENT_DIR)/../../:$(WORKSPACE_COMPONENT_DIR)/../../ \
-		--network=$(TESTING_DB_NETWORK) \
-		-v $(COMPONENT_DIR):$(WORKSPACE_COMPONENT_DIR):delegated \
-		--env PIPELINE_BUILD=1 --env GO111MODULE=on \
-		--env TC_HOST=host.docker.internal \
-		--add-host host.docker.internal:host-gateway \
-		$(DOCKER_CREATE_OPTS) go test ./... -timeout 15m
-	@docker network rm $(TESTING_DB_NETWORK)
+.PHONY: envtest
+envtest: $(ENVTEST) ## Download envtest-setup locally if necessary.
+$(ENVTEST): $(LOCALBIN)
+	test -s $(LOCALBIN)/setup-envtest || GOBIN=$(LOCALBIN) go install sigs.k8s.io/controller-runtime/tools/setup-envtest@latest
 
-clean-up:
-	@docker network rm $(TESTING_DB_NETWORK)
+test-really-local: envtest
+	KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test ./... -timeout 15m
diff --git a/components/provisioner/README.md b/components/provisioner/README.md
@@ -21,7 +21,7 @@ Runtime Provisioner also needs a kubeconfig for a garden and auth data for Direc
 ## Development
 
 ### Testing
-Use `make verify` in the `/components/provisioner/` directory to run the unit and integration tests, or `/components/provisioner/e2e_test/test.sh` to run the e2e test. For the e2e, you must set up a few environmental variables, and the test output will guide you on what's missing. 
+Use `make test-really-local` in the `/components/provisioner/` directory to run the unit and integration tests, or `/components/provisioner/e2e_test/test.sh` to run the e2e test. For the e2e, you must set up a few environmental variables, and the test output will guide you on what's missing. 
 
 ### GraphQL schema
 

diff --git a/components/provisioner/e2e_test/main_test.go b/components/provisioner/e2e_test/main_test.go
@@ -165,7 +165,7 @@ func (gql GQLClient) waitForOp(ctx context.Context, operationID string) (resp Op
 type testConfig struct {
 	ProviderSecret    string `envconfig:"GARDENER_SECRET_NAME"`
 	Provider          string `envconfig:"GARDENER_PROVIDER,default=gcp"`
-	KubernetesVersion string `envconfig:"default=1.26.11"`
+	KubernetesVersion string `envconfig:"default=1.29.2"`
 	MachineType       string `envconfig:"default=e2-medium"`
 	DiskType          string `envconfig:"default=pd-balanced"`
 	Region            string `envconfig:"default=europe-west3"`

diff --git a/components/provisioner/e2e_test/requests/provision.graphql b/components/provisioner/e2e_test/requests/provision.graphql
@@ -1,42 +1,42 @@
 mutation ($name: String!
-  , $provider: String!
-  , $providerSecret: String!
-  , $kubernetesVersion: String!
-  , $machineType: String!
-  , $diskType: String!
-  , $region: String!) {
-    provisionRuntime(
-      config: {
+	, $provider: String!
+	, $providerSecret: String!
+	, $kubernetesVersion: String!
+	, $machineType: String!
+	, $diskType: String!
+	, $region: String!) {
+		provisionRuntime(
+			config: {
 	runtimeInput: { name: $name, description: "" }
 	clusterConfig: {
-	  gardenerConfig: {
-	    name: $name
-	    kubernetesVersion: $kubernetesVersion
-	    diskType: $diskType
-	    volumeSizeGB: 32
-	    machineType: $machineType
-	    region: $region
-	    provider: $provider
-	    targetSecret: $providerSecret
-	    workerCidr: "10.250.0.0/19"
-        podsCidr: "10.64.0.0/11"
-        servicesCidr: "10.243.0.0/16"
-	    autoScalerMin: 2
-	    autoScalerMax: 4
-	    maxSurge: 4
-	    maxUnavailable: 1
-	    providerSpecificConfig: {
-	      azureConfig: { vnetCidr: "10.250.0.0/19" }
-	      gcpConfig: {zones: ["europe-west3-b"]}
-	    }
-	  }
+		gardenerConfig: {
+			name: $name
+			kubernetesVersion: $kubernetesVersion
+			diskType: $diskType
+			volumeSizeGB: 32
+			machineType: $machineType
+			region: $region
+			provider: $provider
+			targetSecret: $providerSecret
+			podsCidr: "10.60.0.0/16"
+			workerCidr: "10.250.0.0/16"
+			servicesCidr: "10.240.0.0/16"
+			autoScalerMin: 2
+			autoScalerMax: 4
+			maxSurge: 4
+			maxUnavailable: 1
+			providerSpecificConfig: {
+				azureConfig: { vnetCidr: "10.250.0.0/19" }
+				gcpConfig: {zones: ["europe-west3-b"]}
+			}
+		}
+	}
+			}
+		) {
+			id
+			operation
+			state
+			message
+			runtimeID
+		}
 	}
-      }
-    ) {
-      id
-      operation
-      state
-      message
-      runtimeID
-    }
-  }
diff --git a/components/provisioner/internal/api/resolver_integration_with_gardener_test.go b/components/provisioner/internal/api/resolver_integration_with_gardener_test.go
@@ -99,16 +99,14 @@ users:
 `
 )
 
+const schemaFilePath = "../../assets/database/provisioner.sql"
+
 func TestProvisioning_ProvisionRuntimeWithDatabase(t *testing.T) {
 	//given
 	ctx := context.WithValue(context.Background(), middlewares.Tenant, tenant)
 	ctx = context.WithValue(ctx, middlewares.SubAccountID, subAccountId)
 
-	cleanupNetwork, err := testutils.EnsureTestNetworkForDB(t, ctx)
-	require.NoError(t, err)
-	defer cleanupNetwork()
-
-	containerCleanupFunc, connString, err := testutils.InitTestDBContainer(t, ctx, "postgres_database_2")
+	containerCleanupFunc, connString, err := testutils.InitTestDBContainer(t, ctx)
 	require.NoError(t, err)
 	defer containerCleanupFunc()
 
@@ -117,7 +115,7 @@ func TestProvisioning_ProvisionRuntimeWithDatabase(t *testing.T) {
 	require.NotNil(t, connection)
 	defer testutils.CloseDatabase(t, connection)
 
-	err = database.SetupSchema(connection, testutils.SchemaFilePath)
+	err = database.SetupSchema(connection, schemaFilePath)
 	require.NoError(t, err)
 
 	directorServiceMock := &directormock.DirectorClient{}

diff --git a/components/provisioner/internal/persistence/database/schema_initializer_test.go b/components/provisioner/internal/persistence/database/schema_initializer_test.go
@@ -9,17 +9,15 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
+const schemaFilePath = "../../../assets/database/provisioner.sql"
+
 func TestSchemaInitializer(t *testing.T) {
 
 	ctx := context.Background()
 
-	cleanupNetwork, err := testutils.EnsureTestNetworkForDB(t, ctx)
-	require.NoError(t, err)
-	defer cleanupNetwork()
-
 	t.Run("Should initialize database when schema not applied", func(t *testing.T) {
 		// given
-		containerCleanupFunc, connString, err := testutils.InitTestDBContainer(t, ctx, "test_DB_1")
+		containerCleanupFunc, connString, err := testutils.InitTestDBContainer(t, ctx)
 		require.NoError(t, err)
 
 		defer containerCleanupFunc()
@@ -31,7 +29,7 @@ func TestSchemaInitializer(t *testing.T) {
 
 		defer testutils.CloseDatabase(t, connection)
 
-		err = SetupSchema(connection, testutils.SchemaFilePath)
+		err = SetupSchema(connection, schemaFilePath)
 		require.NoError(t, err)
 
 		// then
@@ -42,7 +40,7 @@ func TestSchemaInitializer(t *testing.T) {
 
 	t.Run("Should skip database initialization when schema already applied", func(t *testing.T) {
 		//given
-		containerCleanupFunc, connString, err := testutils.InitTestDBContainer(t, ctx, "test_DB_2")
+		containerCleanupFunc, connString, err := testutils.InitTestDBContainer(t, ctx)
 		require.NoError(t, err)
 
 		defer containerCleanupFunc()
@@ -54,10 +52,10 @@ func TestSchemaInitializer(t *testing.T) {
 
 		defer testutils.CloseDatabase(t, connection)
 
-		err = SetupSchema(connection, testutils.SchemaFilePath)
+		err = SetupSchema(connection, schemaFilePath)
 		require.NoError(t, err)
 
-		err = SetupSchema(connection, testutils.SchemaFilePath)
+		err = SetupSchema(connection, schemaFilePath)
 		require.NoError(t, err)
 
 		// then
@@ -68,7 +66,7 @@ func TestSchemaInitializer(t *testing.T) {
 
 	t.Run("Should return error when failed to connect to the database", func(t *testing.T) {
 
-		containerCleanupFunc, _, err := testutils.InitTestDBContainer(t, ctx, "test_DB_3")
+		containerCleanupFunc, _, err := testutils.InitTestDBContainer(t, ctx)
 		require.NoError(t, err)
 
 		defer containerCleanupFunc()
@@ -86,7 +84,7 @@ func TestSchemaInitializer(t *testing.T) {
 
 	t.Run("Should return error when failed to read database schema", func(t *testing.T) {
 		//given
-		containerCleanupFunc, connString, err := testutils.InitTestDBContainer(t, ctx, "test_DB_4")
+		containerCleanupFunc, connString, err := testutils.InitTestDBContainer(t, ctx)
 		require.NoError(t, err)
 
 		defer containerCleanupFunc()

diff --git a/components/provisioner/internal/persistence/testutils/database.go b/components/provisioner/internal/persistence/testutils/database.go
@@ -43,14 +43,7 @@ var (
 
 func makeConnectionString(hostname string, port string) string {
 
-	host := "localhost"
-
-	if os.Getenv(EnvPipelineBuild) != "" {
-		host = hostname
-		port = DbPort
-	}
-
-	return fmt.Sprintf(connStringFormat, host, port, DbUser,
+	return fmt.Sprintf(connStringFormat, hostname, port, DbUser,
 		DbPass, DbName, "disable")
 }
 
@@ -62,22 +55,16 @@ func CloseDatabase(t *testing.T, connection *dbr.Connection) {
 	}
 }
 
-func InitTestDBContainer(t *testing.T, ctx context.Context, hostname string) (func(), string, error) {
-
+func InitTestDBContainer(t *testing.T, ctx context.Context) (func(), string, error) {
 	_, err := isDockerTestNetworkPresent(ctx)
 
 	if err != nil {
 		return nil, "", err
 	}
 
 	req := testcontainers.ContainerRequest{
-		Image:        "postgres:11",
-		SkipReaper:   true,
-		ExposedPorts: []string{fmt.Sprintf("%s", DbPort)},
-		Networks:     []string{DockerUserNetwork},
-		NetworkAliases: map[string][]string{
-			DockerUserNetwork: {hostname},
-		},
+		Image:      "postgres:11",
+		SkipReaper: true,
 		Env: map[string]string{
 			"POSTGRES_USER":     DbUser,
 			"POSTGRES_PASSWORD": DbPass,
@@ -106,13 +93,23 @@ func InitTestDBContainer(t *testing.T, ctx context.Context, hostname string) (fu
 		return nil, "", err
 	}
 
+	ip, err := postgresContainer.Host(ctx)
+	if err != nil {
+		t.Logf("Failed to get ip for container %s : %s", postgresContainer.GetContainerID(), err.Error())
+		errTerminate := postgresContainer.Terminate(ctx)
+		if errTerminate != nil {
+			t.Logf("Failed to terminate container %s after failing to get ip: %s", postgresContainer.GetContainerID(), err.Error())
+		}
+		return nil, "", err
+	}
+
 	cleanupFunc := func() {
 		err := postgresContainer.Terminate(ctx)
 		assert.NoError(t, err)
 		time.Sleep(2 * time.Second)
 	}
 
-	connString := makeConnectionString(hostname, port.Port())
+	connString := makeConnectionString(ip, port.Port())
 
 	return cleanupFunc, connString, nil
 }