From a798597b204fd55f51075dd91a3495074b9c26f4 Mon Sep 17 00:00:00 2001 From: Wojciech Nawa Date: Mon, 22 Apr 2024 09:44:50 +0200 Subject: [PATCH] Refactor provisioner tests env (#3424) * Bump Go in github action, remove redundant caching * Bump buildpack * Change test environment Now tests are run without buildpack by default. This changes DB code to remove special handling of test environment. * Update e2e provision request * Improve Makefile aesthetically * Fix test command in docs --- .github/workflows/provisioner-e2e-test.yaml | 17 ++--- components/provisioner/Makefile | 38 +++++----- components/provisioner/README.md | 2 +- components/provisioner/e2e_test/main_test.go | 2 +- .../e2e_test/requests/provision.graphql | 76 +++++++++---------- ...resolver_integration_with_gardener_test.go | 10 +-- .../database/schema_initializer_test.go | 20 +++-- .../persistence/testutils/database.go | 33 ++++---- 8 files changed, 92 insertions(+), 106 deletions(-) diff --git a/.github/workflows/provisioner-e2e-test.yaml b/.github/workflows/provisioner-e2e-test.yaml index f34e12a3ae..55db3ce72c 100644 --- a/.github/workflows/provisioner-e2e-test.yaml +++ b/.github/workflows/provisioner-e2e-test.yaml @@ -14,15 +14,12 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v3 - - name: Set up cache - uses: actions/cache@v3 + - name: Set up go environment + uses: actions/setup-go@v5 with: - path: | - ~/.cache/go-build - ~/go/pkg/mod - /home/runner/work/common/bin - key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-go- + go-version-file: 'components/provisioner/go.mod' + cache-dependency-path: 'components/provisioner/go.mod' - name: Run unit tests - run: make -C components/provisioner verify + run: | + mkdir -p /home/runner/go/pkg/mod + make -C components/provisioner verify diff --git a/components/provisioner/Makefile b/components/provisioner/Makefile index 03c233574c..a1715d39ac 100644 --- a/components/provisioner/Makefile +++ b/components/provisioner/Makefile @@ -1,7 +1,7 @@ APP_NAME = provisioner APP_PATH = components/provisioner ENTRYPOINT = cmd/main.go -BUILDPACK = eu.gcr.io/kyma-project/test-infra/buildpack-golang:v20220809-002bc8cf3 +BUILDPACK = europe-docker.pkg.dev/kyma-project/prod/testimages/buildpack-go:v20240404-de49794e SCRIPTS_DIR = $(realpath $(shell pwd)/../..)/scripts DOCKER_SOCKET = /var/run/docker.sock TESTING_DB_NETWORK = test_network @@ -10,9 +10,9 @@ export PIPELINE_BUILD include $(SCRIPTS_DIR)/generic_make_go.mk -.PHONY: gqlgen check-gqlgen testing-with-database-network clean-up mod-verify go-mod-check +.PHONY: gqlgen check-gqlgen test-really-local mod-verify go-mod-check -verify:: gqlgen check-gqlgen testing-with-database-network mod-verify go-mod-check +verify:: gqlgen check-gqlgen test-really-local mod-verify go-mod-check resolve-local: GO111MODULE=on go mod vendor -v @@ -38,10 +38,12 @@ go-mod-check-local: exit 1; \ fi; +# Check imports doesn't work with current buildpack, +# and the build won't pass anyway, if imports aren't correct +check-imports-local: ; # We have to override test-local and errcheck, because we need to run provisioner with database -#as docker container connected with custom network and the buildpack container itsefl has to be connected to the network - +# as docker container connected with custom network and the buildpack container itsefl has to be connected to the network test-local: ; errcheck-local: ; @@ -50,21 +52,15 @@ errcheck-local: ; # -v $(COMPONENT_DIR):$(WORKSPACE_COMPONENT_DIR):delegated \ # $(DOCKER_CREATE_OPTS) errcheck -blank -asserts -ignorepkg '$$($(DIRS_TO_CHECK) | tr '\n' ',')' -ignoregenerated ./... +LOCALBIN ?= $(shell pwd)/bin +ENVTEST ?= $(LOCALBIN)/setup-envtest +$(LOCALBIN): + mkdir -p $(LOCALBIN) -testing-with-database-network: - @echo testing-with-database-network - @docker network inspect $(TESTING_DB_NETWORK) >/dev/null 2>&1 || \ - docker network create --driver bridge $(TESTING_DB_NETWORK) - @docker run $(DOCKER_INTERACTIVE) \ - -v $(DOCKER_SOCKET):$(DOCKER_SOCKET) \ - -v $(COMPONENT_DIR)/../../:$(WORKSPACE_COMPONENT_DIR)/../../ \ - --network=$(TESTING_DB_NETWORK) \ - -v $(COMPONENT_DIR):$(WORKSPACE_COMPONENT_DIR):delegated \ - --env PIPELINE_BUILD=1 --env GO111MODULE=on \ - --env TC_HOST=host.docker.internal \ - --add-host host.docker.internal:host-gateway \ - $(DOCKER_CREATE_OPTS) go test ./... -timeout 15m - @docker network rm $(TESTING_DB_NETWORK) +.PHONY: envtest +envtest: $(ENVTEST) ## Download envtest-setup locally if necessary. +$(ENVTEST): $(LOCALBIN) + test -s $(LOCALBIN)/setup-envtest || GOBIN=$(LOCALBIN) go install sigs.k8s.io/controller-runtime/tools/setup-envtest@latest -clean-up: - @docker network rm $(TESTING_DB_NETWORK) +test-really-local: envtest + KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test ./... -timeout 15m diff --git a/components/provisioner/README.md b/components/provisioner/README.md index ded7de2c5c..26e19815b1 100644 --- a/components/provisioner/README.md +++ b/components/provisioner/README.md @@ -21,7 +21,7 @@ Runtime Provisioner also needs a kubeconfig for a garden and auth data for Direc ## Development ### Testing -Use `make verify` in the `/components/provisioner/` directory to run the unit and integration tests, or `/components/provisioner/e2e_test/test.sh` to run the e2e test. For the e2e, you must set up a few environmental variables, and the test output will guide you on what's missing. +Use `make test-really-local` in the `/components/provisioner/` directory to run the unit and integration tests, or `/components/provisioner/e2e_test/test.sh` to run the e2e test. For the e2e, you must set up a few environmental variables, and the test output will guide you on what's missing. ### GraphQL schema diff --git a/components/provisioner/e2e_test/main_test.go b/components/provisioner/e2e_test/main_test.go index 2daa5754a9..60ad0fb772 100644 --- a/components/provisioner/e2e_test/main_test.go +++ b/components/provisioner/e2e_test/main_test.go @@ -165,7 +165,7 @@ func (gql GQLClient) waitForOp(ctx context.Context, operationID string) (resp Op type testConfig struct { ProviderSecret string `envconfig:"GARDENER_SECRET_NAME"` Provider string `envconfig:"GARDENER_PROVIDER,default=gcp"` - KubernetesVersion string `envconfig:"default=1.26.11"` + KubernetesVersion string `envconfig:"default=1.29.2"` MachineType string `envconfig:"default=e2-medium"` DiskType string `envconfig:"default=pd-balanced"` Region string `envconfig:"default=europe-west3"` diff --git a/components/provisioner/e2e_test/requests/provision.graphql b/components/provisioner/e2e_test/requests/provision.graphql index 142128ea37..8aaf2194d5 100644 --- a/components/provisioner/e2e_test/requests/provision.graphql +++ b/components/provisioner/e2e_test/requests/provision.graphql @@ -1,42 +1,42 @@ mutation ($name: String! - , $provider: String! - , $providerSecret: String! - , $kubernetesVersion: String! - , $machineType: String! - , $diskType: String! - , $region: String!) { - provisionRuntime( - config: { + , $provider: String! + , $providerSecret: String! + , $kubernetesVersion: String! + , $machineType: String! + , $diskType: String! + , $region: String!) { + provisionRuntime( + config: { runtimeInput: { name: $name, description: "" } clusterConfig: { - gardenerConfig: { - name: $name - kubernetesVersion: $kubernetesVersion - diskType: $diskType - volumeSizeGB: 32 - machineType: $machineType - region: $region - provider: $provider - targetSecret: $providerSecret - workerCidr: "10.250.0.0/19" - podsCidr: "10.64.0.0/11" - servicesCidr: "10.243.0.0/16" - autoScalerMin: 2 - autoScalerMax: 4 - maxSurge: 4 - maxUnavailable: 1 - providerSpecificConfig: { - azureConfig: { vnetCidr: "10.250.0.0/19" } - gcpConfig: {zones: ["europe-west3-b"]} - } - } + gardenerConfig: { + name: $name + kubernetesVersion: $kubernetesVersion + diskType: $diskType + volumeSizeGB: 32 + machineType: $machineType + region: $region + provider: $provider + targetSecret: $providerSecret + podsCidr: "10.60.0.0/16" + workerCidr: "10.250.0.0/16" + servicesCidr: "10.240.0.0/16" + autoScalerMin: 2 + autoScalerMax: 4 + maxSurge: 4 + maxUnavailable: 1 + providerSpecificConfig: { + azureConfig: { vnetCidr: "10.250.0.0/19" } + gcpConfig: {zones: ["europe-west3-b"]} + } + } + } + } + ) { + id + operation + state + message + runtimeID + } } - } - ) { - id - operation - state - message - runtimeID - } - } diff --git a/components/provisioner/internal/api/resolver_integration_with_gardener_test.go b/components/provisioner/internal/api/resolver_integration_with_gardener_test.go index 9e26b7a8ef..d6a724339b 100644 --- a/components/provisioner/internal/api/resolver_integration_with_gardener_test.go +++ b/components/provisioner/internal/api/resolver_integration_with_gardener_test.go @@ -99,16 +99,14 @@ users: ` ) +const schemaFilePath = "../../assets/database/provisioner.sql" + func TestProvisioning_ProvisionRuntimeWithDatabase(t *testing.T) { //given ctx := context.WithValue(context.Background(), middlewares.Tenant, tenant) ctx = context.WithValue(ctx, middlewares.SubAccountID, subAccountId) - cleanupNetwork, err := testutils.EnsureTestNetworkForDB(t, ctx) - require.NoError(t, err) - defer cleanupNetwork() - - containerCleanupFunc, connString, err := testutils.InitTestDBContainer(t, ctx, "postgres_database_2") + containerCleanupFunc, connString, err := testutils.InitTestDBContainer(t, ctx) require.NoError(t, err) defer containerCleanupFunc() @@ -117,7 +115,7 @@ func TestProvisioning_ProvisionRuntimeWithDatabase(t *testing.T) { require.NotNil(t, connection) defer testutils.CloseDatabase(t, connection) - err = database.SetupSchema(connection, testutils.SchemaFilePath) + err = database.SetupSchema(connection, schemaFilePath) require.NoError(t, err) directorServiceMock := &directormock.DirectorClient{} diff --git a/components/provisioner/internal/persistence/database/schema_initializer_test.go b/components/provisioner/internal/persistence/database/schema_initializer_test.go index f7d3b33d90..8d1e0ac0fa 100644 --- a/components/provisioner/internal/persistence/database/schema_initializer_test.go +++ b/components/provisioner/internal/persistence/database/schema_initializer_test.go @@ -9,17 +9,15 @@ import ( "github.com/stretchr/testify/require" ) +const schemaFilePath = "../../../assets/database/provisioner.sql" + func TestSchemaInitializer(t *testing.T) { ctx := context.Background() - cleanupNetwork, err := testutils.EnsureTestNetworkForDB(t, ctx) - require.NoError(t, err) - defer cleanupNetwork() - t.Run("Should initialize database when schema not applied", func(t *testing.T) { // given - containerCleanupFunc, connString, err := testutils.InitTestDBContainer(t, ctx, "test_DB_1") + containerCleanupFunc, connString, err := testutils.InitTestDBContainer(t, ctx) require.NoError(t, err) defer containerCleanupFunc() @@ -31,7 +29,7 @@ func TestSchemaInitializer(t *testing.T) { defer testutils.CloseDatabase(t, connection) - err = SetupSchema(connection, testutils.SchemaFilePath) + err = SetupSchema(connection, schemaFilePath) require.NoError(t, err) // then @@ -42,7 +40,7 @@ func TestSchemaInitializer(t *testing.T) { t.Run("Should skip database initialization when schema already applied", func(t *testing.T) { //given - containerCleanupFunc, connString, err := testutils.InitTestDBContainer(t, ctx, "test_DB_2") + containerCleanupFunc, connString, err := testutils.InitTestDBContainer(t, ctx) require.NoError(t, err) defer containerCleanupFunc() @@ -54,10 +52,10 @@ func TestSchemaInitializer(t *testing.T) { defer testutils.CloseDatabase(t, connection) - err = SetupSchema(connection, testutils.SchemaFilePath) + err = SetupSchema(connection, schemaFilePath) require.NoError(t, err) - err = SetupSchema(connection, testutils.SchemaFilePath) + err = SetupSchema(connection, schemaFilePath) require.NoError(t, err) // then @@ -68,7 +66,7 @@ func TestSchemaInitializer(t *testing.T) { t.Run("Should return error when failed to connect to the database", func(t *testing.T) { - containerCleanupFunc, _, err := testutils.InitTestDBContainer(t, ctx, "test_DB_3") + containerCleanupFunc, _, err := testutils.InitTestDBContainer(t, ctx) require.NoError(t, err) defer containerCleanupFunc() @@ -86,7 +84,7 @@ func TestSchemaInitializer(t *testing.T) { t.Run("Should return error when failed to read database schema", func(t *testing.T) { //given - containerCleanupFunc, connString, err := testutils.InitTestDBContainer(t, ctx, "test_DB_4") + containerCleanupFunc, connString, err := testutils.InitTestDBContainer(t, ctx) require.NoError(t, err) defer containerCleanupFunc() diff --git a/components/provisioner/internal/persistence/testutils/database.go b/components/provisioner/internal/persistence/testutils/database.go index 3a779375ff..601284cebd 100644 --- a/components/provisioner/internal/persistence/testutils/database.go +++ b/components/provisioner/internal/persistence/testutils/database.go @@ -43,14 +43,7 @@ var ( func makeConnectionString(hostname string, port string) string { - host := "localhost" - - if os.Getenv(EnvPipelineBuild) != "" { - host = hostname - port = DbPort - } - - return fmt.Sprintf(connStringFormat, host, port, DbUser, + return fmt.Sprintf(connStringFormat, hostname, port, DbUser, DbPass, DbName, "disable") } @@ -62,8 +55,7 @@ func CloseDatabase(t *testing.T, connection *dbr.Connection) { } } -func InitTestDBContainer(t *testing.T, ctx context.Context, hostname string) (func(), string, error) { - +func InitTestDBContainer(t *testing.T, ctx context.Context) (func(), string, error) { _, err := isDockerTestNetworkPresent(ctx) if err != nil { @@ -71,13 +63,8 @@ func InitTestDBContainer(t *testing.T, ctx context.Context, hostname string) (fu } req := testcontainers.ContainerRequest{ - Image: "postgres:11", - SkipReaper: true, - ExposedPorts: []string{fmt.Sprintf("%s", DbPort)}, - Networks: []string{DockerUserNetwork}, - NetworkAliases: map[string][]string{ - DockerUserNetwork: {hostname}, - }, + Image: "postgres:11", + SkipReaper: true, Env: map[string]string{ "POSTGRES_USER": DbUser, "POSTGRES_PASSWORD": DbPass, @@ -106,13 +93,23 @@ func InitTestDBContainer(t *testing.T, ctx context.Context, hostname string) (fu return nil, "", err } + ip, err := postgresContainer.Host(ctx) + if err != nil { + t.Logf("Failed to get ip for container %s : %s", postgresContainer.GetContainerID(), err.Error()) + errTerminate := postgresContainer.Terminate(ctx) + if errTerminate != nil { + t.Logf("Failed to terminate container %s after failing to get ip: %s", postgresContainer.GetContainerID(), err.Error()) + } + return nil, "", err + } + cleanupFunc := func() { err := postgresContainer.Terminate(ctx) assert.NoError(t, err) time.Sleep(2 * time.Second) } - connString := makeConnectionString(hostname, port.Port()) + connString := makeConnectionString(ip, port.Port()) return cleanupFunc, connString, nil }