diff --git a/Makefile b/Makefile index 1d150ded..777efcca 100644 --- a/Makefile +++ b/Makefile @@ -89,7 +89,8 @@ TEST_DIR := $(shell pwd)/test ARTIFACTS ?= $(shell pwd)/_artifacts GINKGO_FOCUS ?= GINKGO_SKIP ?= -GINKGO_NODES ?= 1 +GINKGO_NODES ?= 1 # GINKGO_NODES is the number of parallel nodes to run + # when running the e2e tests, 1 means no parallelism GINKGO_TIMEOUT ?= 2h GINKGO_POLL_PROGRESS_AFTER ?= 60m GINKGO_POLL_PROGRESS_INTERVAL ?= 5m diff --git a/bootstrap/config/default/manager_auth_proxy_patch.yaml b/bootstrap/config/default/manager_auth_proxy_patch.yaml index 77e743d1..b7d4b825 100644 --- a/bootstrap/config/default/manager_auth_proxy_patch.yaml +++ b/bootstrap/config/default/manager_auth_proxy_patch.yaml @@ -22,4 +22,4 @@ spec: - name: manager args: - "--metrics-addr=127.0.0.1:8080" - - "--enable-leader-election" + - "--leader-elect" diff --git a/bootstrap/config/manager/kustomization.yaml b/bootstrap/config/manager/kustomization.yaml index c63f1425..293343b0 100644 --- a/bootstrap/config/manager/kustomization.yaml +++ b/bootstrap/config/manager/kustomization.yaml @@ -5,4 +5,5 @@ kind: Kustomization images: - name: controller newName: ghcr.io/k3s-io/cluster-api-k3s/bootstrap-controller + # WARNING: the tag "dev" is used in e2e test, change to a different value will break e2e test! newTag: dev diff --git a/bootstrap/config/manager/manager.yaml b/bootstrap/config/manager/manager.yaml index 36980726..4b866e84 100644 --- a/bootstrap/config/manager/manager.yaml +++ b/bootstrap/config/manager/manager.yaml @@ -26,7 +26,7 @@ spec: - command: - /manager args: - - --enable-leader-election + - --leader-elect image: controller:latest name: manager terminationGracePeriodSeconds: 10 diff --git a/bootstrap/main.go b/bootstrap/main.go index 9d95b8c1..5b48bcf7 100644 --- a/bootstrap/main.go +++ b/bootstrap/main.go @@ -55,7 +55,7 @@ func main() { var syncPeriod time.Duration flag.StringVar(&metricsAddr, "metrics-addr", ":8080", "The address the metric endpoint binds to.") - flag.BoolVar(&enableLeaderElection, "enable-leader-election", false, + flag.BoolVar(&enableLeaderElection, "leader-elect", false, "Enable leader election for controller manager. "+ "Enabling this will ensure there is only one active controller manager.") diff --git a/controlplane/config/default/manager_auth_proxy_patch.yaml b/controlplane/config/default/manager_auth_proxy_patch.yaml index 77e743d1..b7d4b825 100644 --- a/controlplane/config/default/manager_auth_proxy_patch.yaml +++ b/controlplane/config/default/manager_auth_proxy_patch.yaml @@ -22,4 +22,4 @@ spec: - name: manager args: - "--metrics-addr=127.0.0.1:8080" - - "--enable-leader-election" + - "--leader-elect" diff --git a/controlplane/config/manager/kustomization.yaml b/controlplane/config/manager/kustomization.yaml index 08b4c838..c6e86d8f 100644 --- a/controlplane/config/manager/kustomization.yaml +++ b/controlplane/config/manager/kustomization.yaml @@ -5,4 +5,5 @@ kind: Kustomization images: - name: controller newName: ghcr.io/k3s-io/cluster-api-k3s/controlplane-controller + # WARNING: the tag "dev" is used in e2e test, change to a different value will break e2e test! newTag: dev diff --git a/controlplane/config/manager/manager.yaml b/controlplane/config/manager/manager.yaml index b6c85a52..23b85a29 100644 --- a/controlplane/config/manager/manager.yaml +++ b/controlplane/config/manager/manager.yaml @@ -26,7 +26,7 @@ spec: - command: - /manager args: - - --enable-leader-election + - --leader-elect image: controller:latest name: manager resources: diff --git a/controlplane/main.go b/controlplane/main.go index b97c6ef0..0a2bbea2 100644 --- a/controlplane/main.go +++ b/controlplane/main.go @@ -61,7 +61,7 @@ func main() { var etcdCallTimeout time.Duration flag.StringVar(&metricsAddr, "metrics-addr", ":8080", "The address the metric endpoint binds to.") - flag.BoolVar(&enableLeaderElection, "enable-leader-election", false, + flag.BoolVar(&enableLeaderElection, "leader-elect", false, "Enable leader election for controller manager. "+ "Enabling this will ensure there is only one active controller manager.") diff --git a/samples/deployment/bootstrap-k3s/v1.2.2/bootstrap-components.yaml b/samples/deployment/bootstrap-k3s/v1.2.2/bootstrap-components.yaml index 7e796b5c..491a6592 100644 --- a/samples/deployment/bootstrap-k3s/v1.2.2/bootstrap-components.yaml +++ b/samples/deployment/bootstrap-k3s/v1.2.2/bootstrap-components.yaml @@ -740,7 +740,7 @@ spec: name: https - args: - --metrics-addr=127.0.0.1:8080 - - --enable-leader-election + - --leader-elect command: - /manager image: ghcr.io/zawachte/cluster-api-k3s/bootstrap-controller:v0.1.5 diff --git a/samples/deployment/control-plane-k3s/v1.2.2/control-plane-components.yaml b/samples/deployment/control-plane-k3s/v1.2.2/control-plane-components.yaml index 96cda9c4..77413c63 100644 --- a/samples/deployment/control-plane-k3s/v1.2.2/control-plane-components.yaml +++ b/samples/deployment/control-plane-k3s/v1.2.2/control-plane-components.yaml @@ -650,7 +650,7 @@ spec: name: https - args: - --metrics-addr=127.0.0.1:8080 - - --enable-leader-election + - --leader-elect command: - /manager image: ghcr.io/zawachte/cluster-api-k3s/controlplane-controller:v0.1.5 diff --git a/test/e2e/README.md b/test/e2e/README.md index 821969e3..9c73c6b3 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -10,9 +10,9 @@ make docker-build-e2e # should be run everytime you change the controller code make test-e2e # run all e2e tests ``` ### Run a specific e2e test -To run a specific e2e test, you can use the `GINKGO_FOCUS` environment variable to specify the test you want to run. For example: +To run a specific e2e test, such as "Scaling a cluster", use the `GINKGO_FOCUS` environment variable as shown below: ```shell -make GINKGO_FOCUS="Creating a cluster" test-e2e # only run the "Creating a cluster" e2e test +make GINKGO_FOCUS="Scaling a cluster" test-e2e # only run the "Scaling a cluster" e2e test ``` ### Run the e2e test with tilt It is quite useful to run the e2e test with [tilt](https://cluster-api.sigs.k8s.io/developer/tilt), so that you will not need to rebuild docker image with `make docker-build-e2e` everytime. Also you will not need to wait a new cluster creation and setup. If you have set up your tilt cluster and made the current context points to this cluster, you could run: @@ -23,5 +23,12 @@ make USE_EXISTING_CLUSTER=true test-e2e ## Develop an e2e test You could refer to [Developing E2E tests](https://cluster-api.sigs.k8s.io/developer/e2e) for a complete guide for developing e2e tests. +A guide for developing a k3s e2e test: + +* Group test specs by scenarios (e.g., `create_test`, `node_scale_test`, `upgrade_test`). Create a new file under `test/e2e/` for new scenarios. +* If a different docker cluster template is needed, create one under `test/e2e/infrastructure-docker/` and link it in `test/e2e/config/k3s-docker.yaml`. +* Define tunable variables in the cluster template as environment variables under `variables` in `test/e2e/config/k3s-docker.yaml`. Enable necessary feature flags here as well (e.g., `EXP_CLUSTER_RESOURCE_SET: "true"`). +* If reusing a [cluster-api test spec](https://github.com/kubernetes-sigs/cluster-api/tree/main/test/e2e), note that they assume the use of `KubeadmControlPlane`. For customization, copy code into `test/e2e/helpers.go`. + ## Troubleshooting * [Cluster API with Docker - "too many open files".](https://cluster-api.sigs.k8s.io/user/troubleshooting.html?highlight=too%20many#cluster-api-with-docker----too-many-open-files) \ No newline at end of file diff --git a/test/e2e/config/k3s-docker.yaml b/test/e2e/config/k3s-docker.yaml index b4c3bb36..7b5fe99b 100644 --- a/test/e2e/config/k3s-docker.yaml +++ b/test/e2e/config/k3s-docker.yaml @@ -4,8 +4,6 @@ # - bootstrap k3s # - control-plane k3s # - docker -managementClusterName: k3s-e2e - images: # Use local dev images built source tree; - name: ghcr.io/k3s-io/cluster-api-k3s/controlplane-controller:dev @@ -28,7 +26,9 @@ providers: - name: docker type: InfrastructureProvider versions: - - name: v1.6.2 # used during e2e-test + # By default, will use the latest version defined in ../data/shared/v1beta1/metadata.yaml + # to init the management cluster + - name: v1.6.2 # used during e2e-test value: https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.6.2/infrastructure-components-development.yaml type: url files: @@ -36,8 +36,13 @@ providers: replacements: - old: "imagePullPolicy: Always" new: "imagePullPolicy: IfNotPresent" + + # Add v1.7.99 to support tilt (not presented in ../data/shared/v1beta1/metadata.yaml) + # when bootstrapping with tilt, it will use + # the defaultProviderVersion in https://github.com/kubernetes-sigs/cluster-api/blob/main/hack/tools/internal/tilt-prepare/main.go as + # default version for docker infrastructure provider + # name here should match defaultProviderVersion - name: v1.7.99 # next; use manifest from source files - # added to support tilt value: https://github.com/kubernetes-sigs/cluster-api/releases/latest/download/infrastructure-components-development.yaml type: url files: @@ -50,6 +55,11 @@ providers: - name: k3s type: BootstrapProvider versions: + # Could add older release version for upgrading test, but + # by default, will only use the latest version defined in + # ${ProjectRoot}/metadata.yaml to init the management cluster + # this version should be updated when ${ProjectRoot}/metadata.yaml + # is modified - name: v0.1.99 # next; use manifest from source files value: "../../../bootstrap/config/default" files: @@ -64,14 +74,13 @@ providers: - sourcePath: "../../../metadata.yaml" targetName: "metadata.yaml" - variables: KUBERNETES_VERSION: "v1.28.6" KUBERNETES_VERSION_MANAGEMENT: "v1.28.6" IP_FAMILY: "IPv4" intervals: - # the array is defined as [timeout, polling interval] + # The array is defined as [timeout, polling interval] # copied from https://github.com/kubernetes-sigs/cluster-api/blob/main/test/e2e/config/docker.yaml default/wait-controllers: ["3m", "10s"] default/wait-cluster: ["5m", "10s"] diff --git a/test/e2e/data/infrastructure-docker/cluster-template.yaml b/test/e2e/data/infrastructure-docker/cluster-template.yaml index d9338d69..a6e796a2 100644 --- a/test/e2e/data/infrastructure-docker/cluster-template.yaml +++ b/test/e2e/data/infrastructure-docker/cluster-template.yaml @@ -1,3 +1,6 @@ +# TODO: copied from https://github.com/k3s-io/cluster-api-k3s/pull/93/files#diff-c4a336ec56832a2ff7aed26c94d0d67ae3a0e6139d30701cc53c0f0962fe8cca +# should be the same as samples/docker/quickstart.yaml in the future +# for testing the quickstart scenario apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: diff --git a/test/e2e/data/shared/v1beta1/metadata.yaml b/test/e2e/data/shared/v1beta1/metadata.yaml index df0b02a1..e156c339 100644 --- a/test/e2e/data/shared/v1beta1/metadata.yaml +++ b/test/e2e/data/shared/v1beta1/metadata.yaml @@ -1,33 +1,9 @@ +# maps release series of major.minor to cluster-api contract version, +# update this file only when you update the version for cluster-api +# CoreProvider and docker InfrastructureProvider apiVersion: clusterctl.cluster.x-k8s.io/v1alpha3 kind: Metadata releaseSeries: - major: 1 minor: 6 - contract: v1beta1 - - major: 1 - minor: 5 - contract: v1beta1 - - major: 1 - minor: 4 - contract: v1beta1 - - major: 1 - minor: 3 - contract: v1beta1 - - major: 1 - minor: 2 - contract: v1beta1 - - major: 1 - minor: 1 - contract: v1beta1 - - major: 1 - minor: 0 - contract: v1beta1 - - major: 0 - minor: 4 - contract: v1alpha4 - - major: 0 - minor: 3 - contract: v1alpha3 - - major: 0 - minor: 2 - contract: v1alpha2 \ No newline at end of file + contract: v1beta1 \ No newline at end of file diff --git a/test/e2e/e2e_test.go b/test/e2e/node_scale_test.go similarity index 76% rename from test/e2e/e2e_test.go rename to test/e2e/node_scale_test.go index c4746344..5d55935d 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/node_scale_test.go @@ -22,7 +22,6 @@ package e2e import ( "context" "fmt" - "os" "path/filepath" . "github.com/onsi/ginkgo/v2" @@ -35,10 +34,10 @@ import ( "sigs.k8s.io/cluster-api/util" ) -var _ = Describe("Workload cluster creation", func() { +var _ = Describe("Workload cluster scaling", func() { var ( ctx = context.TODO() - specName = "workload-cluster-creation" + specName = "workload-cluster-scaling" namespace *corev1.Namespace cancelWatches context.CancelFunc result *ApplyClusterTemplateAndWaitResult @@ -48,14 +47,9 @@ var _ = Describe("Workload cluster creation", func() { ) BeforeEach(func() { - Expect(e2eConfig).ToNot(BeNil(), "Invalid argument. e2eConfig can't be nil when calling %s spec", specName) - Expect(clusterctlConfigPath).To(BeAnExistingFile(), "Invalid argument. clusterctlConfigPath must be an existing file when calling %s spec", specName) - Expect(bootstrapClusterProxy).ToNot(BeNil(), "Invalid argument. bootstrapClusterProxy can't be nil when calling %s spec", specName) - Expect(os.MkdirAll(artifactFolder, 0755)).To(Succeed(), "Invalid argument. artifactFolder can't be created for %s spec", specName) - Expect(e2eConfig.Variables).To(HaveKey(KubernetesVersion)) - clusterName = fmt.Sprintf("capik3s-e2e-%s", util.RandomString(6)) + clusterName = fmt.Sprintf("capik3s-node-scale-%s", util.RandomString(6)) infrastructureProvider = "docker" // Setup a Namespace where to host objects for this spec and create a watcher for the namespace events. @@ -81,30 +75,9 @@ var _ = Describe("Workload cluster creation", func() { dumpSpecResourcesAndCleanup(ctx, cleanInput) }) - Context("Creating a cluster", func() { - It("Should create a simple workload cluster with 1 controlplane node and 3 worker nodes", func() { - ApplyClusterTemplateAndWait(ctx, ApplyClusterTemplateAndWaitInput{ - ClusterProxy: bootstrapClusterProxy, - ConfigCluster: clusterctl.ConfigClusterInput{ - LogFolder: clusterctlLogFolder, - ClusterctlConfigPath: clusterctlConfigPath, - KubeconfigPath: bootstrapClusterProxy.GetKubeconfigPath(), - InfrastructureProvider: infrastructureProvider, - Namespace: namespace.Name, - ClusterName: clusterName, - KubernetesVersion: e2eConfig.GetVariable(KubernetesVersion), - ControlPlaneMachineCount: pointer.Int64Ptr(1), - WorkerMachineCount: pointer.Int64Ptr(3), - }, - WaitForClusterIntervals: e2eConfig.GetIntervals(specName, "wait-cluster"), - WaitForControlPlaneIntervals: e2eConfig.GetIntervals(specName, "wait-control-plane"), - WaitForMachineDeployments: e2eConfig.GetIntervals(specName, "wait-worker-nodes"), - }, result) - }) - }) - Context("Scaling a cluster", func() { It("Should create a workload cluster with 1 worker node and scale", func() { + By("Creating a workload cluster with 1 control plane and 1 worker node") ApplyClusterTemplateAndWait(ctx, ApplyClusterTemplateAndWaitInput{ ClusterProxy: bootstrapClusterProxy, ConfigCluster: clusterctl.ConfigClusterInput{ @@ -163,6 +136,27 @@ var _ = Describe("Workload cluster creation", func() { WaitForMachineDeployments: e2eConfig.GetIntervals(specName, "wait-worker-nodes"), }, result) + // TODO: enable after scaling down control planes working + // By("Scaling down control planes to 1") + + // ApplyClusterTemplateAndWait(ctx, ApplyClusterTemplateAndWaitInput{ + // ClusterProxy: bootstrapClusterProxy, + // ConfigCluster: clusterctl.ConfigClusterInput{ + // LogFolder: clusterctlLogFolder, + // ClusterctlConfigPath: clusterctlConfigPath, + // KubeconfigPath: bootstrapClusterProxy.GetKubeconfigPath(), + // InfrastructureProvider: infrastructureProvider, + // Namespace: namespace.Name, + // ClusterName: clusterName, + // KubernetesVersion: e2eConfig.GetVariable(KubernetesVersion), + // ControlPlaneMachineCount: pointer.Int64Ptr(1), + // WorkerMachineCount: pointer.Int64Ptr(3), + // }, + // WaitForClusterIntervals: e2eConfig.GetIntervals(specName, "wait-cluster"), + // WaitForControlPlaneIntervals: e2eConfig.GetIntervals(specName, "wait-control-plane"), + // WaitForMachineDeployments: e2eConfig.GetIntervals(specName, "wait-worker-nodes"), + // }, result) + By("Scaling down worker nodes to 1") ApplyClusterTemplateAndWait(ctx, ApplyClusterTemplateAndWaitInput{ @@ -175,7 +169,7 @@ var _ = Describe("Workload cluster creation", func() { Namespace: namespace.Name, ClusterName: clusterName, KubernetesVersion: e2eConfig.GetVariable(KubernetesVersion), - ControlPlaneMachineCount: pointer.Int64Ptr(3), + ControlPlaneMachineCount: pointer.Int64Ptr(3), // TODO: change to 1 after scaling down control planes working WorkerMachineCount: pointer.Int64Ptr(1), }, WaitForClusterIntervals: e2eConfig.GetIntervals(specName, "wait-cluster"),