fix

Signed-off-by: nasusoba <[email protected]>

Makefile

@@ -89,7 +89,8 @@ TEST_DIR := $(shell pwd)/test
 ARTIFACTS ?= $(shell pwd)/_artifacts
 GINKGO_FOCUS ?=
 GINKGO_SKIP ?=
-GINKGO_NODES ?= 1
+GINKGO_NODES ?= 1 # GINKGO_NODES is the number of parallel nodes to run 
+                  # when running the e2e tests, 1 means no parallelism
 GINKGO_TIMEOUT ?= 2h
 GINKGO_POLL_PROGRESS_AFTER ?= 60m
 GINKGO_POLL_PROGRESS_INTERVAL ?= 5m

bootstrap/config/default/manager_auth_proxy_patch.yaml

@@ -22,4 +22,4 @@ spec:
       - name: manager
         args:
         - "--metrics-addr=127.0.0.1:8080"
-        - "--enable-leader-election"
+        - "--leader-elect"

bootstrap/config/manager/kustomization.yaml

@@ -5,4 +5,5 @@ kind: Kustomization
 images:
 - name: controller
   newName: ghcr.io/k3s-io/cluster-api-k3s/bootstrap-controller
+  # WARNING: the tag "dev" is used in e2e test, change to a different value will break e2e test!
   newTag: dev

bootstrap/config/manager/manager.yaml

@@ -26,7 +26,7 @@ spec:
       - command:
         - /manager
         args:
-        - --enable-leader-election
+        - --leader-elect
         image: controller:latest
         name: manager
       terminationGracePeriodSeconds: 10

bootstrap/main.go

@@ -55,7 +55,7 @@ func main() {
 	var syncPeriod time.Duration
 
 	flag.StringVar(&metricsAddr, "metrics-addr", ":8080", "The address the metric endpoint binds to.")
-	flag.BoolVar(&enableLeaderElection, "enable-leader-election", false,
+	flag.BoolVar(&enableLeaderElection, "leader-elect", false,
 		"Enable leader election for controller manager. "+
 			"Enabling this will ensure there is only one active controller manager.")
 

controlplane/config/default/manager_auth_proxy_patch.yaml

@@ -22,4 +22,4 @@ spec:
       - name: manager
         args:
         - "--metrics-addr=127.0.0.1:8080"
-        - "--enable-leader-election"
+        - "--leader-elect"

controlplane/config/manager/kustomization.yaml

@@ -5,4 +5,5 @@ kind: Kustomization
 images:
 - name: controller
   newName: ghcr.io/k3s-io/cluster-api-k3s/controlplane-controller
+  # WARNING: the tag "dev" is used in e2e test, change to a different value will break e2e test!
   newTag: dev

controlplane/config/manager/manager.yaml

@@ -26,7 +26,7 @@ spec:
       - command:
         - /manager
         args:
-        - --enable-leader-election
+        - --leader-elect
         image: controller:latest
         name: manager
         resources:

controlplane/main.go

@@ -61,7 +61,7 @@ func main() {
 	var etcdCallTimeout time.Duration
 
 	flag.StringVar(&metricsAddr, "metrics-addr", ":8080", "The address the metric endpoint binds to.")
-	flag.BoolVar(&enableLeaderElection, "enable-leader-election", false,
+	flag.BoolVar(&enableLeaderElection, "leader-elect", false,
 		"Enable leader election for controller manager. "+
 			"Enabling this will ensure there is only one active controller manager.")
 

samples/deployment/bootstrap-k3s/v1.2.2/bootstrap-components.yaml

@@ -740,7 +740,7 @@ spec:
           name: https
       - args:
         - --metrics-addr=127.0.0.1:8080
-        - --enable-leader-election
+        - --leader-elect
         command:
         - /manager
         image: ghcr.io/zawachte/cluster-api-k3s/bootstrap-controller:v0.1.5

samples/deployment/control-plane-k3s/v1.2.2/control-plane-components.yaml

@@ -650,7 +650,7 @@ spec:
           name: https
       - args:
         - --metrics-addr=127.0.0.1:8080
-        - --enable-leader-election
+        - --leader-elect
         command:
         - /manager
         image: ghcr.io/zawachte/cluster-api-k3s/controlplane-controller:v0.1.5

test/e2e/README.md

@@ -10,9 +10,9 @@ make docker-build-e2e   # should be run everytime you change the controller code
 make test-e2e   # run all e2e tests
 ```
 ### Run a specific e2e test
-To run a specific e2e test, you can use the `GINKGO_FOCUS` environment variable to specify the test you want to run. For example:
+To run a specific e2e test, such as "Scaling a cluster", use the `GINKGO_FOCUS` environment variable as shown below:
 ```shell
-make GINKGO_FOCUS="Creating a cluster" test-e2e  # only run the "Creating a cluster" e2e test
+make GINKGO_FOCUS="Scaling a cluster" test-e2e  # only run the "Scaling a cluster" e2e test
 ```
 ### Run the e2e test with tilt
 It is quite useful to run the e2e test with [tilt](https://cluster-api.sigs.k8s.io/developer/tilt), so that you will not need to rebuild docker image with `make docker-build-e2e` everytime. Also you will not need to wait a new cluster creation and setup. If you have set up your tilt cluster and made the current context points to this cluster, you could run:
@@ -23,5 +23,12 @@ make USE_EXISTING_CLUSTER=true test-e2e
 ## Develop an e2e test
 You could refer to [Developing E2E tests](https://cluster-api.sigs.k8s.io/developer/e2e) for a complete guide for developing e2e tests.
 
+A guide for developing a k3s e2e test:
+
+* Group test specs by scenarios (e.g., `create_test`, `node_scale_test`, `upgrade_test`). Create a new file under `test/e2e/` for new scenarios.
+* If a different docker cluster template is needed, create one under `test/e2e/infrastructure-docker/` and link it in `test/e2e/config/k3s-docker.yaml`.
+* Define tunable variables in the cluster template as environment variables under `variables` in `test/e2e/config/k3s-docker.yaml`. Enable necessary feature flags here as well (e.g., `EXP_CLUSTER_RESOURCE_SET: "true"`).
+* If reusing a [cluster-api test spec](https://github.com/kubernetes-sigs/cluster-api/tree/main/test/e2e), note that they assume the use of `KubeadmControlPlane`. For customization, copy code into `test/e2e/helpers.go`.
+
 ## Troubleshooting
 * [Cluster API with Docker - "too many open files".](https://cluster-api.sigs.k8s.io/user/troubleshooting.html?highlight=too%20many#cluster-api-with-docker----too-many-open-files)

test/e2e/config/k3s-docker.yaml

@@ -4,8 +4,6 @@
 # - bootstrap k3s
 # - control-plane k3s
 # - docker
-managementClusterName: k3s-e2e
-
 images:
   # Use local dev images built source tree;
   - name: ghcr.io/k3s-io/cluster-api-k3s/controlplane-controller:dev
@@ -28,16 +26,23 @@ providers:
   - name: docker
     type: InfrastructureProvider
     versions:
-      - name: v1.6.2  # used during e2e-test
+      # By default, will use the latest version defined in ../data/shared/v1beta1/metadata.yaml
+      # to init the management cluster
+      - name: v1.6.2 # used during e2e-test
         value: https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.6.2/infrastructure-components-development.yaml
         type: url
         files:
           - sourcePath: "../data/shared/v1beta1/metadata.yaml"
         replacements:
           - old: "imagePullPolicy: Always"
             new: "imagePullPolicy: IfNotPresent"
+
+      # Add v1.7.99 to support tilt (not presented in ../data/shared/v1beta1/metadata.yaml)
+      # when bootstrapping with tilt, it will use
+      # the defaultProviderVersion in https://github.com/kubernetes-sigs/cluster-api/blob/main/hack/tools/internal/tilt-prepare/main.go as 
+      # default version for docker infrastructure provider
+      # name here should match defaultProviderVersion
       - name: v1.7.99 # next; use manifest from source files
-                      # added to support tilt
         value: https://github.com/kubernetes-sigs/cluster-api/releases/latest/download/infrastructure-components-development.yaml
         type: url
         files:
@@ -50,6 +55,11 @@ providers:
   - name: k3s
     type: BootstrapProvider
     versions:
+      # Could add older release version for upgrading test, but
+      # by default, will only use the latest version defined in
+      # ${ProjectRoot}/metadata.yaml to init the management cluster
+      # this version should be updated when ${ProjectRoot}/metadata.yaml
+      # is modified
       - name: v0.1.99 # next; use manifest from source files
         value: "../../../bootstrap/config/default"
     files:
@@ -64,14 +74,13 @@ providers:
       - sourcePath: "../../../metadata.yaml"
         targetName: "metadata.yaml"
 
-
 variables:
   KUBERNETES_VERSION: "v1.28.6"
   KUBERNETES_VERSION_MANAGEMENT: "v1.28.6"
   IP_FAMILY: "IPv4"
 
 intervals:
-  # the array is defined as [timeout, polling interval]
+  # The array is defined as [timeout, polling interval]
   # copied from https://github.com/kubernetes-sigs/cluster-api/blob/main/test/e2e/config/docker.yaml
   default/wait-controllers: ["3m", "10s"]
   default/wait-cluster: ["5m", "10s"]

test/e2e/data/infrastructure-docker/cluster-template.yaml

@@ -1,3 +1,6 @@
+# TODO: copied from https://github.com/k3s-io/cluster-api-k3s/pull/93/files#diff-c4a336ec56832a2ff7aed26c94d0d67ae3a0e6139d30701cc53c0f0962fe8cca
+# should be the same as samples/docker/quickstart.yaml in the future
+# for testing the quickstart scenario
 apiVersion: cluster.x-k8s.io/v1beta1
 kind: Cluster 
 metadata:

test/e2e/data/shared/v1beta1/metadata.yaml

@@ -1,33 +1,9 @@
+# maps release series of major.minor to cluster-api contract version,
+# update this file only when you update the version for cluster-api
+# CoreProvider and docker InfrastructureProvider
 apiVersion: clusterctl.cluster.x-k8s.io/v1alpha3
 kind: Metadata
 releaseSeries:
   - major: 1
     minor: 6
-    contract: v1beta1
-  - major: 1
-    minor: 5
-    contract: v1beta1
-  - major: 1
-    minor: 4
-    contract: v1beta1
-  - major: 1
-    minor: 3
-    contract: v1beta1
-  - major: 1
-    minor: 2
-    contract: v1beta1
-  - major: 1
-    minor: 1
-    contract: v1beta1
-  - major: 1
-    minor: 0
-    contract: v1beta1
-  - major: 0
-    minor: 4
-    contract: v1alpha4
-  - major: 0
-    minor: 3
-    contract: v1alpha3
-  - major: 0
-    minor: 2
-    contract: v1alpha2
+    contract: v1beta1

test/e2e/e2e_test.go → test/e2e/node_scale_test.go

@@ -22,7 +22,6 @@ package e2e
 import (
 	"context"
 	"fmt"
-	"os"
 	"path/filepath"
 
 	. "github.com/onsi/ginkgo/v2"
@@ -35,10 +34,10 @@ import (
 	"sigs.k8s.io/cluster-api/util"
 )
 
-var _ = Describe("Workload cluster creation", func() {
+var _ = Describe("Workload cluster scaling", func() {
 	var (
 		ctx                    = context.TODO()
-		specName               = "workload-cluster-creation"
+		specName               = "workload-cluster-scaling"
 		namespace              *corev1.Namespace
 		cancelWatches          context.CancelFunc
 		result                 *ApplyClusterTemplateAndWaitResult
@@ -48,14 +47,9 @@ var _ = Describe("Workload cluster creation", func() {
 	)
 
 	BeforeEach(func() {
-		Expect(e2eConfig).ToNot(BeNil(), "Invalid argument. e2eConfig can't be nil when calling %s spec", specName)
-		Expect(clusterctlConfigPath).To(BeAnExistingFile(), "Invalid argument. clusterctlConfigPath must be an existing file when calling %s spec", specName)
-		Expect(bootstrapClusterProxy).ToNot(BeNil(), "Invalid argument. bootstrapClusterProxy can't be nil when calling %s spec", specName)
-		Expect(os.MkdirAll(artifactFolder, 0755)).To(Succeed(), "Invalid argument. artifactFolder can't be created for %s spec", specName)
-
 		Expect(e2eConfig.Variables).To(HaveKey(KubernetesVersion))
 
-		clusterName = fmt.Sprintf("capik3s-e2e-%s", util.RandomString(6))
+		clusterName = fmt.Sprintf("capik3s-node-scale-%s", util.RandomString(6))
 		infrastructureProvider = "docker"
 
 		// Setup a Namespace where to host objects for this spec and create a watcher for the namespace events.
@@ -81,30 +75,9 @@ var _ = Describe("Workload cluster creation", func() {
 		dumpSpecResourcesAndCleanup(ctx, cleanInput)
 	})
 
-	Context("Creating a cluster", func() {
-		It("Should create a simple workload cluster with 1 controlplane node and 3 worker nodes", func() {
-			ApplyClusterTemplateAndWait(ctx, ApplyClusterTemplateAndWaitInput{
-				ClusterProxy: bootstrapClusterProxy,
-				ConfigCluster: clusterctl.ConfigClusterInput{
-					LogFolder:                clusterctlLogFolder,
-					ClusterctlConfigPath:     clusterctlConfigPath,
-					KubeconfigPath:           bootstrapClusterProxy.GetKubeconfigPath(),
-					InfrastructureProvider:   infrastructureProvider,
-					Namespace:                namespace.Name,
-					ClusterName:              clusterName,
-					KubernetesVersion:        e2eConfig.GetVariable(KubernetesVersion),
-					ControlPlaneMachineCount: pointer.Int64Ptr(1),
-					WorkerMachineCount:       pointer.Int64Ptr(3),
-				},
-				WaitForClusterIntervals:      e2eConfig.GetIntervals(specName, "wait-cluster"),
-				WaitForControlPlaneIntervals: e2eConfig.GetIntervals(specName, "wait-control-plane"),
-				WaitForMachineDeployments:    e2eConfig.GetIntervals(specName, "wait-worker-nodes"),
-			}, result)
-		})
-	})
-
 	Context("Scaling a cluster", func() {
 		It("Should create a workload cluster with 1 worker node and scale", func() {
+			By("Creating a workload cluster with 1 control plane and 1 worker node")
 			ApplyClusterTemplateAndWait(ctx, ApplyClusterTemplateAndWaitInput{
 				ClusterProxy: bootstrapClusterProxy,
 				ConfigCluster: clusterctl.ConfigClusterInput{
@@ -163,6 +136,27 @@ var _ = Describe("Workload cluster creation", func() {
 				WaitForMachineDeployments:    e2eConfig.GetIntervals(specName, "wait-worker-nodes"),
 			}, result)
 
+			// TODO: enable after scaling down control planes working
+			// By("Scaling down control planes to 1")
+
+			// ApplyClusterTemplateAndWait(ctx, ApplyClusterTemplateAndWaitInput{
+			// 	ClusterProxy: bootstrapClusterProxy,
+			// 	ConfigCluster: clusterctl.ConfigClusterInput{
+			// 		LogFolder:                clusterctlLogFolder,
+			// 		ClusterctlConfigPath:     clusterctlConfigPath,
+			// 		KubeconfigPath:           bootstrapClusterProxy.GetKubeconfigPath(),
+			// 		InfrastructureProvider:   infrastructureProvider,
+			// 		Namespace:                namespace.Name,
+			// 		ClusterName:              clusterName,
+			// 		KubernetesVersion:        e2eConfig.GetVariable(KubernetesVersion),
+			// 		ControlPlaneMachineCount: pointer.Int64Ptr(1),
+			// 		WorkerMachineCount:       pointer.Int64Ptr(3),
+			// 	},
+			// 	WaitForClusterIntervals:      e2eConfig.GetIntervals(specName, "wait-cluster"),
+			// 	WaitForControlPlaneIntervals: e2eConfig.GetIntervals(specName, "wait-control-plane"),
+			// 	WaitForMachineDeployments:    e2eConfig.GetIntervals(specName, "wait-worker-nodes"),
+			// }, result)
+
 			By("Scaling down worker nodes to 1")
 
 			ApplyClusterTemplateAndWait(ctx, ApplyClusterTemplateAndWaitInput{
@@ -175,7 +169,7 @@ var _ = Describe("Workload cluster creation", func() {
 					Namespace:                namespace.Name,
 					ClusterName:              clusterName,
 					KubernetesVersion:        e2eConfig.GetVariable(KubernetesVersion),
-					ControlPlaneMachineCount: pointer.Int64Ptr(3),
+					ControlPlaneMachineCount: pointer.Int64Ptr(3), // TODO: change to 1 after scaling down control planes working
 					WorkerMachineCount:       pointer.Int64Ptr(1),
 				},
 				WaitForClusterIntervals:      e2eConfig.GetIntervals(specName, "wait-cluster"),