Add ability to enable Kubernetes feature gates and runtimeconfig in KubeVirt CI

KUBEVIRTCI_LOCAL_TESTING.md

@@ -65,6 +65,18 @@ export KUBEVIRT_DEPLOY_PROMETHEUS_ALERTMANAGER=true
 export KUBEVIRT_DEPLOY_GRAFANA=true
 ```
 
+#### start cluster with Kubernetes feature gates
+To enable/disable Kubernetes feature gates, export the following variable before running `make cluster-up`:
+
+```bash
+export K8S_FEATURE_GATES="<FeatureGate1>=true,<FeatureGate2>=false"
+```
+
+for feature gates that require runtime config the following variable should be exported:
+```bash
+export K8S_API_RUNTIME_CONFIG="<RuntimeConfig>=true"
+```
+
 #### start cluster with swap enabled
 To enable swap, please also export the following variables before running `make cluster-up`:
 ```bash

cluster-provision/gocli/cmd/nodesconfig/nodeconfig.go

@@ -4,6 +4,8 @@ package nodesconfig
 type NodeLinuxConfig struct {
 	NodeIdx         int
 	K8sVersion      string
+	FeatureGates    string
+	RuntimeConfig   string
 	FipsEnabled     bool
 	DockerProxy     string
 	EtcdInMemory    bool

cluster-provision/gocli/cmd/nodesconfig/opts.go

@@ -88,6 +88,18 @@ func WithSwap(swap bool) LinuxConfigFunc {
 	}
 }
 
+func WithFeatureGates(featureGates string) LinuxConfigFunc {
+	return func(n *NodeLinuxConfig) {
+		n.FeatureGates = featureGates
+	}
+}
+
+func WithRuntimeConfig(runtimeConfig string) LinuxConfigFunc {
+	return func(n *NodeLinuxConfig) {
+		n.RuntimeConfig = runtimeConfig
+	}
+}
+
 func WithKsmEnabled(ksmEnabled bool) LinuxConfigFunc {
 	return func(n *NodeLinuxConfig) {
 		n.KsmEnabled = ksmEnabled

cluster-provision/gocli/cmd/run.go

@@ -38,6 +38,7 @@ import (
 	dockerproxy "kubevirt.io/kubevirtci/cluster-provision/gocli/opts/docker-proxy"
 	etcdinmemory "kubevirt.io/kubevirtci/cluster-provision/gocli/opts/etcd"
 	"kubevirt.io/kubevirtci/cluster-provision/gocli/opts/istio"
+	"kubevirt.io/kubevirtci/cluster-provision/gocli/opts/k8scomponents"
 	"kubevirt.io/kubevirtci/cluster-provision/gocli/opts/ksm"
 	"kubevirt.io/kubevirtci/cluster-provision/gocli/opts/multus"
 	"kubevirt.io/kubevirtci/cluster-provision/gocli/opts/nfscsi"
@@ -80,8 +81,8 @@ EOF
 	etcdDataDir         = "/var/lib/etcd"
 	nvmeDiskImagePrefix = "/nvme"
 	scsiDiskImagePrefix = "/scsi"
-	QEMU_DEVICE_S390X  = "virtio-net-ccw"
-	QEMU_DEVICE_X86_64 = "virtio-net-pci"
+	QEMU_DEVICE_S390X   = "virtio-net-ccw"
+	QEMU_DEVICE_X86_64  = "virtio-net-pci"
 )
 
 var soundcardPCIIDs = []string{"8086:2668", "8086:2415"}
@@ -112,6 +113,8 @@ func NewRunCommand() *cobra.Command {
 	run.Flags().UintP("secondary-nics", "", 0, "number of secondary nics to add")
 	run.Flags().String("qemu-args", "", "additional qemu args to pass through to the nodes")
 	run.Flags().String("kernel-args", "", "additional kernel args to pass through to the nodes")
+	run.Flags().String("feature-gates", "", "k8s feature gates to enable")
+	run.Flags().String("runtime-config", "", "k8s api runtime config")
 	run.Flags().BoolP("background", "b", true, "go to background after nodes are up")
 	run.Flags().Bool("random-ports", true, "expose all ports on random localhost ports")
 	run.Flags().Bool("slim", false, "use the slim flavor")
@@ -225,6 +228,16 @@ func run(cmd *cobra.Command, args []string) (retErr error) {
 		return err
 	}
 
+	featureGates, err := cmd.Flags().GetString("feature-gates")
+	if err != nil {
+		return err
+	}
+
+	runtimeConfig, err := cmd.Flags().GetString("runtime-config")
+	if err != nil {
+		return err
+	}
+
 	cpu, err := cmd.Flags().GetUint("cpu")
 	if err != nil {
 		return err
@@ -578,7 +591,7 @@ func run(cmd *cobra.Command, args []string) (retErr error) {
 	qemuArgs += " -serial pty"
 
 	var qemuNetDevice = getNetDeviceByArch()
-	
+
 	wg := sync.WaitGroup{}
 	wg.Add(int(nodes))
 	// start one vm after each other
@@ -592,8 +605,8 @@ func run(cmd *cobra.Command, args []string) (retErr error) {
 			netSuffix := fmt.Sprintf("%d-%d", x, i)
 			macSuffix := fmt.Sprintf("%02x", macCounter)
 			macCounter++
-			// Secondary network devices are added after VM is started (hot-plug) using qemu monitor to avoid 
-			// primary network interface to be named other than eth0. This is mainly required for s390x, as 
+			// Secondary network devices are added after VM is started (hot-plug) using qemu monitor to avoid
+			// primary network interface to be named other than eth0. This is mainly required for s390x, as
 			// otherwise if primary interface is other than eth0, it can't get the IP from dhcp server.
 			if qemuNetDevice == QEMU_DEVICE_S390X {
 				nodeQemuMonitorArgs = fmt.Sprintf("%s netdev_add tap,id=secondarynet%s,ifname=stap%s,script=no,downscript=no; device_add %s,netdev=secondarynet%s,mac=52:55:00:d1:56:%s;", nodeQemuMonitorArgs, netSuffix, netSuffix, qemuNetDevice, netSuffix, macSuffix)
@@ -815,6 +828,8 @@ func run(cmd *cobra.Command, args []string) (retErr error) {
 			nodesconfig.WithSwapiness(int(swapiness)),
 			nodesconfig.WithSwapSize(int(swapSize)),
 			nodesconfig.WithUnlimitedSwap(unlimitedSwap),
+			nodesconfig.WithFeatureGates(featureGates),
+			nodesconfig.WithRuntimeConfig(runtimeConfig),
 		}
 
 		n := nodesconfig.NewNodeLinuxConfig(x+1, prefix, linuxConfigFuncs)
@@ -1021,6 +1036,11 @@ func provisionNode(sshClient libssh.Client, n *nodesconfig.NodeLinuxConfig) erro
 		opts = append(opts, swapOpt)
 	}
 
+	if n.FeatureGates != "" || n.RuntimeConfig != "" {
+		featureGatesOpt := k8scomponents.NewK8sComponentsOpt(sshClient, n.FeatureGates, n.RuntimeConfig)
+		opts = append(opts, featureGatesOpt)
+	}
+
 	for _, o := range opts {
 		if err := o.Exec(); err != nil {
 			return err

cluster-provision/gocli/opts/k8scomponents/components_helper.go

@@ -0,0 +1,126 @@
+package k8scomponents
+
+import (
+	"fmt"
+	"strings"
+	"time"
+
+	"kubevirt.io/kubevirtci/cluster-provision/gocli/pkg/libssh"
+)
+
+type componentName string
+
+const (
+	componentKubeAPIServer     componentName = "kube-apiserver"
+	componentKubeControllerMgr componentName = "kube-controller-manager"
+	componentKubeScheduler     componentName = "kube-scheduler"
+
+	searchFeatureGatesInFileFormat                    = "awk '/feature-gates/' %s"
+	getComponentCommandFormat                         = "kubectl --kubeconfig=/etc/kubernetes/admin.conf get pods -n kube-system -l component=%s -o jsonpath='{.items[0].spec.containers[*].command}'"
+	getComponentReadyContainersFormat                 = "kubectl --kubeconfig=/etc/kubernetes/admin.conf get pods -n kube-system -l component=%s -o=jsonpath='{.items[0].status.containerStatuses[*].ready}'"
+	getNodeReadyStatusCommand                         = "kubectl --kubeconfig=/etc/kubernetes/admin.confget nodes -o=jsonpath='{.items[*].status.conditions[?(@.type==\"Ready\")].status}'"
+	addFlagsToComponentCommandFormat                  = `sudo sed -i '/- %s/a\    - %s' /etc/kubernetes/manifests/%s.yaml`
+	searchComponentsFilesCommand                      = "ls /etc/kubernetes/manifests"
+	addFeatureGatesFieldToKubeletConfigCommand        = "sudo echo -e 'featureGates:' >> /var/lib/kubelet/config.yaml"
+	addFeatureGatesToKubeletConfigCommandFormatFormat = `sudo sed -i 's/featureGates:/featureGates:\n  %s/g' /var/lib/kubelet/config.yaml`
+	kubeletRestartCommand                             = "sudo systemctl restart kubelet"
+
+	featureGateExistInKubeletError                = "feature gates should not exist in kubelet by default"
+	featureGateExistInComponentCommandErrorFormat = "feature gates should not exist in %v command by default"
+)
+
+type component interface {
+	verifyComponent() error
+	prepareCommandsForConfiguration() error
+	runCommandsToConfigure() error
+	waitForComponentReady() error
+	requiredOnlyForMaster() bool
+}
+
+func componentReady(component componentName, sshClient libssh.Client, waitingForFeatureGate bool, waitingForRuntimeConfig bool) error {
+	ticker := time.NewTicker(3 * time.Second)
+	defer ticker.Stop()
+	timeoutChannel := time.After(5 * time.Minute)
+
+	select {
+	case <-timeoutChannel:
+		return fmt.Errorf("timed out after 5 minutes waiting for %v to be ready", component)
+	case <-ticker.C:
+		ready, reason := componentReadyHelper(component, sshClient, waitingForFeatureGate, waitingForRuntimeConfig)
+		if ready {
+			return nil
+		}
+		fmt.Printf(reason)
+	}
+
+	return nil
+}
+
+func componentReadyHelper(component componentName, sshClient libssh.Client, waitingForFeatureGate bool, waitingForRuntimeConfig bool) (bool, string) {
+	if waitingForFeatureGate || waitingForRuntimeConfig {
+		output, err := sshClient.CommandWithNoStdOut(getComponentCommand(component))
+		if err != nil {
+			return false, fmt.Sprintf("modifying flags, waiting for apiserver to respord after %v restart err:%v\n", component, err)
+		}
+
+		if waitingForFeatureGate && !strings.Contains(output, "feature-gate") {
+			return false, fmt.Sprintf("modifying flags, waiting for %v pods to restart\n", component)
+		}
+
+		if waitingForRuntimeConfig && !strings.Contains(output, "runtime-config") {
+			return false, fmt.Sprintf("modifying flags, waiting for %v pods to restart\n", component)
+		}
+	}
+
+	output, err := sshClient.CommandWithNoStdOut(getComponentReadyContainers(component))
+	if err != nil {
+		return false, fmt.Sprintf("modifying flags, waiting for apiserver to respord after %v restart err:%v\n", component, err)
+	}
+	if strings.Contains(output, "false") {
+		return false, fmt.Sprintf("modifying flags, waiting for %v pods to be ready\n", component)
+	}
+
+	return true, ""
+}
+
+func runCommands(commands []string, sshClient libssh.Client) error {
+	for _, cmd := range commands {
+		err := sshClient.Command(cmd)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func verifyComponentCommandHelper(sshClient libssh.Client, c componentName) error {
+	output, err := sshClient.CommandWithNoStdOut(searchFeatureGatesInFile(fmt.Sprintf("/etc/kubernetes/manifests/%s.yaml", c)))
+	if err != nil {
+		return err
+	}
+	fgsExist := len(output) > 0
+	if fgsExist {
+		return fmt.Errorf(fmt.Sprintf(featureGateExistInComponentCommandErrorFormat, c))
+	}
+	return nil
+}
+
+func searchFeatureGatesInFile(file string) string {
+	return fmt.Sprintf(searchFeatureGatesInFileFormat, file)
+}
+
+func getComponentCommand(component componentName) string {
+	return fmt.Sprintf(getComponentCommandFormat, component)
+}
+
+func getComponentReadyContainers(component componentName) string {
+	return fmt.Sprintf(getComponentReadyContainersFormat, component)
+}
+
+func addFlagsToComponentCommand(component componentName, flag string) string {
+	return fmt.Sprintf(addFlagsToComponentCommandFormat, component, flag, component)
+}
+
+func addFeatureGatesToKubeletConfigCommand(feature string) string {
+	return fmt.Sprintf(addFeatureGatesToKubeletConfigCommandFormatFormat, feature)
+}

cluster-provision/gocli/opts/k8scomponents/k8scomponents.go

@@ -0,0 +1,53 @@
+package k8scomponents
+
+import (
+	"kubevirt.io/kubevirtci/cluster-provision/gocli/pkg/libssh"
+)
+
+type k8sComponentsOpt struct {
+	sshClient     libssh.Client
+	kubeletSetup  bool
+	runtimeConfig string
+	featureGates  string
+	component     []component
+}
+
+func NewK8sComponentsOpt(sc libssh.Client, featureGates string, runtimeConfig string) *k8sComponentsOpt {
+	return &k8sComponentsOpt{
+		sshClient:     sc,
+		runtimeConfig: runtimeConfig,
+		featureGates:  featureGates,
+		component: []component{
+			newKubeletComponent(sc, featureGates),
+			newKubeAPIComponent(sc, featureGates, runtimeConfig),
+			newKubeSchedulerComponent(sc, featureGates),
+			newKubeControllerMgrComponent(sc, featureGates),
+		},
+	}
+}
+
+func (o *k8sComponentsOpt) Exec() error {
+	output, err := o.sshClient.CommandWithNoStdOut(searchComponentsFilesCommand)
+	if err != nil {
+		return err
+	}
+	onMasterNode := len(output) > 0
+	for _, c := range o.component {
+		if c.requiredOnlyForMaster() && !onMasterNode {
+			continue
+		}
+		if err := c.verifyComponent(); err != nil {
+			return err
+		}
+		if err := c.prepareCommandsForConfiguration(); err != nil {
+			return err
+		}
+		if err := c.runCommandsToConfigure(); err != nil {
+			return err
+		}
+		if err := c.waitForComponentReady(); err != nil {
+			return err
+		}
+	}
+	return nil
+}