diff --git a/KUBEVIRTCI_LOCAL_TESTING.md b/KUBEVIRTCI_LOCAL_TESTING.md index 9ec67a8d90..e6dd078f92 100644 --- a/KUBEVIRTCI_LOCAL_TESTING.md +++ b/KUBEVIRTCI_LOCAL_TESTING.md @@ -65,6 +65,18 @@ export KUBEVIRT_DEPLOY_PROMETHEUS_ALERTMANAGER=true export KUBEVIRT_DEPLOY_GRAFANA=true ``` +#### start cluster with Kubernetes feature gates +To enable/disable Kubernetes feature gates, export the following variable before running `make cluster-up`: + +```bash +export K8S_FEATURE_GATES="=true,=false" +``` + +for feature gates that require runtime config the following variable should be exported: +```bash +export K8S_API_RUNTIME_CONFIG="=true" +``` + #### start cluster with swap enabled To enable swap, please also export the following variables before running `make cluster-up`: ```bash diff --git a/cluster-provision/gocli/cmd/nodesconfig/nodeconfig.go b/cluster-provision/gocli/cmd/nodesconfig/nodeconfig.go index 187cbb0774..aa48c8ffe5 100644 --- a/cluster-provision/gocli/cmd/nodesconfig/nodeconfig.go +++ b/cluster-provision/gocli/cmd/nodesconfig/nodeconfig.go @@ -4,6 +4,8 @@ package nodesconfig type NodeLinuxConfig struct { NodeIdx int K8sVersion string + FeatureGates string + RuntimeConfig string FipsEnabled bool DockerProxy string EtcdInMemory bool diff --git a/cluster-provision/gocli/cmd/nodesconfig/opts.go b/cluster-provision/gocli/cmd/nodesconfig/opts.go index 1994660dd8..05ff689a75 100644 --- a/cluster-provision/gocli/cmd/nodesconfig/opts.go +++ b/cluster-provision/gocli/cmd/nodesconfig/opts.go @@ -88,6 +88,18 @@ func WithSwap(swap bool) LinuxConfigFunc { } } +func WithFeatureGates(featureGates string) LinuxConfigFunc { + return func(n *NodeLinuxConfig) { + n.FeatureGates = featureGates + } +} + +func WithRuntimeConfig(runtimeConfig string) LinuxConfigFunc { + return func(n *NodeLinuxConfig) { + n.RuntimeConfig = runtimeConfig + } +} + func WithKsmEnabled(ksmEnabled bool) LinuxConfigFunc { return func(n *NodeLinuxConfig) { n.KsmEnabled = ksmEnabled diff --git a/cluster-provision/gocli/cmd/run.go b/cluster-provision/gocli/cmd/run.go index 2e032e174a..0a2482e2a0 100644 --- a/cluster-provision/gocli/cmd/run.go +++ b/cluster-provision/gocli/cmd/run.go @@ -38,6 +38,7 @@ import ( dockerproxy "kubevirt.io/kubevirtci/cluster-provision/gocli/opts/docker-proxy" etcdinmemory "kubevirt.io/kubevirtci/cluster-provision/gocli/opts/etcd" "kubevirt.io/kubevirtci/cluster-provision/gocli/opts/istio" + "kubevirt.io/kubevirtci/cluster-provision/gocli/opts/k8scomponents" "kubevirt.io/kubevirtci/cluster-provision/gocli/opts/ksm" "kubevirt.io/kubevirtci/cluster-provision/gocli/opts/multus" "kubevirt.io/kubevirtci/cluster-provision/gocli/opts/nfscsi" @@ -80,8 +81,8 @@ EOF etcdDataDir = "/var/lib/etcd" nvmeDiskImagePrefix = "/nvme" scsiDiskImagePrefix = "/scsi" - QEMU_DEVICE_S390X = "virtio-net-ccw" - QEMU_DEVICE_X86_64 = "virtio-net-pci" + QEMU_DEVICE_S390X = "virtio-net-ccw" + QEMU_DEVICE_X86_64 = "virtio-net-pci" ) var soundcardPCIIDs = []string{"8086:2668", "8086:2415"} @@ -112,6 +113,8 @@ func NewRunCommand() *cobra.Command { run.Flags().UintP("secondary-nics", "", 0, "number of secondary nics to add") run.Flags().String("qemu-args", "", "additional qemu args to pass through to the nodes") run.Flags().String("kernel-args", "", "additional kernel args to pass through to the nodes") + run.Flags().String("feature-gates", "", "k8s feature gates to enable") + run.Flags().String("runtime-config", "", "k8s api runtime config") run.Flags().BoolP("background", "b", true, "go to background after nodes are up") run.Flags().Bool("random-ports", true, "expose all ports on random localhost ports") run.Flags().Bool("slim", false, "use the slim flavor") @@ -225,6 +228,16 @@ func run(cmd *cobra.Command, args []string) (retErr error) { return err } + featureGates, err := cmd.Flags().GetString("feature-gates") + if err != nil { + return err + } + + runtimeConfig, err := cmd.Flags().GetString("runtime-config") + if err != nil { + return err + } + cpu, err := cmd.Flags().GetUint("cpu") if err != nil { return err @@ -578,7 +591,7 @@ func run(cmd *cobra.Command, args []string) (retErr error) { qemuArgs += " -serial pty" var qemuNetDevice = getNetDeviceByArch() - + wg := sync.WaitGroup{} wg.Add(int(nodes)) // start one vm after each other @@ -592,8 +605,8 @@ func run(cmd *cobra.Command, args []string) (retErr error) { netSuffix := fmt.Sprintf("%d-%d", x, i) macSuffix := fmt.Sprintf("%02x", macCounter) macCounter++ - // Secondary network devices are added after VM is started (hot-plug) using qemu monitor to avoid - // primary network interface to be named other than eth0. This is mainly required for s390x, as + // Secondary network devices are added after VM is started (hot-plug) using qemu monitor to avoid + // primary network interface to be named other than eth0. This is mainly required for s390x, as // otherwise if primary interface is other than eth0, it can't get the IP from dhcp server. if qemuNetDevice == QEMU_DEVICE_S390X { nodeQemuMonitorArgs = fmt.Sprintf("%s netdev_add tap,id=secondarynet%s,ifname=stap%s,script=no,downscript=no; device_add %s,netdev=secondarynet%s,mac=52:55:00:d1:56:%s;", nodeQemuMonitorArgs, netSuffix, netSuffix, qemuNetDevice, netSuffix, macSuffix) @@ -815,6 +828,8 @@ func run(cmd *cobra.Command, args []string) (retErr error) { nodesconfig.WithSwapiness(int(swapiness)), nodesconfig.WithSwapSize(int(swapSize)), nodesconfig.WithUnlimitedSwap(unlimitedSwap), + nodesconfig.WithFeatureGates(featureGates), + nodesconfig.WithRuntimeConfig(runtimeConfig), } n := nodesconfig.NewNodeLinuxConfig(x+1, prefix, linuxConfigFuncs) @@ -1021,6 +1036,11 @@ func provisionNode(sshClient libssh.Client, n *nodesconfig.NodeLinuxConfig) erro opts = append(opts, swapOpt) } + if n.FeatureGates != "" || n.RuntimeConfig != "" { + featureGatesOpt := k8scomponents.NewK8sComponentsOpt(sshClient, n.FeatureGates, n.RuntimeConfig) + opts = append(opts, featureGatesOpt) + } + for _, o := range opts { if err := o.Exec(); err != nil { return err diff --git a/cluster-provision/gocli/opts/k8scomponents/components_helper.go b/cluster-provision/gocli/opts/k8scomponents/components_helper.go new file mode 100644 index 0000000000..22a8c15571 --- /dev/null +++ b/cluster-provision/gocli/opts/k8scomponents/components_helper.go @@ -0,0 +1,126 @@ +package k8scomponents + +import ( + "fmt" + "strings" + "time" + + "kubevirt.io/kubevirtci/cluster-provision/gocli/pkg/libssh" +) + +type componentName string + +const ( + componentKubeAPIServer componentName = "kube-apiserver" + componentKubeControllerMgr componentName = "kube-controller-manager" + componentKubeScheduler componentName = "kube-scheduler" + + searchFeatureGatesInFileFormat = "awk '/feature-gates/' %s" + getComponentCommandFormat = "kubectl --kubeconfig=/etc/kubernetes/admin.conf get pods -n kube-system -l component=%s -o jsonpath='{.items[0].spec.containers[*].command}'" + getComponentReadyContainersFormat = "kubectl --kubeconfig=/etc/kubernetes/admin.conf get pods -n kube-system -l component=%s -o=jsonpath='{.items[0].status.containerStatuses[*].ready}'" + getNodeReadyStatusCommand = "kubectl --kubeconfig=/etc/kubernetes/admin.confget nodes -o=jsonpath='{.items[*].status.conditions[?(@.type==\"Ready\")].status}'" + addFlagsToComponentCommandFormat = `sudo sed -i '/- %s/a\ - %s' /etc/kubernetes/manifests/%s.yaml` + searchComponentsFilesCommand = "ls /etc/kubernetes/manifests" + addFeatureGatesFieldToKubeletConfigCommand = "sudo echo -e 'featureGates:' >> /var/lib/kubelet/config.yaml" + addFeatureGatesToKubeletConfigCommandFormatFormat = `sudo sed -i 's/featureGates:/featureGates:\n %s/g' /var/lib/kubelet/config.yaml` + kubeletRestartCommand = "sudo systemctl restart kubelet" + + featureGateExistInKubeletError = "feature gates should not exist in kubelet by default" + featureGateExistInComponentCommandErrorFormat = "feature gates should not exist in %v command by default" +) + +type component interface { + verifyComponent() error + prepareCommandsForConfiguration() error + runCommandsToConfigure() error + waitForComponentReady() error + requiredOnlyForMaster() bool +} + +func componentReady(component componentName, sshClient libssh.Client, waitingForFeatureGate bool, waitingForRuntimeConfig bool) error { + ticker := time.NewTicker(3 * time.Second) + defer ticker.Stop() + timeoutChannel := time.After(5 * time.Minute) + + select { + case <-timeoutChannel: + return fmt.Errorf("timed out after 5 minutes waiting for %v to be ready", component) + case <-ticker.C: + ready, reason := componentReadyHelper(component, sshClient, waitingForFeatureGate, waitingForRuntimeConfig) + if ready { + return nil + } + fmt.Printf(reason) + } + + return nil +} + +func componentReadyHelper(component componentName, sshClient libssh.Client, waitingForFeatureGate bool, waitingForRuntimeConfig bool) (bool, string) { + if waitingForFeatureGate || waitingForRuntimeConfig { + output, err := sshClient.CommandWithNoStdOut(getComponentCommand(component)) + if err != nil { + return false, fmt.Sprintf("modifying flags, waiting for apiserver to respord after %v restart err:%v\n", component, err) + } + + if waitingForFeatureGate && !strings.Contains(output, "feature-gate") { + return false, fmt.Sprintf("modifying flags, waiting for %v pods to restart\n", component) + } + + if waitingForRuntimeConfig && !strings.Contains(output, "runtime-config") { + return false, fmt.Sprintf("modifying flags, waiting for %v pods to restart\n", component) + } + } + + output, err := sshClient.CommandWithNoStdOut(getComponentReadyContainers(component)) + if err != nil { + return false, fmt.Sprintf("modifying flags, waiting for apiserver to respord after %v restart err:%v\n", component, err) + } + if strings.Contains(output, "false") { + return false, fmt.Sprintf("modifying flags, waiting for %v pods to be ready\n", component) + } + + return true, "" +} + +func runCommands(commands []string, sshClient libssh.Client) error { + for _, cmd := range commands { + err := sshClient.Command(cmd) + if err != nil { + return err + } + } + return nil +} + +func verifyComponentCommandHelper(sshClient libssh.Client, c componentName) error { + output, err := sshClient.CommandWithNoStdOut(searchFeatureGatesInFile(fmt.Sprintf("/etc/kubernetes/manifests/%s.yaml", c))) + if err != nil { + return err + } + fgsExist := len(output) > 0 + if fgsExist { + return fmt.Errorf(fmt.Sprintf(featureGateExistInComponentCommandErrorFormat, c)) + } + return nil +} + +func searchFeatureGatesInFile(file string) string { + return fmt.Sprintf(searchFeatureGatesInFileFormat, file) +} + +func getComponentCommand(component componentName) string { + return fmt.Sprintf(getComponentCommandFormat, component) +} + +func getComponentReadyContainers(component componentName) string { + return fmt.Sprintf(getComponentReadyContainersFormat, component) +} + +func addFlagsToComponentCommand(component componentName, flag string) string { + return fmt.Sprintf(addFlagsToComponentCommandFormat, component, flag, component) +} + +func addFeatureGatesToKubeletConfigCommand(feature string) string { + return fmt.Sprintf(addFeatureGatesToKubeletConfigCommandFormatFormat, feature) +} diff --git a/cluster-provision/gocli/opts/k8scomponents/k8scomponents.go b/cluster-provision/gocli/opts/k8scomponents/k8scomponents.go new file mode 100644 index 0000000000..e3213d88ac --- /dev/null +++ b/cluster-provision/gocli/opts/k8scomponents/k8scomponents.go @@ -0,0 +1,53 @@ +package k8scomponents + +import ( + "kubevirt.io/kubevirtci/cluster-provision/gocli/pkg/libssh" +) + +type k8sComponentsOpt struct { + sshClient libssh.Client + kubeletSetup bool + runtimeConfig string + featureGates string + component []component +} + +func NewK8sComponentsOpt(sc libssh.Client, featureGates string, runtimeConfig string) *k8sComponentsOpt { + return &k8sComponentsOpt{ + sshClient: sc, + runtimeConfig: runtimeConfig, + featureGates: featureGates, + component: []component{ + newKubeletComponent(sc, featureGates), + newKubeAPIComponent(sc, featureGates, runtimeConfig), + newKubeSchedulerComponent(sc, featureGates), + newKubeControllerMgrComponent(sc, featureGates), + }, + } +} + +func (o *k8sComponentsOpt) Exec() error { + output, err := o.sshClient.CommandWithNoStdOut(searchComponentsFilesCommand) + if err != nil { + return err + } + onMasterNode := len(output) > 0 + for _, c := range o.component { + if c.requiredOnlyForMaster() && !onMasterNode { + continue + } + if err := c.verifyComponent(); err != nil { + return err + } + if err := c.prepareCommandsForConfiguration(); err != nil { + return err + } + if err := c.runCommandsToConfigure(); err != nil { + return err + } + if err := c.waitForComponentReady(); err != nil { + return err + } + } + return nil +} diff --git a/cluster-provision/gocli/opts/k8scomponents/k8scomponents_test.go b/cluster-provision/gocli/opts/k8scomponents/k8scomponents_test.go new file mode 100644 index 0000000000..96d86e2a1b --- /dev/null +++ b/cluster-provision/gocli/opts/k8scomponents/k8scomponents_test.go @@ -0,0 +1,134 @@ +package k8scomponents + +import ( + "fmt" + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "go.uber.org/mock/gomock" + kubevirtcimocks "kubevirt.io/kubevirtci/cluster-provision/gocli/utils/mock" +) + +func TestFeatureGatesOpt(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "FeatureGatesOpt Suite") +} + +type cmdOutPut struct { + output string + err error +} + +var _ = Describe("featureGatesOpt", func() { + var sshClient *kubevirtcimocks.MockSSHClient + + BeforeEach(func() { + sshClient = kubevirtcimocks.NewMockSSHClient(gomock.NewController(GinkgoT())) + }) + + DescribeTable("when ", func(featureGates string, runtimeConfig string, cmds []string, cmdsWithOutPut map[string]cmdOutPut, expectedMsg string) { + opt := NewK8sComponentsOpt(sshClient, featureGates, runtimeConfig) + for cmd, output := range cmdsWithOutPut { + sshClient.EXPECT().CommandWithNoStdOut(cmd).Return(output.output, output.err) + } + + for _, cmd := range cmds { + sshClient.EXPECT().Command(cmd) + } + + err := opt.Exec() + if expectedMsg != "" { + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring(expectedMsg)) + } else { + Expect(err).NotTo(HaveOccurred()) + } + }, + Entry("should execute FeatureGatesOpt successfully with only featureGates", + "FG1=true,FG2=true", "", + []string{addFeatureGatesFieldToKubeletConfigCommand, + addFeatureGatesToKubeletConfigCommand("FG1: true"), + addFeatureGatesToKubeletConfigCommand("FG2: true"), + kubeletRestartCommand, + addFlagsToComponentCommand(componentKubeAPIServer, "--feature-gates=FG1=true,FG2=true"), + addFlagsToComponentCommand(componentKubeControllerMgr, "--feature-gates=FG1=true,FG2=true"), + addFlagsToComponentCommand(componentKubeScheduler, "--feature-gates=FG1=true,FG2=true"), + }, map[string]cmdOutPut{ + searchFeatureGatesInFile("/var/lib/kubelet/config.yaml"): {"", nil}, + getNodeReadyStatusCommand: {"true", nil}, + searchComponentsFilesCommand: {"nonEmptyOutput", nil}, + searchFeatureGatesInFile(fmt.Sprintf("/etc/kubernetes/manifests/%s.yaml", componentKubeAPIServer)): {"", nil}, + getComponentCommand(componentKubeAPIServer): {"feature-gate", nil}, + getComponentReadyContainers(componentKubeAPIServer): {"true", nil}, + searchFeatureGatesInFile(fmt.Sprintf("/etc/kubernetes/manifests/%s.yaml", componentKubeControllerMgr)): {"", nil}, + getComponentCommand(componentKubeControllerMgr): {"feature-gate", nil}, + getComponentReadyContainers(componentKubeControllerMgr): {"true", nil}, + searchFeatureGatesInFile(fmt.Sprintf("/etc/kubernetes/manifests/%s.yaml", componentKubeScheduler)): {"", nil}, + getComponentCommand(componentKubeScheduler): {"feature-gate", nil}, + getComponentReadyContainers(componentKubeScheduler): {"true", nil}, + }, ""), + Entry("should execute FeatureGatesOpt successfully with only runtimeConfig", + "", "runtimeConfig", + []string{addFlagsToComponentCommand(componentKubeAPIServer, "--runtime-config=runtimeConfig")}, + map[string]cmdOutPut{ + searchFeatureGatesInFile("/var/lib/kubelet/config.yaml"): {"", nil}, + getNodeReadyStatusCommand: {"true", nil}, + searchComponentsFilesCommand: {"nonEmptyOutput", nil}, + searchFeatureGatesInFile(fmt.Sprintf("/etc/kubernetes/manifests/%s.yaml", componentKubeAPIServer)): {"", nil}, + getComponentCommand(componentKubeAPIServer): {"runtime-config", nil}, + getComponentReadyContainers(componentKubeAPIServer): {"true", nil}, + searchFeatureGatesInFile(fmt.Sprintf("/etc/kubernetes/manifests/%s.yaml", componentKubeControllerMgr)): {"", nil}, + getComponentReadyContainers(componentKubeControllerMgr): {"true", nil}, + searchFeatureGatesInFile(fmt.Sprintf("/etc/kubernetes/manifests/%s.yaml", componentKubeScheduler)): {"", nil}, + getComponentReadyContainers(componentKubeScheduler): {"true", nil}, + }, ""), + Entry("should fail when kubelet feature gate already exists", + "FG1=true,FG2=true", "", + []string{}, map[string]cmdOutPut{ + searchComponentsFilesCommand: {"nonEmptyOutput", nil}, + searchFeatureGatesInFile("/var/lib/kubelet/config.yaml"): {"featureGates", nil}, + }, featureGateExistInKubeletError), + Entry("should fail when feature gate already exists in components", + "FG1=true,FG2=true", "runtimeConfig", + []string{ + addFeatureGatesFieldToKubeletConfigCommand, + addFeatureGatesToKubeletConfigCommand("FG1: true"), + addFeatureGatesToKubeletConfigCommand("FG2: true"), + kubeletRestartCommand, + }, + map[string]cmdOutPut{ + searchComponentsFilesCommand: {"nonEmptyOutput", nil}, + getNodeReadyStatusCommand: {"true", nil}, + searchFeatureGatesInFile("/var/lib/kubelet/config.yaml"): {"", nil}, + searchFeatureGatesInFile(fmt.Sprintf("/etc/kubernetes/manifests/%s.yaml", componentKubeAPIServer)): {"featureGates", nil}, + }, fmt.Sprintf(featureGateExistInComponentCommandErrorFormat, componentKubeAPIServer)), + Entry("should retry when API server does not respond or if changes are not propagated to components", + "FG1=true,FG2=true", "", + []string{ + addFeatureGatesFieldToKubeletConfigCommand, + addFeatureGatesToKubeletConfigCommand("FG1: true"), + addFeatureGatesToKubeletConfigCommand("FG2: true"), + kubeletRestartCommand, + addFlagsToComponentCommand(componentKubeAPIServer, "--feature-gates=FG1=true,FG2=true"), + addFlagsToComponentCommand(componentKubeControllerMgr, "--feature-gates=FG1=true,FG2=true"), + addFlagsToComponentCommand(componentKubeScheduler, "--feature-gates=FG1=true,FG2=true"), + }, map[string]cmdOutPut{ + searchComponentsFilesCommand: {"nonEmptyOutput", nil}, + searchFeatureGatesInFile("/var/lib/kubelet/config.yaml"): {"", nil}, + getNodeReadyStatusCommand: {"true", nil}, + searchFeatureGatesInFile(fmt.Sprintf("/etc/kubernetes/manifests/%s.yaml", componentKubeAPIServer)): {"", nil}, + searchFeatureGatesInFile(fmt.Sprintf("/etc/kubernetes/manifests/%s.yaml", componentKubeControllerMgr)): {"", nil}, + searchFeatureGatesInFile(fmt.Sprintf("/etc/kubernetes/manifests/%s.yaml", componentKubeScheduler)): {"", nil}, + // First 2 attempts fail + getComponentCommand(componentKubeAPIServer): {"some-output", nil}, + getComponentCommand(componentKubeAPIServer): {"", fmt.Errorf("API server not responding")}, + // Third attempt succeeds + getComponentCommand(componentKubeAPIServer): {"feature-gate", nil}, + getComponentReadyContainers(componentKubeAPIServer): {"true", nil}, + getComponentCommand(componentKubeControllerMgr): {"feature-gate", nil}, + getComponentReadyContainers(componentKubeControllerMgr): {"true", nil}, + getComponentCommand(componentKubeScheduler): {"feature-gate", nil}, + getComponentReadyContainers(componentKubeScheduler): {"true", nil}, + }, "")) +}) diff --git a/cluster-provision/gocli/opts/k8scomponents/kube-scheduler.go b/cluster-provision/gocli/opts/k8scomponents/kube-scheduler.go new file mode 100644 index 0000000000..99d821e65f --- /dev/null +++ b/cluster-provision/gocli/opts/k8scomponents/kube-scheduler.go @@ -0,0 +1,42 @@ +package k8scomponents + +import ( + "fmt" + "kubevirt.io/kubevirtci/cluster-provision/gocli/pkg/libssh" +) + +type kubeSchedulerComponent struct { + featureGates string + cmds []string + sshClient libssh.Client +} + +func newKubeSchedulerComponent(sc libssh.Client, featureGates string) *kubeSchedulerComponent { + return &kubeSchedulerComponent{ + sshClient: sc, + featureGates: featureGates, + } +} + +func (k *kubeSchedulerComponent) verifyComponent() error { + return verifyComponentCommandHelper(k.sshClient, componentKubeScheduler) +} + +func (k *kubeSchedulerComponent) prepareCommandsForConfiguration() error { + if k.featureGates != "" { + k.cmds = append(k.cmds, addFlagsToComponentCommand(componentKubeScheduler, fmt.Sprintf("--feature-gates=%s", k.featureGates))) + } + return nil +} + +func (k *kubeSchedulerComponent) runCommandsToConfigure() error { + return runCommands(k.cmds, k.sshClient) +} + +func (k *kubeSchedulerComponent) waitForComponentReady() error { + return componentReady(componentKubeScheduler, k.sshClient, k.featureGates != "", false) +} + +func (k *kubeSchedulerComponent) requiredOnlyForMaster() bool { + return true +} diff --git a/cluster-provision/gocli/opts/k8scomponents/kubeapi.go b/cluster-provision/gocli/opts/k8scomponents/kubeapi.go new file mode 100644 index 0000000000..cce3e17c4a --- /dev/null +++ b/cluster-provision/gocli/opts/k8scomponents/kubeapi.go @@ -0,0 +1,47 @@ +package k8scomponents + +import ( + "fmt" + "kubevirt.io/kubevirtci/cluster-provision/gocli/pkg/libssh" +) + +type kubeAPIComponent struct { + runtimeConfig string + featureGates string + cmds []string + sshClient libssh.Client +} + +func newKubeAPIComponent(sc libssh.Client, featureGates string, runtimeCofig string) *kubeAPIComponent { + return &kubeAPIComponent{ + sshClient: sc, + featureGates: featureGates, + runtimeConfig: runtimeCofig, + } +} + +func (k *kubeAPIComponent) verifyComponent() error { + return verifyComponentCommandHelper(k.sshClient, componentKubeAPIServer) +} + +func (k *kubeAPIComponent) prepareCommandsForConfiguration() error { + if k.featureGates != "" { + k.cmds = append(k.cmds, addFlagsToComponentCommand(componentKubeAPIServer, fmt.Sprintf("--feature-gates=%s", k.featureGates))) + } + if k.runtimeConfig != "" { + k.cmds = append(k.cmds, addFlagsToComponentCommand(componentKubeAPIServer, fmt.Sprintf("--runtime-config=%s", k.runtimeConfig))) + } + return nil +} + +func (k *kubeAPIComponent) runCommandsToConfigure() error { + return runCommands(k.cmds, k.sshClient) +} + +func (k *kubeAPIComponent) waitForComponentReady() error { + return componentReady(componentKubeAPIServer, k.sshClient, k.featureGates != "", k.runtimeConfig != "") +} + +func (k *kubeAPIComponent) requiredOnlyForMaster() bool { + return true +} diff --git a/cluster-provision/gocli/opts/k8scomponents/kubecontrollermanager.go b/cluster-provision/gocli/opts/k8scomponents/kubecontrollermanager.go new file mode 100644 index 0000000000..4c60e8f741 --- /dev/null +++ b/cluster-provision/gocli/opts/k8scomponents/kubecontrollermanager.go @@ -0,0 +1,42 @@ +package k8scomponents + +import ( + "fmt" + "kubevirt.io/kubevirtci/cluster-provision/gocli/pkg/libssh" +) + +type kubeControllerMgrComponent struct { + featureGates string + cmds []string + sshClient libssh.Client +} + +func newKubeControllerMgrComponent(sc libssh.Client, featureGates string) *kubeControllerMgrComponent { + return &kubeControllerMgrComponent{ + sshClient: sc, + featureGates: featureGates, + } +} + +func (k *kubeControllerMgrComponent) verifyComponent() error { + return verifyComponentCommandHelper(k.sshClient, componentKubeControllerMgr) +} + +func (k *kubeControllerMgrComponent) prepareCommandsForConfiguration() error { + if k.featureGates != "" { + k.cmds = append(k.cmds, addFlagsToComponentCommand(componentKubeControllerMgr, fmt.Sprintf("--feature-gates=%s", k.featureGates))) + } + return nil +} + +func (k *kubeControllerMgrComponent) runCommandsToConfigure() error { + return runCommands(k.cmds, k.sshClient) +} + +func (k *kubeControllerMgrComponent) waitForComponentReady() error { + return componentReady(componentKubeControllerMgr, k.sshClient, k.featureGates != "", false) +} + +func (k *kubeControllerMgrComponent) requiredOnlyForMaster() bool { + return true +} diff --git a/cluster-provision/gocli/opts/k8scomponents/kubelet.go b/cluster-provision/gocli/opts/k8scomponents/kubelet.go new file mode 100644 index 0000000000..ae22cd3bfb --- /dev/null +++ b/cluster-provision/gocli/opts/k8scomponents/kubelet.go @@ -0,0 +1,78 @@ +package k8scomponents + +import ( + "fmt" + "strings" + "time" + + "kubevirt.io/kubevirtci/cluster-provision/gocli/pkg/libssh" +) + +type kubeletComponent struct { + featureGates string + cmds []string + sshClient libssh.Client +} + +func newKubeletComponent(sc libssh.Client, featureGates string) *kubeletComponent { + return &kubeletComponent{ + sshClient: sc, + featureGates: featureGates, + } +} + +func (k *kubeletComponent) verifyComponent() error { + output, err := k.sshClient.CommandWithNoStdOut(searchFeatureGatesInFile("/var/lib/kubelet/config.yaml")) + if err != nil { + return err + } + fgsExist := len(output) > 0 + if fgsExist { + return fmt.Errorf(featureGateExistInKubeletError) + } + return nil +} + +func (k *kubeletComponent) prepareCommandsForConfiguration() error { + if k.featureGates != "" { + k.cmds = append(k.cmds, addFeatureGatesFieldToKubeletConfigCommand) + keyValuePairs := strings.Split(k.featureGates, ",") + var formattedFeatureGates []string + for _, pair := range keyValuePairs { + formattedFeatureGates = append(formattedFeatureGates, strings.Replace(pair, "=", ": ", 1)) + } + for _, fg := range formattedFeatureGates { + k.cmds = append(k.cmds, addFeatureGatesToKubeletConfigCommand(fg)) + } + k.cmds = append(k.cmds, kubeletRestartCommand) + } + + return nil +} + +func (k *kubeletComponent) runCommandsToConfigure() error { + return runCommands(k.cmds, k.sshClient) +} + +func (k *kubeletComponent) waitForComponentReady() error { + ticker := time.NewTicker(1 * time.Second) + defer ticker.Stop() + timeoutChannel := time.After(3 * time.Minute) + select { + case <-timeoutChannel: + return fmt.Errorf("timed out after 3 minutes waiting for node to be ready") + case <-ticker.C: + output, err := k.sshClient.CommandWithNoStdOut(getNodeReadyStatusCommand) + if err == nil && !strings.Contains(output, "false") { + return nil + } + if err != nil { + fmt.Printf("Modifying kubelet configuration, API server not responding yet, err: %v\n", err) + } + } + return nil +} + +func (k *kubeletComponent) requiredOnlyForMaster() bool { + return false +} diff --git a/cluster-up/cluster/ephemeral-provider-common.sh b/cluster-up/cluster/ephemeral-provider-common.sh index d7d83f4583..c9348fd5e5 100644 --- a/cluster-up/cluster/ephemeral-provider-common.sh +++ b/cluster-up/cluster/ephemeral-provider-common.sh @@ -225,6 +225,14 @@ function _add_common_params() { params=" --ksm-page-count=$KUBEVIRT_KSM_PAGES_TO_SCAN $params" fi + if [ ! -z $K8S_FEATURE_GATES ]; then + params=" --feature-gates=$K8S_FEATURE_GATES $params" + fi + + if [ ! -z $K8S_API_RUNTIME_CONFIG ]; then + params=" --runtime-config=$K8S_API_RUNTIME_CONFIG $params" + fi + if [ "$KUBEVIRT_SWAP_ON" == "true" ]; then params=" --enable-swap $params" fi