feat: K8S 1.30 provider provisioning script changes to support s390x

- Updated username to be obtained conditionally based on arch. - Removed CPU manager related kubelet arguments, as CPU manager args aren’t supported on s390x. - Installing openvswitch for s390x from kojipkgs.fedoraproject.org, as for s390x same isn’t available from default centos repos - Explicitly installing gettext rpm to make envsubst command available for s390x environments. Signed-off-by: chandramerla <[email protected]>
kubevirt · Aug 12, 2024 · 71595fe · 71595fe
1 parent 28fa857
commit 71595fe
Show file tree

Hide file tree

Showing 4 changed files with 57 additions and 14 deletions.
diff --git a/cluster-provision/k8s/1.30/k8s_provision.sh b/cluster-provision/k8s/1.30/k8s_provision.sh
@@ -2,6 +2,8 @@
 
 set -ex
 
+arch=$(uname -m)
+
 source /var/lib/kubevirtci/shared_vars.sh
 
 function getKubernetesClosestStableVersion() {
@@ -62,15 +64,25 @@ function pull_container_retry() {
     fi
 }
 
-export CRIO_VERSION=1.30
-cat << EOF >/etc/yum.repos.d/devel_kubic_libcontainers_stable_cri-o_${CRIO_VERSION}.repo
+if [ "$arch" == "s390x" ]; then
+  export CRIO_VERSION=1.28
+  #As crio version available in kubevirtci-crio-mirror/isv_kubernetes_addons_cri-o_stable_v1.30 is broken with issue https://github.com/containers/crun/issues/1494, which is fixed, but yet to part of crio release
+  BASEURL="https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable:/cri-o:/${CRIO_VERSION}:/${CRIO_VERSION}.4/CentOS_9_Stream/"
+else
+  export CRIO_VERSION=1.30
+  BASEURL="https://storage.googleapis.com/kubevirtci-crio-mirror/isv_kubernetes_addons_cri-o_stable_v${CRIO_VERSION}"
+fi
+
+REPO_CONTENT=$(cat << EOF
 [isv_kubernetes_addons_cri-o_stable_v${CRIO_VERSION}]
 name=CRI-O v${CRIO_VERSION} (Stable) (rpm)
 type=rpm-md
-baseurl=https://storage.googleapis.com/kubevirtci-crio-mirror/isv_kubernetes_addons_cri-o_stable_v${CRIO_VERSION}
+baseurl=${BASEURL}
 gpgcheck=0
 enabled=1
 EOF
+)
+echo "$REPO_CONTENT" | tee /etc/yum.repos.d/devel_kubic_libcontainers_stable_cri-o_${CRIO_VERSION}.repo > /dev/null
 
 dnf install -y cri-o
 
@@ -202,10 +214,13 @@ sysctl --system
 
 systemctl restart NetworkManager
 
-nmcli connection modify "System eth0" \
-   ipv6.method auto \
-   ipv6.addr-gen-mode eui64
-nmcli connection up "System eth0"
+# No need to modify the ethernet connection incase of s390x Architecture.
+if [ "$arch" != "s390x" ]; then
+  nmcli connection modify "System eth0" \
+    ipv6.method auto \
+    ipv6.addr-gen-mode eui64
+  nmcli connection up "System eth0"
+fi
 
 kubeadmn_patches_path="/provision/kubeadm-patches"
 mkdir -p $kubeadmn_patches_path

diff --git a/cluster-provision/k8s/1.30/node01.sh b/cluster-provision/k8s/1.30/node01.sh
@@ -2,14 +2,20 @@
 
 set -ex
 
+ARCH=$(uname -m)
+SSH_USER="vagrant"
+if [ "$ARCH" == "s390x" ]; then
+   SSH_USER="cloud-user"
+fi
+
 kubeadm_conf="/etc/kubernetes/kubeadm.conf"
 cni_manifest="/provision/cni.yaml"
-if [ -f /home/vagrant/single_stack ]; then
+if [ -f /home/$SSH_USER/single_stack ]; then
     kubeadm_conf="/etc/kubernetes/kubeadm_ipv6.conf"
     cni_manifest="/provision/cni_ipv6.yaml"
 fi
 
-if [ -f /home/vagrant/enable_audit ]; then
+if [ -f /home/$SSH_USER/enable_audit ]; then
     apiVer=$(head -1 /etc/kubernetes/audit/adv-audit.yaml)
     echo $apiVer > /etc/kubernetes/audit/adv-audit.yaml
 

diff --git a/cluster-provision/k8s/1.30/nodes.sh b/cluster-provision/k8s/1.30/nodes.sh
@@ -4,13 +4,25 @@ set -ex
 
 source /var/lib/kubevirtci/shared_vars.sh
 
+ARCH=$(uname -m)
+SSH_USER="vagrant"
+if [ "$ARCH" == "s390x" ]; then
+   SSH_USER="cloud-user"
+fi
+
 nodeip=
 control_ip=192.168.66.101
-if [ -f /home/vagrant/single_stack ]; then
+if [ -f /home/$SSH_USER/single_stack ]; then
     nodeip="--node-ip=::"
     control_ip=[fd00::101]
 fi
 
+KUBELET_EXTRA_ARGS_ARCH="--fail-swap-on=false ${nodeip} --feature-gates=CPUManager=true,NodeSwap=true --cpu-manager-policy=static --kube-reserved=cpu=500m --system-reserved=cpu=500m"
+if [ "$ARCH" == "s390x" ]; then
+   # cpu manager feature is not yet supported on s390x.
+   KUBELET_EXTRA_ARGS_ARCH="--fail-swap-on=false ${nodeip} --feature-gates=NodeSwap=true"
+fi
+
 timeout=30
 interval=5
 while ! hostnamectl  |grep Transient ; do
@@ -51,11 +63,11 @@ done
 if [ -f /etc/sysconfig/kubelet ]; then
     # TODO use config file! this is deprecated
     cat <<EOT >>/etc/sysconfig/kubelet
-KUBELET_EXTRA_ARGS=${KUBELET_CGROUP_ARGS} --fail-swap-on=false ${nodeip} --feature-gates=CPUManager=true,NodeSwap=true --cpu-manager-policy=static --kube-reserved=cpu=500m --system-reserved=cpu=500m
+KUBELET_EXTRA_ARGS=${KUBELET_CGROUP_ARGS} $KUBELET_EXTRA_ARGS_ARCH
 EOT
 else
     cat <<EOT >>/etc/systemd/system/kubelet.service.d/09-kubeadm.conf
-Environment="KUBELET_CPUMANAGER_ARGS=--fail-swap-on=false --feature-gates=CPUManager=true,NodeSwap=true ${nodeip} --cpu-manager-policy=static --kube-reserved=cpu=500m --system-reserved=cpu=500m"
+Environment="KUBELET_CPUMANAGER_ARGS=$KUBELET_EXTRA_ARGS_ARCH"
 EOT
 sed -i 's/$KUBELET_EXTRA_ARGS/$KUBELET_EXTRA_ARGS $KUBELET_CPUMANAGER_ARGS/' /etc/systemd/system/kubelet.service.d/10-kubeadm.conf
 fi

diff --git a/cluster-provision/k8s/1.30/provision.sh b/cluster-provision/k8s/1.30/provision.sh
@@ -2,6 +2,8 @@
 
 set -ex
 
+ARCH=$(uname -m)
+
 KUBEVIRTCI_SHARED_DIR=/var/lib/kubevirtci
 mkdir -p $KUBEVIRTCI_SHARED_DIR
 export ISTIO_VERSION=1.15.0
@@ -58,8 +60,16 @@ dnf install -y container-selinux
 
 dnf install -y libseccomp-devel
 
-dnf install -y centos-release-nfv-openvswitch
-dnf install -y openvswitch2.16
+#openvswitch for s390x is not available from the centos default repos.
+if [ "$ARCH" == "s390x" ]; then
+  dnf install -y https://kojipkgs.fedoraproject.org//packages/openvswitch/2.16.0/2.fc36/s390x/openvswitch-2.16.0-2.fc36.s390x.rpm
+  systemctl enable openvswitch
+else
+  dnf install -y centos-release-nfv-openvswitch
+  dnf install -y openvswitch2.16
+fi 
 
 dnf install -y NetworkManager NetworkManager-ovs NetworkManager-config-server
 
+# envsubst pkg is not available by default in s390x Architecture, so explicitly installing it as part of gettext
+dnf install -y gettext