Bootc provisioner

cluster-provision/gocli/bootc/k8s-container/config.toml

@@ -0,0 +1,5 @@
+[[customizations.user]]
+name = "vagrant"
+password = "vagrant"
+key = "ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA6NF8iallvQVp22WDkTkyrtvp9eWW6A8YVr+kz4TjGYe7gHzIw+niNltGEFHzD8+v1I2YJ6oXevct1YeS0o9HZyN1Q9qgCgzUFtdOKLv6IedplqoPkcmF0aYet2PkEDo3MlTBckFXPITAMzF8dJSIFo9D8HfdOV0IAdx4O7PtixWKn5y2hMNG0zQPyUecp4pzC6kivAIhyfHilFR61RGL+GPXQ2MWZWFYbAGjyiYJnAmCP3NOTd0jMZEnDkbUvxhMmBYSdETk1rRgm+R4LOzFUGaHqHDLKLX+FIPKcF96hrucXzcWyLbIbEgE98OHlnVYCzRdK8jlqm8tehUc9c9WhQ=="
+groups = ["wheel"]

cluster-provision/gocli/bootc/k8s-container/k8s.Containerfile

@@ -0,0 +1,32 @@
+FROM LINUX_BASE
+
+ARG VERSION
+
+RUN echo -e "[isv_kubernetes_addons_cri-o_stable_v1.28]\n\
+name=CRI-O v1.28 (Stable) (rpm)\n\
+type=rpm-md\n\
+baseurl=https://storage.googleapis.com/kubevirtci-crio-mirror/isv_kubernetes_addons_cri-o_stable_v1.28\n\
+gpgcheck=0\n\
+enabled=1" > /etc/yum.repos.d/devel_kubic_libcontainers_stable_cri-o_v1.28.repo
+
+RUN MAJOR_MINOR=$(echo $VERSION | awk -F. '{print $1"."$2}') && \
+    echo -e "[kubernetes]\n\
+name=Kubernetes Release\n\
+baseurl=https://pkgs.k8s.io/core:/stable:/v${MAJOR_MINOR}/rpm\n\
+enabled=1\n\
+gpgcheck=0\n\
+repo_gpgcheck=0" > /etc/yum.repos.d/kubernetes.repo
+
+RUN dnf install --nobest --nogpgcheck --disableexcludes=kubernetes -y \
+    kubectl-${VERSION} \
+    kubeadm-${VERSION} \
+    kubelet-${VERSION} \
+    kubernetes-cni
+
+
+RUN dnf install -y cri-o patch
+
+RUN mkdir -p /provision/kubeadm-patches
+
+# COPY manifests /opt/
+# COPY patches /provision/kubeadm-patches

cluster-provision/gocli/bootc/k8s-container/linux.Containerfile

@@ -0,0 +1,47 @@
+FROM quay.io/centos-bootc/centos-bootc:stream9
+
+ENV KUBEVIRTCI_SHARED_DIR=/var/lib/kubevirtci
+ENV ISTIO_VERSION=1.15.0
+ENV ISTIO_BIN_DIR=/opt/istio-${ISTIO_VERSION}/bin
+
+RUN dnf update -y
+
+RUN mkdir -p /opt/scripts
+
+COPY provision-system.sh /opt/scripts/provision-system.sh
+RUN chmod 755 /opt/scripts/provision-system.sh
+COPY provision-system.service /etc/systemd/system/provision-system.service
+
+RUN echo "vagrant ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers
+
+RUN mkdir -p $KUBEVIRTCI_SHARED_DIR \
+    && echo '#!/bin/bash\n' \
+         'set -ex\n' \
+         'export KUBELET_CGROUP_ARGS="--cgroup-driver=systemd --runtime-cgroups=/systemd/system.slice --kubelet-cgroups=/systemd/system.slice"\n' \
+         'export ISTIO_VERSION=${ISTIO_VERSION}\n' \
+         'export ISTIO_BIN_DIR="/opt/istio-${ISTIO_VERSION}/bin"\n' \
+         > $KUBEVIRTCI_SHARED_DIR/shared_vars.sh \
+    && chmod +x $KUBEVIRTCI_SHARED_DIR/shared_vars.sh
+
+RUN dnf install -y "kernel-modules-5.14.0-480.el9.x86_64" \
+    && dnf install -y patch \
+    && dnf install -y pciutils \
+    && systemctl enable provision-system.service \
+    && dnf -y remove firewalld \
+    && dnf -y install iscsi-initiator-utils \
+    && dnf -y install nftables \
+    && dnf -y install lvm2 \
+    && echo 'ACTION=="add|change", SUBSYSTEM=="block", KERNEL=="vd[a-z]", ATTR{queue/rotational}="0"' > /etc/udev/rules.d/60-force-ssd-rotational.rules \
+    && dnf install -y iproute-tc \
+    && mkdir -p "$ISTIO_BIN_DIR" \
+    && curl "https://storage.googleapis.com/kubevirtci-istioctl-mirror/istio-${ISTIO_VERSION}/bin/istioctl" -o "$ISTIO_BIN_DIR/istioctl" \
+    && chmod +x "$ISTIO_BIN_DIR/istioctl" \
+    && dnf install -y container-selinux \
+    && dnf install -y libseccomp-devel \
+    && dnf install -y centos-release-nfv-openvswitch \
+    && dnf install -y openvswitch2.16 \
+    && dnf install -y --skip-broken NetworkManager NetworkManager-ovs NetworkManager-config-server \
+    && dnf clean all \
+    && rm -rf /lib/systemd/system/[email protected]
+
+ENV PATH="$ISTIO_BIN_DIR:$PATH"

cluster-provision/gocli/bootc/k8s-container/patches/add-security-context-deployment-patch.yaml

@@ -0,0 +1,6 @@
+spec:
+  template:
+    spec:
+      securityContext:
+        seLinuxOptions:
+          type: spc_t

cluster-provision/gocli/bootc/k8s-container/patches/etcd.yaml

@@ -0,0 +1,4 @@
+spec:
+  securityContext:
+    seLinuxOptions:
+      type: spc_t

cluster-provision/gocli/bootc/k8s-container/patches/kube-apiserver.yaml

@@ -0,0 +1,4 @@
+spec:
+  securityContext:
+    seLinuxOptions:
+      type: spc_t

cluster-provision/gocli/bootc/k8s-container/patches/kube-controller-manager.yaml

@@ -0,0 +1,4 @@
+spec:
+  securityContext:
+    seLinuxOptions:
+      type: spc_t

cluster-provision/gocli/bootc/k8s-container/patches/kube-scheduler.yaml

@@ -0,0 +1,4 @@
+spec:
+  securityContext:
+    seLinuxOptions:
+      type: spc_t

cluster-provision/gocli/bootc/k8s-container/provision-system.service

@@ -0,0 +1,11 @@
+[Unit]
+Description=KubevirtCI System Setup
+After=NetworkManager-wait-online.service
+Requires=NetworkManager-wait-online.service
+
+[Service]
+ExecStart=/opt/scripts/provision-system.sh
+RemainAfterExit=yes
+
+[Install]
+WantedBy=multi-user.target

cluster-provision/gocli/bootc/k8s-container/provision-system.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+nmcli connection add type ethernet ifname enp0s2 con-name enp0s2 ipv4.method auto ipv6.method auto
+nmcli connection modify enp0s2 connection.autoconnect yes
+nmcli connection up enp0s2
+sudo ostree admin unlock --hotfix
+sudo mkdir -p /var/opt_writable
+sudo cp -r /opt/* /var/opt_writable
+sudo mount --bind /var/opt_writable /opt

cluster-provision/gocli/bootc/main.go

@@ -0,0 +1,134 @@
+package bootc
+
+import (
+	"embed"
+	"strings"
+
+	"os"
+
+	"kubevirt.io/kubevirtci/cluster-provision/gocli/cri"
+)
+
+//go:embed k8s-container/k8s.Containerfile
+var k8sContainerfile []byte
+
+//go:embed k8s-container/linux.Containerfile
+var linuxContainerfile []byte
+
+//go:embed k8s-container/provision-system.sh
+var provisionSystem []byte
+
+//go:embed k8s-container/provision-system.service
+var provisionSystemService []byte
+
+//go:embed k8s-container/config.toml
+var configToml []byte
+
+//go:embed k8s-container/patches/*
+var patches embed.FS
+
+type BootcProvisioner struct {
+	cri cri.ContainerClient
+}
+
+func NewBootcProvisioner(cri cri.ContainerClient) *BootcProvisioner {
+	return &BootcProvisioner{
+		cri: cri,
+	}
+}
+
+func (b *BootcProvisioner) BuildLinuxBase(tag string) error {
+	fileName := "provision-system.sh"
+	containerFile, err := os.Create(fileName)
+	if err != nil {
+		return err
+	}
+	_, err = containerFile.Write(provisionSystem)
+	if err != nil {
+		return err
+	}
+
+	fileName = "provision-system.service"
+	containerFile, err = os.Create(fileName)
+	if err != nil {
+		return err
+	}
+	_, err = containerFile.Write(provisionSystemService)
+	if err != nil {
+		return err
+	}
+
+	fileName = "linux.Containerfile"
+	containerFile, err = os.Create(fileName)
+	if err != nil {
+		return err
+	}
+	_, err = containerFile.Write(linuxContainerfile)
+	if err != nil {
+		return err
+	}
+
+	err = b.cri.Build(tag, fileName, map[string]string{})
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (b *BootcProvisioner) BuildK8sBase(tag, k8sVersion, baseImage string) error {
+	fileName := "k8s.Containerfile"
+	fileWithBase := strings.Replace(string(k8sContainerfile), "LINUX_BASE", baseImage, 1)
+
+	containerFile, err := os.Create(fileName)
+	if err != nil {
+		return err
+	}
+	_, err = containerFile.Write([]byte(fileWithBase))
+	if err != nil {
+		return err
+	}
+	_ = os.Mkdir("patches", 0777)
+
+	err = b.cri.Build(tag, fileName, map[string]string{"VERSION": k8sVersion})
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (b *BootcProvisioner) GenerateQcow(image string) error {
+	_ = os.Mkdir("output", 0777)
+
+	configFileName := "config.toml"
+	conf, err := os.Create(configFileName)
+	if err != nil {
+		return err
+	}
+	_, err = conf.Write(configToml)
+	if err != nil {
+		return err
+	}
+
+	runArgs := []string{"--rm",
+		"--privileged",
+		"--security-opt",
+		"label=type:unconfined_t",
+		"-v",
+		"./output:/output",
+		"-v",
+		"/var/lib/containers/storage:/var/lib/containers/storage",
+		"-v",
+		"./config.toml:/config.toml:ro",
+		"quay.io/centos-bootc/bootc-image-builder:latest",
+		"--type",
+		"qcow2",
+		"--local",
+		"localhost/" + image}
+
+	err = b.cri.Run(runArgs)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}