From c7f72424bf6d27740af4d363ca80beed9c04c23f Mon Sep 17 00:00:00 2001 From: Florian Wilhelm <2292245+fwilhe@users.noreply.github.com> Date: Wed, 18 Oct 2023 11:23:49 +0000 Subject: [PATCH] Sync from gl upstream --- gardenlinux/bin/README.md | 7 ++ gardenlinux/build | 2 +- gardenlinux/container/Makefile | 41 +------ gardenlinux/container/base-test/Dockerfile | 2 +- gardenlinux/container/needslim | 9 +- gardenlinux/features/_boot/exec.late | 18 +-- gardenlinux/features/_boot/file.exclude | 2 +- .../usr/local/sbin/update-syslinux | 105 ++++++++++++------ gardenlinux/features/_dev/pkg.include | 1 + gardenlinux/features/base/fstab | 2 +- .../base/test/test_sgid_suid_files.py | 4 +- .../systemd/system/ignition-disable.service | 2 +- .../file.include/etc/update-motd.d/05-logo | 2 +- .../systemd/system/ignition-disable.service | 2 +- sync-gl-from-upstream.sh | 1 + 15 files changed, 106 insertions(+), 94 deletions(-) diff --git a/gardenlinux/bin/README.md b/gardenlinux/bin/README.md index c19b064..06e5c56 100644 --- a/gardenlinux/bin/README.md +++ b/gardenlinux/bin/README.md @@ -72,6 +72,13 @@ This script allows to inject a SSH pubkey to a final Garden Linux image to ensur ### start-vm This script starts a given `.raw` or `.qcow2` image in a local QEMU/KVM VM and supports `amd64` and `arm64 builds`. Keep in mind, that running different architectures may be very slow. However, it may still be useful for validating and running unit tests. A spawned VM runs in `textmode` which a `hostfwd` (portforward) for SSH on `tcp/2222`. By the given options this allows the user to user copy/paste in the terminal, as well as connecting to the sshd. *(Hint: Custom SSH pubkeys can be injected with `inject-sshkey`.)* +**UEFI ARM64 Files** +Running ARM64 based images requires ARM64 UEFI firmware. This can be installed for QEMU on Debian systems by installing the following packages: + +``` +apt-get install ovmf qemu-efi-aarch64 +``` + **Acceleration Support:** Currently, `start-vm` supports `KVM` and `HVF` acceleration. While `HVF` is only supported on macOS, `KVM` will mostly be used. When using `KVM` acceleration you need to ensure that `/dev/kvm` can be used by your user account. However, if `/dev/kvm` is not usable it will fallback to a non accelerated support that may still work but may be slower. Setting permissions on `/dev/kvm` can be don is several ways; for example: diff --git a/gardenlinux/build b/gardenlinux/build index 6d56def..90727c6 100755 --- a/gardenlinux/build +++ b/gardenlinux/build @@ -3,7 +3,7 @@ set -euo pipefail shopt -s nullglob -container_image=ghcr.io/gardenlinux/builder:301ce9f70045c001c5d724c2f9d1a9503e1d5ccc +container_image=ghcr.io/gardenlinux/builder:002a2ff4d1dc6a39d04543b6a6e92a1465d9e226 container_engine=podman target_dir=.build diff --git a/gardenlinux/container/Makefile b/gardenlinux/container/Makefile index d2a80d3..1c35f38 100644 --- a/gardenlinux/container/Makefile +++ b/gardenlinux/container/Makefile @@ -1,44 +1,22 @@ VERSION=`../bin/garden-version` -VERSION_NUMBER_MAJOR=$(shell ../bin/garden-version --major) -VERSION_NUMBER_MINOR=$(shell ../bin/garden-version --minor) -VERSION_NUMBER=$(VERSION_NUMBER_MAJOR).$(VERSION_NUMBER_MINOR) ALTNAME= ALTNAME_INTERNAL=$(shell [ -n "$(ALTNAME)" ] && printf "%s %s" "-t" "$(ALTNAME)" ) PATH_KERNEL_PACKAGES="../.packages/main/l/linux" GARDENLINUX_BUILD_CRE ?= sudo podman -all: build-image build-cert build-integration-test +all: build-integration-test .PHONY: needslim needslim: @./needslim -.PHONY: build-image -build-image: needslim - cp -p ../gardenlinux.asc build-image/gardenlinux.asc - if [ -n "$$($(GARDENLINUX_BUILD_CRE) image ls gardenlinux/build-image:$(VERSION_NUMBER) --format "{{.Repository}}:{{.Tag}}")" ]; then \ - $(GARDENLINUX_BUILD_CRE) image rm --force gardenlinux/build-image:$(VERSION_NUMBER) || true; \ - fi - @$(GARDENLINUX_BUILD_CRE) build --build-arg VERSION=$(VERSION) -t gardenlinux/build-image:$(VERSION) -t gardenlinux/build-image:$(VERSION_NUMBER) $(ALTNAME_INTERNAL) build-image - rm build-image/gardenlinux.asc - -.PHONY: build-cert -build-cert: needslim - cp -p ../gardenlinux.asc build-cert/gardenlinux.asc - @$(GARDENLINUX_BUILD_CRE) build --build-arg VERSION=$(VERSION) -t gardenlinux/build-cert:$(VERSION) $(ALTNAME_INTERNAL) build-cert - rm build-cert/gardenlinux.asc - .PHONY: build build: needslim cp ../checksums.sha256 build/checksums.sha256 @$(GARDENLINUX_BUILD_CRE) build --build-arg BUILDARCH="$$([ "$$(uname -m)" = "aarch64" ] && echo "arm64" || echo "amd64")" -t gardenlinux/build $(ALTNAME_INTERNAL) build rm build/checksums.sha256 -.PHONY: build-deb -build-deb: build - @$(GARDENLINUX_BUILD_CRE) build -t gardenlinux/build-deb $(ALTNAME_INTERNAL) build-deb - .PHONY: build-base-test build-base-test: needslim cp -p ../gardenlinux.asc base-test/gardenlinux.asc @@ -55,26 +33,9 @@ build-integration-test: build-base-test rm integration-test/checksums.sha256 rm integration-test/cloud.google.gpg -.PHONY: build-kernelmodule -build-kernelmodule: - cp -p ../gardenlinux.asc build-kernelmodule/gardenlinux.asc - @$(GARDENLINUX_BUILD_CRE) build \ - --build-arg VERSION=$(VERSION) \ - --build-arg ARCH="arm64" \ - --build-arg GNU_TYPE_PACKAGE="aarch64-linux-gnu"\ - -t gardenlinux/build-kernelmodule-arm64:$(VERSION) build-kernelmodule - @$(GARDENLINUX_BUILD_CRE) build \ - --build-arg VERSION=$(VERSION) \ - --build-arg ARCH="amd64" \ - --build-arg GNU_TYPE_PACKAGE="x86-64-linux-gnu" \ - -t gardenlinux/build-kernelmodule-amd64:$(VERSION) build-kernelmodule - rm build-kernelmodule/gardenlinux.asc - .PHONY: clean clean: rm -rf integration-test/_pipfiles -@[ -n "$$($(GARDENLINUX_BUILD_CRE) image ls gardenlinux/integration-test --format "{{.ID}}")" ] && $(GARDENLINUX_BUILD_CRE) image rm --force $$($(GARDENLINUX_BUILD_CRE) image ls gardenlinux/integration-test --format "{{.Repository}}:{{.Tag}}"); true - -@[ -n "$$($(GARDENLINUX_BUILD_CRE) image ls gardenlinux/build-image --format "{{.ID}}")" ] && $(GARDENLINUX_BUILD_CRE) image rm --force $$($(GARDENLINUX_BUILD_CRE) image ls gardenlinux/build-image --format "{{.Repository}}:{{.Tag}}"); true - -@[ -n "$$($(GARDENLINUX_BUILD_CRE) image ls gardenlinux/build-deb --format "{{.ID}}")" ] && $(GARDENLINUX_BUILD_CRE) image rm --force $$($(GARDENLINUX_BUILD_CRE) image ls gardenlinux/build-deb --format "{{.Repository}}:{{.Tag}}"); true -@[ -n "$$($(GARDENLINUX_BUILD_CRE) image ls gardenlinux/build --format "{{.ID}}")" ] && $(GARDENLINUX_BUILD_CRE) image rm --force $$($(GARDENLINUX_BUILD_CRE) image ls gardenlinux/build --format "{{.Repository}}:{{.Tag}}"); true -@[ -n "$$($(GARDENLINUX_BUILD_CRE) image ls gardenlinux/slim --format "{{.ID}}")" ] && $(GARDENLINUX_BUILD_CRE) image rm --force $$($(GARDENLINUX_BUILD_CRE) image ls gardenlinux/slim --format "{{.Repository}}:{{.Tag}}"); true diff --git a/gardenlinux/container/base-test/Dockerfile b/gardenlinux/container/base-test/Dockerfile index 7e40050..0f594df 100644 --- a/gardenlinux/container/base-test/Dockerfile +++ b/gardenlinux/container/base-test/Dockerfile @@ -51,5 +51,5 @@ ENV PATH="$VIRTUAL_ENV/bin:$PATH" COPY Pipfile "$VIRTUAL_ENV_PARENT" # Do not use --system, we want the pip from the virtual env -RUN cd "$VIRTUAL_ENV_PARENT" && pipenv install --dev --skip-lock +RUN cd "$VIRTUAL_ENV_PARENT" && pipenv install --dev WORKDIR /gardenlinux/tests diff --git a/gardenlinux/container/needslim b/gardenlinux/container/needslim index 1550e67..e46ecaf 100755 --- a/gardenlinux/container/needslim +++ b/gardenlinux/container/needslim @@ -11,10 +11,11 @@ if [ "$(${gardenlinux_build_cre} image ls gardenlinux/slim --format \"{{.Reposit echo echo "Please run 'make slim' afterwards" echo - ${gardenlinux_build_cre} pull debian:testing-slim - ${gardenlinux_build_cre} tag debian:testing-slim gardenlinux/slim - ${gardenlinux_build_cre} tag debian:testing-slim gardenlinux/slim:$VERSION - ${gardenlinux_build_cre} tag debian:testing-slim gardenlinux/slim:latest + base_container="ghcr.io/gardenlinux/gardenlinux:nightly" + ${gardenlinux_build_cre} pull "$base_container" + ${gardenlinux_build_cre} tag "$base_container" gardenlinux/slim + ${gardenlinux_build_cre} tag "$base_container" gardenlinux/slim:$VERSION + ${gardenlinux_build_cre} tag "$base_container" gardenlinux/slim:latest else if [ "$(${gardenlinux_build_cre} image ls gardenlinux/slim:latest --format \"{{.ID}}\")" == \ "$(${gardenlinux_build_cre} image ls debian:testing-slim --format \"{{.ID}}\")" ]; then diff --git a/gardenlinux/features/_boot/exec.late b/gardenlinux/features/_boot/exec.late index d7f4122..fc08204 100755 --- a/gardenlinux/features/_boot/exec.late +++ b/gardenlinux/features/_boot/exec.late @@ -4,7 +4,7 @@ set -euo pipefail update-kernel-cmdline -mkdir -p /boot/efi/Default +mkdir -p /efi/Default for kernel in /boot/vmlinuz-*; do unshare --mount bash -c 'mount -t tmpfs none /sys && mount --bind /usr/bin/false /usr/bin/systemd-detect-virt && "$@"' \ @@ -16,20 +16,20 @@ for kernel in /boot/vmlinuz-*; do --reproducible \ "/boot/initrd.img-${kernel#*-}" - SYSTEMD_ESP_PATH=/boot/efi kernel-install add "${kernel#*-}" "${kernel}" + SYSTEMD_ESP_PATH=/efi kernel-install add "${kernel#*-}" "${kernel}" done -sed 's/boot\/efi\///' -i /boot/efi/loader/entries/*.conf +sed 's/efi\///' -i /efi/loader/entries/*.conf -SYSTEMD_ESP_PATH=/boot/efi bootctl --no-variables install +SYSTEMD_ESP_PATH=/efi bootctl --no-variables install -mkdir -p /boot/efi/syslinux +mkdir -p /efi/syslinux # syslinux if [ -f "/usr/bin/syslinux" ]; then - mkdir -p /boot/efi/syslinux - cp /usr/lib/syslinux/modules/bios/menu.c32 /boot/efi/syslinux/ - cp /usr/lib/syslinux/modules/bios/libutil.c32 /boot/efi/syslinux/ + mkdir -p /efi/syslinux + cp /usr/lib/syslinux/modules/bios/menu.c32 /efi/syslinux/ + cp /usr/lib/syslinux/modules/bios/libutil.c32 /efi/syslinux/ - update-syslinux + ESP_PATH=/efi update-syslinux fi diff --git a/gardenlinux/features/_boot/file.exclude b/gardenlinux/features/_boot/file.exclude index 2e0f9df..2ea54f5 100644 --- a/gardenlinux/features/_boot/file.exclude +++ b/gardenlinux/features/_boot/file.exclude @@ -1 +1 @@ -/boot/efi/loader/random-seed +/efi/loader/random-seed diff --git a/gardenlinux/features/_boot/file.include/usr/local/sbin/update-syslinux b/gardenlinux/features/_boot/file.include/usr/local/sbin/update-syslinux index cb51948..c7bab93 100755 --- a/gardenlinux/features/_boot/file.include/usr/local/sbin/update-syslinux +++ b/gardenlinux/features/_boot/file.include/usr/local/sbin/update-syslinux @@ -2,69 +2,110 @@ set -uoeE pipefail -bootDir="/boot/efi" -kernelDir="${bootDir}/Default" configDir="/etc/kernel/cmdline.d" -configFile="${bootDir}/syslinux/syslinux.cfg" +configFile="syslinux/syslinux.cfg" -check_version() { - local v=$1 - if [ ! -d "$kernelDir/$v" ]; then +check_config() { + local f=$1 + if [ ! -f "$ESP_PATH/$(get_linux "$f" | cut -c 3-)" ]; then + echo "ce plm" return 1 fi - if [ ! -f "$kernelDir/$v/linux" ]; then - return 1 + for i in $(get_initrd "$f" | tr ',' ' '); do + if [ ! -f "$ESP_PATH/$(echo "$i" | cut -c 3-)" ]; then + return 1 + fi + done + return 0 +} + +err() { + echo "${@}" >&2 + exit 1 +} + +get_options() { + local f=$1 + options=$(awk '/^options/{ $1=""; print $0 }' "$f") + if [ -z "$options" ]; then + err "no options detected for $f" fi - if [ ! -f "$kernelDir/$v/initrd.img-$v" ]; then - return 1 + echo "$options" +} + +get_version() { + local f=$1 + version=$(awk '/^version/{ print $2 }' "$f") + if [ -z "$version" ]; then + err "no version field detected for $f" fi - return 0 + echo "$version" +} + +get_linux() { + local f=$1 + linux=$(awk '/^linux/{ print ".."$2 }' "$f") + if [ -z "$linux" ]; then + err "no kernel field detected for $f" + fi + echo "$linux" +} + +get_initrd() { + local f=$1 + initrd=$(awk '/^initrd/{ print ".."$2 }' "$f" | paste -s -d ',') + if [ -z "$initrd" ]; then + err "no initrd field detected for $f" + fi + echo "$initrd" } if ! which syslinux &> /dev/null; then exit 0 fi -#TODO: detect if anything other than bootDir/Default is used -if [ ! -d "$kernelDir" ]; then - exit 0 -fi +if [ -z "${ESP_PATH:-}" ]; then + ESP_PATH=$(bootctl -p || err "cannot detect ESP, check if ESP is mounted on /efi or /boot/efi") +fi -# load extras for i in "${configDir}"/*-*.cfg; do [ -e "$i" ] || continue + # shellcheck disable=SC1090 source "$i" done -versions=() +configs=() # kernel / initrd -for kernel in /boot/vmlinuz-*; do - if check_version "${kernel#*-}"; then - versions+=("${kernel#*-}") +for config in "${ESP_PATH}"/loader/entries/*.conf*; do + if check_config "$config"; then + configs+=("${config}") + else + err "kernel or initrd doesn't exist for $config" fi done -if [ "${#versions[@]}" == "0" ]; then - echo "no valid kernels found" 1>&2 - exit 1 + +if [ "${#configs[@]}" == "0" ]; then + err "no valid configs found under $ESP_PATH/loader/entries" fi -readarray -t vSorted < <(printf '%s\n' "${versions[@]}" | sort -rV) +readarray -t vSorted < <(printf '%s\n' "${configs[@]}" | sort -rV) { + echo "# automatically generated by update-syslinux" echo "UI menu.c32" echo "PROMPT 0" echo - echo "MENU TITLE Gardenlinux" + echo "MENU TITLE Gardenlinux" echo "TIMEOUT $TIMEOUT" - echo "DEFAULT ${vSorted[0]}" + echo "DEFAULT Linux-$(get_version "${vSorted[0]}")" echo for v in "${vSorted[@]}"; do - echo "LABEL Linux $v" - echo " LINUX ../Default/$v/linux" - echo " APPEND root=${DEVICE} ${CMDLINE_LINUX}" - echo " INITRD ../Default/${v}/initrd.img-$v" + echo "LABEL Linux-$(get_version "$v")" + echo " LINUX $(get_linux "$v")" + echo " APPEND $(get_options "$v")" + echo " INITRD $(get_initrd "$v")" echo done -} > "${configFile}.new" +} > "${ESP_PATH}/${configFile}.new" -mv "${configFile}.new" "${configFile}" +mv "${ESP_PATH}/${configFile}.new" "${ESP_PATH}/${configFile}" diff --git a/gardenlinux/features/_dev/pkg.include b/gardenlinux/features/_dev/pkg.include index f027e0d..8ed1818 100644 --- a/gardenlinux/features/_dev/pkg.include +++ b/gardenlinux/features/_dev/pkg.include @@ -1 +1,2 @@ vim +neofetch diff --git a/gardenlinux/features/base/fstab b/gardenlinux/features/base/fstab index 3070317..844dbaf 100644 --- a/gardenlinux/features/base/fstab +++ b/gardenlinux/features/base/fstab @@ -1,3 +1,3 @@ # -LABEL=EFI /boot/efi vfat umask=0077 type=uefi +LABEL=EFI /efi vfat umask=0077 type=uefi LABEL=ROOT / ext4 rw,errors=remount-ro,prjquota,discard diff --git a/gardenlinux/features/base/test/test_sgid_suid_files.py b/gardenlinux/features/base/test/test_sgid_suid_files.py index 6e86bd3..1fc3220 100644 --- a/gardenlinux/features/base/test/test_sgid_suid_files.py +++ b/gardenlinux/features/base/test/test_sgid_suid_files.py @@ -13,7 +13,7 @@ "/usr/bin/chage,root,shadow", "/usr/bin/ssh-agent,root,_ssh", "/usr/sbin/unix_chkpwd,root,shadow", - "/usr/lib/systemd-cron/crontab_setgid,root,crontab", + "/usr/libexec/systemd-cron/crontab_setgid,root,crontab", ] ), ("suid", [ @@ -36,5 +36,5 @@ # Run the test unit to perform the # final tests by the given artifact. -def test_sgid_suid_files(client, test_type, whitelist_files, non_vhost): +def test_sgid_suid_files(client, test_type, whitelist_files): sgid_suid_files(client, test_type, whitelist_files) diff --git a/gardenlinux/features/kvm/file.include/etc/systemd/system/ignition-disable.service b/gardenlinux/features/kvm/file.include/etc/systemd/system/ignition-disable.service index 1ad9530..49786ab 100644 --- a/gardenlinux/features/kvm/file.include/etc/systemd/system/ignition-disable.service +++ b/gardenlinux/features/kvm/file.include/etc/systemd/system/ignition-disable.service @@ -4,7 +4,7 @@ Description=Ignition disable after first boot ConditionFirstBoot=true ConditionPathExists=/run/ignition.env -RequiresMountsFor=/boot/efi +RequiresMountsFor=/efi DefaultDependencies=no Before=sysinit.target diff --git a/gardenlinux/features/server/file.include/etc/update-motd.d/05-logo b/gardenlinux/features/server/file.include/etc/update-motd.d/05-logo index 24aa438..7b1f003 100755 --- a/gardenlinux/features/server/file.include/etc/update-motd.d/05-logo +++ b/gardenlinux/features/server/file.include/etc/update-motd.d/05-logo @@ -5,5 +5,5 @@ echo ' / ___| __ _ _ __ __| | ___ _ __ | | _ _ __ _ ___ __' echo '| | _ / _` | '\''__/ _` |/ _ \ '\''_ \ | | | | '\''_ \| | | \ \/ /' echo '| |_| | (_| | | | (_| | __/ | | | | |___| | | | | |_| |> < ' echo ' \____|\__,_|_| \__,_|\___|_| |_| |_____|_|_| |_|\__,_/_/\_\' -echo 'Garden Linux @VERSION@ (based on Debian GNU/Linux bookworm) ' +echo 'Garden Linux @VERSION@ (based on Debian GNU/Linux trixie) ' echo diff --git a/gardenlinux/features/vmware/file.include/etc/systemd/system/ignition-disable.service b/gardenlinux/features/vmware/file.include/etc/systemd/system/ignition-disable.service index 1ad9530..49786ab 100644 --- a/gardenlinux/features/vmware/file.include/etc/systemd/system/ignition-disable.service +++ b/gardenlinux/features/vmware/file.include/etc/systemd/system/ignition-disable.service @@ -4,7 +4,7 @@ Description=Ignition disable after first boot ConditionFirstBoot=true ConditionPathExists=/run/ignition.env -RequiresMountsFor=/boot/efi +RequiresMountsFor=/efi DefaultDependencies=no Before=sysinit.target diff --git a/sync-gl-from-upstream.sh b/sync-gl-from-upstream.sh index f8824fb..ef01aca 100755 --- a/sync-gl-from-upstream.sh +++ b/sync-gl-from-upstream.sh @@ -29,3 +29,4 @@ cp -r $TEMP_DIR/gardenlinux-main/features/ssh/* gardenlinux/features/ssh/ cp -r $TEMP_DIR/gardenlinux-main/features/vmware/* gardenlinux/features/vmware/ cp $TEMP_DIR/gardenlinux-main/build gardenlinux/build +sed -i '/ - sap/d' gardenlinux/features/server/info.yaml