From ae9fd0699df150f5dc79367e60d399f572732aec Mon Sep 17 00:00:00 2001 From: Tiago Castro Date: Fri, 17 Jan 2025 14:53:43 +0000 Subject: [PATCH] test: improve ux with cli arguments Use cli arguments to control cleanup rather than using env variables which are more obscure. Signed-off-by: Tiago Castro --- .github/workflows/build_and_push.yml | 2 +- .github/workflows/pull_request.yml | 2 +- Makefile | 10 +- ci/ci-test.sh | 269 +++++++++++++++++++-------- docs/developer-setup.md | 23 ++- shell.nix | 5 +- vm.nix | 11 +- 7 files changed, 232 insertions(+), 90 deletions(-) diff --git a/.github/workflows/build_and_push.yml b/.github/workflows/build_and_push.yml index 390fa468..daba1f33 100644 --- a/.github/workflows/build_and_push.yml +++ b/.github/workflows/build_and_push.yml @@ -106,7 +106,7 @@ jobs: - name: Running tests env: OPENEBS_NAMESPACE: "openebs" - run: ./ci/ci-test.sh + run: ./ci/ci-test.sh run - name: Upload Coverage Report uses: codecov/codecov-action@v4 diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 9614b20e..9f754712 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -137,7 +137,7 @@ jobs: - name: Running tests env: OPENEBS_NAMESPACE: "openebs" - run: ./ci/ci-test.sh + run: ./ci/ci-test.sh run - name: Upload Coverage Report uses: codecov/codecov-action@v4 diff --git a/Makefile b/Makefile index ace8f651..a3d9aedb 100644 --- a/Makefile +++ b/Makefile @@ -86,7 +86,7 @@ clean: @echo "--> Cleaning Directory" ; go clean -testcache rm -rf bin - CLEANUP_ONLY=1 ./ci/ci-test.sh + CLEANUP_ONLY=1 ./ci/ci-test.sh run chmod -R u+w ${GOPATH}/bin/${CSI_DRIVER} 2>/dev/null || true chmod -R u+w ${GOPATH}/pkg/* 2>/dev/null || true rm -rf ${GOPATH}/bin/${CSI_DRIVER} @@ -231,6 +231,14 @@ lvm-driver-image: lvm-driver cd buildscripts/${CSI_DRIVER} && docker build -t ${IMAGE_ORG}/${CSI_DRIVER}:${IMAGE_TAG} ${DBUILD_ARGS} . && docker tag ${IMAGE_ORG}/${CSI_DRIVER}:${IMAGE_TAG} quay.io/${IMAGE_ORG}/${CSI_DRIVER}:${IMAGE_TAG} @rm buildscripts/${CSI_DRIVER}/${CSI_DRIVER} +.PHONY: image-tag +image-tag: + @echo ${IMAGE_TAG} + +.PHONY: image-ref +image-ref: + @echo docker.io/${IMAGE_ORG}/${CSI_DRIVER}:${IMAGE_TAG} + .PHONY: ci ci: @echo "--> Running ci test"; diff --git a/ci/ci-test.sh b/ci/ci-test.sh index 8e34811b..87479f81 100755 --- a/ci/ci-test.sh +++ b/ci/ci-test.sh @@ -8,27 +8,53 @@ set -e +SCRIPT_DIR="$(dirname "$(realpath "${BASH_SOURCE[0]:-"$0"}")")" SNAP_CLASS="$(realpath deploy/sample/lvmsnapclass.yaml)" export OPENEBS_NAMESPACE=${OPENEBS_NAMESPACE:-openebs} export TEST_DIR="tests" -# allow override -if [ -z "${KUBECONFIG}" ] -then - export KUBECONFIG="${HOME}/.kube/config" -fi - -if [ -n "${SETUP_IMAGE}" ]; then - make lvm-driver-image - docker save openebs/lvm-driver | ctr images import - -fi - # foreign systemid for the testing environment. FOREIGN_LVM_SYSTEMID="openebs-ci-test-system" FOREIGN_LVM_CONFIG="global{system_id_source=lvmlocal}local{system_id=${FOREIGN_LVM_SYSTEMID}}" CRDS_TO_DELETE_ON_CLEANUP="lvmnodes.local.openebs.io lvmsnapshots.local.openebs.io lvmvolumes.local.openebs.io volumesnapshotclasses.snapshot.storage.k8s.io volumesnapshotcontents.snapshot.storage.k8s.io volumesnapshots.snapshot.storage.k8s.io" +help() { + cat <&2 +Usage: $(basename "${0}") [COMMAND] [OPTIONS] + +Commands: + run Run the tests. + clean Clean the leftovers. + +Options: + -h, --help Display this text. + +Options for run: + -r, --reset Clean before running the tests. + -x, --no-cleanup Don't cleanup after running the tests. + -b, --build-always Build and load the images before running the tests. [ By default image is built if not present only ] + +Examples: + $(basename "${0}") run -rcb +EOF +} + +echo_err() { + echo -e "ERROR: $1" >&2 +} + +needs_help() { + [ -n "$1" ] && echo_err "$1\n" + help + exit 1 +} + +die() { + echo_err "FATAL: $1" + exit 1 +} + # Clean up generated resources for successive tests. cleanup_loopdev() { losetup -l | grep '(deleted)' | awk '{print $1}' \ @@ -72,37 +98,14 @@ cleanup() { kubectl delete pvc -n "$OPENEBS_NAMESPACE" lvmpv-pvc kubectl delete -f "${SNAP_CLASS}" - helm uninstall lvm-localpv -n "$OPENEBS_NAMESPACE" || true - # shellcheck disable=SC2086 - kubectl delete crds $CRDS_TO_DELETE_ON_CLEANUP + if helm uninstall lvm-localpv -n "$OPENEBS_NAMESPACE" --timeout 1m --wait; then + # shellcheck disable=SC2086 + kubectl delete crds $CRDS_TO_DELETE_ON_CLEANUP + fi fi - # always return true - return 0 -} -# trap "cleanup 2>/dev/null" EXIT -[ -n "${CLEANUP_ONLY}" ] && cleanup && exit 0 -[ -n "${RESET}" ] && cleanup - -# setup a foreign lvm to test -cleanup_foreign_lvmvg -truncate -s 100G /tmp/openebs_ci_foreign_disk.img -foreign_disk="$(sudo losetup -f /tmp/openebs_ci_foreign_disk.img --show)" -sudo pvcreate "${foreign_disk}" -sudo vgcreate foreign_lvmvg "${foreign_disk}" --config="${FOREIGN_LVM_CONFIG}" - -# install snapshot and thin volume module for lvm -sudo modprobe dm-snapshot -sudo modprobe dm_thin_pool - -# Set the configuration for thin pool autoextend in lvm.conf -# WARNING: this is modifying the host's settings!!! -sudo sed -i '/^[^#]*thin_pool_autoextend_threshold/ s/= .*/= 50/' /etc/lvm/lvm.conf -sudo sed -i '/^[^#]*thin_pool_autoextend_percent/ s/= .*/= 20/' /etc/lvm/lvm.conf - -# Prepare env for running BDD tests -# Minikube is already running -helm install lvm-localpv ./deploy/helm/charts -n "$OPENEBS_NAMESPACE" --create-namespace --set lvmPlugin.image.pullPolicy=Never --set analytics.enabled=false -kubectl apply -f "${SNAP_CLASS}" + + set -e +} dumpAgentLogs() { NR=$1 @@ -122,6 +125,38 @@ dumpControllerLogs() { printf "\n\n" } +dump_logs() { + sudo pvscan --cache + + sudo lvdisplay + + sudo vgdisplay + + echo "******************** LVM Controller logs***************************** " + dumpControllerLogs 1000 + + echo "********************* LVM Agent logs *********************************" + dumpAgentLogs 1000 + + echo "get all the pods" + kubectl get pods -owide --all-namespaces + + echo "get pvc and pv details" + kubectl get pvc,pv -oyaml --all-namespaces + + echo "get snapshot details" + kubectl get volumesnapshot.snapshot -oyaml --all-namespaces + + echo "get sc details" + kubectl get sc --all-namespaces -oyaml + + echo "get lvm volume details" + kubectl get lvmvolumes.local.openebs.io -n "$OPENEBS_NAMESPACE" -oyaml + + echo "get lvm snapshot details" + kubectl get lvmsnapshots.local.openebs.io -n "$OPENEBS_NAMESPACE" -oyaml +} + isPodReady(){ [ "$(kubectl get po "$1" -o 'jsonpath={.status.conditions[?(@.type=="Ready")].status}' -n "$OPENEBS_NAMESPACE")" = 'True' ] } @@ -152,53 +187,137 @@ waitForLVMDriver() { return 1 } -# wait for lvm-driver to be up -waitForLVMDriver - -cd $TEST_DIR - -kubectl get po -n "$OPENEBS_NAMESPACE" - -set +e +run() { + # setup a foreign lvm to test + cleanup_foreign_lvmvg + truncate -s 100G /tmp/openebs_ci_foreign_disk.img + foreign_disk="$(sudo losetup -f /tmp/openebs_ci_foreign_disk.img --show)" + sudo pvcreate "${foreign_disk}" + sudo vgcreate foreign_lvmvg "${foreign_disk}" --config="${FOREIGN_LVM_CONFIG}" -echo "running ginkgo test case" + # install snapshot and thin volume module for lvm + sudo modprobe dm-snapshot + sudo modprobe dm_thin_pool -if ! ginkgo -v -coverprofile=bdd_coverage.txt -covermode=atomic; then + # Set the configuration for thin pool autoextend in lvm.conf + # WARNING: this is modifying the host's settings!!! + sudo sed -i '/^[^#]*thin_pool_autoextend_threshold/ s/= .*/= 50/' /etc/lvm/lvm.conf + sudo sed -i '/^[^#]*thin_pool_autoextend_percent/ s/= .*/= 20/' /etc/lvm/lvm.conf - sudo pvscan --cache + # Prepare env for running BDD tests + helm install lvm-localpv ./deploy/helm/charts -n "$OPENEBS_NAMESPACE" --create-namespace --set lvmPlugin.image.pullPolicy=Never --set analytics.enabled=false + kubectl apply -f "${SNAP_CLASS}" - sudo lvdisplay + # wait for lvm-driver to be up + waitForLVMDriver - sudo vgdisplay + cd $TEST_DIR - echo "******************** LVM Controller logs***************************** " - dumpControllerLogs 1000 + kubectl get po -n "$OPENEBS_NAMESPACE" - echo "********************* LVM Agent logs *********************************" - dumpAgentLogs 1000 + echo "running ginkgo test case" - echo "get all the pods" - kubectl get pods -owide --all-namespaces + if ! ginkgo -v -coverprofile=bdd_coverage.txt -covermode=atomic; then + dump_logs + [ "$CLEAN_AFTER" = "true" ] && cleanup + exit 1 + fi - echo "get pvc and pv details" - kubectl get pvc,pv -oyaml --all-namespaces + printf "\n\n######### All test cases passed #########\n\n" +} - echo "get snapshot details" - kubectl get volumesnapshot.snapshot -oyaml --all-namespaces +load_image() { + make lvm-driver-image + docker save openebs/lvm-driver | ctr images import - +} - echo "get sc details" - kubectl get sc --all-namespaces -oyaml +maybe_load_image() { + if [ "$BUILD_ALWAYS" = "true" ]; then + load_image + return 0 + fi - echo "get lvm volume details" - kubectl get lvmvolumes.local.openebs.io -n "$OPENEBS_NAMESPACE" -oyaml + local id + id=$(crictl image --output json | jq --arg image "$(make image-ref -s -C "$SCRIPT_DIR"/.. 2>/dev/null)" '.images[]|select(.repoTags[0] == $image)|.id') + if [ -n "$id" ]; then + return 0 + fi - echo "get lvm snapshot details" - kubectl get lvmsnapshots.local.openebs.io -n "$OPENEBS_NAMESPACE" -oyaml + load_image +} - exit 1 +# allow override +if [ -z "${KUBECONFIG}" ] +then + export KUBECONFIG="${HOME}/.kube/config" fi -printf "\n\n######### All test cases passed #########\n\n" - -# last statement formatted to always return true -[ -z "${CLEANUP}" ] || cleanup 2>/dev/null +COMMAND= +CLEAN_BEFORE="false" +CLEAN_AFTER="true" +BUILD_ALWAYS="false" + +while test $# -gt 0; do + arg="$1" + case "$arg" in + run | clean) + [ -n "$COMMAND" ] && needs_help "Can't specify two commands" + COMMAND="$1" + ;; + -r | --reset) + CLEAN_BEFORE="true" + ;; + -x | --no-cleanup) + CLEAN_AFTER="false" + ;; + -b | --build-always) + BUILD_ALWAYS="true" + ;; + -h | --help) + needs_help + ;; + -*) + singleLetterOpts="${1:1}" + shift + while [ -n "$singleLetterOpts" ]; do + case "${singleLetterOpts:0:1}" in + r) + CLEAN_BEFORE="true" + ;; + x) + CLEAN_AFTER="false" + ;; + b) + BUILD_ALWAYS="true" + ;; + *) + needs_help "Unrecognized argument $singleLetterOpts" + ;; + esac + singleLetterOpts="${singleLetterOpts:1}" + done + ;; + *) + needs_help "Unrecognized argument $1" + ;; + esac + shift +done + +case "$COMMAND" in + clean) + cleanup + ;; + run) + # trap "cleanup 2>/dev/null" EXIT + [ "$CLEAN_BEFORE" = "true" ] && cleanup + + maybe_load_image + run + + [ "$CLEAN_AFTER" = "true" ] && cleanup + ;; + *) + die "Unknown Command" + ;; +esac diff --git a/docs/developer-setup.md b/docs/developer-setup.md index b8547930..6dcca0e4 100644 --- a/docs/developer-setup.md +++ b/docs/developer-setup.md @@ -244,27 +244,32 @@ Integration tests are written in ginkgo and run against a [K8s] cluster. > *NOTE*: For nixos-shell, remember to run these command within the nixos-shell vm! -Before running the tests, please ensure the test image is built and deployed in the cluster. \ +Then you can run the tests: ```sh -make lvm-driver-image -docker save openebs/lvm-driver | ctr images import - +./ci/ci-test.sh run ``` -Then you can run the tests: +> *WARNING*: Each individual tests don't currently clean up after themselves properly, so a failed test may affect subsequent tests and even new test runs! + +If the test script was killed before cleaning up, you can issue a cleanup as such: ```sh -./ci/ci-test.sh +./ci/ci-test.sh clean ``` -> *WARNING*: The tests don't currently clean up after themselves properly, so a failed test may affect subsequent tests and even new test runs! - -You can try to reset the test, between runs by either running the script with the env var `CLEANUP_ONLY=1` or `RESET=1` to cleanup and run, example: +You can also request one before running the tests: ```sh -RESET=1 ./ci/ci-test.sh +./ci/ci-test.sh run --reset ``` +> *WARNING*: If you modify the code, remember to rebuild and load the new image, example: +> +> ```sh +> ./ci/ci-test.sh run --build +> ``` + If this doesn't work, you might need to dig a little deeper, or, if you're on nixos-shell, simply start over again: ```sh diff --git a/shell.nix b/shell.nix index d8c6cac0..3617fb68 100644 --- a/shell.nix +++ b/shell.nix @@ -1,6 +1,6 @@ let sources = import ./nix/sources.nix; - pkgs = import sources.nixpkgs {}; + pkgs = import sources.nixpkgs { }; in pkgs.mkShell { name = "scripts-shell"; @@ -25,6 +25,9 @@ pkgs.mkShell { lvm2_dmeventd nixos-shell ] ++ pkgs.lib.optional (builtins.getEnv "IN_NIX_SHELL" == "pure") [ docker-client ]; + + PRE_COMMIT_ALLOW_NO_CONFIG = 1; + shellHook = '' export GOPATH=$(pwd)/nix/.go export GOCACHE=$(pwd)/nix/.go/cache diff --git a/vm.nix b/vm.nix index 4ee4a150..33c1adc4 100644 --- a/vm.nix +++ b/vm.nix @@ -1,7 +1,7 @@ { ... }: let sources = import ./nix/sources.nix; - pkgs = import sources.nixpkgs {}; + pkgs = import sources.nixpkgs { }; in { nix.nixPath = [ @@ -11,6 +11,13 @@ in mountHome = true; mountNixProfile = false; cache = "none"; # default is "loose" + + extraMounts = { + "/root" = { + target = ./.; + cache = "none"; + }; + }; }; virtualisation = { @@ -52,7 +59,7 @@ in }; shellAliases = { - k = "kubectl"; + k = "kubectl"; ke = "kubectl -n openebs"; };