test: improve ux with cli arguments

Use cli arguments to control cleanup rather than using env variables which are more obscure. Signed-off-by: Tiago Castro <[email protected]>
openebs · Jan 17, 2025 · ae9fd06 · ae9fd06
1 parent 4c7fe5d
commit ae9fd06
Show file tree

Hide file tree

Showing 7 changed files with 232 additions and 90 deletions.
diff --git a/.github/workflows/build_and_push.yml b/.github/workflows/build_and_push.yml
@@ -106,7 +106,7 @@ jobs:
       - name: Running tests
         env:
           OPENEBS_NAMESPACE: "openebs"
-        run: ./ci/ci-test.sh
+        run: ./ci/ci-test.sh run
 
       - name: Upload Coverage Report
         uses: codecov/codecov-action@v4

diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml
@@ -137,7 +137,7 @@ jobs:
       - name: Running tests
         env:
           OPENEBS_NAMESPACE: "openebs"
-        run: ./ci/ci-test.sh
+        run: ./ci/ci-test.sh run
 
       - name: Upload Coverage Report
         uses: codecov/codecov-action@v4

diff --git a/Makefile b/Makefile
@@ -86,7 +86,7 @@ clean:
 	@echo "--> Cleaning Directory" ;
 	go clean -testcache
 	rm -rf bin
-	CLEANUP_ONLY=1 ./ci/ci-test.sh
+	CLEANUP_ONLY=1 ./ci/ci-test.sh run
 	chmod -R u+w ${GOPATH}/bin/${CSI_DRIVER} 2>/dev/null || true
 	chmod -R u+w ${GOPATH}/pkg/* 2>/dev/null || true
 	rm -rf ${GOPATH}/bin/${CSI_DRIVER}
@@ -231,6 +231,14 @@ lvm-driver-image: lvm-driver
 	cd buildscripts/${CSI_DRIVER} && docker build -t ${IMAGE_ORG}/${CSI_DRIVER}:${IMAGE_TAG} ${DBUILD_ARGS} . && docker tag ${IMAGE_ORG}/${CSI_DRIVER}:${IMAGE_TAG} quay.io/${IMAGE_ORG}/${CSI_DRIVER}:${IMAGE_TAG}
 	@rm buildscripts/${CSI_DRIVER}/${CSI_DRIVER}
 
+.PHONY: image-tag
+image-tag:
+	@echo ${IMAGE_TAG}
+
+.PHONY: image-ref
+image-ref:
+	@echo docker.io/${IMAGE_ORG}/${CSI_DRIVER}:${IMAGE_TAG}
+
 .PHONY: ci
 ci:
 	@echo "--> Running ci test";

diff --git a/ci/ci-test.sh b/ci/ci-test.sh
@@ -8,27 +8,53 @@
 
 set -e
 
+SCRIPT_DIR="$(dirname "$(realpath "${BASH_SOURCE[0]:-"$0"}")")"
 SNAP_CLASS="$(realpath deploy/sample/lvmsnapclass.yaml)"
 export OPENEBS_NAMESPACE=${OPENEBS_NAMESPACE:-openebs}
 
 export TEST_DIR="tests"
 
-# allow override
-if [ -z "${KUBECONFIG}" ]
-then
-  export KUBECONFIG="${HOME}/.kube/config"
-fi
-
-if [ -n "${SETUP_IMAGE}" ]; then
-  make lvm-driver-image
-  docker save openebs/lvm-driver | ctr images import -
-fi
-
 # foreign systemid for the testing environment.
 FOREIGN_LVM_SYSTEMID="openebs-ci-test-system"
 FOREIGN_LVM_CONFIG="global{system_id_source=lvmlocal}local{system_id=${FOREIGN_LVM_SYSTEMID}}"
 CRDS_TO_DELETE_ON_CLEANUP="lvmnodes.local.openebs.io lvmsnapshots.local.openebs.io lvmvolumes.local.openebs.io volumesnapshotclasses.snapshot.storage.k8s.io volumesnapshotcontents.snapshot.storage.k8s.io volumesnapshots.snapshot.storage.k8s.io"
 
+help() {
+  cat <<EOF >&2
+Usage: $(basename "${0}") [COMMAND] [OPTIONS]
+
+Commands:
+  run                          Run the tests.
+  clean                        Clean the leftovers.
+
+Options:
+  -h, --help                   Display this text.
+
+Options for run:
+  -r, --reset                  Clean before running the tests.
+  -x, --no-cleanup             Don't cleanup after running the tests.
+  -b, --build-always           Build and load the images before running the tests. [ By default image is built if not present only ]
+
+Examples:
+  $(basename "${0}") run -rcb
+EOF
+}
+
+echo_err() {
+  echo -e "ERROR: $1" >&2
+}
+
+needs_help() {
+  [ -n "$1" ] && echo_err "$1\n"
+  help
+  exit 1
+}
+
+die() {
+  echo_err "FATAL: $1"
+  exit 1
+}
+
 # Clean up generated resources for successive tests.
 cleanup_loopdev() {
   losetup -l | grep '(deleted)' | awk '{print $1}' \
@@ -72,37 +98,14 @@ cleanup() {
     kubectl delete pvc -n "$OPENEBS_NAMESPACE" lvmpv-pvc
     kubectl delete -f "${SNAP_CLASS}"
 
-    helm uninstall lvm-localpv -n "$OPENEBS_NAMESPACE" || true
-    # shellcheck disable=SC2086
-    kubectl delete crds $CRDS_TO_DELETE_ON_CLEANUP
+    if helm uninstall lvm-localpv -n "$OPENEBS_NAMESPACE" --timeout 1m --wait; then
+      # shellcheck disable=SC2086
+      kubectl delete crds $CRDS_TO_DELETE_ON_CLEANUP
+    fi
   fi
-  # always return true
-  return 0
-}
-# trap "cleanup 2>/dev/null" EXIT
-[ -n "${CLEANUP_ONLY}" ] && cleanup && exit 0
-[ -n "${RESET}" ] && cleanup
-
-# setup a foreign lvm to test
-cleanup_foreign_lvmvg
-truncate -s 100G /tmp/openebs_ci_foreign_disk.img
-foreign_disk="$(sudo losetup -f /tmp/openebs_ci_foreign_disk.img --show)"
-sudo pvcreate "${foreign_disk}"
-sudo vgcreate foreign_lvmvg "${foreign_disk}" --config="${FOREIGN_LVM_CONFIG}"
-
-# install snapshot and thin volume module for lvm
-sudo modprobe dm-snapshot
-sudo modprobe dm_thin_pool
-
-# Set the configuration for thin pool autoextend in lvm.conf
-# WARNING: this is modifying the host's settings!!!
-sudo sed -i '/^[^#]*thin_pool_autoextend_threshold/ s/= .*/= 50/' /etc/lvm/lvm.conf
-sudo sed -i '/^[^#]*thin_pool_autoextend_percent/ s/= .*/= 20/' /etc/lvm/lvm.conf
-
-# Prepare env for running BDD tests
-# Minikube is already running
-helm install lvm-localpv ./deploy/helm/charts -n "$OPENEBS_NAMESPACE" --create-namespace --set lvmPlugin.image.pullPolicy=Never --set analytics.enabled=false
-kubectl apply -f "${SNAP_CLASS}"
+
+  set -e
+}
 
 dumpAgentLogs() {
   NR=$1
@@ -122,6 +125,38 @@ dumpControllerLogs() {
   printf "\n\n"
 }
 
+dump_logs() {
+  sudo pvscan --cache
+
+  sudo lvdisplay
+
+  sudo vgdisplay
+
+  echo "******************** LVM Controller logs***************************** "
+  dumpControllerLogs 1000
+
+  echo "********************* LVM Agent logs *********************************"
+  dumpAgentLogs 1000
+
+  echo "get all the pods"
+  kubectl get pods -owide --all-namespaces
+
+  echo "get pvc and pv details"
+  kubectl get pvc,pv -oyaml --all-namespaces
+
+  echo "get snapshot details"
+  kubectl get volumesnapshot.snapshot -oyaml --all-namespaces
+
+  echo "get sc details"
+  kubectl get sc --all-namespaces -oyaml
+
+  echo "get lvm volume details"
+  kubectl get lvmvolumes.local.openebs.io -n "$OPENEBS_NAMESPACE" -oyaml
+
+  echo "get lvm snapshot details"
+  kubectl get lvmsnapshots.local.openebs.io -n "$OPENEBS_NAMESPACE" -oyaml
+}
+
 isPodReady(){
   [ "$(kubectl get po "$1" -o 'jsonpath={.status.conditions[?(@.type=="Ready")].status}' -n "$OPENEBS_NAMESPACE")" = 'True' ]
 }
@@ -152,53 +187,137 @@ waitForLVMDriver() {
   return 1
 }
 
-# wait for lvm-driver to be up
-waitForLVMDriver
-
-cd $TEST_DIR
-
-kubectl get po -n "$OPENEBS_NAMESPACE"
-
-set +e
+run() {
+  # setup a foreign lvm to test
+  cleanup_foreign_lvmvg
+  truncate -s 100G /tmp/openebs_ci_foreign_disk.img
+  foreign_disk="$(sudo losetup -f /tmp/openebs_ci_foreign_disk.img --show)"
+  sudo pvcreate "${foreign_disk}"
+  sudo vgcreate foreign_lvmvg "${foreign_disk}" --config="${FOREIGN_LVM_CONFIG}"
 
-echo "running ginkgo test case"
+  # install snapshot and thin volume module for lvm
+  sudo modprobe dm-snapshot
+  sudo modprobe dm_thin_pool
 
-if ! ginkgo -v -coverprofile=bdd_coverage.txt -covermode=atomic; then
+  # Set the configuration for thin pool autoextend in lvm.conf
+  # WARNING: this is modifying the host's settings!!!
+  sudo sed -i '/^[^#]*thin_pool_autoextend_threshold/ s/= .*/= 50/' /etc/lvm/lvm.conf
+  sudo sed -i '/^[^#]*thin_pool_autoextend_percent/ s/= .*/= 20/' /etc/lvm/lvm.conf
 
-  sudo pvscan --cache
+  # Prepare env for running BDD tests
+  helm install lvm-localpv ./deploy/helm/charts -n "$OPENEBS_NAMESPACE" --create-namespace --set lvmPlugin.image.pullPolicy=Never --set analytics.enabled=false
+  kubectl apply -f "${SNAP_CLASS}"
 
-  sudo lvdisplay
+  # wait for lvm-driver to be up
+  waitForLVMDriver
 
-  sudo vgdisplay
+  cd $TEST_DIR
 
-  echo "******************** LVM Controller logs***************************** "
-  dumpControllerLogs 1000
+  kubectl get po -n "$OPENEBS_NAMESPACE"
 
-  echo "********************* LVM Agent logs *********************************"
-  dumpAgentLogs 1000
+  echo "running ginkgo test case"
 
-  echo "get all the pods"
-  kubectl get pods -owide --all-namespaces
+  if ! ginkgo -v -coverprofile=bdd_coverage.txt -covermode=atomic; then
+    dump_logs
+    [ "$CLEAN_AFTER" = "true" ] && cleanup
+    exit 1
+  fi
 
-  echo "get pvc and pv details"
-  kubectl get pvc,pv -oyaml --all-namespaces
+  printf "\n\n######### All test cases passed #########\n\n"
+}
 
-  echo "get snapshot details"
-  kubectl get volumesnapshot.snapshot -oyaml --all-namespaces
+load_image() {
+  make lvm-driver-image
+  docker save openebs/lvm-driver | ctr images import -
+}
 
-  echo "get sc details"
-  kubectl get sc --all-namespaces -oyaml
+maybe_load_image() {
+  if [ "$BUILD_ALWAYS" = "true" ]; then
+    load_image
+    return 0
+  fi
 
-  echo "get lvm volume details"
-  kubectl get lvmvolumes.local.openebs.io -n "$OPENEBS_NAMESPACE" -oyaml
+  local id
+  id=$(crictl image --output json | jq --arg image "$(make image-ref -s -C "$SCRIPT_DIR"/.. 2>/dev/null)" '.images[]|select(.repoTags[0] == $image)|.id')
+  if [ -n "$id" ]; then
+    return 0
+  fi
 
-  echo "get lvm snapshot details"
-  kubectl get lvmsnapshots.local.openebs.io -n "$OPENEBS_NAMESPACE" -oyaml
+  load_image
+}
 
-  exit 1
+# allow override
+if [ -z "${KUBECONFIG}" ]
+then
+  export KUBECONFIG="${HOME}/.kube/config"
 fi
 
-printf "\n\n######### All test cases passed #########\n\n"
-
-# last statement formatted to always return true
-[ -z "${CLEANUP}" ] || cleanup 2>/dev/null
+COMMAND=
+CLEAN_BEFORE="false"
+CLEAN_AFTER="true"
+BUILD_ALWAYS="false"
+
+while test $# -gt 0; do
+  arg="$1"
+  case "$arg" in
+    run | clean)
+      [ -n "$COMMAND" ] && needs_help "Can't specify two commands"
+      COMMAND="$1"
+      ;;
+    -r | --reset)
+      CLEAN_BEFORE="true"
+      ;;
+    -x | --no-cleanup)
+      CLEAN_AFTER="false"
+      ;;
+    -b | --build-always)
+      BUILD_ALWAYS="true"
+      ;;
+    -h | --help)
+      needs_help
+      ;;
+    -*)
+      singleLetterOpts="${1:1}"
+      shift
+      while [ -n "$singleLetterOpts" ]; do
+        case "${singleLetterOpts:0:1}" in
+          r)
+            CLEAN_BEFORE="true"
+            ;;
+          x)
+            CLEAN_AFTER="false"
+            ;;
+          b)
+            BUILD_ALWAYS="true"
+            ;;
+          *)
+            needs_help "Unrecognized argument $singleLetterOpts"
+            ;;
+        esac
+        singleLetterOpts="${singleLetterOpts:1}"
+      done
+      ;;
+    *)
+      needs_help "Unrecognized argument $1"
+      ;;
+  esac
+  shift
+done
+
+case "$COMMAND" in
+  clean)
+    cleanup
+    ;;
+  run)
+    # trap "cleanup 2>/dev/null" EXIT
+    [ "$CLEAN_BEFORE" = "true" ] && cleanup
+
+    maybe_load_image
+    run
+
+    [ "$CLEAN_AFTER" = "true" ] && cleanup
+    ;;
+  *)
+    die "Unknown Command"
+    ;;
+esac