Prepare scripts for external execution outside of wrapper

.gitignore

@@ -0,0 +1 @@
+.history/

checks/alertmanager

@@ -2,6 +2,8 @@
 
 # https://access.redhat.com/solutions/4250221
 
+[ -z ${UTILSFILE} ] && source $(echo "$(dirname ${0})/../utils")
+
 if oc auth can-i get routes -n openshift-monitoring >/dev/null 2>&1; then
   alert_url=$(oc -n openshift-monitoring get routes/alertmanager-main -o json | jq -r .spec.host)
   alerts=$(curl -s -k -H "Authorization: Bearer $(oc -n openshift-monitoring sa get-token prometheus-k8s)" https://$alert_url/api/v1/alerts | jq '.data[] | {alert:.labels.alertname, severity:.labels.severity, namespace:.labels.namespace, instance:.labels.instance, message:.annotations.message} | select((.severity == "warning") or (.severity == "critical"))')

checks/bz1948052

@@ -1,5 +1,7 @@
 #!/usr/bin/env bash
 
+[ -z ${UTILSFILE} ] && source $(echo "$(dirname ${0})/../utils")
+
 BADKERNEL="4.18.0-193.24.1.el8_2.dt1.x86_64"
 
 if oc auth can-i get nodes >/dev/null 2>&1; then

checks/chronyc

@@ -1,5 +1,7 @@
 #!/usr/bin/env bash
 
+[ -z ${UTILSFILE} ] && source $(echo "$(dirname ${0})/../utils")
+
 if oc auth can-i debug node >/dev/null 2>&1; then
   msg "Collecting NTP data... (${BLUE}using oc debug, it can take a while${NOCOLOR})"
   # shellcheck disable=SC2016

checks/clusterversion_errors

@@ -1,5 +1,7 @@
 #!/usr/bin/env bash
 
+[ -z ${UTILSFILE} ] && source $(echo "$(dirname ${0})/../utils")
+
 if oc auth can-i get clusterversion >/dev/null 2>&1; then
   clusterversion_msgs=$(oc get clusterversion -o json | jq '.items[].status.conditions[] | select ((.status == "True") and (.type == "Failing") and (.message != null)) | { message: .message }')
   count_errors=$(echo "${clusterversion_msgs}" | jq .message | wc -l)

checks/csr

@@ -1,5 +1,7 @@
 #!/usr/bin/env bash
 
+[ -z ${UTILSFILE} ] && source $(echo "$(dirname ${0})/../utils")
+
 if oc auth can-i get csr >/dev/null 2>&1; then
   pending_csr=$(oc get csr --no-headers --ignore-not-found=true | grep -ci 'pending')
   if [[ ${pending_csr} -ge 1 ]]; then

checks/ctrlnodes

@@ -1,5 +1,7 @@
 #!/usr/bin/env bash
 
+[ -z ${UTILSFILE} ] && source $(echo "$(dirname ${0})/../utils")
+
 if oc auth can-i get nodes >/dev/null 2>&1; then
   scheduable_controllers=$(oc get nodes -o json | jq '.items[] | { name: .metadata.name, scheduable: .spec.taints, control: .metadata.labels."node-role.kubernetes.io/master" } | select((.control == "") and (.scheduable == null))')
   if [[ -n ${scheduable_controllers} ]]; then

checks/entropy

@@ -1,5 +1,7 @@
 #!/usr/bin/env bash
 
+[ -z ${UTILSFILE} ] && source $(echo "$(dirname ${0})/../utils")
+
 if oc auth can-i debug node >/dev/null 2>&1; then
   msg "Collecting entropy data... (${BLUE}using oc debug, it can take a while${NOCOLOR})"
   # shellcheck disable=SC2016

checks/iptables-22623-22624

@@ -9,6 +9,8 @@
 # We concatenate all commands with || meaning it will stop if
 # some command fails (returns 0, so if the rule exist)
 
+[ -z ${UTILSFILE} ] && source $(echo "$(dirname ${0})/../utils")
+
 if oc auth can-i debug node >/dev/null 2>&1; then
   msg "Checking if ports 22623/tcp and 22624/tcp are blocked (${BLUE}using oc debug, it can take a while${NOCOLOR})"
   # shellcheck disable=SC2016

checks/mcp

@@ -1,5 +1,7 @@
 #!/usr/bin/env bash
 
+[ -z ${UTILSFILE} ] && source $(echo "$(dirname ${0})/../utils")
+
 if oc auth can-i get mcp >/dev/null 2>&1; then
   degrated_mcps=$(oc get mcp -o json | jq '.items[] | { name: .metadata.name, status: .status } | select (.status.degradedMachineCount >= 1) | { name: .name, status: .status.degradedMachineCount}')
   if [[ -n $degrated_mcps ]]; then

checks/nodes

@@ -1,5 +1,7 @@
 #!/usr/bin/env bash
 
+[ -z ${UTILSFILE} ] && source $(echo "$(dirname ${0})/../utils")
+
 if oc auth can-i get nodes >/dev/null 2>&1; then
   nodes_not_ready=$(oc get nodes -o json | jq '.items[] | { name: .metadata.name, type: .status.conditions[] } | select ((.type.type == "Ready") and (.type.status != "True"))')
   if [[ -n ${nodes_not_ready} ]]; then

checks/notrunningpods

@@ -1,5 +1,7 @@
 #!/usr/bin/env bash
 
+[ -z ${UTILSFILE} ] && source $(echo "$(dirname ${0})/../utils")
+
 if oc auth can-i get pods -A >/dev/null 2>&1; then
   # Get all nonrunning pods with headers even if they are not found
   notrunning=$(oc get pods -A --field-selector=status.phase!=Running,status.phase!=Succeeded --ignore-not-found=true)

checks/operators

@@ -1,5 +1,7 @@
 #!/usr/bin/env bash
 
+[ -z ${UTILSFILE} ] && source $(echo "$(dirname ${0})/../utils")
+
 if oc auth can-i get co >/dev/null 2>&1; then
   bad_operators=$(oc get co --no-headers | grep -E -civ 'True.*False.*False')
   if [[ ${bad_operators} -ge 1 ]]; then

checks/port-thrasing

@@ -3,6 +3,8 @@
 THRASINGMSG="Changing chassis for lport"
 NAMESPACE="openshift-ovn-kubernetes"
 
+[ -z ${UTILSFILE} ] && source $(echo "$(dirname ${0})/../utils")
+
 if [[ $(oc get network/cluster -o jsonpath={.spec.networkType}) != "OVNKubernetes" ]]; then
   msg "This check only works for OVNKubernetes SDN"
 else

checks/restarts

@@ -1,5 +1,7 @@
 #!/usr/bin/env bash
 
+[ -z ${UTILSFILE} ] && source $(echo "$(dirname ${0})/../utils")
+
 if oc auth can-i get pods -A >/dev/null 2>&1; then
   restarts=$(oc get pods -o json -A | jq -r ".items[] | { name: .metadata.name, project: .metadata.namespace, restarts: .status.containerStatuses[].restartCount } | select(.restarts > $RESTART_THRESHOLD)" 2>/dev/null)
   if [[ -n $restarts ]]; then

checks/terminating

@@ -1,5 +1,7 @@
 #!/usr/bin/env bash
 
+[ -z ${UTILSFILE} ] && source $(echo "$(dirname ${0})/../utils")
+
 if oc auth can-i get pods -A >/dev/null 2>&1; then
   terminating_pods=$(oc get pods -A | grep -c 'Terminating')
   if [[ $terminating_pods -ge 1 ]]; then

info/00-clusterversion

@@ -1,5 +1,7 @@
 #!/usr/bin/env bash
 
+[ -z ${UTILSFILE} ] && source $(echo "$(dirname ${0})/../utils")
+
 if oc auth can-i get clusterversion >/dev/null 2>&1; then
   msg "Cluster version:\n$(oc get clusterversion/version)"
 else

info/01-clusteroperators

@@ -1,5 +1,7 @@
 #!/usr/bin/env bash
 
+[ -z ${UTILSFILE} ] && source $(echo "$(dirname ${0})/../utils")
+
 if oc auth can-i get clusteroperators >/dev/null 2>&1; then
   msg "Cluster operators:\n$(oc get clusteroperators)"
 else

info/02-nodes

@@ -1,5 +1,7 @@
 #!/usr/bin/env bash
 
+[ -z ${UTILSFILE} ] && source $(echo "$(dirname ${0})/../utils")
+
 if oc auth can-i get nodes -A >/dev/null 2>&1; then
   msg "Nodes:\n$(oc get nodes -o wide)"
   msg "Masters: $(oc get nodes -o name --no-headers --selector='node-role.kubernetes.io/master' | wc -l)"

info/03-pods

@@ -1,5 +1,7 @@
 #!/usr/bin/env bash
 
+[ -z ${UTILSFILE} ] && source $(echo "$(dirname ${0})/../utils")
+
 if oc auth can-i get pods -A >/dev/null 2>&1; then
   msg "Total pods: $(oc get pods -A --no-headers | wc -l)"
 else

info/biosversion

@@ -1,5 +1,7 @@
 #!/usr/bin/env bash
 
+[ -z ${UTILSFILE} ] && source $(echo "$(dirname ${0})/../utils")
+
 # Check BIOS version to begin with
 if oc auth can-i debug node >/dev/null 2>&1; then
   msg "Checking bios versions (${BLUE}using oc debug, it can take a while${NOCOLOR})"

info/container-images-running

@@ -1,5 +1,7 @@
 #!/usr/bin/env bash
 
+[ -z ${UTILSFILE} ] && source $(echo "$(dirname ${0})/../utils")
+
 if oc auth can-i get pods -A >/dev/null 2>&1; then
   IMAGES=$(oc get pods -A -o go-template --template='{{range .items}}{{range .spec.containers}}{{printf "%s\n" .image -}} {{end}}{{end}}' | sort -u)
   msg "Images:\n${IMAGES}"

info/container-images-stored

@@ -1,5 +1,7 @@
 #!/usr/bin/env bash
 
+[ -z ${UTILSFILE} ] && source $(echo "$(dirname ${0})/../utils")
+
 if oc auth can-i debug node >/dev/null 2>&1; then
   msg "Checking container images stored in the cluster (${BLUE}using oc debug, it can take a while${NOCOLOR})"
   # shellcheck disable=SC2016

info/ethtool-firmware-version

@@ -1,5 +1,7 @@
 #!/usr/bin/env bash
 
+[ -z ${UTILSFILE} ] && source $(echo "$(dirname ${0})/../utils")
+
 if oc auth can-i debug node >/dev/null 2>&1; then
   msg "Checking NIC firmware version using ethtool (${BLUE}using oc debug, it can take a while${NOCOLOR})"
   # shellcheck disable=SC2016

info/intel-firmware-version

@@ -4,6 +4,8 @@
 # Intel XXV710 25GbE SFP28 = 8086:158b
 INTEL_IDS="${INTEL_IDS:=8086:158b}"
 
+[ -z ${UTILSFILE} ] && source $(echo "$(dirname ${0})/../utils")
+
 if oc auth can-i debug node >/dev/null 2>&1; then
   msg "Checking Intel firmware version (${BLUE}using oc debug, it can take a while${NOCOLOR})"
   # shellcheck disable=SC2016

info/mellanox-firmware-version

@@ -11,6 +11,8 @@
 
 IDS="15b3:1015 15b3:1017 15b3:1013 15b3:101b"
 
+[ -z ${UTILSFILE} ] && source $(echo "$(dirname ${0})/../utils")
+
 if oc auth can-i debug node >/dev/null 2>&1; then
   msg "Checking Mellanox firmware version (${BLUE}using oc debug, it can take a while${NOCOLOR})"
   # shellcheck disable=SC2016

info/mtu

@@ -1,5 +1,7 @@
 #!/usr/bin/env bash
 
+[ -z ${UTILSFILE} ] && source $(echo "$(dirname ${0})/../utils")
+
 if oc auth can-i get network/cluster >/dev/null 2>&1; then
   network_type=$(oc get network/cluster -ojson | jq -r .status.networkType)
   if [[ $network_type != "OVNKubernetes" ]]; then

info/node-versions

@@ -1,5 +1,7 @@
 #!/usr/bin/env bash
 
+[ -z ${UTILSFILE} ] && source $(echo "$(dirname ${0})/../utils")
+
 if oc auth can-i get nodes -A >/dev/null 2>&1; then
   KUBELETVERSIONS=$(oc get nodes -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.status.nodeInfo.kubeletVersion}{"\n"}{end}' | column -t -N "NODE,KUBELET")
   CRIOVERSIONS=$(oc get nodes -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.status.nodeInfo.containerRuntimeVersion}{"\n"}{end}' | column -t -N "NODE,CRIO")

info/ovs-hostnames

@@ -1,5 +1,7 @@
 #!/usr/bin/env bash
 
+[ -z ${UTILSFILE} ] && source $(echo "$(dirname ${0})/../utils")
+
 if oc auth can-i exec pod -n openshift-ovn-kubernetes >/dev/null 2>&1 && oc get ns openshift-ovn-kubernetes >/dev/null 2>&1; then
   OVSHOSTNAMES=$(oc -n openshift-ovn-kubernetes exec pod/"$(oc -n openshift-ovn-kubernetes get pod -l app=ovnkube-master,component=network -o jsonpath='{.items[0].metadata.name}')" -c northd -- ovn-sbctl --no-leader-only list chassis | awk '/hostname/ { print $3 }' | sort -n)
   if [ -n "${OVSHOSTNAMES}" ]; then

openshift-checks.sh

@@ -11,7 +11,8 @@ IFS=$'\n\t'
 cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1
 
 # shellcheck disable=SC1091
-source ./utils
+export UTILSFILE="$(pwd)/utils"
+source ${UTILSFILE}
 
 #trap cleanup SIGINT SIGTERM ERR EXIT
 

pre/00-install-config-valid-yaml

@@ -1,5 +1,7 @@
 #!/usr/bin/env bash
 
+[ -z ${UTILSFILE} ] && source $(echo "$(dirname ${0})/../utils")
+
 if yq eval ${INSTALL_CONFIG_PATH} >/dev/null; then
   msg "${INSTALL_CONFIG_PATH} seems valid"
 else

pre/dns-hostnames

@@ -1,15 +1,17 @@
 #!/usr/bin/env bash
 
-BASEDOMAIN=$(yq e '.baseDomain' ${INSTALL_CONFIG_PATH} 2> /dev/null)
+[ -z ${UTILSFILE} ] && source $(echo "$(dirname ${0})/../utils")
+
+BASEDOMAIN=$(yq e '.baseDomain' ${INSTALL_CONFIG_PATH} 2>/dev/null)
 
 if [ -z ${BASEDOMAIN} ]; then
-  errors=$(("${errors}"+1))
+  errors=$(("${errors}" + 1))
   msg ".baseDomain not found in ${INSTALL_CONFIG_PATH}"
 fi
 
-CLUSTERNAME=$(yq e '.metadata.name' ${INSTALL_CONFIG_PATH} 2> /dev/null)
+CLUSTERNAME=$(yq e '.metadata.name' ${INSTALL_CONFIG_PATH} 2>/dev/null)
 if [ -z ${CLUSTERNAME} ]; then
-  errors=$(("${errors}"+1))
+  errors=$(("${errors}" + 1))
   msg ".metadata.name not found in ${INSTALL_CONFIG_PATH}"
 fi
 
@@ -22,23 +24,23 @@ WILDCARD="foobar.apps."${CLUSTERNAME}"."${BASEDOMAIN}"."
 
 IP_API=$(dig +short ${API})
 if [ -z ${IP_API} ]; then
-  errors=$(("${errors}"+1))
+  errors=$(("${errors}" + 1))
   msg "${RED}${API} doesn't resolve${NOCOLOR}"
 fi
 
 IP_WILDCARD=$(dig +short ${WILDCARD})
 if [ -z ${IP_WILDCARD} ]; then
-  errors=$(("${errors}"+1))
+  errors=$(("${errors}" + 1))
   msg "${RED}${WILDCARD} doesn't resolve${NOCOLOR}"
 fi
 
 IP_API_REVERSE=$(dig +short -x ${IP_API})
 if [ -z ${IP_API_REVERSE} ]; then
   msg "${YELLOW}api reverse not found${NOCOLOR}"
-  else
-    if [ ${IP_API_REVERSE} != ${API} ]; then
-      msg "${YELLOW}${API} doesn't match the reverse ${IP_API_REVERSE}${NOCOLOR}"
-    fi
+else
+  if [ ${IP_API_REVERSE} != ${API} ]; then
+    msg "${YELLOW}${API} doesn't match the reverse ${IP_API_REVERSE}${NOCOLOR}"
+  fi
 fi
 
 # Wildcard reverse DNS doesn't seem to be a thing