diff --git a/ci/jenkins/jobs/macros.yaml b/ci/jenkins/jobs/macros.yaml index d13c442d815..9295fa7d4f6 100644 --- a/ci/jenkins/jobs/macros.yaml +++ b/ci/jenkins/jobs/macros.yaml @@ -37,6 +37,16 @@ #!/bin/bash set -ex sudo ./ci/test-conformance-aks.sh --cluster-name "${CLUSTERNAME}" --cleanup-only + +- builder: + name: builder-cluster-cleanup + builders: + - shell: |- + #!/bin/bash + set -ex + sudo ./ci/test-conformance-aks.sh --cleanup-all + sudo ./ci/test-conformance-gke.sh --gcloud-sdk-path "${GCLOUD_SDK_PATH}" --cleanup-all + sudo ./ci/test-conformance-eks.sh --cleanup-all - builder: name: builder-workload-cluster-garbage-collection diff --git a/ci/jenkins/jobs/projects-cloud.yaml b/ci/jenkins/jobs/projects-cloud.yaml index c54b7f49469..1a738c70f41 100644 --- a/ci/jenkins/jobs/projects-cloud.yaml +++ b/ci/jenkins/jobs/projects-cloud.yaml @@ -785,6 +785,20 @@ - text: credential-id: WORKFORCE_POOL # Jenkins secret that stores the cloud resource pool id variable: WORKFORCE_POOL + - 'cloud-{name}-cleanup-period': + description: This is for deleting remaining clusters on all cloud providers. + builders: + - builder-cluster-cleanup + concurrent: false + disabled: false + node: antrea-cloud + branches: + - '${{ANTREA_GIT_REVISION}}' + repo_url: '${{ANTREA_REPO}}' + publishers: + triggers: + - timed: H 22 * * * + wrappers: [] - 'cloud-{name}-{test_name}-cleanup': test_name: gke description: This is for deleting GKE test clusters. diff --git a/ci/test-conformance-aks.sh b/ci/test-conformance-aks.sh index c4738373f40..f4a5aecb2cb 100755 --- a/ci/test-conformance-aks.sh +++ b/ci/test-conformance-aks.sh @@ -34,7 +34,7 @@ KUBE_CONFORMANCE_IMAGE_VERSION=auto _usage="Usage: $0 [--cluster-name ] [--kubeconfig ] [--k8s-version ]\ [--azure-app-id ] [--azure-tenant-id ] [--azure-password ] \ - [--aks-region ] [--log-mode ] [--setup-only] [--cleanup-only] + [--aks-region ] [--log-mode ] [--setup-only] [--cleanup-only] [--cleanup-all] Setup a AKS cluster to run K8s e2e community tests (Conformance & Network Policy). @@ -47,7 +47,8 @@ Setup a AKS cluster to run K8s e2e community tests (Conformance & Network Policy --aks-region The Azure region where the cluster will be initiated. Defaults to westus. --log-mode Use the flag to set either 'report', 'detail', or 'dump' level data for sonobuoy results. --setup-only Only perform setting up the cluster and run test. - --cleanup-only Only perform cleaning up the cluster." + --cleanup-only Only perform cleaning up the cluster. + --cleanup-all Cleaning up all clusters without protected tag." function print_usage { echoerr "$_usage" @@ -104,6 +105,10 @@ case $key in RUN_ALL=false shift ;; + --cleanup-all) + RUN_CLEANUP_ALL=true + shift + ;; -h|--help) print_usage exit 0 @@ -300,6 +305,29 @@ function cleanup_cluster() { echo "=== Cleanup cluster ${CLUSTER} succeeded ===" } +function cleanup_all_clusters() { + echo '=== Cleaning up all AKS clusters without tag protected ===' + clusters=$(az aks list \ + --query "[!(tags.protected && tags.protected=='true') && resourceGroup=='${RESOURCE_GROUP}'].{name:name,rg:resourceGroup}" \ + -o tsv) + if [[ -z "$clusters" ]]; then + echo "Unprotected cluster not found." + exit + fi + while read -r clusterName resourceGroup; do + [[ -z "$clusterName" ]] && continue + echo "Deleting Cluster: $clusterName in $resourceGroup" + az aks delete --name "$clusterName" --resource-group "$resourceGroup" --yes + done <<< "$clusters" + resource=$(az aks list \ + --query "[resourceGroup=='${RESOURCE_GROUP}'].{name:name,rg:resourceGroup}" \ + -o tsv) + if [[ -z "$resource" ]]; then + az group delete --name ${RESOURCE_GROUP} --yes --no-wait + fi + echo "=== Cleanup AKS clusters succeeded ===" +} + # ensures that the script can be run from anywhere THIS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" GIT_CHECKOUT_DIR=${THIS_DIR}/.. @@ -317,6 +345,10 @@ if [[ "$RUN_ALL" == true || "$RUN_CLEANUP_ONLY" == true ]]; then cleanup_cluster fi +if [[ "$RUN_CLEANUP_ALL" == true ]]; then + cleanup_all_clusters +fi + if [[ "$RUN_CLEANUP_ONLY" == false && $TEST_SCRIPT_RC -ne 0 ]]; then exit 1 fi diff --git a/ci/test-conformance-eks.sh b/ci/test-conformance-eks.sh index a2fc300aaab..30016befd1f 100755 --- a/ci/test-conformance-eks.sh +++ b/ci/test-conformance-eks.sh @@ -40,7 +40,7 @@ AWS_SERVICE_USER_NAME="" _usage="Usage: $0 [--cluster-name ] [--kubeconfig ] [--k8s-version ]\ [--aws-access-key ] [--aws-secret-key ] [--aws-region ] [--aws-service-user ]\ [--aws-service-user-role-arn ] [--ssh-key ]\ - [--setup-only] [--cleanup-only] + [--setup-only] [--cleanup-only] [--cleanup-all] Setup a EKS cluster to run K8s e2e community tests (Conformance & Network Policy). @@ -56,6 +56,7 @@ Setup a EKS cluster to run K8s e2e community tests (Conformance & Network Policy --log-mode Use the flag to set either 'report', 'detail', or 'dump' level data for sonobuoy results. --setup-only Only perform setting up the cluster and run test. --cleanup-only Only perform cleaning up the cluster. + --cleanup-all Cleaning up all clusters without protected tag. --skip-eksctl-install Do not install the latest eksctl version. Eksctl must be installed already." function print_usage { @@ -125,6 +126,10 @@ case $key in RUN_ALL=false shift ;; + --cleanup-all) + RUN_CLEANUP_ALL=true + shift + ;; --skip-eksctl-install) INSTALL_EKSCTL=false shift @@ -354,6 +359,32 @@ function cleanup_cluster() { echo "=== Cleanup cluster ${CLUSTER} succeeded ===" } +function cleanup_all_clusters() { + echo '=== Cleaning up all EKS clusters without tag protected ===' + clusters=$(eksctl get cluster --output json | jq -r '.[].metadata.name') + if [[ -z "$clusters" ]]; then + echo "Unprotected cluster not found." + exit + fi + for cluster in $clusters; do + cluster_arn=$(aws eks describe-cluster --name "$cluster" --query "cluster.arn" --output text 2>/dev/null) + if [[ "$cluster_arn" == "None" ]] || [[ -z "$cluster_arn" ]]; then + log "Warning: Unable to retrieve ARN for cluster '$cluster'. Skipping this cluster." + continue + fi + tags=$(aws eks list-tags-for-resource --resource-arn "$cluster_arn" --query "tags" --output json 2>/dev/null) + if [[ $? -ne 0 ]]; then + echo "Warning: Unable to retrieve tags for cluster '$cluster'. Skipping this cluster." + continue + fi + has_tag=$(echo "$tags" | jq -r --arg key "protected" '. | has($key)') + if [[ "$has_tag" == "false" ]]; then + eksctl delete cluster --name ${cluster} --region $REGION + fi + done + echo "=== Cleanup cluster ${cluster} succeeded ===" +} + # ensures that the script can be run from anywhere THIS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" GIT_CHECKOUT_DIR=${THIS_DIR}/.. @@ -371,6 +402,10 @@ if [[ "$RUN_ALL" == true || "$RUN_CLEANUP_ONLY" == true ]]; then cleanup_cluster fi +if [[ "$RUN_CLEANUP_ALL" == true ]]; then + cleanup_all_clusters +fi + if [[ "$RUN_CLEANUP_ONLY" == false && $TEST_SCRIPT_RC -ne 0 ]]; then exit 1 fi diff --git a/ci/test-conformance-gke.sh b/ci/test-conformance-gke.sh index 750ff1926c5..912384e03f7 100755 --- a/ci/test-conformance-gke.sh +++ b/ci/test-conformance-gke.sh @@ -128,6 +128,10 @@ case $key in RUN_ALL=false shift ;; + --cleanup-all) + RUN_CLEANUP_ALL=true + shift + ;; -h|--help) print_usage exit 0 @@ -329,6 +333,29 @@ function cleanup_cluster() { echo "=== Cleanup cluster ${CLUSTER} succeeded ===" } +function cleanup_all_clusters() { + echo '=== Cleaning up all unprotected GKE clusters ===' + gcloud container clusters list --zone "${GKE_ZONE}" --format="json(name, location, resourceLabels)" | \ + jq -r '.[] | select(.resourceLabels.protected != "true") | "\(.name) \(.location)"' | \ + while read CLUSTER_NAME ZONE; do + echo "Deleting cluster ${CLUSTER_NAME} in zone '${GKE_ZONE}'..." + retry=5 + while [[ "${retry}" -gt 0 ]]; do + gcloud container clusters delete ${CLUSTER_NAME} --zone ${GKE_ZONE} + if [[ $? -eq 0 ]]; then + break + fi + sleep 10 + retry=$((retry-1)) + done + if [[ "${retry}" -eq 0 ]]; then + echo "=== Failed to delete GKE cluster ${CLUSTER}! ===" + continue + fi + done + echo "=== Cleanup GKE clusters succeeded ===" +} + if [[ "$RUN_ALL" == true || "$RUN_SETUP_ONLY" == true ]]; then setup_gke deliver_antrea_to_gke @@ -339,6 +366,10 @@ if [[ "$RUN_ALL" == true || "$RUN_CLEANUP_ONLY" == true ]]; then cleanup_cluster fi +if [[ "$RUN_CLEANUP_ALL" == true ]]; then + cleanup_all_clusters +fi + if [[ "$RUN_CLEANUP_ONLY" == false && $TEST_SCRIPT_RC -ne 0 ]]; then exit 1 fi