HDDS-12290. Move custom logic from ci.yml into the check scripts (#7841)

apache · Feb 10, 2025 · fa30000 · fa30000
1 parent 743d24d
commit fa30000
Show file tree

Hide file tree

Showing 7 changed files with 29 additions and 26 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -479,11 +479,6 @@ jobs:
       - name: Summary of failures
         run: hadoop-ozone/dev-support/checks/_summary.sh target/${{ github.job }}/summary.txt
         if: ${{ failure() }}
-      - name: Install diffoscope
-        run: |
-          sudo apt update -q
-          sudo apt install -y diffoscope
-        if: ${{ failure() }}
       - name: Check artifact differences
         run: |
           hadoop-ozone/dev-support/checks/_diffoscope.sh
@@ -534,14 +529,10 @@ jobs:
           rm ozone*.tar.gz
       - name: Execute tests
         run: |
-          pushd hadoop-ozone/dist/target/ozone-*
-          sudo mkdir .aws && sudo chmod 777 .aws && sudo chown 1000 .aws
-          popd
           ./hadoop-ozone/dev-support/checks/acceptance.sh
         env:
           KEEP_IMAGE: false
           OZONE_ACCEPTANCE_SUITE: ${{ matrix.suite }}
-          OZONE_VOLUME_OWNER: 1000
       - name: Summary of failures
         run: hadoop-ozone/dev-support/checks/_summary.sh target/${{ github.job }}/summary.txt
         if: ${{ failure() }}
@@ -586,9 +577,6 @@ jobs:
           tar xzvf ozone*.tar.gz -C hadoop-ozone/dist/target
       - name: Execute tests
         run: |
-          pushd hadoop-ozone/dist/target/ozone-*
-          sudo mkdir .aws && sudo chmod 777 .aws && sudo chown 1000 .aws
-          popd
           ./hadoop-ozone/dev-support/checks/kubernetes.sh
       - name: Summary of failures
         run: hadoop-ozone/dev-support/checks/_summary.sh target/${{ github.job }}/summary.txt
@@ -639,15 +627,7 @@ jobs:
           java-version: ${{ env.TEST_JAVA_VERSION }}
       - name: Execute tests
         run: |
-          args="${{ inputs.ratis_args }}"
-          if [[ "${{ matrix.profile }}" == "flaky" ]]; then
-            args="$args -Dsurefire.rerunFailingTestsCount=5 -Dsurefire.fork.timeout=3600"
-          fi
-          if [[ "${{ matrix.profile }}" != "filesystem" ]]; then
-            args="$args -DskipShade"
-          fi
-
-          hadoop-ozone/dev-support/checks/integration.sh -Ptest-${{ matrix.profile }} ${args}
+          hadoop-ozone/dev-support/checks/integration.sh -Ptest-${{ matrix.profile }} ${{ inputs.ratis_args }}
         env:
           DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }}
       - name: Summary of failures

diff --git a/.github/workflows/repeat-acceptance.yml b/.github/workflows/repeat-acceptance.yml
@@ -145,13 +145,9 @@ jobs:
           sudo chmod -R a+rwX hadoop-ozone/dist/target
       - name: Execute tests
         run: |
-          pushd hadoop-ozone/dist/target/ozone-*
-          sudo mkdir .aws && sudo chmod 777 .aws && sudo chown 1000 .aws
-          popd
           ./hadoop-ozone/dev-support/checks/acceptance.sh
         env:
           KEEP_IMAGE: false
-          OZONE_VOLUME_OWNER: 1000
         continue-on-error: true
       - name: Summary of failures
         run: hadoop-ozone/dev-support/checks/_summary.sh target/${{ github.job }}/summary.txt

diff --git a/hadoop-ozone/dev-support/checks/_diffoscope.sh b/hadoop-ozone/dev-support/checks/_diffoscope.sh
@@ -24,6 +24,11 @@ cd "$DIR/../../.." || exit 1
 BASE_DIR="$(pwd -P)"
 : ${OUTPUT_LOG:="${BASE_DIR}/target/repro/output.log"}
 
+if [[ "${GITHUB_ACTIONS:-}" == "true" ]]; then
+  sudo apt update -q
+  sudo apt install -y diffoscope
+fi
+
 for jar in $(grep -o "investigate with diffoscope [^ ]*\.jar [^ ]*\.jar" "${OUTPUT_LOG}" | awk '{ print $NF }'); do
   jarname=$(basename "$jar")
   if [[ ! -e "$jar" ]]; then

diff --git a/hadoop-ozone/dev-support/checks/_lib.sh b/hadoop-ozone/dev-support/checks/_lib.sh
@@ -63,3 +63,12 @@ _install_tool() {
     fi
   fi
 }
+
+create_aws_dir() {
+  if [[ "${CI:-}" == "true" ]]; then
+    export OZONE_VOLUME_OWNER=1000 # uid (from ozone-runner image)
+    pushd hadoop-ozone/dist/target/ozone-*
+    sudo mkdir .aws && sudo chmod 777 .aws && sudo chown ${OZONE_VOLUME_OWNER} .aws
+    popd
+  fi
+}
diff --git a/hadoop-ozone/dev-support/checks/acceptance.sh b/hadoop-ozone/dev-support/checks/acceptance.sh
@@ -40,6 +40,8 @@ if [ ! -d "$DIST_DIR" ]; then
     "$DIR/build.sh" -Pcoverage
 fi
 
+create_aws_dir
+
 mkdir -p "$REPORT_DIR"
 
 if [[ "${OZONE_ACCEPTANCE_SUITE}" == "s3a" ]]; then

diff --git a/hadoop-ozone/dev-support/checks/integration.sh b/hadoop-ozone/dev-support/checks/integration.sh
@@ -16,4 +16,13 @@
 
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 CHECK=integration
-source "${DIR}/junit.sh" "$@"
+
+args=""
+if [[ "$@" =~ "-Ptest-flaky" ]]; then
+  args="$args -Dsurefire.rerunFailingTestsCount=5 -Dsurefire.fork.timeout=3600"
+fi
+if [[ "$@" =~ "-Ptest-" ]] && [[ ! "$@" =~ "-Ptest-filesystem" ]]; then
+  args="$args -DskipShade"
+fi
+
+source "${DIR}/junit.sh" $args "$@"
diff --git a/hadoop-ozone/dev-support/checks/kubernetes.sh b/hadoop-ozone/dev-support/checks/kubernetes.sh
@@ -41,6 +41,8 @@ if [ ! -d "$DIST_DIR" ]; then
     "$DIR/build.sh" -Pcoverage
 fi
 
+create_aws_dir
+
 mkdir -p "$REPORT_DIR"
 
 cd "$DIST_DIR/kubernetes/examples" || exit 1