diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fa852843e7d..7ddc1b749c1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -479,11 +479,6 @@ jobs: - name: Summary of failures run: hadoop-ozone/dev-support/checks/_summary.sh target/${{ github.job }}/summary.txt if: ${{ failure() }} - - name: Install diffoscope - run: | - sudo apt update -q - sudo apt install -y diffoscope - if: ${{ failure() }} - name: Check artifact differences run: | hadoop-ozone/dev-support/checks/_diffoscope.sh @@ -534,14 +529,10 @@ jobs: rm ozone*.tar.gz - name: Execute tests run: | - pushd hadoop-ozone/dist/target/ozone-* - sudo mkdir .aws && sudo chmod 777 .aws && sudo chown 1000 .aws - popd ./hadoop-ozone/dev-support/checks/acceptance.sh env: KEEP_IMAGE: false OZONE_ACCEPTANCE_SUITE: ${{ matrix.suite }} - OZONE_VOLUME_OWNER: 1000 - name: Summary of failures run: hadoop-ozone/dev-support/checks/_summary.sh target/${{ github.job }}/summary.txt if: ${{ failure() }} @@ -586,9 +577,6 @@ jobs: tar xzvf ozone*.tar.gz -C hadoop-ozone/dist/target - name: Execute tests run: | - pushd hadoop-ozone/dist/target/ozone-* - sudo mkdir .aws && sudo chmod 777 .aws && sudo chown 1000 .aws - popd ./hadoop-ozone/dev-support/checks/kubernetes.sh - name: Summary of failures run: hadoop-ozone/dev-support/checks/_summary.sh target/${{ github.job }}/summary.txt @@ -639,15 +627,7 @@ jobs: java-version: ${{ env.TEST_JAVA_VERSION }} - name: Execute tests run: | - args="${{ inputs.ratis_args }}" - if [[ "${{ matrix.profile }}" == "flaky" ]]; then - args="$args -Dsurefire.rerunFailingTestsCount=5 -Dsurefire.fork.timeout=3600" - fi - if [[ "${{ matrix.profile }}" != "filesystem" ]]; then - args="$args -DskipShade" - fi - - hadoop-ozone/dev-support/checks/integration.sh -Ptest-${{ matrix.profile }} ${args} + hadoop-ozone/dev-support/checks/integration.sh -Ptest-${{ matrix.profile }} ${{ inputs.ratis_args }} env: DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} - name: Summary of failures diff --git a/.github/workflows/repeat-acceptance.yml b/.github/workflows/repeat-acceptance.yml index c36a841817e..b4e7b42fcb5 100644 --- a/.github/workflows/repeat-acceptance.yml +++ b/.github/workflows/repeat-acceptance.yml @@ -145,13 +145,9 @@ jobs: sudo chmod -R a+rwX hadoop-ozone/dist/target - name: Execute tests run: | - pushd hadoop-ozone/dist/target/ozone-* - sudo mkdir .aws && sudo chmod 777 .aws && sudo chown 1000 .aws - popd ./hadoop-ozone/dev-support/checks/acceptance.sh env: KEEP_IMAGE: false - OZONE_VOLUME_OWNER: 1000 continue-on-error: true - name: Summary of failures run: hadoop-ozone/dev-support/checks/_summary.sh target/${{ github.job }}/summary.txt diff --git a/hadoop-ozone/dev-support/checks/_diffoscope.sh b/hadoop-ozone/dev-support/checks/_diffoscope.sh index cc7cc700c82..7ab22b42796 100755 --- a/hadoop-ozone/dev-support/checks/_diffoscope.sh +++ b/hadoop-ozone/dev-support/checks/_diffoscope.sh @@ -24,6 +24,11 @@ cd "$DIR/../../.." || exit 1 BASE_DIR="$(pwd -P)" : ${OUTPUT_LOG:="${BASE_DIR}/target/repro/output.log"} +if [[ "${GITHUB_ACTIONS:-}" == "true" ]]; then + sudo apt update -q + sudo apt install -y diffoscope +fi + for jar in $(grep -o "investigate with diffoscope [^ ]*\.jar [^ ]*\.jar" "${OUTPUT_LOG}" | awk '{ print $NF }'); do jarname=$(basename "$jar") if [[ ! -e "$jar" ]]; then diff --git a/hadoop-ozone/dev-support/checks/_lib.sh b/hadoop-ozone/dev-support/checks/_lib.sh index ec59c16b632..1ae2d73a9d6 100644 --- a/hadoop-ozone/dev-support/checks/_lib.sh +++ b/hadoop-ozone/dev-support/checks/_lib.sh @@ -63,3 +63,12 @@ _install_tool() { fi fi } + +create_aws_dir() { + if [[ "${CI:-}" == "true" ]]; then + export OZONE_VOLUME_OWNER=1000 # uid (from ozone-runner image) + pushd hadoop-ozone/dist/target/ozone-* + sudo mkdir .aws && sudo chmod 777 .aws && sudo chown ${OZONE_VOLUME_OWNER} .aws + popd + fi +} diff --git a/hadoop-ozone/dev-support/checks/acceptance.sh b/hadoop-ozone/dev-support/checks/acceptance.sh index 1e877ade9f1..4a46efe6d61 100755 --- a/hadoop-ozone/dev-support/checks/acceptance.sh +++ b/hadoop-ozone/dev-support/checks/acceptance.sh @@ -40,6 +40,8 @@ if [ ! -d "$DIST_DIR" ]; then "$DIR/build.sh" -Pcoverage fi +create_aws_dir + mkdir -p "$REPORT_DIR" if [[ "${OZONE_ACCEPTANCE_SUITE}" == "s3a" ]]; then diff --git a/hadoop-ozone/dev-support/checks/integration.sh b/hadoop-ozone/dev-support/checks/integration.sh index 72f8b2ddb80..e918e1dd9ab 100755 --- a/hadoop-ozone/dev-support/checks/integration.sh +++ b/hadoop-ozone/dev-support/checks/integration.sh @@ -16,4 +16,13 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" CHECK=integration -source "${DIR}/junit.sh" "$@" + +args="" +if [[ "$@" =~ "-Ptest-flaky" ]]; then + args="$args -Dsurefire.rerunFailingTestsCount=5 -Dsurefire.fork.timeout=3600" +fi +if [[ "$@" =~ "-Ptest-" ]] && [[ ! "$@" =~ "-Ptest-filesystem" ]]; then + args="$args -DskipShade" +fi + +source "${DIR}/junit.sh" $args "$@" diff --git a/hadoop-ozone/dev-support/checks/kubernetes.sh b/hadoop-ozone/dev-support/checks/kubernetes.sh index 1f9772f8f43..ef5f12c4776 100755 --- a/hadoop-ozone/dev-support/checks/kubernetes.sh +++ b/hadoop-ozone/dev-support/checks/kubernetes.sh @@ -41,6 +41,8 @@ if [ ! -d "$DIST_DIR" ]; then "$DIR/build.sh" -Pcoverage fi +create_aws_dir + mkdir -p "$REPORT_DIR" cd "$DIST_DIR/kubernetes/examples" || exit 1