Optimize listing objects for s3 api

.github/workflows/checkstyle.yml

@@ -1,4 +1,4 @@
-name: Checkstyle, Findbugs, Doc Check, etc.
+name: Checkstyle, Findbugs, Doc Check, SortPom, etc.
 
 on: [pull_request]
 
@@ -8,40 +8,39 @@ jobs:
 
     steps:
       - name: checkout repo
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       - name: Use Node.js ${{ matrix.node-version }}
         uses: actions/setup-node@v1
         with:
           node-version: '10.11.0'
 
       - name: Cache local Maven repository
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         with:
           path: ~/.m2/repository
           key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
           restore-keys: |
             ${{ runner.os }}-maven-
 
       - name: Cache local Go modules
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         with:
           path: ~/go/pkg/mod
           key: ${{ runner.os }}-gomod-${{ hashFiles('**/go.mod') }}
 
-      - name: Execute license check, checkstyle, findbugs
+      - name: Execute license check, checkstyle, findbugs, sortpom
         run: |
           mkdir -p ~/.m2
           ALLUXIO_DOCKER_NO_TTY=true \
           ALLUXIO_DOCKER_GIT_CLEAN=true \
-          ALLUXIO_DOCKER_MVN_RUNTOEND=true \
           ALLUXIO_CHECKSTYLE=true \
-          dev/github/run_docker.sh "\"-Dtest=${{ matrix.modules }}\"" -pl tests
+          dev/github/run_docker.sh
         timeout-minutes: 60
 
       - name: Archive artifacts
         continue-on-error: true
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         if: always()
         with:
           name: artifact

.github/workflows/java8_container_tests.yml

@@ -0,0 +1,66 @@
+name: Java 8 Container Tests
+
+# This build runs integration tests using testcontainers
+# The user executing the test command must have access to a running docker daemon
+
+on: [pull_request]
+
+jobs:
+  build:
+    name: "modules: "
+
+    strategy:
+      fail-fast: false
+      matrix:
+        modules:
+          - >-
+            alluxio.membership.**
+
+    runs-on: ubuntu-latest
+    if: "!contains(github.event.pull_request.title, 'DOCFIX') &&
+        !contains(github.event.pull_request.title, 'SKIPCI')"
+
+    steps:
+      - name: checkout repo
+        uses: actions/checkout@v3
+
+      - name: Use Node.js ${{ matrix.node-version }}
+        uses: actions/setup-node@v1
+        with:
+          node-version: '10.11.0'
+
+      - name: Cache local Maven repository
+        uses: actions/cache@v3
+        with:
+          path: ~/.m2/repository
+          key: ${{ runner.os }}-maven-java11-${{ hashFiles('**/pom.xml') }}
+
+      - name: Cache local Go modules
+        uses: actions/cache@v3
+        with:
+          path: ~/go/pkg/mod
+          key: ${{ runner.os }}-gomod-${{ hashFiles('**/go.mod') }}
+
+      - name: Run tests
+        id: test0
+        run: |
+          mkdir -p ~/.m2
+          ALLUXIO_DOCKER_NO_TTY=true \
+          ALLUXIO_DOCKER_GIT_CLEAN=true \
+          ALLUXIO_DOCKER_ID=0 \
+          ALLUXIO_DOCKER_MVN_PROJECT_LIST=dora/tests/testcontainers \
+          ALLUXIO_DOCKER_MVN_TESTS=${{ matrix.modules }} \
+          dev/github/run_docker.sh
+        timeout-minutes: 60
+
+      - name: Archive artifacts
+        continue-on-error: true
+        uses: actions/upload-artifact@v3
+        if: always()
+        with:
+          name: artifact
+          path: |
+            **/target/surefire-reports/*
+            **/target/artifacts/*
+            **/target/logs/*
+          retention-days: 7

.github/workflows/java8_integration_tests.yml

@@ -1,5 +1,11 @@
 name: Java 8 Integration Tests
 
+# This build runs the default integration tests in the dora/tests module
+# It runs all the tests besides those categorized as:
+# - Fault tolerance = alluxio.server.ft
+# - Web UI = alluxio.web
+# - FUSE = alluxio.client.fuse
+
 on: [pull_request]
 
 jobs:
@@ -11,15 +17,11 @@ jobs:
       matrix:
         modules:
           - >-
-            alluxio.client.cli.**,!alluxio.client.cli.fs.**
-          - >-
-            alluxio.client.cli.fs.**
-          - >-
-            alluxio.client.fs.**,!alluxio.client.fs.concurrent.**,!alluxio.client.fs.io.**
+            alluxio.client.cli.**
           - >-
-            alluxio.client.fs.concurrent.**,alluxio.client.fs.io.**
+            alluxio.client.fs.**
           - >-
-            alluxio.client.**,!alluxio.client.fs.**,!alluxio.client.cli.**,!alluxio.client.fuse.**
+            alluxio.client.**,!alluxio.client.fs.**,!alluxio.client.cli.**
           - >-
             alluxio.job.**,alluxio.master.**,alluxio.stress.**
           - >-
@@ -31,21 +33,21 @@ jobs:
 
     steps:
       - name: checkout repo
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       - name: Use Node.js ${{ matrix.node-version }}
         uses: actions/setup-node@v1
         with:
           node-version: '10.11.0'
 
       - name: Cache local Maven repository
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         with:
           path: ~/.m2/repository
-          key: ${{ runner.os }}-maven-java8-${{ hashFiles('**/pom.xml') }}
+          key: ${{ runner.os }}-maven-java11-${{ hashFiles('**/pom.xml') }}
 
       - name: Cache local Go modules
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         with:
           path: ~/go/pkg/mod
           key: ${{ runner.os }}-gomod-${{ hashFiles('**/go.mod') }}
@@ -56,14 +58,13 @@ jobs:
           mkdir -p ~/.m2
           ALLUXIO_DOCKER_NO_TTY=true \
           ALLUXIO_DOCKER_GIT_CLEAN=true \
-          ALLUXIO_DOCKER_MVN_RUNTOEND=true \
-          ALLUXIO_DOCKER_MVN_PROJECT_LIST=!webui,!shaded/client,!integration/tools/hms,!integration/yarn,!assembly/client,!assembly/server,!table/server/underdb/glue,!underfs/hdfs,!underfs/ozone,!underfs/adl,!underfs/abfs,!underfs/cosn,!underfs/wasb,!underfs/cos,!underfs/kodo,!underfs/oss,!underfs/swift \
-          dev/github/run_docker.sh "\"-Dtest=${{ matrix.modules }}\"" -pl tests
+          ALLUXIO_DOCKER_MVN_TESTS=${{ matrix.modules }} \
+          dev/github/run_docker.sh
         timeout-minutes: 60
 
       - name: Archive artifacts
         continue-on-error: true
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         if: always()
         with:
           name: artifact

.github/workflows/java8_integration_tests_ft.yml

@@ -1,5 +1,10 @@
 name: Java 8 Fault Tolerant Integration Tests
 
+# This build runs the fault tolerance specific integration tests in the dora/tests module
+# The key differences between this build and the default integration test build are
+# - Only runs integration tests with prefix alluxio.server.ft
+# - Sets fork count to 1 to prevent flakey errors due to running concurrent tests; default count is 2
+
 on: [pull_request]
 
 jobs:
@@ -21,21 +26,21 @@ jobs:
 
     steps:
       - name: checkout repo
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       - name: Use Node.js ${{ matrix.node-version }}
         uses: actions/setup-node@v1
         with:
           node-version: '10.11.0'
 
       - name: Cache local Maven repository
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         with:
           path: ~/.m2/repository
-          key: ${{ runner.os }}-maven-java8-${{ hashFiles('**/pom.xml') }}
+          key: ${{ runner.os }}-maven-java11-${{ hashFiles('**/pom.xml') }}
 
       - name: Cache local Go modules
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         with:
           path: ~/go/pkg/mod
           key: ${{ runner.os }}-gomod-${{ hashFiles('**/go.mod') }}
@@ -48,14 +53,13 @@ jobs:
           ALLUXIO_DOCKER_FORK_COUNT=1 \
           ALLUXIO_DOCKER_NO_TTY=true \
           ALLUXIO_DOCKER_GIT_CLEAN=true \
-          ALLUXIO_DOCKER_MVN_RUNTOEND=true \
-          ALLUXIO_DOCKER_MVN_PROJECT_LIST=!webui,!shaded/client,!integration/tools/hms,!integration/yarn,!assembly/client,!assembly/server,!table/server/underdb/glue,!underfs/hdfs,!underfs/ozone,!underfs/adl,!underfs/abfs,!underfs/cosn,!underfs/wasb,!underfs/cos,!underfs/kodo,!underfs/oss,!underfs/swift \
-          dev/github/run_docker.sh "\"-Dtest=${{ matrix.modules }}\"" -pl tests
+          ALLUXIO_DOCKER_MVN_TESTS=${{ matrix.modules }} \
+          dev/github/run_docker.sh
         timeout-minutes: 60
 
       - name: Archive artifacts
         continue-on-error: true
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         if: always()
         with:
           name: artifact

.github/workflows/java8_integration_tests_webui.yml

@@ -1,5 +1,10 @@
 name: Java 8 Integration Tests (w/ webui)
 
+# This build runs the integration tests in the dora/tests module that require the web UI
+# The key differences between this build and the default integration test build are
+# - Only runs integration tests with prefix alluxio.web
+# - Adds webui to the project list
+
 on: [pull_request]
 
 jobs:
@@ -19,21 +24,21 @@ jobs:
 
     steps:
       - name: checkout repo
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       - name: Use Node.js ${{ matrix.node-version }}
         uses: actions/setup-node@v1
         with:
           node-version: '10.11.0'
 
       - name: Cache local Maven repository
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         with:
           path: ~/.m2/repository
-          key: ${{ runner.os }}-maven-java8-${{ hashFiles('**/pom.xml') }}
+          key: ${{ runner.os }}-maven-java11-${{ hashFiles('**/pom.xml') }}
 
       - name: Cache local Go modules
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         with:
           path: ~/go/pkg/mod
           key: ${{ runner.os }}-gomod-${{ hashFiles('**/go.mod') }}
@@ -44,14 +49,13 @@ jobs:
           mkdir -p ~/.m2
           ALLUXIO_DOCKER_NO_TTY=true \
           ALLUXIO_DOCKER_GIT_CLEAN=true \
-          ALLUXIO_DOCKER_MVN_RUNTOEND=true \
-          ALLUXIO_DOCKER_MVN_PROJECT_LIST=!shaded/client,!integration/tools/hms,!integration/yarn,!assembly/client,!assembly/server \
-          dev/github/run_docker.sh "\"-Dtest=${{ matrix.modules }}\"" -pl tests
+          ALLUXIO_DOCKER_MVN_TESTS=${{ matrix.modules }} \
+          dev/github/run_docker.sh
         timeout-minutes: 60
-      
+
       - name: Archive artifacts
         continue-on-error: true
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         if: always()
         with:
           name: artifact