[MINOR][DNM][TESTING] Flink bundle testing 5.6 #33409
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Java CI | |
| on: | |
| push: | |
| branches: | |
| - master | |
| - 'release-*' | |
| - branch-0.x | |
| pull_request: | |
| paths-ignore: | |
| - '**.bmp' | |
| - '**.gif' | |
| - '**.jpg' | |
| - '**.jpeg' | |
| - '**.md' | |
| - '**.pdf' | |
| - '**.png' | |
| - '**.svg' | |
| - '**.yaml' | |
| - '.gitignore' | |
| branches: | |
| - master | |
| - 'release-*' | |
| - branch-0.x | |
| concurrency: | |
| group: ${{ github.ref }} | |
| cancel-in-progress: ${{ !contains(github.ref, 'master') && !contains(github.ref, 'branch-0.x') }} | |
| env: | |
| MVN_ARGS: -e -ntp -B -V -Dgpg.skip -Djacoco.skip -Pwarn-log -Dorg.slf4j.simpleLogger.log.org.apache.maven.plugins.shade=warn -Dorg.slf4j.simpleLogger.log.org.apache.maven.plugins.dependency=warn -Dmaven.wagon.httpconnectionManager.ttlSeconds=25 -Dmaven.wagon.http.retryHandler.count=5 | |
| SPARK_COMMON_MODULES: hudi-spark-datasource/hudi-spark,hudi-spark-datasource/hudi-spark-common | |
| jobs: | |
| validate-source: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Set up JDK 8 | |
| uses: actions/setup-java@v3 | |
| with: | |
| java-version: '8' | |
| distribution: 'adopt' | |
| architecture: x64 | |
| cache: maven | |
| - name: Check Binary Files | |
| run: ./scripts/release/validate_source_binary_files.sh | |
| - name: Check Copyright | |
| run: | | |
| ./scripts/release/create_source_directory.sh hudi-tmp-repo | |
| cd hudi-tmp-repo | |
| ./scripts/release/validate_source_copyright.sh | |
| - name: RAT check | |
| run: ./scripts/release/validate_source_rat.sh | |
| test-flink: | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| include: | |
| - flinkProfile: "flink1.14" | |
| - flinkProfile: "flink1.15" | |
| - flinkProfile: "flink1.16" | |
| - flinkProfile: "flink1.17" | |
| - flinkProfile: "flink1.18" | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Set up JDK 8 | |
| uses: actions/setup-java@v3 | |
| with: | |
| java-version: '8' | |
| distribution: 'adopt' | |
| architecture: x64 | |
| cache: maven | |
| - name: Build Project | |
| env: | |
| SCALA_PROFILE: 'scala-2.12' | |
| FLINK_PROFILE: ${{ matrix.flinkProfile }} | |
| run: | |
| mvn clean install -T 2 -D"$SCALA_PROFILE" -D"$FLINK_PROFILE" -pl hudi-examples/hudi-examples-flink -am -Davro.version=1.10.0 -DskipTests=true $MVN_ARGS | |
| - name: Quickstart Test | |
| env: | |
| SCALA_PROFILE: 'scala-2.12' | |
| FLINK_PROFILE: ${{ matrix.flinkProfile }} | |
| run: | |
| mvn test -Punit-tests -D"$SCALA_PROFILE" -D"$FLINK_PROFILE" -pl hudi-examples/hudi-examples-flink $MVN_ARGS | |
| - name: Integration Test | |
| env: | |
| SCALA_PROFILE: 'scala-2.12' | |
| FLINK_PROFILE: ${{ matrix.flinkProfile }} | |
| if: ${{ endsWith(env.FLINK_PROFILE, '1.18') }} | |
| run: | | |
| mvn clean install -T 2 -Pintegration-tests -D"$SCALA_PROFILE" -D"$FLINK_PROFILE" -pl hudi-flink-datasource/hudi-flink -am -Davro.version=1.10.0 -DskipTests=true $MVN_ARGS | |
| mvn verify -Pintegration-tests -D"$SCALA_PROFILE" -D"$FLINK_PROFILE" -pl hudi-flink-datasource/hudi-flink $MVN_ARGS | |
| validate-bundles: | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| include: | |
| - flinkProfile: 'flink1.18' | |
| sparkProfile: 'spark3.5' | |
| sparkRuntime: 'spark3.5.0' | |
| - flinkProfile: 'flink1.18' | |
| sparkProfile: 'spark3.4' | |
| sparkRuntime: 'spark3.4.0' | |
| - flinkProfile: 'flink1.17' | |
| sparkProfile: 'spark3.3' | |
| sparkRuntime: 'spark3.3.2' | |
| - flinkProfile: 'flink1.16' | |
| sparkProfile: 'spark3.3' | |
| sparkRuntime: 'spark3.3.1' | |
| - flinkProfile: 'flink1.15' | |
| sparkProfile: 'spark3.2' | |
| sparkRuntime: 'spark3.2.3' | |
| - flinkProfile: 'flink1.14' | |
| sparkProfile: 'spark3.1' | |
| sparkRuntime: 'spark3.1.3' | |
| - flinkProfile: 'flink1.14' | |
| sparkProfile: 'spark3.0' | |
| sparkRuntime: 'spark3.0.2' | |
| - flinkProfile: 'flink1.14' | |
| sparkProfile: 'spark2.4' | |
| sparkRuntime: 'spark2.4.8' | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Set up JDK 8 | |
| uses: actions/setup-java@v3 | |
| with: | |
| java-version: '8' | |
| distribution: 'adopt' | |
| architecture: x64 | |
| cache: maven | |
| - name: Build Project | |
| env: | |
| FLINK_PROFILE: ${{ matrix.flinkProfile }} | |
| SPARK_PROFILE: ${{ matrix.sparkProfile }} | |
| SCALA_PROFILE: 'scala-2.12' | |
| run: | | |
| mvn clean package -T 2 -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -DdeployArtifacts=true -DskipTests=true $MVN_ARGS | |
| # TODO remove the sudo below. It's a needed workaround as detailed in HUDI-5708. | |
| sudo chown -R "$USER:$(id -g -n)" hudi-platform-service/hudi-metaserver/target/generated-sources | |
| mvn clean package -T 2 -D"$SCALA_PROFILE" -D"$FLINK_PROFILE" -DdeployArtifacts=true -DskipTests=true $MVN_ARGS -pl packaging/hudi-flink-bundle -am -Davro.version=1.10.0 | |
| - name: IT - Bundle Validation - OpenJDK 8 | |
| env: | |
| FLINK_PROFILE: ${{ matrix.flinkProfile }} | |
| SPARK_PROFILE: ${{ matrix.sparkProfile }} | |
| SPARK_RUNTIME: ${{ matrix.sparkRuntime }} | |
| SCALA_PROFILE: 'scala-2.12' | |
| if: ${{ env.SPARK_PROFILE >= 'spark3' }} # Only run validation on Spark 3 | |
| run: | | |
| HUDI_VERSION=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout) | |
| ./packaging/bundle-validation/ci_run.sh hudi_docker_java8 $HUDI_VERSION openjdk8 | |
| - name: IT - Bundle Validation - OpenJDK 11 | |
| env: | |
| FLINK_PROFILE: ${{ matrix.flinkProfile }} | |
| SPARK_PROFILE: ${{ matrix.sparkProfile }} | |
| SPARK_RUNTIME: ${{ matrix.sparkRuntime }} | |
| SCALA_PROFILE: 'scala-2.12' | |
| if: ${{ env.SPARK_PROFILE >= 'spark3' }} # Only run validation on Spark 3 | |
| run: | | |
| HUDI_VERSION=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout) | |
| ./packaging/bundle-validation/ci_run.sh hudi_docker_java11 $HUDI_VERSION openjdk11 | |
| - name: IT - Bundle Validation - OpenJDK 17 | |
| env: | |
| FLINK_PROFILE: ${{ matrix.flinkProfile }} | |
| SPARK_PROFILE: ${{ matrix.sparkProfile }} | |
| SPARK_RUNTIME: ${{ matrix.sparkRuntime }} | |
| SCALA_PROFILE: 'scala-2.12' | |
| if: ${{ env.SPARK_PROFILE >= 'spark3.3' }} # Only Spark 3.3 and above support Java 17 | |
| run: | | |
| HUDI_VERSION=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout) | |
| ./packaging/bundle-validation/ci_run.sh hudi_docker_java17 $HUDI_VERSION openjdk17 |