inspection-pipeline.yml

name: INSPECTION PIPELINE

on:
  schedule:
    - cron: "0 0 * * 1-5"
    - cron: "30 4 * * 1-5"
    - cron: "30 9 * * 1-5"
    - cron: "0 11 * * 1,3,5"
    - cron: "0 11 * * 2,4"
  workflow_dispatch:
    inputs:
      BRANCH:
        description: 'BRANCH'
        required: true
        type: string
      COMMIT_ID:
        description: 'COMMIT ID'
        required: true
        type: string
      TAR_PATH:
        description: 'TAR PATH(Release)'
        required: false
        type: string
      ASAN_TAR_PATH:
        description: 'TAR PATH(ASAN)'
        required: false
        type: string
      UBUNTU_TAR_PATH:
        description: 'TAR PATH(Release & Ubuntu)'
        required: false
        type: string
      UBUNTU_ASAN_TAR_PATH:
        description: 'TAR PATH(ASAN & Ubuntu)'
        required: false
        type: string
      IS_REBUILD:
        description: 'REBUILD'
        type: boolean
        default: false
      INCLUDE_ADMIT:
        description: 'RUN ADMIT?'
        type: string
        default: 'true'
      ALL_LINUX:
        description: 'Centos7 && Release only?'
        type: string
        default: 'true'
      RETAIN_ENV:
        description: 'RETAIN THE ENVIRONMENT?'
        type: boolean
        default: false

permissions:
  checks: write
  actions: write
  contents: write
  deployments: write
  discussions: write
  issues: write
  packages: write
  pages: write
  pull-requests: write
  repository-projects: write
  security-events: write
  statuses: write

env:
  IS_INSPECTION: true

jobs:
  info:
    runs-on: [self-hosted, normal]
    name: INFO
    env:
      REPO: ${{ github.repository }}
      GH_TOKEN: ${{ github.token }}
    outputs:
      BRANCH: ${{ steps.param.outputs.BRANCH }}
      PR_NUMBER: ${{ steps.param.outputs.PR_NUMBER }}
      TAR_PATH: ${{ steps.param.outputs.TAR_PATH }}
      ASAN_TAR_PATH: ${{ steps.param.outputs.ASAN_TAR_PATH }}
      UBUNTU_TAR_PATH: ${{ steps.param.outputs.UBUNTU_TAR_PATH }}
      UBUNTU_ASAN_TAR_PATH: ${{ steps.param.outputs.UBUNTU_ASAN_TAR_PATH }}
      BE_UT_LINUX: ${{ steps.param.outputs.BE_UT_LINUX }}
      RETAIN_ENV: ${{ steps.param.outputs.RETAIN_ENV }}
      ALL_LINUX: ${{ steps.param.outputs.ALL_LINUX }}
      INCLUDE_ADMIT: ${{ steps.param.outputs.INCLUDE_ADMIT }}
    steps:
      - name: CRON PARAM
        id: param
        run: |
          BE_UT_LINUX=centos7
          if [[ "${{ github.event_name }}" == "schedule" ]]; then
            if [[ "${{ github.event.schedule }}" == "0 11 * * 1,3,5" ]]; then
              branch=branch-3.3
              if [[ "$GITHUB_REPOSITORY" == 'StarRocks/starrocks' && `date +%u` -eq 1 ]]; then
                all_linux=true
              fi
            elif [[ "${{ github.event.schedule }}" == "0 11 * * 2,4" ]]; then
              branch=branch-3.2
              all_linux=false
            elif [[ "${{ github.event.schedule }}" == "0 0 * * 1-5" ]]; then
              branch=main
              if [[ `date +%u` -eq 1 || `date +%u` -eq 4 ]]; then
                all_linux=true
              fi
            else
              branch=main
            fi
            [[ $((`date +%e` % 2)) -eq 1 ]] && BE_UT_LINUX=ubuntu
            base_sha=$(gh api /repos/${REPO}/branches/${branch} | jq -r .commit.sha)
            echo "::notice::${branch}(${BE_UT_LINUX} ${base_sha})"
            [[ "${base_sha}" == "null" ]] && (echo "::error::Get HEAD SHA error, please check." && exit -1);
            echo "BRANCH=${branch}" >> $GITHUB_OUTPUT
            echo "PR_NUMBER=${base_sha}" >> $GITHUB_OUTPUT
            echo "ALL_LINUX=${all_linux}" >> $GITHUB_OUTPUT
            echo "INCLUDE_ADMIT=true" >> $GITHUB_OUTPUT
          else
            echo "BRANCH=${{ inputs.BRANCH }}" >> $GITHUB_OUTPUT
            echo "PR_NUMBER=${{ inputs.COMMIT_ID }}" >> $GITHUB_OUTPUT
            echo "TAR_PATH=${{ inputs.TAR_PATH }}" >> $GITHUB_OUTPUT
            echo "ASAN_TAR_PATH=${{ inputs.ASAN_TAR_PATH }}" >> $GITHUB_OUTPUT
            echo "UBUNTU_TAR_PATH=${{ inputs.UBUNTU_TAR_PATH }}" >> $GITHUB_OUTPUT
            echo "UBUNTU_ASAN_TAR_PATH=${{ inputs.UBUNTU_ASAN_TAR_PATH }}" >> $GITHUB_OUTPUT
            echo "RETAIN_ENV=${{ inputs.RETAIN_ENV }}" >> $GITHUB_OUTPUT
            echo "ALL_LINUX=${{ inputs.ALL_LINUX }}" >> $GITHUB_OUTPUT
            echo "INCLUDE_ADMIT=false" >> $GITHUB_OUTPUT
          fi
          
          echo "BE_UT_LINUX=${BE_UT_LINUX}" >> $GITHUB_OUTPUT

  be-ut:
    runs-on: [self-hosted, normal]
    needs:
      - info
    timeout-minutes: 90
    name: BE UT
    continue-on-error: true
    env:
      BRANCH: ${{ needs.info.outputs.BRANCH }}
      PR_NUMBER: ${{ needs.info.outputs.PR_NUMBER }}
      LINUX: ${{ needs.info.outputs.BE_UT_LINUX }}
    strategy:
      fail-fast: false
      matrix:
        build_type: [ Release, ASAN ]
    steps:
      - name: UPDATE ECI & RUN UT
        id: run_ut
        shell: bash
        timeout-minutes: 90
        env:
          build_type: ${{ matrix.build_type }}
        run: |
          rm -rf ./ci-tool && cp -rf /var/lib/ci-tool ./ci-tool && cd ci-tool && git pull && source lib/init.sh
          ./bin/elastic-ut.sh --repository ${{ github.repository }} --branch ${BRANCH} --pr ${PR_NUMBER} --module be --build ${build_type} --linuxdistro ${LINUX}

      - name: clean ECI
        if: always()
        run: |
          echo ${{ steps.run_ut.outputs.ECI_ID }}
          eci rm ${{ steps.run_ut.outputs.ECI_ID }}
          echo "${{ steps.run_ut.outcome }}" > ./${{ matrix.build_type }}.txt

      - name: Upload Log
        uses: actions/upload-artifact@v4
        if: always() && steps.run_ut.outcome == 'failure'
        with:
          name: BE UT LOG
          path: ${{ steps.run_ut.outputs.BE_LOG }}
          retention-days: 1
          overwrite: true

      - name: Upload BE UT Result
        uses: actions/upload-artifact@v4
        if: always()
        with:
          name: BE-UT-RESULT-${{ matrix.build_type }}
          path: ${{ matrix.build_type }}.txt
          retention-days: 1
          overwrite: true

      - name: Clean ENV
        if: always()
        run: |
          rm -f ${{ steps.run_ut.outputs.RES_FILE }}
          rm -f ${{ steps.run_ut.outputs.RES_LOG }}
          rm -f ${{ steps.run_ut.outputs.BE_LOG }}
          rm -rf ${{ github.workspace }}/*

  fe-ut:
    runs-on: [self-hosted, normal]
    name: FE UT
    needs:
      - info
    env:
      GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      BRANCH: ${{ needs.info.outputs.BRANCH }}
      PR_NUMBER: ${{ needs.info.outputs.PR_NUMBER }}
    steps:
      - name: clean
        run: |
          rm -rf ${{ github.workspace }}
          mkdir -p ${{ github.workspace }}

      - name: BRANCH INFO
        id: branch
        env:
          repo: ${{ github.repository }}
        run: |
          bucket_prefix=`echo ${repo%/*} | tr '[:upper:]' '[:lower:]'` 
          echo "bucket_prefix=${bucket_prefix}" >> $GITHUB_OUTPUT

      - name: UPDATE ECI & RUN UT
        id: run_ut
        shell: bash
        timeout-minutes: 60
        run: |
          rm -rf ./ci-tool && cp -rf /var/lib/ci-tool ./ci-tool && cd ci-tool && git pull && source lib/init.sh
          ./bin/elastic-ut.sh --repository ${{ github.repository }} --branch ${BRANCH} --pr ${PR_NUMBER} --module fe --build Release

      - name: Clean ECI
        if: always()
        run: |
          echo ${{ steps.run_ut.outputs.ECI_ID }}
          echo ">>> Dmesg info:"
          eci exec ${{ steps.run_ut.outputs.ECI_ID }} bash -c "dmesg -T"
          eci rm ${{ steps.run_ut.outputs.ECI_ID }}

      - name: Upload log
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: FE UT LOG
          path: ${{ steps.run_ut.outputs.RES_LOG }}
          retention-days: 1
          overwrite: true

      - name: Download FE UT XML
        if: always()
        id: download-ut-xml
        env:
          oss_path: ${{ steps.run_ut.outputs.oss_path }}
        run: |
          mkdir fe && cd fe
          ossutil64 --config-file ~/.ossutilconfig cp ${oss_path}/ . --recursive
          tar zxf fe_ut_report.tar.gz

      - name: Publish UT Report
        uses: mikepenz/action-junit-report@v4
        if: always()
        id: publish_report
        env:
          github_token: ${{ secrets.GITHUB_TOKEN }}
          token: ${{ secrets.GITHUB_TOKEN }}
        with:
          check_name: 'FE UT Report'
          detailed_summary: true
          fail_on_failure: true
          report_paths: ./fe/fe-core/target/surefire-reports/*.xml

      - name: Clean ENV
        if: always()
        run: |
          rm -rf ${{ steps.run_ut.outputs.FE_REPORT_DIR }}
          rm -f ${{ steps.run_ut.outputs.RES_FILE }}
          rm -f ${{ steps.run_ut.outputs.RES_LOG }}
          rm -rf ${{ steps.run_ut.outputs.COV_DIR }}
          rm -rf ${{ github.workspace }}/*

  build:
    runs-on: [self-hosted, normal]
    name: BUILD
    needs:
      - info
    if: github.event_name == 'schedule' || inputs.IS_REBUILD
    continue-on-error: true
    strategy:
      fail-fast: false
      matrix:
        build_type: [ Release, ASAN ]
        linux: [ centos7, ubuntu ]
    env:
      BRANCH: ${{ needs.info.outputs.BRANCH }}
      PR_NUMBER: ${{ needs.info.outputs.PR_NUMBER }}
      ALL_LINUX: ${{ needs.info.outputs.ALL_LINUX }}
    steps:
      - name: CLEAN
        run: |
          rm -rf ${{ github.workspace }} && mkdir -p ${{ github.workspace }}

      - name: UPDATE ECI & RUN BUILD
        id: run_build
        shell: bash
        timeout-minutes: 90
        if: (matrix.build_type == 'Release' && matrix.linux == 'centos7') || env.ALL_LINUX == 'true'
        run: |
          rm -rf ./ci-tool && cp -rf /var/lib/ci-tool ./ci-tool && cd ci-tool && git pull && source lib/init.sh
          ./bin/elastic-build.sh --repository ${{ github.repository }} --branch ${BRANCH} --pr ${PR_NUMBER} --build ${{ matrix.build_type }} --linuxdistro ${{ matrix.linux }}

      - name: BUILD RESULT
        run: |
          echo ${{ steps.run_build.outputs.ABSOLUTE_OUTPUT_TAR }}
          echo "Package: ${{ steps.run_build.outputs.ABSOLUTE_OUTPUT_TAR }}" >> $GITHUB_STEP_SUMMARY
          echo ${{ steps.run_build.outputs.ABSOLUTE_OUTPUT_TAR }} > tar.txt

      - uses: actions/upload-artifact@v4
        with:
          name: BUILD-RESULT-${{ matrix.build_type }}-${{ matrix.linux }}
          path: tar.txt
          retention-days: 1
          overwrite: true

      - name: clean ECI
        if: always()
        run: |
          echo ${{ steps.run_build.outputs.ECI_ID }}
          eci rm ${{ steps.run_build.outputs.ECI_ID }}

      - name: Clean ENV
        if: always()
        run: |
          rm -f ${{ steps.run_build.outputs.RES_FILE }}
          rm -f ${{ steps.run_build.outputs.RES_LOG }}
          rm -rf ${{ github.workspace }}

  test_filter:
    runs-on: [self-hosted, normal]
    name: FILTER
    needs:
      - info
      - build
    if: always() && needs.info.result == 'success'
    env:
      ALL_LINUX: needs.info.outputs.ALL_LINUX
    outputs:
      RELEASE_TAR_PATH: ${{ steps.set_output.outputs.CENTOS_RELEASE_TAR_PATH }}
      ASAN_TAR_PATH: ${{ steps.set_output.outputs.CENTOS_ASAN_TAR_PATH }}
      UBUNTU_RELEASE_TAR_PATH: ${{ steps.set_output.outputs.UBUNTU_RELEASE_TAR_PATH }}
      UBUNTU_ASAN_TAR_PATH: ${{ steps.set_output.outputs.UBUNTU_ASAN_TAR_PATH }}
      IS_SELF_BUILD: ${{ steps.set_output.outputs.IS_SELF_BUILD }}
    steps:
      - name: Clean Workspace
        uses: AutoModality/action-clean@v1.1.0

      - name: Download Build Artifact
        if: github.event_name == 'schedule' || inputs.IS_REBUILD
        uses: actions/download-artifact@v4
        with:
          pattern: BUILD-RESULT-*
          path: outputs

      - name: Analyse Build Artifact
        id: analyse_artifact
        if: github.event_name == 'schedule' || inputs.IS_REBUILD
        run: |
          if [[ -e "./outputs/BUILD-RESULT-Release-centos7/tar.txt" ]]; then
            build_result=$(cat "./outputs/BUILD-RESULT-Release-centos7/tar.txt")
            echo "CENTOS_RELEASE_TAR_PATH=${build_result}" >> $GITHUB_OUTPUT
          else
            has_failure=true
          fi
          
          if [[ -e "./outputs/BUILD-RESULT-ASAN-centos7/tar.txt" ]]; then
            build_result=$(cat "./outputs/BUILD-RESULT-ASAN-centos7/tar.txt")
            echo "CENTOS_ASAN_TAR_PATH=${build_result}" >> $GITHUB_OUTPUT
          else
            has_failure=true
          fi
          
          if [[ -e "./outputs/BUILD-RESULT-Release-ubuntu/tar.txt" ]]; then
            build_result=$(cat "./outputs/BUILD-RESULT-Release-ubuntu/tar.txt")
            echo "UBUNTU_RELEASE_TAR_PATH=${build_result}" >> $GITHUB_OUTPUT
          else
            has_failure=true
          fi
          
          if [[ -e "./outputs/BUILD-RESULT-ASAN-ubuntu/tar.txt" ]]; then
            build_result=$(cat "./outputs/BUILD-RESULT-ASAN-ubuntu/tar.txt")
            echo "UBUNTU_ASAN_TAR_PATH=${build_result}" >> $GITHUB_OUTPUT
          else
            has_failure=true
          fi
          
          if [[ "${has_failure}" == "true" && "${ALL_LINUX}" == "true" ]]; then
            exit 1
          fi

      - name: SET OUTPUT
        id: set_output
        if: always()
        run: |
          if [[ "${{steps.analyse_artifact.outcome}}" == "skipped" ]]; then
            echo "CENTOS_RELEASE_TAR_PATH=${{ needs.info.outputs.TAR_PATH }}" >> $GITHUB_OUTPUT
            echo "CENTOS_ASAN_TAR_PATH=${{ needs.info.outputs.ASAN_TAR_PATH }}" >> $GITHUB_OUTPUT
            echo "IS_SELF_BUILD=false" >> $GITHUB_OUTPUT
          else
            echo "CENTOS_RELEASE_TAR_PATH=${{ steps.analyse_artifact.outputs.CENTOS_RELEASE_TAR_PATH }}" >> $GITHUB_OUTPUT
            echo "CENTOS_ASAN_TAR_PATH=${{ steps.analyse_artifact.outputs.CENTOS_ASAN_TAR_PATH }}" >> $GITHUB_OUTPUT
            echo "UBUNTU_RELEASE_TAR_PATH=${{ steps.analyse_artifact.outputs.UBUNTU_RELEASE_TAR_PATH }}" >> $GITHUB_OUTPUT
            echo "UBUNTU_ASAN_TAR_PATH=${{ steps.analyse_artifact.outputs.UBUNTU_ASAN_TAR_PATH }}" >> $GITHUB_OUTPUT
            echo "IS_SELF_BUILD=true" >> $GITHUB_OUTPUT
          fi

  system-test-release:
    name: 'Release/Centos'
    uses: ./.github/workflows/inspection-reusable-pipeline.yml
    needs:
      - info
      - test_filter
    secrets: inherit
    if: always() && needs.test_filter.outputs.RELEASE_TAR_PATH != ''
    with:
      BRANCH: ${{ needs.info.outputs.BRANCH }}
      COMMIT_ID: ${{ needs.info.outputs.PR_NUMBER }}
      TAR_PATH: ${{ needs.test_filter.outputs.RELEASE_TAR_PATH }}
      BUILD_TYPE: Release
      LINUX_DISTRO: centos7
      IS_SELF_BUILD: ${{ needs.test_filter.outputs.IS_SELF_BUILD }}
      INCLUDE_ADMIT: ${{ needs.info.outputs.INCLUDE_ADMIT }}

  system-test-asan:
    name: 'ASAN/Centos'
    uses: ./.github/workflows/inspection-reusable-pipeline.yml
    needs:
      - info
      - test_filter
    secrets: inherit
    if: always() && needs.test_filter.outputs.ASAN_TAR_PATH != '' && needs.info.outputs.ALL_LINUX == 'true'
    with:
      BRANCH: ${{ needs.info.outputs.BRANCH }}
      COMMIT_ID: ${{ needs.info.outputs.PR_NUMBER }}
      TAR_PATH: ${{ needs.test_filter.outputs.ASAN_TAR_PATH }}
      BUILD_TYPE: ASAN
      LINUX_DISTRO: centos7
      IS_SELF_BUILD: ${{ needs.test_filter.outputs.IS_SELF_BUILD }}
      INCLUDE_ADMIT: ${{ needs.info.outputs.INCLUDE_ADMIT }}

  system-test-release-ubuntu:
    name: 'Release/Ubuntu'
    uses: ./.github/workflows/inspection-reusable-pipeline.yml
    needs:
      - info
      - test_filter
    secrets: inherit
    if: always() && needs.test_filter.outputs.UBUNTU_RELEASE_TAR_PATH != '' && needs.info.outputs.ALL_LINUX == 'true'
    with:
      BRANCH: ${{ needs.info.outputs.BRANCH }}
      COMMIT_ID: ${{ needs.info.outputs.PR_NUMBER }}
      TAR_PATH: ${{ needs.test_filter.outputs.UBUNTU_RELEASE_TAR_PATH }}
      BUILD_TYPE: Release
      LINUX_DISTRO: ubuntu
      IS_SELF_BUILD: ${{ needs.test_filter.outputs.IS_SELF_BUILD }}
      INCLUDE_ADMIT: ${{ needs.info.outputs.INCLUDE_ADMIT }}

  system-test-asan-ubuntu:
    name: 'ASAN/Ubuntu'
    uses: ./.github/workflows/inspection-reusable-pipeline.yml
    needs:
      - info
      - test_filter
    secrets: inherit
    if: always() && needs.test_filter.outputs.UBUNTU_ASAN_TAR_PATH != '' && needs.info.outputs.ALL_LINUX == 'true'
    with:
      BRANCH: ${{ needs.info.outputs.BRANCH }}
      COMMIT_ID: ${{ needs.info.outputs.PR_NUMBER }}
      TAR_PATH: ${{ needs.test_filter.outputs.UBUNTU_ASAN_TAR_PATH }}
      BUILD_TYPE: ASAN
      LINUX_DISTRO: ubuntu
      IS_SELF_BUILD: ${{ needs.test_filter.outputs.IS_SELF_BUILD }}
      INCLUDE_ADMIT: ${{ needs.info.outputs.INCLUDE_ADMIT }}

  Teardown:
    runs-on: [self-hosted, normal]
    name: Teardown
    needs:
      - info
      - be-ut
      - fe-ut
      - system-test-release
      - system-test-asan
      - system-test-release-ubuntu
      - system-test-asan-ubuntu
    if: always()
    env:
      BRANCH: ${{ needs.info.outputs.BRANCH }}
      PR_NUMBER: ${{ needs.info.outputs.PR_NUMBER }}
      GH_TOKEN: ${{ github.token }}
      RELEASE_CONF_FILE: ${{ needs.system-test-release.outputs.CONF_FILE }}
      ASAN_CONF_FILE: ${{ needs.system-test-asan.outputs.CONF_FILE }}
      RELEASE_MYSQL_ECI_ID: ${{ needs.system-test-release.outputs.MYSQL_ECI_ID }}
      ASAN_MYSQL_ECI_ID: ${{ needs.system-test-asan.outputs.MYSQL_ECI_ID }}
      UBUNTU_RELEASE_CONF_FILE: ${{ needs.system-test-release-ubuntu.outputs.CONF_FILE }}
      UBUNTU_ASAN_CONF_FILE: ${{ needs.system-test-asan-ubuntu.outputs.CONF_FILE }}
      UBUNTU_RELEASE_MYSQL_ECI_ID: ${{ needs.system-test-release-ubuntu.outputs.MYSQL_ECI_ID }}
      UBUNTU_ASAN_MYSQL_ECI_ID: ${{ needs.system-test-asan-ubuntu.outputs.MYSQL_ECI_ID }}
      BE_UT_LINUX: ${{ needs.info.outputs.BE_UT_LINUX }}
      RETAIN_ENV: ${{ needs.info.outputs.RETAIN_ENV }}
    steps:
      - name: init
        run: |
          rm -rf ./ci-tool && cp -rf /var/lib/ci-tool ./ci-tool && cd ci-tool && git pull

      - name: Backup SR Info (Release & Centos)
        if: env.RELEASE_CONF_FILE != ''
        id: backup_release_centos
        env:
          build_type: Release
          linuxdistro: centos7
        run: |
          cd ci-tool && source lib/init.sh
          ./bin/backup_log_cores.sh --branch "${BRANCH}" --pr "${PR_NUMBER}" --build "${build_type}" --conf "${RELEASE_CONF_FILE}" --repository "${GITHUB_REPOSITORY}" || true
          
          if [[ "${RETAIN_ENV}" != "true" ]]; then
            ./bin/elastic-cluster.sh --delete
          fi

      - name: Backup SR Info (ASAN & Centos)
        if: env.ASAN_CONF_FILE != ''
        id: backup_asan_centos
        env:
          build_type: ASAN
          linuxdistro: centos7
        run: |
          cd ci-tool && source lib/init.sh
          ./bin/backup_log_cores.sh --branch "${BRANCH}" --pr "${PR_NUMBER}" --build "${build_type}" --conf "${ASAN_CONF_FILE}" --repository "${GITHUB_REPOSITORY}" || true
          
          if [[ "${RETAIN_ENV}" != "true" ]]; then
            ./bin/elastic-cluster.sh --delete
          fi

      - name: Backup SR Info (Release & Ubuntu)
        if: env.UBUNTU_RELEASE_CONF_FILE != ''
        id: backup_release_ubuntu
        env:
          build_type: Release
          linuxdistro: ubuntu
        run: |
          cd ci-tool && source lib/init.sh
          ./bin/backup_log_cores.sh --branch "${BRANCH}" --pr "${PR_NUMBER}" --build "${build_type}" --conf "${UBUNTU_RELEASE_CONF_FILE}" --repository "${GITHUB_REPOSITORY}" || true
          
          if [[ "${RETAIN_ENV}" != "true" ]]; then
            ./bin/elastic-cluster.sh --delete
          fi

      - name: Backup SR Info (ASAN & Ubuntu)
        if: env.UBUNTU_ASAN_CONF_FILE != ''
        id: backup_asan_ubuntu
        env:
          build_type: ASAN
          linuxdistro: ubuntu
        run: |
          cd ci-tool && source lib/init.sh
          ./bin/backup_log_cores.sh --branch "${BRANCH}" --pr "${PR_NUMBER}" --build "${build_type}" --conf "${UBUNTU_ASAN_CONF_FILE}" --repository "${GITHUB_REPOSITORY}" || true
          
          if [[ "${RETAIN_ENV}" != "true" ]]; then
            ./bin/elastic-cluster.sh --delete
          fi

      - name: save unstable cases
        if: always() && (github.event_name == 'schedule' || (github.event_name != 'schedule' && env.BRANCH != 'main'))
        run: |
          cd ci-tool && source lib/init.sh
          ./bin/save_unstable_cases.sh

      - name: clean ECI
        if: always()
        run: |
          if [[ "${RELEASE_MYSQL_ECI_ID}" != "" ]]; then
            eci rm ${RELEASE_MYSQL_ECI_ID}
          fi
          
          if [[ "${ASAN_MYSQL_ECI_ID}" != "" ]]; then
            eci rm ${ASAN_MYSQL_ECI_ID}
          fi
          
          if [[ "${UBUNTU_RELEASE_MYSQL_ECI_ID}" != "" ]]; then
            eci rm ${UBUNTU_RELEASE_MYSQL_ECI_ID}
          fi
          
          if [[ "${UBUNTU_ASAN_MYSQL_ECI_ID}" != "" ]]; then
            eci rm ${UBUNTU_ASAN_MYSQL_ECI_ID}
          fi

      - name: Clean
        if: always()
        run: |
          rm -f $RELEASE_CONF_FILE
          rm -f $ASAN_CONF_FILE
          rm -f $UBUNTU_RELEASE_CONF_FILE
          rm -f $UBUNTU_ASAN_CONF_FILE
          rm -rf ${{ github.workspace }}/*