Building 2011/merge on pull_request by vasil-pashov #8111
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build and Test | |
| on: | |
| push: | |
| tags: ["v**"] | |
| branches: ["master"] | |
| pull_request: | |
| branches: ["**"] | |
| workflow_dispatch: | |
| inputs: | |
| persistent_storage: | |
| description: Run the persistent storage tests | |
| type: boolean | |
| pypi_publish: | |
| type: boolean | |
| publish_env: | |
| description: Environment to publish to | |
| type: environment | |
| cmake_preset_type: | |
| description: Override CMAKE preset type | |
| type: choice | |
| options: ["-", debug, release] | |
| run-name: Building ${{github.ref_name}} on ${{github.event_name}} by ${{github.actor}} | |
| concurrency: | |
| group: ${{github.ref}} | |
| cancel-in-progress: true | |
| jobs: | |
| cibw_docker_image: | |
| uses: ./.github/workflows/cibw_docker_image.yml | |
| permissions: {packages: write} | |
| with: | |
| cibuildwheel_ver: "2.12.1" | |
| force_update: false | |
| common_config: | |
| needs: [cibw_docker_image] | |
| runs-on: ubuntu-22.04 | |
| strategy: | |
| matrix: | |
| include: | |
| # Use a matrix to build the common hierarchical structure used by multiple matrix jobs in build_steps.yaml | |
| # Please declare any key you added below in build_steps.yaml's dummy matrix as well to aid the linting tools | |
| - linux_matrix: | |
| - os: linux | |
| distro: ubuntu-22.04 | |
| cmake_preset_prefix: linux | |
| cibw_build_suffix: manylinux_x86_64 | |
| build_dir: /tmp/cpp_build | |
| vcpkg_packages_dir: "" | |
| symbols: "*.gz" # Please update publish.yml when changing this!! | |
| do_not_archive: ["*.so", "*.[ao]", "vcpkg_installed"] | |
| test_services: | |
| mongodb: | |
| image: "mongo:4.4" | |
| container: | |
| image: ${{needs.cibw_docker_image.outputs.tag}} | |
| volumes: | |
| - /:/mnt | |
| windows_matrix: | |
| - os: windows | |
| distro: windows-latest | |
| cmake_preset_prefix: windows-cl | |
| cibw_build_suffix: win_amd64 | |
| build_dir: C:/cpp_build | |
| vcpkg_packages_dir: C:/vcpkg_packages | |
| symbols: "*.pdb" # Please update publish.yml when changing this!! | |
| do_not_archive: ["*.lib", "*.ilk", "*.pyd", "*.dll", "CMakeFiles/*.dir", "vcpkg_installed"] | |
| test_services: {} | |
| container: null | |
| steps: | |
| - run: | | |
| if ${{startsWith(github.ref, 'refs/tags/v')}}; then | |
| # Can only upload to Pypi once per version, so only auto upload on tag builds: | |
| echo -e "PUBLISH_ENV=ProdPypi\nCMAKE_PRESET_TYPE=release\nPYPI_PUBLISH=1" | tee -a $GITHUB_ENV | |
| elif $GITHUB_REF_PROTECTED || ${{github.ref == 'refs/heads/master'}} ; then | |
| echo -e "PUBLISH_ENV=TestPypi\nCMAKE_PRESET_TYPE=release" | tee -a $GITHUB_ENV | |
| else | |
| echo -e "PUBLISH_ENV=${{vars.DEFAULT_PUBLISH_ENV}}\nCMAKE_PRESET_TYPE=$DEFAULT_PRESET" | tee -a $GITHUB_ENV | |
| fi | |
| env: | |
| DEFAULT_PRESET: ${{startsWith(github.repository, 'man-group/ArcticDB') && 'release' || vars.DEFAULT_CMAKE_PRESET_TYPE || 'debug'}} | |
| outputs: | |
| cibuildwheel_ver: ${{needs.cibw_docker_image.outputs.cibuildwheel_ver}} | |
| publish_env: ${{inputs.publish_env || env.PUBLISH_ENV || ''}} | |
| pypi_publish: ${{inputs.pypi_publish || env.PYPI_PUBLISH}} | |
| cmake_preset_type_resolved: ${{inputs.cmake_preset_type != '-' && inputs.cmake_preset_type || env.CMAKE_PRESET_TYPE}} | |
| linux_matrix: ${{toJson(matrix.linux_matrix)}} | |
| windows_matrix: ${{toJson(matrix.windows_matrix)}} | |
| pre_seed_cleanup: | |
| if: inputs.persistent_storage == true | |
| name: Cleanup persistent storages | |
| uses: ./.github/workflows/persistent_storage.yml | |
| secrets: inherit | |
| with: | |
| job_type: cleanup | |
| persistent_storage_seed_linux: | |
| needs: [common_config, pre_seed_cleanup] | |
| if: inputs.persistent_storage == true | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python3: ${{fromJson(vars.LINUX_PYTHON_VERSIONS || '[6, 7, 8, 9, 10, 11]')}} | |
| arcticdb_version: ["oldest", "latest"] | |
| include: | |
| - python_deps_ids: [""] | |
| matrix_override: ${{fromJson(needs.common_config.outputs.linux_matrix)}} | |
| name: Seed Persistent Storage for 3.${{matrix.python3}} Linux with ${{ matrix.arcticdb_version }} ArcticDB package version | |
| uses: ./.github/workflows/persistent_storage.yml | |
| secrets: inherit | |
| permissions: {packages: write} | |
| with: | |
| job_type: seed | |
| python3: ${{matrix.python3}} | |
| arcticdb_version: ${{matrix.arcticdb_version}} | |
| matrix: ${{toJson(matrix.matrix_override)}} | |
| python_deps_ids: ${{toJson(matrix.python_deps_ids)}} | |
| persistent_storage_seed_windows: | |
| needs: [common_config, pre_seed_cleanup] | |
| if: inputs.persistent_storage == true | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python3: ${{fromJson(vars.WINDOWS_PYTHON_VERSIONS || '[7, 8, 9, 10, 11]')}} | |
| arcticdb_version: ["oldest", "latest"] | |
| include: | |
| - python_deps_ids: [""] | |
| matrix_override: ${{fromJson(needs.common_config.outputs.windows_matrix)}} | |
| name: Seed Persistent Storage for 3.${{matrix.python3}} Windows with ${{ matrix.arcticdb_version }} ArcticDB package version | |
| uses: ./.github/workflows/persistent_storage.yml | |
| secrets: inherit | |
| permissions: {packages: write} | |
| with: | |
| job_type: seed | |
| python3: ${{matrix.python3}} | |
| arcticdb_version: ${{matrix.arcticdb_version}} | |
| matrix: ${{toJson(matrix.matrix_override)}} | |
| python_deps_ids: ${{toJson(matrix.python_deps_ids)}} | |
| cpp-test-linux: | |
| needs: [cibw_docker_image, common_config] | |
| name: Linux C++ Tests | |
| uses: ./.github/workflows/build_steps.yml | |
| secrets: inherit | |
| with: | |
| job_type: cpp-tests | |
| cmake_preset_type: ${{needs.common_config.outputs.cmake_preset_type_resolved}} | |
| cibw_image_tag: ${{needs.cibw_docker_image.outputs.tag}} | |
| matrix: ${{needs.common_config.outputs.linux_matrix}} | |
| build-python-wheels-linux: | |
| # Then use the cached compilation artifacts to build other python versions concurrently in cibuildwheels | |
| needs: [cibw_docker_image, common_config] | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python3: ${{fromJson(vars.LINUX_PYTHON_VERSIONS || '[6, 7, 8, 9, 10, 11]')}} | |
| include: | |
| - python_deps_ids: [""] | |
| matrix_override: ${{fromJson(needs.common_config.outputs.linux_matrix)}} | |
| pytest_xdist_mode: "--dist worksteal" | |
| - python3: 6 | |
| python_deps_ids: ["", -compat36] | |
| matrix_override: | |
| - ${{fromJson(needs.common_config.outputs.linux_matrix)[0]}} | |
| - python_deps_id: -compat36 | |
| python_deps: requirements-compatibility-py36.txt | |
| pytest_xdist_mode: "" # worksteal Not supported on Python 3.6 | |
| - python3: 8 | |
| python_deps_ids: ["", -compat38] | |
| matrix_override: | |
| - ${{fromJson(needs.common_config.outputs.linux_matrix)[0]}} | |
| - python_deps_id: -compat38 | |
| python_deps: requirements-compatibility-py38.txt | |
| name: 3.${{matrix.python3}} Linux | |
| uses: ./.github/workflows/build_steps.yml | |
| secrets: inherit | |
| permissions: {packages: write} | |
| with: | |
| job_type: build-python-wheels | |
| python3: ${{matrix.python3}} | |
| cibw_image_tag: ${{needs.cibw_docker_image.outputs.tag}} | |
| cibw_version: ${{needs.common_config.outputs.cibuildwheel_ver}} | |
| cmake_preset_type: ${{needs.common_config.outputs.cmake_preset_type_resolved}} | |
| matrix: ${{toJson(matrix.matrix_override)}} | |
| python_deps_ids: ${{toJson(matrix.python_deps_ids)}} | |
| persistent_storage: ${{inputs.persistent_storage}} | |
| pytest_xdist_mode: ${{matrix.pytest_xdist_mode}} | |
| cpp-test-windows: | |
| needs: [common_config] | |
| name: Windows C++ Tests | |
| uses: ./.github/workflows/build_steps.yml | |
| secrets: inherit | |
| with: | |
| job_type: cpp-tests | |
| cmake_preset_type: ${{needs.common_config.outputs.cmake_preset_type_resolved}} | |
| matrix: ${{needs.common_config.outputs.windows_matrix}} | |
| build-python-wheels-windows: | |
| needs: [common_config] | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python3: ${{fromJson(vars.WINDOWS_PYTHON_VERSIONS || '[7, 8, 9, 10, 11]')}} | |
| name: 3.${{matrix.python3}} Windows | |
| uses: ./.github/workflows/build_steps.yml | |
| secrets: inherit | |
| permissions: {packages: write} | |
| with: | |
| job_type: build-python-wheels | |
| python3: ${{matrix.python3}} | |
| cibw_version: ${{needs.common_config.outputs.cibuildwheel_ver}} | |
| cmake_preset_type: ${{needs.common_config.outputs.cmake_preset_type_resolved}} | |
| matrix: ${{needs.common_config.outputs.windows_matrix}} | |
| persistent_storage: ${{ inputs.persistent_storage }} | |
| persistent_storage_verify_linux: | |
| needs: [common_config, build-python-wheels-linux, build-python-wheels-windows] | |
| if: inputs.persistent_storage == true | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python3: ${{fromJson(vars.LINUX_PYTHON_VERSIONS || '[6, 7, 8, 9, 10, 11]')}} | |
| arcticdb_version: ["oldest", "latest"] | |
| include: | |
| - python_deps_ids: [""] | |
| matrix_override: ${{fromJson(needs.common_config.outputs.linux_matrix)}} | |
| name: Verify Persistent Storage for 3.${{matrix.python3}} Linux with ${{ matrix.arcticdb_version }} ArcticDB package version | |
| uses: ./.github/workflows/persistent_storage.yml | |
| secrets: inherit | |
| permissions: {packages: write} | |
| with: | |
| job_type: verify | |
| python3: ${{matrix.python3}} | |
| arcticdb_version: ${{matrix.arcticdb_version}} | |
| matrix: ${{toJson(matrix.matrix_override)}} | |
| python_deps_ids: ${{toJson(matrix.python_deps_ids)}} | |
| persistent_storage_verify_windows: | |
| needs: [common_config, build-python-wheels-windows, build-python-wheels-linux] | |
| if: inputs.persistent_storage == true | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python3: ${{fromJson(vars.WINDOWS_PYTHON_VERSIONS || '[7, 8, 9, 10, 11]')}} | |
| arcticdb_version: ["oldest", "latest"] | |
| include: | |
| - python_deps_ids: [""] | |
| matrix_override: ${{fromJson(needs.common_config.outputs.windows_matrix)}} | |
| name: Verify Persistent Storage for 3.${{matrix.python3}} Windows with ${{ matrix.arcticdb_version }} ArcticDB package version | |
| uses: ./.github/workflows/persistent_storage.yml | |
| secrets: inherit | |
| permissions: {packages: write} | |
| with: | |
| job_type: verify | |
| python3: ${{matrix.python3}} | |
| arcticdb_version: ${{matrix.arcticdb_version}} | |
| matrix: ${{toJson(matrix.matrix_override)}} | |
| python_deps_ids: ${{toJson(matrix.python_deps_ids)}} | |
| post_verify_cleanup: | |
| needs: [persistent_storage_verify_windows, persistent_storage_verify_linux] | |
| if: inputs.persistent_storage == true | |
| name: Cleanup persistent storages | |
| uses: ./.github/workflows/persistent_storage.yml | |
| secrets: inherit | |
| with: | |
| job_type: cleanup | |
| can_merge: | |
| needs: [cpp-test-linux, cpp-test-windows, build-python-wheels-linux, build-python-wheels-windows, persistent_storage_verify_linux, persistent_storage_verify_windows] | |
| if: | | |
| always() && | |
| !failure() && | |
| !cancelled() | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - run: echo Dummy job to simplify PR merge checks configuration | |
| # FUTURE: add some test stats/reporting | |
| publish: | |
| needs: [common_config, can_merge] | |
| if: | | |
| always() && | |
| !failure() && | |
| !cancelled() && | |
| (needs.common_config.outputs.pypi_publish && needs.common_config.outputs.publish_env) | |
| uses: ./.github/workflows/publish.yml | |
| secrets: inherit | |
| permissions: {contents: write} | |
| with: | |
| environment: ${{needs.common_config.outputs.publish_env}} |