cid.yaml

name: CID

on:
  push:
    branches:
      - main
      - beta
    paths-ignore:
      [
        "conda/**",
        "docs/**",
        "scripts/**",
        "**.md",
        "*",
        "!pyproject.toml",
        "!flake.nix",
        "!*.lock",
      ]
  pull_request:
    types: [opened, synchronize, labeled, reopened, ready_for_review]
    paths-ignore:
      [
        "conda/**",
        "docs/**",
        "scripts/**",
        "**.md",
        "*",
        "!pyproject.toml",
        "!flake.nix",
        "!*.lock",
      ]

  workflow_dispatch:
    inputs:
      debug_enabled:
        description: "Run with tmate.io debugging enabled"
        required: true
        type: boolean
        default: false
      run_build_images:
        description: "Run build-images job"
        required: false
        type: boolean
        default: false
      run_execute_workflow:
        description: "Run execute-workflow job"
        required: false
        type: boolean
        default: false
      workflow_execution_mode:
        description: "Workflow execution mode"
        required: false
        type: string
        default: "prod"

defaults:
  run:
    shell: bash

permissions:
  actions: write
  contents: read

jobs:
  set-variables:
    runs-on: ubuntu-latest
    outputs:
      debug: ${{ steps.set-variables.outputs.debug }}
      skip_ci: ${{ steps.set-variables.outputs.skip_ci }}
      skip_tests: ${{ steps.set-variables.outputs.skip_tests }}
      mode: ${{ steps.set-variables.outputs.mode }}

    steps:
      - name: Set action variables
        id: set-variables
        run: |
          DEBUG="false"
          MODE="prod"
          SKIP_CI="false"
          SKIP_TESTS="false"

          if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then
            DEBUG="${{ inputs.debug_enabled }}"
            MODE="${{ inputs.workflow_execution_mode }}"
          fi

          if [[ "${{ github.event_name }}" == "pull_request" ]]; then
            if ${{ contains(github.event.pull_request.labels.*.name, 'skip-ci') }}; then
              SKIP_CI="true"
            fi
            if ${{ contains(github.event.pull_request.labels.*.name, 'skip-tests') }}; then
              SKIP_TESTS="true"
            fi
            if ${{ contains(github.event.pull_request.labels.*.name, 'actions-debug') }}; then
              DEBUG="true"
            fi
            if ${{ contains(github.event.pull_request.labels.*.name, 'workflow-mode-dev') }}; then
              MODE="dev"
            fi
          fi

          echo "DEBUG=$DEBUG"
          echo "MODE=$MODE"
          echo "SKIP_CI=$SKIP_CI"
          echo "SKIP_TESTS=$SKIP_TESTS"

          echo "DEBUG=$DEBUG" >> $GITHUB_OUTPUT
          echo "MODE=$MODE" >> $GITHUB_OUTPUT
          echo "SKIP_CI=$SKIP_CI" >> $GITHUB_OUTPUT
          echo "SKIP_TESTS=$SKIP_TESTS" >> $GITHUB_OUTPUT

  config-workflows:
    needs: [set-variables]
    if: ${{ needs.set-variables.outputs.skip_ci != 'true' }}
    runs-on: ubuntu-latest
    outputs:
      config-path: ${{ steps.config-output.outputs.path }}

    steps:
      - name: Checkout code
        uses: actions/checkout@v4
        with:
          ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.ref || github.ref }}

      - name: Create Flyte config from YAML template
        id: yq-process
        uses: mikefarah/yq@master
        with:
          cmd: "yq e \
            '.admin.endpoint = strenv(FLYTE_CLUSTER_ENDPOINT) | \
            .storage.stow.config.project_id = strenv(GCP_PROJECT_ID) | \
            .storage.stow.config.scopes = strenv(GCP_STORAGE_SCOPES) | \
            .storage.container = strenv(GCP_STORAGE_CONTAINER)' \
            .flyte/config-template.yaml > .flyte/config.yaml"
        env:
          FLYTE_CLUSTER_ENDPOINT: ${{ secrets.FLYTE_CLUSTER_ENDPOINT }}
          GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }}
          GCP_STORAGE_SCOPES: ${{ secrets.GCP_STORAGE_SCOPES }}
          GCP_STORAGE_CONTAINER: ${{ secrets.GCP_STORAGE_CONTAINER }}

      - name: Upload Flyte config as an artifact
        id: config-output
        uses: actions/upload-artifact@v3
        with:
          name: flyte-config
          path: ${{ secrets.FLYTECTL_CONFIG }}

  test:
    runs-on: ubuntu-latest
    needs: [set-variables, config-workflows]
    if: ${{ needs.set-variables.outputs.skip_ci != 'true' && needs.set-variables.outputs.skip_tests != 'true' }}
    concurrency:
      group: test-${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.event.pull_request.number || github.ref }}-${{ needs.set-variables.outputs.mode }}
      cancel-in-progress: true
    strategy:
      matrix:
        python_version: ["3.10"]

    steps:
      - name: Check Variables
        run: |
          echo "SKIP_CI=${{ needs.set-variables.outputs.skip_ci }}"
          echo "SKIP_TESTS=${{ needs.set-variables.outputs.skip_tests }}"
          echo "DEBUG=${{ needs.set-variables.outputs.debug }}"
          echo "MODE=${{ needs.set-variables.outputs.mode }}"
      - name: Checkout code
        uses: actions/checkout@v4
        with:
          ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.ref || github.ref }}

      - name: Setup environment
        uses: ./.github/actions/setup_environment
        with:
          python_version: ${{ matrix.python_version }}
          debug_enabled: ${{ needs.set-variables.outputs.debug }}

      - name: Download Flyte config
        uses: actions/download-artifact@v3
        with:
          name: flyte-config
          path: .flyte/

      - name: "Setup tmate debug session"
        uses: mxschmitt/action-tmate@v3
        if: ${{ inputs.debug_enabled }}

      - name: Lint and typecheck
        run: |
          make lint-check

      - name: Run tests
        run: |
          make test-cov-xml

      - name: Upload coverage
        uses: codecov/codecov-action@v3
        with:
          token: ${{ secrets.CODECOV_TOKEN }}
          fail_ci_if_error: false
          verbose: true

  test-nix:
    runs-on: ubuntu-latest
    needs: [set-variables, config-workflows]
    if: ${{ needs.set-variables.outputs.skip_ci != 'true' && needs.set-variables.outputs.skip_tests != 'true' }}
    concurrency:
      group: test-nix-${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.event.pull_request.number || github.ref }}-${{ needs.set-variables.outputs.mode }}
      cancel-in-progress: true
    permissions:
      contents: read
    steps:
      - uses: actions/checkout@v4
        with:
          ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.ref || github.ref }}
      - uses: DeterminateSystems/nix-installer-action@v9
      - uses: DeterminateSystems/magic-nix-cache-action@v2
      - run: nix flake check --impure --accept-flake-config
      - run: |
          ./scripts/flake pytest
      # It is also possible to build the default package with a checkPhase
      # - run: |
      #     export NUMBA_CACHE_DIR=$(mktemp -d -p $RUNNER_TEMP numba-cache-XXXXX)
      #     chmod 777 $NUMBA_CACHE_DIR
      #     echo "NUMBA_CACHE_DIR: $NUMBA_CACHE_DIR"
      #     nix build . --accept-flake-config --impure

  build-docker-images:
    needs: [set-variables]
    if: ${{ needs.set-variables.outputs.skip_ci != 'true' && (contains(github.event.pull_request.labels.*.name, 'build-images') || contains(github.event.pull_request.labels.*.name, 'execute-workflow') || (github.event_name == 'workflow_dispatch' && inputs.run_build_images)) }}
    uses: ./.github/workflows/build-images.yaml
    concurrency:
      group: bi-${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.event.pull_request.number || github.ref }}-${{ needs.set-variables.outputs.mode }}
      cancel-in-progress: true
    secrets:
      GCP_ARTIFACT_REGISTRY_PATH: ${{ secrets.GCP_ARTIFACT_REGISTRY_PATH }}
    with:
      debug_enabled: ${{ needs.set-variables.outputs.debug }}
      mode: ${{ needs.set-variables.outputs.mode }}

  build-nix-images:
    runs-on: ubuntu-latest
    needs: [set-variables]
    if: ${{ needs.set-variables.outputs.skip_ci != 'true' && ( contains(github.event.pull_request.labels.*.name, 'build-images') || contains(github.event.pull_request.labels.*.name, 'execute-workflow') || (github.event_name == 'workflow_dispatch' && inputs.run_build_images) || (github.event_name == 'workflow_dispatch' && inputs.run_execute_workflow) ) }}
    concurrency:
      group: bni-${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.event.pull_request.number || github.ref }}-${{ needs.set-variables.outputs.mode }}
      cancel-in-progress: true
    permissions:
      contents: read
      packages: write
    environment:
      name: release
      url: https://ghcr.io/pinellolab/pyrovelocitydev
    steps:
      - uses: actions/checkout@v4
        with:
          ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.ref || github.ref }}
      - uses: docker/setup-qemu-action@v3
        with:
          platforms: arm64
      - uses: DeterminateSystems/nix-installer-action@v9
        with:
          extra-conf: |
            extra-platforms = aarch64-linux
      - uses: DeterminateSystems/magic-nix-cache-action@v2
      - uses: rlespinasse/github-slug-action@v4
        with:
          prefix: CI_
      - name: Set git variables
        run: |
          echo "GIT_REPO_NAME=$CI_GITHUB_REPOSITORY_NAME_PART" >> $GITHUB_ENV
          echo "GIT_REF=$CI_GITHUB_REF_NAME" >> $GITHUB_ENV
          if [ "${{ github.event_name }}" == "pull_request" ]; then
            echo "GIT_SHA=$CI_GITHUB_EVENT_PULL_REQUEST_HEAD_SHA" >> $GITHUB_ENV
            echo "GIT_SHA_SHORT=$CI_GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT" >> $GITHUB_ENV
          else
            echo "GIT_SHA=$CI_GITHUB_SHA" >> $GITHUB_ENV
            echo "GIT_SHA_SHORT=$CI_GITHUB_SHA_SHORT" >> $GITHUB_ENV
          fi
      - name: Set nix variables
        run: |
          echo "NIX_IMAGE_SYSTEMS=x86_64-linux" >> $GITHUB_ENV
      - name: Build nix images
        run: |
          echo "Using Git Repository Name: $GIT_REPO_NAME"
          echo "Using Git Reference: $GIT_REF"
          echo "Using Git SHA: $GIT_SHA"
          echo "Using Git SHA Short: $GIT_SHA_SHORT"
          nix run .#devcontainerManifest --impure --accept-flake-config
        env:
          GH_TOKEN: ${{ github.token }}

  execute-workflow:
    needs: [config-workflows, build-nix-images, set-variables]
    if: ${{ needs.set-variables.outputs.skip_ci != 'true' && (contains(github.event.pull_request.labels.*.name, 'execute-workflow') || (github.event_name == 'workflow_dispatch' && inputs.run_execute_workflow)) }}
    runs-on: ubuntu-latest
    concurrency:
      group: ef-${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.event.pull_request.number || github.ref }}-${{ needs.set-variables.outputs.mode }}
      cancel-in-progress: true
    strategy:
      matrix:
        python_version: ["3.10"]
    env:
      FLYTECTL_CONFIG: ${{ secrets.FLYTECTL_CONFIG }}
      FLYTE_OAUTH_CLIENT_SECRET: ${{ secrets.FLYTE_OAUTH_CLIENT_SECRET }}
      WORKFLOW_IMAGE: ${{ vars.WORKFLOW_IMAGE }}

    steps:
      - name: Checkout code
        uses: actions/checkout@v4
        with:
          ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.ref || github.ref }}

      - name: Setup environment
        uses: ./.github/actions/setup_environment
        with:
          python_version: ${{ matrix.python_version }}
          debug_enabled: ${{ needs.set-variables.outputs.debug }}

      - name: Download Flyte config
        uses: actions/download-artifact@v3
        with:
          name: flyte-config
          path: .flyte/

      - name: Setup tmate debug session
        if: ${{ inputs.debug_enabled == 'true' }}
        uses: mxschmitt/action-tmate@v3

      - name: Execute workflow
        id: execute
        run: |
          make run_${{ needs.set-variables.outputs.mode }}

      - name: Create config tarball
        id: save-hydra-outputs
        run: |
          TAR_FILENAME="hydra_outputs_${GITHUB_SHA_SHORT}.tar.gz"
          tar -czf $TAR_FILENAME ./outputs/
          tar -tzf $TAR_FILENAME
          echo "HYDRA_OUTPUTS_TAR=$TAR_FILENAME" >> $GITHUB_ENV

      - name: Upload config artifact
        uses: actions/upload-artifact@v3
        with:
          name: hydra-outputs
          path: ${{ env.HYDRA_OUTPUTS_TAR }}

  release:
    runs-on: ubuntu-latest
    needs: [test, test-nix]
    if: ${{ github.repository_owner == 'pinellolab' && github.event_name == 'push' }}
    environment:
      name: release
      url: https://github.com/pinellolab/pyrovelocity/releases/tag/${{ steps.semanticrelease.outputs.git-tag }}
    permissions:
      contents: write
    outputs:
      version: ${{ steps.semanticrelease.outputs.version }}
      released: ${{ steps.semanticrelease.outputs.released }}
      git-head: ${{ steps.semanticrelease.outputs.git-head }}
    steps:
      - uses: actions/checkout@v4
        with:
          ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.ref || github.ref }}
      - uses: DeterminateSystems/nix-installer-action@v9
      - uses: DeterminateSystems/magic-nix-cache-action@v2
      - run: nix profile install .#releaseEnv
      - uses: cihelper/action-semanticrelease-poetry@v1
        id: semanticrelease
      - uses: actions/upload-artifact@v3
        if: ${{ steps.semanticrelease.outputs.released == 'true' }}
        with:
          name: poetry-build
          path: ./dist

  publish-release-images:
    runs-on: ubuntu-latest
    needs: [set-variables, release]
    if: ${{ needs.release.outputs.released == 'true' }}
    concurrency:
      group: bri-${{ github.workflow }}-${{ github.ref }}-${{ github.sha }}
      cancel-in-progress: true
    permissions:
      contents: read
      packages: write
    environment:
      name: release
      url: https://ghcr.io/pinellolab/pyrovelocitydev
    steps:
      - uses: actions/checkout@v4
        with:
          ref: ${{ needs.release.outputs.git-head }}
      - uses: docker/setup-qemu-action@v3
        with:
          platforms: arm64
      - uses: DeterminateSystems/nix-installer-action@v9
        with:
          extra-conf: |
            extra-platforms = aarch64-linux
      - uses: DeterminateSystems/magic-nix-cache-action@v2
      - uses: rlespinasse/github-slug-action@v4
        with:
          prefix: CI_
      - name: Set git variables
        run: |
          echo "GIT_REPO_NAME=$CI_GITHUB_REPOSITORY_NAME_PART" >> $GITHUB_ENV
          echo "GIT_REF=$CI_GITHUB_REF_NAME" >> $GITHUB_ENV
          if [ "${{ github.event_name }}" == "pull_request" ]; then
            echo "GIT_SHA=$CI_GITHUB_EVENT_PULL_REQUEST_HEAD_SHA" >> $GITHUB_ENV
            echo "GIT_SHA_SHORT=$CI_GITHUB_EVENT_PULL_REQUEST_HEAD_SHA_SHORT" >> $GITHUB_ENV
          else
            echo "GIT_SHA=$CI_GITHUB_SHA" >> $GITHUB_ENV
            echo "GIT_SHA_SHORT=$CI_GITHUB_SHA_SHORT" >> $GITHUB_ENV
          fi
      - name: Set nix variables
        run: |
          echo "NIX_IMAGE_SYSTEMS=x86_64-linux" >> $GITHUB_ENV
      - name: Build nix images
        run: |
          echo "Using Git Repository Name: $GIT_REPO_NAME"
          echo "Using Git Reference: $GIT_REF"
          echo "Using Git SHA: $GIT_SHA"
          echo "Using Git SHA Short: $GIT_SHA_SHORT"
          nix run .#devcontainerManifest --impure --accept-flake-config
        env:
          GH_TOKEN: ${{ github.token }}
          VERSION: ${{ needs.release.outputs.version }}

  publish-pypi:
    runs-on: ubuntu-latest
    needs: release
    if: ${{ needs.release.outputs.released == 'true' }}
    permissions:
      id-token: write
    environment:
      name: release
      url: https://pypi.org/project/pyrovelocity/${{needs.release.outputs.version}}/
    steps:
      - uses: actions/download-artifact@v3
        with:
          name: poetry-build
          path: ./dist
      - uses: pypa/gh-action-pypi-publish@release/v1

  test-docs-build:
    runs-on: ubuntu-latest
    needs: set-variables
    if: ${{ needs.set-variables.outputs.skip_ci != 'true' && needs.set-variables.outputs.skip_tests != 'true' }}
    strategy:
      matrix:
        python_version: ["3.10"]
    permissions:
      contents: read
      pages: read
    steps:
      - uses: actions/checkout@v4
        with:
          ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.ref || github.ref }}
      - name: Setup environment
        uses: ./.github/actions/setup_environment
        with:
          python_version: ${{ matrix.python_version }}
          debug_enabled: ${{ needs.set-variables.outputs.debug }}
      - name: Build
        run: make docs-build

  build-release-docs:
    runs-on: ubuntu-latest
    needs: [set-variables, release]
    if: ${{ needs.release.outputs.released == 'true' }}
    strategy:
      matrix:
        python_version: ["3.10"]
    permissions:
      contents: read
      pages: read
    environment: github-pages
    steps:
      - uses: actions/checkout@v4
        with:
          ref: ${{ needs.release.outputs.git-head }}
      - name: Setup environment
        uses: ./.github/actions/setup_environment
        with:
          python_version: ${{ matrix.python_version }}
          debug_enabled: ${{ needs.set-variables.outputs.debug }}
      - name: Build
        run: make docs-build
      - name: Upload artifact
        uses: actions/upload-pages-artifact@v2
        with:
          path: ./site

  publish-docs:
    runs-on: ubuntu-latest
    needs: [build-release-docs, release]
    if: ${{ needs.release.outputs.released == 'true' }}
    permissions:
      id-token: write
      pages: write
    environment:
      name: github-pages
      url: ${{ steps.deployment.outputs.page_url }}
    steps:
      - name: Deploy to GitHub Pages
        id: deployment
        uses: actions/deploy-pages@v2