ibis-backends-cloud.yml

name: Cloud Backends

on:
  push:
    # Skip the backend suite if all changes are in the docs directory
    paths-ignore:
      - "docs/**"
      - "**/*.md"
      - "**/*.qmd"
      - "codecov.yml"
      - ".envrc"
    branches:
      - main
  pull_request_target:
    types:
      - labeled

env:
  FORCE_COLOR: "1"
  SQLALCHEMY_WARN_20: "1"
  HYPOTHESIS_PROFILE: "ci"

jobs:
  test_backends:
    name: ${{ matrix.backend.title }} python-${{ matrix.python-version }}-${{ matrix.os }}
    # only a single bigquery or snowflake run at a time, otherwise test data is
    # clobbered by concurrent runs
    concurrency:
      group: ${{ matrix.backend.title }}-${{ matrix.python-version }}-${{ github.event.label.name || 'ci-run-cloud' }}
      cancel-in-progress: false

    runs-on: ${{ matrix.os }}
    if: github.event_name == 'push' || github.event.label.name == 'ci-run-cloud'
    strategy:
      fail-fast: false
      matrix:
        os:
          - ubuntu-latest
        python-version:
          - "3.10"
          - "3.13"
        backend:
          - name: snowflake
            title: Snowflake
            extras:
              - --extra snowflake
          - name: databricks
            title: Databricks
            extras:
              - --extra databricks
          - name: athena
            title: Amazon Athena
            extras:
              - --extra athena
        include:
          - os: ubuntu-latest
            python-version: "3.10"
            backend:
              name: bigquery
              title: BigQuery
              extras:
                - --extra bigquery
          - os: ubuntu-latest
            python-version: "3.13"
            backend:
              name: bigquery
              title: BigQuery
              extras:
                - --extra bigquery
                - --extra geospatial
          - os: ubuntu-latest
            python-version: "3.10"
            backend:
              name: snowflake
              title: Snowflake + Snowpark
              key: snowpark
              extras:
                - --extra snowflake
          - os: ubuntu-latest
            python-version: "3.11"
            backend:
              name: snowflake
              title: Snowflake + Snowpark
              key: snowpark
              extras:
                - --extra snowflake
          - os: windows-latest
            python-version: "3.12"
            backend:
              name: snowflake
              title: Snowflake
              extras:
                - --extra snowflake
    # this allows extractions/setup-just to list releases for `just` at a higher
    # rate limit while restricting GITHUB_TOKEN permissions elsewhere
    permissions:
      contents: "read"
      # required for workload identity federation
      id-token: "write"

    steps:
      - name: checkout
        uses: actions/checkout@v4
        if: github.event.label.name != 'ci-run-cloud'

      - name: checkout
        if: github.event.label.name == 'ci-run-cloud'
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
          ref: ${{ github.event.pull_request.head.sha }}

      - uses: actions/create-github-app-token@v1.11.6
        id: generate_token
        with:
          app-id: ${{ secrets.DOCS_BOT_APP_ID }}
          private-key: ${{ secrets.DOCS_BOT_APP_PRIVATE_KEY }}

      - name: reset cloud ci run label
        uses: actions-ecosystem/action-remove-labels@v1
        if: github.event.label.name == 'ci-run-cloud'
        with:
          labels: ci-run-cloud
          github_token: ${{ steps.generate_token.outputs.token }}

      - name: install python
        uses: actions/setup-python@v5
        id: install_python
        with:
          python-version: ${{ matrix.python-version }}

      - name: install uv
        uses: astral-sh/setup-uv@v5.3.1

      - name: install additional deps
        if: matrix.backend.key == 'snowpark'
        run: uv add --no-sync --optional snowflake snowflake-snowpark-python

      - uses: extractions/setup-just@v2
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

      - name: download backend data
        run: just download-data

      - uses: google-github-actions/auth@v2
        if: matrix.backend.name == 'bigquery'
        with:
          project_id: "ibis-gbq"
          workload_identity_provider: "${{ vars.WIF_PROVIDER_NAME }}"

      - name: setup databricks credentials
        if: matrix.backend.name == 'databricks'
        shell: bash
        run: |
          {
            echo "DATABRICKS_HTTP_PATH=${DATABRICKS_HTTP_PATH}"
            echo "DATABRICKS_SERVER_HOSTNAME=${DATABRICKS_SERVER_HOSTNAME}"
            echo "DATABRICKS_TOKEN=${DATABRICKS_TOKEN}"
          } >> "$GITHUB_ENV"
        env:
          DATABRICKS_HTTP_PATH: ${{ secrets.DATABRICKS_HTTP_PATH }}
          DATABRICKS_SERVER_HOSTNAME: ${{ secrets.DATABRICKS_SERVER_HOSTNAME }}
          DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }}

      - name: setup snowflake credentials
        if: matrix.backend.name == 'snowflake'
        shell: bash
        run: |
          pyversion="${{ matrix.python-version }}"
          {
            echo "SNOWFLAKE_USER=${SNOWFLAKE_USER}"
            echo "SNOWFLAKE_PASSWORD=${SNOWFLAKE_PASSWORD}"
            echo "SNOWFLAKE_ACCOUNT=${SNOWFLAKE_ACCOUNT}"
            echo "SNOWFLAKE_DATABASE=${SNOWFLAKE_DATABASE}"
            echo "SNOWFLAKE_SCHEMA=${SNOWFLAKE_SCHEMA}_python${pyversion//./}_${{ matrix.backend.key }}"
            echo "SNOWFLAKE_WAREHOUSE=${SNOWFLAKE_WAREHOUSE}"
            if ${{ matrix.backend.key == 'snowpark' }}; then
              echo "SNOWFLAKE_SNOWPARK=1" >> "$GITHUB_ENV"
            fi
          } >> "$GITHUB_ENV"
        env:
          SNOWFLAKE_USER: ${{ secrets.SNOWFLAKE_USER }}
          SNOWFLAKE_PASSWORD: ${{ secrets.SNOWFLAKE_PASSWORD }}
          SNOWFLAKE_ACCOUNT: ${{ secrets.SNOWFLAKE_ACCOUNT }}
          SNOWFLAKE_DATABASE: ${{ secrets.SNOWFLAKE_DATABASE }}
          SNOWFLAKE_SCHEMA: ${{ secrets.SNOWFLAKE_SCHEMA }}
          SNOWFLAKE_WAREHOUSE: ${{ secrets.SNOWFLAKE_WAREHOUSE }}

      - name: setup aws credentials
        if: matrix.backend.name == 'athena'
        uses: aws-actions/configure-aws-credentials@v4
        with:
          aws-region: us-east-2
          role-to-assume: arn:aws:iam::070284473168:role/ibis-project-athena

      - name: "run parallel tests: ${{ matrix.backend.name }}"
        timeout-minutes: 60
        run: just ci-check "${{ join(matrix.backend.extras, ' ') }} --extra examples" -m ${{ matrix.backend.name }} --numprocesses auto --dist loadgroup

      - name: upload code coverage
        if: success()
        continue-on-error: true
        uses: codecov/codecov-action@v5
        with:
          flags: backend,${{ matrix.backend.name }},${{ runner.os }},python-${{ steps.install_python.outputs.python-version }}
          token: ${{ secrets.CODECOV_TOKEN }}