ci.yaml

name: CI

on:
  push:
    branches: [ master ]
    paths:
      - '**.py'
      - '!airflow_dbt_python/__version__.py'
    tags:
      - "v*"
  pull_request:
    branches:
      - master
    paths:
      - '**.py'
      - '!airflow_dbt_python/__version__.py'

jobs:
  test:
    name: Test on Python ${{ matrix.python-version }} and Airflow ${{ matrix.airflow-version }} and dbt ${{ matrix.dbt-version }}
    strategy:
      fail-fast: false
      matrix:
        python-version:
          - '3.12'
          - '3.11'
          - '3.10'
          - '3.9'
          - '3.8'
        airflow-version:
          - '2.9.2'
          - '2.8.4'
          - '2.7.3'
        dbt-version:
          - 1.8
          - 1.7
        exclude:
          # Incompatible combinations
          - python-version: 3.12
            airflow-version: '2.8.4'

          - python-version: 3.12
            airflow-version: '2.7.3'

    runs-on: ubuntu-latest
    steps:
      - name: Harden Runner
        uses: step-security/harden-runner@v2.8.1
        with:
          egress-policy: block
          allowed-endpoints: >
            api.github.com:443
            files.pythonhosted.org:443
            hub.getdbt.com:443
            github.com:80
            github.com:443
            gitlab.com:80
            gitlab.com:443
            objects.githubusercontent.com:443
            raw.githubusercontent.com:443
            pypi.org:443
            archive.ubuntu.com:80
            azure.archive.ubuntu.com:80
            esm.ubuntu.com:443
            motd.ubuntu.com:443
            packages.microsoft.com:80
            ppa.launchpadcontent.net:443
            security.ubuntu.com:80

      - run: |
          sudo apt-get update
          sudo apt-get install --yes --no-install-recommends postgresql

      - uses: actions/checkout@v4.1.7
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5.1.0
        with:
          python-version: ${{ matrix.python-version }}

      - name: Install Poetry
        uses: abatilo/actions-poetry@v2.3.0
        with:
          poetry-version: 1.8.3

      - name: Install Airflow & dbt
        run: |
          poetry env use ${{ matrix.python-version }}
          poetry add "apache-airflow==${{ matrix.airflow-version }}" \
                     "dbt-core~=${{ matrix.dbt-version }}.0" \
                     "dbt-postgres~=${{ matrix.dbt-version }}.0" \
                 --python ${{ matrix.python-version }}
          poetry install -E postgres --with dev
          poetry run airflow db migrate
          poetry run airflow connections create-default-connections

      - name: Linting with ruff
        run: poetry run ruff check .

      - name: Static type checking with mypy
        # We only run mypy on the latest supported versions of Airflow & dbt,
        # so it is currently impossible to write conditions for that depend on package versions.
        if: matrix.python-version == '3.12' && matrix.airflow-version == '2.9.2' && matrix.dbt-version == '1.8'
        run: poetry run mypy .

      - name: Code formatting with black
        run: poetry run black --check .

      - name: Set COVERAGE_FILE in environment
        run: echo "COVERAGE_FILE=.coverage.${{ matrix.python-version }}.${{ matrix.airflow-version }}" >> $GITHUB_ENV

      - name: Run tests with pytest
        run: poetry run coverage run -m pytest -v tests/ airflow_dbt_python/utils/
        env:
          GITLAB_READ_TOKEN: ${{ secrets.GITLAB_READ_TOKEN }}
          GITLAB_USERNAME: ${{ secrets.GITLAB_USERNAME }}
          GITHUB_READ_TOKEN: ${{ secrets.GH_READ_TOKEN }}
          GITHUB_USERNAME: ${{ secrets.GH_USERNAME }}

      - name: Upload code coverage
        uses: actions/upload-artifact@v3
        with:
          name: coverage-data
          path: ".coverage.*"
          if-no-files-found: ignore

  coverage:
    name: Combine and check coverage
    runs-on: ubuntu-latest
    needs: test

    steps:
      - name: Harden Runner
        uses: step-security/harden-runner@v2.8.1
        with:
          egress-policy: block
          allowed-endpoints: >
            files.pythonhosted.org:443
            github.com:443
            api.github.com:443
            pypi.org:443

      - uses: actions/checkout@v4.1.7
      - uses: actions/setup-python@v5.1.0
        with:
          python-version: '3.12'

      - name: Install Poetry
        uses: abatilo/actions-poetry@v2.3.0
        with:
          poetry-version: 1.8.3

      - name: Install airflow-dbt-python with Poetry
        run: poetry install --with dev -E airflow-providers

      - name: Download coverage data.
        uses: actions/download-artifact@v3
        with:
          name: coverage-data

      - name: Combine coverage & fail if it's <95%.
        run: |
          poetry run coverage combine
          poetry run coverage html --skip-covered --skip-empty
          poetry run coverage json
          # Save in env variable for badge
          export TOTAL=$(python -c "import json;print(json.load(open('coverage.json'))['totals']['percent_covered_display'])")
          echo "total=$TOTAL" >> $GITHUB_ENV
          # Report and write to summary.
          poetry run coverage report | sed 's/^/    /' >> $GITHUB_STEP_SUMMARY
          # Report again and fail if under 100%.
          poetry run coverage report --fail-under=95

      - name: Upload HTML report if check failed.
        uses: actions/upload-artifact@v3
        if: ${{ failure() }}
        with:
          name: html-report
          path: htmlcov

      - name: "Make coverage badge"
        uses: schneegans/dynamic-badges-action@v1.7.0
        if: github.event_name != 'pull_request'
        with:
          auth: ${{ secrets.GIST_TOKEN }}
          gistID: 81ef37701aa088d18db8a58ce07c79c7 # pragma: allowlist secret
          filename: covbadge.json
          label: Coverage
          message: ${{ env.total }}%
          minColorRange: 50
          maxColorRange: 90
          valColorRange: ${{ env.total }}