tests.yml

# If you change this name also do it in tests_skipper.yml and ci_metrics.yml
name: Tests

on:
  workflow_dispatch: # Activate this workflow manually
  push:
    branches:
      - main
      # release branches have the form v1.9.x
      - "v[0-9].*[0-9].x"
  pull_request:
    types:
      - opened
      - reopened
      - synchronize
      - ready_for_review
    paths:
      - "**.py"
      - "pyproject.toml"
      - "!.github/**/*.py"
      - "!rest_api/**/*.py"
      - "!docs/**/*.py"

env:
  OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
  COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }}
  PYTHON_VERSION: "3.8"

jobs:
  black:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - uses: actions/setup-python@v4
        with:
          python-version: ${{ env.PYTHON_VERSION }}

      - name: Install Black
        run: |
          pip install --upgrade pip
          pip install .[formatting]

      - name: Check status
        run: |
          if ! black . --check; then
            git status
            echo "###################################################################################################"
            echo "# "
            echo "# CHECK FAILED! Black found issues with your code formatting."
            echo "# "
            echo "# Either:"
            echo "# 1. Run Black locally before committing:"
            echo "# "
            echo "#     pip install .[formatting]"
            echo "#     black ."
            echo "# "
            echo "# 2. Install the pre-commit hook:"
            echo "# "
            echo "#     pre-commit install --hook-type pre-push"
            echo "# "
            echo "# 3. See https://github.com/deepset-ai/haystack/blob/main/CONTRIBUTING.md for help."
            echo "# "
            echo "# If you have further problems, please open an issue: https://github.com/deepset-ai/haystack/issues"
            echo "# "
            echo "##################################################################################################"
            exit 1
          fi

      - name: Calculate alert data
        id: calculator
        shell: bash
        if: (success() || failure()) && github.ref_name == 'main'
        run: |
          if [ "${{ job.status }}" = "success" ]; then
            echo "alert_type=success" >> "$GITHUB_OUTPUT";
          else
            echo "alert_type=error" >> "$GITHUB_OUTPUT";
          fi

      - name: Send event to Datadog
        if: (success() || failure()) && github.ref_name == 'main'
        uses: masci/datadog@v1
        with:
          api-key: ${{ secrets.CORE_DATADOG_API_KEY }}
          api-url: https://api.datadoghq.eu
          events: |
            - title: "${{ github.workflow }} workflow"
              text: "Job ${{ github.job }} in branch ${{ github.ref_name }}"
              alert_type: "${{ steps.calculator.outputs.alert_type }}"
              source_type_name: "Github"
              host: ${{ github.repository_owner }}
              tags:
                - "project:${{ github.repository }}"
                - "job:${{ github.job }}"
                - "run_id:${{ github.run_id }}"
                - "workflow:${{ github.workflow }}"
                - "branch:${{ github.ref_name }}"
                - "url:https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"

  unit-tests:
    name: Unit / ${{ matrix.topic }} / ${{ matrix.os }} / ${{ matrix.dependencies }}
    needs: black
    strategy:
      fail-fast: false
      matrix:
        os:
          - ubuntu-latest
          - windows-latest
        topic:
          - document_stores
          - nodes
          - agents
          - cli
          - prompt
          - pipelines
          - utils
          - others
          - modeling
        dependencies:
          - all,dev
        include:
          - topic: document_stores
            os: ubuntu-latest
            dependencies: elasticsearch8,faiss,weaviate,pinecone,opensearch,mongodb,inference,crawler,preprocessing,file-conversion,pdf,ocr,metrics,dev
          - topic: document_stores
            os: windows-latest
            dependencies: elasticsearch8,faiss,weaviate,pinecone,opensearch,mongodb,inference,crawler,preprocessing,file-conversion,pdf,ocr,metrics,dev
    runs-on: ${{ matrix.os }}
    steps:
      - uses: actions/checkout@v4

      - uses: actions/setup-python@v4
        with:
          python-version: ${{ env.PYTHON_VERSION }}

      - name: Install Haystack
        run: pip install .[${{ matrix.dependencies }}]

      - name: Run
        run: pytest --cov-report xml:coverage.xml --cov="haystack" -m "unit" test/${{ matrix.topic }}

      - name: Calculate alert data
        id: calculator
        shell: bash
        if: (success() || failure()) && github.ref_name == 'main'
        run: |
          if [ "${{ job.status }}" = "success" ]; then
            echo "alert_type=success" >> "$GITHUB_OUTPUT";
          else
            echo "alert_type=error" >> "$GITHUB_OUTPUT";
          fi

      - name: Send event to Datadog
        if: (success() || failure()) && github.ref_name == 'main'
        uses: masci/datadog@v1
        with:
          api-key: ${{ secrets.CORE_DATADOG_API_KEY }}
          api-url: https://api.datadoghq.eu
          events: |
            - title: "${{ github.workflow }} workflow"
              text: "Job ${{ github.job }} in branch ${{ github.ref_name }}"
              alert_type: "${{ steps.calculator.outputs.alert_type }}"
              source_type_name: "Github"
              host: ${{ github.repository_owner }}
              tags:
                - "project:${{ github.repository }}"
                - "job:${{ github.job }}"
                - "run_id:${{ github.run_id }}"
                - "workflow:${{ github.workflow }}"
                - "branch:${{ github.ref_name }}"
                - "url:https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"

  integration-tests-elasticsearch7:
    name: Integration / Elasticsearch7 / ${{ matrix.os }}
    needs:
      - unit-tests
    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu-latest]
    runs-on: ${{ matrix.os }}
    services:
      elasticsearch:
        image: elasticsearch:7.17.6
        env:
          discovery.type: "single-node"
          ES_JAVA_OPTS: "-Xms128m -Xmx256m"
        ports:
          - 9200:9200
    steps:
      - uses: actions/checkout@v4

      - uses: actions/setup-python@v4
        with:
          python-version: ${{ env.PYTHON_VERSION }}

      - name: Install Haystack
        run: pip install .[elasticsearch,dev,preprocessing,inference]

      - name: Run tests
        run: |
          pytest --maxfail=5 -m "document_store and integration" test/document_stores/test_elasticsearch.py

      - name: Calculate alert data
        id: calculator
        shell: bash
        if: (success() || failure()) && github.ref_name == 'main'
        run: |
          if [ "${{ job.status }}" = "success" ]; then
            echo "alert_type=success" >> "$GITHUB_OUTPUT";
          else
            echo "alert_type=error" >> "$GITHUB_OUTPUT";
          fi

      - name: Send event to Datadog
        if: (success() || failure()) && github.ref_name == 'main'
        uses: masci/datadog@v1
        with:
          api-key: ${{ secrets.CORE_DATADOG_API_KEY }}
          api-url: https://api.datadoghq.eu
          events: |
            - title: "${{ github.workflow }} workflow"
              text: "Job ${{ github.job }} in branch ${{ github.ref_name }}"
              alert_type: "${{ steps.calculator.outputs.alert_type }}"
              source_type_name: "Github"
              host: ${{ github.repository_owner }}
              tags:
                - "project:${{ github.repository }}"
                - "job:${{ github.job }}"
                - "run_id:${{ github.run_id }}"
                - "workflow:${{ github.workflow }}"
                - "branch:${{ github.ref_name }}"
                - "url:https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"

  integration-tests-elasticsearch8:
    name: Integration / Elasticsearch8 / ${{ matrix.os }}
    needs:
      - unit-tests
    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu-latest]
    runs-on: ${{ matrix.os }}
    services:
      elasticsearch:
        image: elasticsearch:8.10.2
        env:
          discovery.type: "single-node"
          xpack.security.enabled: "false"
          ES_JAVA_OPTS: "-Xms128m -Xmx256m"
        ports:
          - 9200:9200
    steps:
      - uses: actions/checkout@v4

      - uses: actions/setup-python@v4
        with:
          python-version: ${{ env.PYTHON_VERSION }}

      - name: Install Haystack
        run: pip install .[elasticsearch8,dev,preprocessing,inference]

      - name: Make elasticsearch comfortable with a disk almost full
        run: |
          curl -X PUT "localhost:9200/_cluster/settings?pretty" -H 'Content-Type: application/json' -d'
          {
            "persistent": {
              "cluster.routing.allocation.disk.watermark.low": "90%",
              "cluster.routing.allocation.disk.watermark.low.max_headroom": "100GB",
              "cluster.routing.allocation.disk.watermark.high": "95%",
              "cluster.routing.allocation.disk.watermark.high.max_headroom": "20GB",
              "cluster.routing.allocation.disk.watermark.flood_stage": "97%",
              "cluster.routing.allocation.disk.watermark.flood_stage.max_headroom": "5GB",
              "cluster.routing.allocation.disk.watermark.flood_stage.frozen": "97%",
              "cluster.routing.allocation.disk.watermark.flood_stage.frozen.max_headroom": "5GB"
            }
          }
          '
          curl -X PUT "localhost:9200/*/_settings?expand_wildcards=all&pretty" -H 'Content-Type: application/json' -d'
          {
            "index.blocks.read_only_allow_delete": null
          }
          '

      - name: Run tests
        run: |
          pytest -x -m"document_store and integration" test/document_stores/test_elasticsearch.py

      - name: logs
        if: failure()
        run: |
          docker logs "${{ job.services.elasticsearch.id }}"

      - name: Calculate alert data
        id: calculator
        shell: bash
        if: (success() || failure()) && github.ref_name == 'main'
        run: |
          if [ "${{ job.status }}" = "success" ]; then
            echo "alert_type=success" >> "$GITHUB_OUTPUT";
          else
            echo "alert_type=error" >> "$GITHUB_OUTPUT";
          fi

      - name: Send event to Datadog
        if: (success() || failure()) && github.ref_name == 'main'
        uses: masci/datadog@v1
        with:
          api-key: ${{ secrets.CORE_DATADOG_API_KEY }}
          api-url: https://api.datadoghq.eu
          events: |
            - title: "${{ github.workflow }} workflow"
              text: "Job ${{ github.job }} in branch ${{ github.ref_name }}"
              alert_type: "${{ steps.calculator.outputs.alert_type }}"
              source_type_name: "Github"
              host: ${{ github.repository_owner }}
              tags:
                - "project:${{ github.repository }}"
                - "job:${{ github.job }}"
                - "run_id:${{ github.run_id }}"
                - "workflow:${{ github.workflow }}"
                - "branch:${{ github.ref_name }}"
                - "url:https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"

  integration-tests-sql:
    name: Integration / SQL / ${{ matrix.os }}
    needs:
      - unit-tests
    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu-latest, windows-latest]
    runs-on: ${{ matrix.os }}
    steps:
      - uses: actions/checkout@v4

      - uses: actions/setup-python@v4
        with:
          python-version: ${{ env.PYTHON_VERSION }}

      - name: Install Haystack
        run: pip install .[dev,sql,preprocessing]

      - name: Run tests
        run: |
          pytest --maxfail=5 -m "document_store and integration" test/document_stores/test_sql.py

      - name: Calculate alert data
        id: calculator
        shell: bash
        if: (success() || failure()) && github.ref_name == 'main'
        run: |
          if [ "${{ job.status }}" = "success" ]; then
            echo "alert_type=success" >> "$GITHUB_OUTPUT";
          else
            echo "alert_type=error" >> "$GITHUB_OUTPUT";
          fi

      - name: Send event to Datadog
        if: (success() || failure()) && github.ref_name == 'main'
        uses: masci/datadog@v1
        with:
          api-key: ${{ secrets.CORE_DATADOG_API_KEY }}
          api-url: https://api.datadoghq.eu
          events: |
            - title: "${{ github.workflow }} workflow"
              text: "Job ${{ github.job }} in branch ${{ github.ref_name }}"
              alert_type: "${{ steps.calculator.outputs.alert_type }}"
              source_type_name: "Github"
              host: ${{ github.repository_owner }}
              tags:
                - "project:${{ github.repository }}"
                - "job:${{ github.job }}"
                - "run_id:${{ github.run_id }}"
                - "workflow:${{ github.workflow }}"
                - "branch:${{ github.ref_name }}"
                - "url:https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"

  integration-tests-opensearch:
    name: Integration / Opensearch / ${{ matrix.os }}
    needs:
      - unit-tests
    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu-latest]
    runs-on: ${{ matrix.os }}
    services:
      opensearch:
        image: opensearchproject/opensearch:1.3.5
        env:
          discovery.type: "single-node"
          ES_JAVA_OPTS: "-Xms128m -Xmx256m"
        ports:
          - 9200:9200
    steps:
      - uses: actions/checkout@v4

      - uses: actions/setup-python@v4
        with:
          python-version: ${{ env.PYTHON_VERSION }}

      - name: Install Haystack
        run: pip install .[dev,opensearch,preprocessing]

      - name: Run tests
        run: |
          pytest --maxfail=5 -m "document_store and integration" test/document_stores/test_opensearch.py

      - name: Calculate alert data
        id: calculator
        shell: bash
        if: (success() || failure()) && github.ref_name == 'main'
        run: |
          if [ "${{ job.status }}" = "success" ]; then
            echo "alert_type=success" >> "$GITHUB_OUTPUT";
          else
            echo "alert_type=error" >> "$GITHUB_OUTPUT";
          fi

      - name: Send event to Datadog
        if: (success() || failure()) && github.ref_name == 'main'
        uses: masci/datadog@v1
        with:
          api-key: ${{ secrets.CORE_DATADOG_API_KEY }}
          api-url: https://api.datadoghq.eu
          events: |
            - title: "${{ github.workflow }} workflow"
              text: "Job ${{ github.job }} in branch ${{ github.ref_name }}"
              alert_type: "${{ steps.calculator.outputs.alert_type }}"
              source_type_name: "Github"
              host: ${{ github.repository_owner }}
              tags:
                - "project:${{ github.repository }}"
                - "job:${{ github.job }}"
                - "run_id:${{ github.run_id }}"
                - "workflow:${{ github.workflow }}"
                - "branch:${{ github.ref_name }}"
                - "url:https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"

  integration-tests-dc:
    name: Integration / dC / ${{ matrix.os }}
    needs:
      - unit-tests
    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu-latest, windows-latest]
    runs-on: ${{ matrix.os }}
    steps:
      - uses: actions/checkout@v4

      - uses: actions/setup-python@v4
        with:
          python-version: ${{ env.PYTHON_VERSION }}

      - name: Install Haystack
        run: pip install .[dev,preprocessing]

      - name: Run tests
        run: |
          pytest --maxfail=5 -m "document_store and integration" test/document_stores/test_deepsetcloud.py

      - name: Calculate alert data
        id: calculator
        shell: bash
        if: (success() || failure()) && github.ref_name == 'main'
        run: |
          if [ "${{ job.status }}" = "success" ]; then
            echo "alert_type=success" >> "$GITHUB_OUTPUT";
          else
            echo "alert_type=error" >> "$GITHUB_OUTPUT";
          fi

      - name: Send event to Datadog
        if: (success() || failure()) && github.ref_name == 'main'
        uses: masci/datadog@v1
        with:
          api-key: ${{ secrets.CORE_DATADOG_API_KEY }}
          api-url: https://api.datadoghq.eu
          events: |
            - title: "${{ github.workflow }} workflow"
              text: "Job ${{ github.job }} in branch ${{ github.ref_name }}"
              alert_type: "${{ steps.calculator.outputs.alert_type }}"
              source_type_name: "Github"
              host: ${{ github.repository_owner }}
              tags:
                - "project:${{ github.repository }}"
                - "job:${{ github.job }}"
                - "run_id:${{ github.run_id }}"
                - "workflow:${{ github.workflow }}"
                - "branch:${{ github.ref_name }}"
                - "url:https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"

  integration-tests-faiss:
    name: Integration / faiss / ${{ matrix.os }}
    needs:
      - unit-tests
    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu-latest, windows-latest]
    runs-on: ${{ matrix.os }}
    steps:
      - uses: actions/checkout@v4

      - uses: actions/setup-python@v4
        with:
          python-version: ${{ env.PYTHON_VERSION }}

      - name: Install Haystack
        run: pip install .[faiss,dev,preprocessing]

      - name: Run tests
        run: |
          pytest --maxfail=5 -m "document_store and integration" test/document_stores/test_faiss.py

      - name: Calculate alert data
        id: calculator
        shell: bash
        if: (success() || failure()) && github.ref_name == 'main'
        run: |
          if [ "${{ job.status }}" = "success" ]; then
            echo "alert_type=success" >> "$GITHUB_OUTPUT";
          else
            echo "alert_type=error" >> "$GITHUB_OUTPUT";
          fi

      - name: Send event to Datadog
        if: (success() || failure()) && github.ref_name == 'main'
        uses: masci/datadog@v1
        with:
          api-key: ${{ secrets.CORE_DATADOG_API_KEY }}
          api-url: https://api.datadoghq.eu
          events: |
            - title: "${{ github.workflow }} workflow"
              text: "Job ${{ github.job }} in branch ${{ github.ref_name }}"
              alert_type: "${{ steps.calculator.outputs.alert_type }}"
              source_type_name: "Github"
              host: ${{ github.repository_owner }}
              tags:
                - "project:${{ github.repository }}"
                - "job:${{ github.job }}"
                - "run_id:${{ github.run_id }}"
                - "workflow:${{ github.workflow }}"
                - "branch:${{ github.ref_name }}"
                - "url:https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"

  integration-tests-weaviate:
    name: Integration / Weaviate / ${{ matrix.os }}
    needs:
      - unit-tests
    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu-latest]
    runs-on: ${{ matrix.os }}
    services:
      weaviate:
        image: semitechnologies/weaviate:1.17.2
        env:
          AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED: "true"
          PERSISTENCE_DATA_PATH: "/var/lib/weaviate"
          ENABLE_EXPERIMENTAL_BM25: "true"
          DISK_USE_READONLY_PERCENTAGE: 95
        ports:
          - 8080:8080
    steps:
      - uses: actions/checkout@v4

      - uses: actions/setup-python@v4
        with:
          python-version: ${{ env.PYTHON_VERSION }}

      - name: Install Haystack
        run: pip install .[dev,weaviate,preprocessing]

      - name: Run tests
        run: |
          pytest --maxfail=5 -m "document_store and integration" test/document_stores/test_weaviate.py

      - name: Calculate alert data
        id: calculator
        shell: bash
        if: (success() || failure()) && github.ref_name == 'main'
        run: |
          if [ "${{ job.status }}" = "success" ]; then
            echo "alert_type=success" >> "$GITHUB_OUTPUT";
          else
            echo "alert_type=error" >> "$GITHUB_OUTPUT";
          fi

      - name: Send event to Datadog
        if: (success() || failure()) && github.ref_name == 'main'
        uses: masci/datadog@v1
        with:
          api-key: ${{ secrets.CORE_DATADOG_API_KEY }}
          api-url: https://api.datadoghq.eu
          events: |
            - title: "${{ github.workflow }} workflow"
              text: "Job ${{ github.job }} in branch ${{ github.ref_name }}"
              alert_type: "${{ steps.calculator.outputs.alert_type }}"
              source_type_name: "Github"
              host: ${{ github.repository_owner }}
              tags:
                - "project:${{ github.repository }}"
                - "job:${{ github.job }}"
                - "run_id:${{ github.run_id }}"
                - "workflow:${{ github.workflow }}"
                - "branch:${{ github.ref_name }}"
                - "url:https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"

  integration-tests-pinecone:
    name: Integration / pinecone / ${{ matrix.os }}
    needs:
      - unit-tests
    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu-latest, windows-latest]
    runs-on: ${{ matrix.os }}
    steps:
      - uses: actions/checkout@v4

      - uses: actions/setup-python@v4
        with:
          python-version: ${{ env.PYTHON_VERSION }}

      - name: Install Haystack
        run: pip install .[dev,pinecone,preprocessing]

      - name: Run tests
        env:
          PINECONE_API_KEY: ${{ secrets.PINECONE_API_KEY }}
        run: |
          pytest --maxfail=5 -m "document_store and integration" test/document_stores/test_pinecone.py

      - name: Calculate alert data
        id: calculator
        shell: bash
        if: (success() || failure()) && github.ref_name == 'main'
        run: |
          if [ "${{ job.status }}" = "success" ]; then
            echo "alert_type=success" >> "$GITHUB_OUTPUT";
          else
            echo "alert_type=error" >> "$GITHUB_OUTPUT";
          fi

      - name: Send event to Datadog
        if: (success() || failure()) && github.ref_name == 'main'
        uses: masci/datadog@v1
        with:
          api-key: ${{ secrets.CORE_DATADOG_API_KEY }}
          api-url: https://api.datadoghq.eu
          events: |
            - title: "${{ github.workflow }} workflow"
              text: "Job ${{ github.job }} in branch ${{ github.ref_name }}"
              alert_type: "${{ steps.calculator.outputs.alert_type }}"
              source_type_name: "Github"
              host: ${{ github.repository_owner }}
              tags:
                - "project:${{ github.repository }}"
                - "job:${{ github.job }}"
                - "run_id:${{ github.run_id }}"
                - "workflow:${{ github.workflow }}"
                - "branch:${{ github.ref_name }}"
                - "url:https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"

  integration-tests-memory:
    name: Integration / memory / ${{ matrix.os }}
    needs:
      - unit-tests
    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu-latest, windows-latest]
    runs-on: ${{ matrix.os }}
    steps:
      - uses: actions/checkout@v4

      - uses: actions/setup-python@v4
        with:
          python-version: ${{ env.PYTHON_VERSION }}

      - name: Install Haystack
        run: pip install .[dev,preprocessing,inference]

      - name: Run tests
        run: |
          pytest --maxfail=5 -m "document_store and integration" test/document_stores/test_memory.py

      - name: Calculate alert data
        id: calculator
        shell: bash
        if: (success() || failure()) && github.ref_name == 'main'
        run: |
          if [ "${{ job.status }}" = "success" ]; then
            echo "alert_type=success" >> "$GITHUB_OUTPUT";
          else
            echo "alert_type=error" >> "$GITHUB_OUTPUT";
          fi

      - name: Send event to Datadog
        if: (success() || failure()) && github.ref_name == 'main'
        uses: masci/datadog@v1
        with:
          api-key: ${{ secrets.CORE_DATADOG_API_KEY }}
          api-url: https://api.datadoghq.eu
          events: |
            - title: "${{ github.workflow }} workflow"
              text: "Job ${{ github.job }} in branch ${{ github.ref_name }}"
              alert_type: "${{ steps.calculator.outputs.alert_type }}"
              source_type_name: "Github"
              host: ${{ github.repository_owner }}
              tags:
                - "project:${{ github.repository }}"
                - "job:${{ github.job }}"
                - "run_id:${{ github.run_id }}"
                - "workflow:${{ github.workflow }}"
                - "branch:${{ github.ref_name }}"
                - "url:https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"

  integration-tests-promptnode:
    name: Integration / PromptNode / ${{ matrix.os }}
    needs:
      - unit-tests
    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu-latest, windows-latest]
    runs-on: ${{ matrix.os }}
    steps:
      - uses: actions/checkout@v4

      - uses: actions/setup-python@v4
        with:
          python-version: ${{ env.PYTHON_VERSION }}

      - name: Install Haystack
        run: pip install .[dev,preprocessing,inference]

      - name: Run tests
        run: |
          pytest --maxfail=5 -m "integration" test/prompt

      - name: Calculate alert data
        id: calculator
        shell: bash
        if: (success() || failure()) && github.ref_name == 'main'
        run: |
          if [ "${{ job.status }}" = "success" ]; then
            echo "alert_type=success" >> "$GITHUB_OUTPUT";
          else
            echo "alert_type=error" >> "$GITHUB_OUTPUT";
          fi

      - name: Send event to Datadog
        if: (success() || failure()) && github.ref_name == 'main'
        uses: masci/datadog@v1
        with:
          api-key: ${{ secrets.CORE_DATADOG_API_KEY }}
          api-url: https://api.datadoghq.eu
          events: |
            - title: "${{ github.workflow }} workflow"
              text: "Job ${{ github.job }} in branch ${{ github.ref_name }}"
              alert_type: "${{ steps.calculator.outputs.alert_type }}"
              source_type_name: "Github"
              host: ${{ github.repository_owner }}
              tags:
                - "project:${{ github.repository }}"
                - "job:${{ github.job }}"
                - "run_id:${{ github.run_id }}"
                - "workflow:${{ github.workflow }}"
                - "branch:${{ github.ref_name }}"
                - "url:https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"

  integration-tests-agents:
    name: Integration / Agents / ${{ matrix.os }}
    needs:
      - unit-tests
    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu-latest, windows-latest]
    runs-on: ${{ matrix.os }}
    steps:
      - uses: actions/checkout@v4

      - uses: actions/setup-python@v4
        with:
          python-version: ${{ env.PYTHON_VERSION }}

      - name: Install Haystack
        run: pip install .[dev,preprocessing,inference]

      - name: Run tests
        run: |
          pytest --maxfail=5 -m "integration" test/agents

      - name: Calculate alert data
        id: calculator
        shell: bash
        if: (success() || failure()) && github.ref_name == 'main'
        run: |
          if [ "${{ job.status }}" = "success" ]; then
            echo "alert_type=success" >> "$GITHUB_OUTPUT";
          else
            echo "alert_type=error" >> "$GITHUB_OUTPUT";
          fi

      - name: Send event to Datadog
        if: (success() || failure()) && github.ref_name == 'main'
        uses: masci/datadog@v1
        with:
          api-key: ${{ secrets.CORE_DATADOG_API_KEY }}
          api-url: https://api.datadoghq.eu
          events: |
            - title: "${{ github.workflow }} workflow"
              text: "Job ${{ github.job }} in branch ${{ github.ref_name }}"
              alert_type: "${{ steps.calculator.outputs.alert_type }}"
              source_type_name: "Github"
              host: ${{ github.repository_owner }}
              tags:
                - "project:${{ github.repository }}"
                - "job:${{ github.job }}"
                - "run_id:${{ github.run_id }}"
                - "workflow:${{ github.workflow }}"
                - "branch:${{ github.ref_name }}"
                - "url:https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"

  catch-all:
    name: Catch-all check
    runs-on: ubuntu-latest
    # This job will be executed only after all the other tests
    # are successful.
    # This way we'll be able to mark only this test as required
    # and skip it accordingly.
    needs:
      - integration-tests-elasticsearch7
      - integration-tests-elasticsearch8
      - integration-tests-sql
      - integration-tests-opensearch
      - integration-tests-dc
      - integration-tests-faiss
      - integration-tests-weaviate
      - integration-tests-pinecone
      - integration-tests-memory
      - integration-tests-promptnode
      - integration-tests-agents

    steps:
      - name: Finisher
        run: echo "Finish him!"