Merge branch 'microsoft:main' into main

KylinMountain · Nov 6, 2024 · 4bd5f88 · 4bd5f88
2 parents 6360a3e + a6d9b0c
commit 4bd5f88
Show file tree

Hide file tree

Showing 460 changed files with 19,459 additions and 15,694 deletions.
diff --git a/.gitattributes b/.gitattributes
@@ -0,0 +1,11 @@
+*.txt text eol=lf
+*.md text eol=lf
+*.yml text eol=lf
+*.html text eol=lf
+*.py text eol=lf
+*.toml text eol=lf
+.gitattributes text eol=lf
+.gitignore text eol=lf
+*.lock
+CODEOWNERS text eol=lf
+LICENSE text eol=lf
diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
diff --git a/.github/ISSUE_TEMPLATE/general_issue.yml b/.github/ISSUE_TEMPLATE/general_issue.yml
@@ -7,11 +7,12 @@ body:
   - type: checkboxes
     id: existingcheck
     attributes:
-      label: Is there an existing issue for this?
-      description: Please search to see if an issue already exists for the bug you encountered.
+      label: Do you need to file an issue?
+      description: Please help us manage our time by avoiding duplicates and common questions with the steps below.
       options:
-        - label: I have searched the existing issues
-        - label: I have checked [#657](https://github.com/microsoft/graphrag/issues/657) to validate if my issue is covered by community support
+        - label: I have searched the existing issues and this bug is not already filed.
+        - label: My model is hosted on OpenAI or Azure. If not, please look at the "model providers" issue and don't file a new one here.
+        - label: I believe this is a legitimate bug, not just a question. If this is a question, please use the Discussions area.
   - type: textarea
     id: description
     attributes:

diff --git a/.github/dependabot.yml b/.github/dependabot.yml
@@ -4,10 +4,6 @@
 # https://docs.github.com/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file
 version: 2
 updates:
-  - package-ecosystem: "npm" # See documentation for possible values
-    directory: "docsite/" # Location of package manifests
-    schedule:
-      interval: "weekly"
   - package-ecosystem: "pip" # See documentation for possible values
     directory: "/" # Location of package manifests
     schedule:

diff --git a/.github/workflows/gh-pages.yml b/.github/workflows/gh-pages.yml
@@ -2,44 +2,22 @@ name: gh-pages
 on:
   push:
     branches: [main]
-
 permissions:
   contents: write
 
 env:
-  POETRY_VERSION: 1.8.3
-  PYTHON_VERSION: "3.11"
-  NODE_VERSION: 18.x
+  POETRY_VERSION: '1.8.3'
+  PYTHON_VERSION: '3.11'
 
 jobs:
   build:
     runs-on: ubuntu-latest
     env:
       GH_PAGES: 1
       DEBUG: 1
-      GRAPHRAG_LLM_TYPE: "azure_openai_chat"
-      GRAPHRAG_EMBEDDING_TYPE: "azure_openai_embedding"
-      GRAPHRAG_API_KEY: ${{ secrets.OPENAI_API_KEY }}
-      GRAPHRAG_API_BASE: ${{ secrets.GRAPHRAG_API_BASE }}
-      GRAPHRAG_API_VERSION: ${{ secrets.GRAPHRAG_API_VERSION }}
-      GRAPHRAG_LLM_DEPLOYMENT_NAME: ${{ secrets.GRAPHRAG_LLM_DEPLOYMENT_NAME }}
-      GRAPHRAG_EMBEDDING_DEPLOYMENT_NAME: ${{ secrets.GRAPHRAG_EMBEDDING_DEPLOYMENT_NAME }}
-      GRAPHRAG_CACHE_TYPE: "blob"
-      GRAPHRAG_CACHE_CONNECTION_STRING: ${{ secrets.BLOB_STORAGE_CONNECTION_STRING }}
-      GRAPHRAG_CACHE_CONTAINER_NAME: "cicache"
-      GRAPHRAG_CACHE_BASE_DIR": "cache"
-      GRAPHRAG_LLM_MODEL: gpt-3.5-turbo-16k
-      GRAPHRAG_EMBEDDING_MODEL: text-embedding-ada-002
-      # We have Windows + Linux runners in 3.10 and 3.11, so we need to divide the rate limits by 4
-      GRAPHRAG_LLM_TPM: 45_000 # 180,000 / 4
-      GRAPHRAG_LLM_RPM: 270 # 1,080 / 4
-      GRAPHRAG_EMBEDDING_TPM: 87_500 # 350,000 / 4
-      GRAPHRAG_EMBEDDING_RPM: 525 # 2,100 / 4
-      GRAPHRAG_CHUNK_SIZE: 1200
-      GRAPHRAG_CHUNK_OVERLAP: 0
-      # Azure AI Search config
-      AZURE_AI_SEARCH_URL_ENDPOINT: ${{ secrets.AZURE_AI_SEARCH_URL_ENDPOINT }}
-      AZURE_AI_SEARCH_API_KEY: ${{ secrets.AZURE_AI_SEARCH_API_KEY }}
+      GRAPHRAG_API_KEY: ${{ secrets.OPENAI_NOTEBOOK_KEY }}
+      GRAPHRAG_LLM_MODEL: ${{ secrets.GRAPHRAG_LLM_MODEL }}
+      GRAPHRAG_EMBEDDING_MODEL: ${{ secrets.GRAPHRAG_EMBEDDING_MODEL }}
 
     steps:
       - uses: actions/checkout@v4
@@ -56,42 +34,20 @@ jobs:
         with:
           poetry-version: ${{ env.POETRY_VERSION }}
 
-      - name: Use Node ${{ env.NODE_VERSION }}
-        uses: actions/setup-node@v4
-        with:
-          node-version: ${{ env.NODE_VERSION }}
-
-      - name: Install Yarn dependencies
-        run: yarn install
-        working-directory: docsite
-
-      - name: Install Poetry dependencies
+      - name: poetry intsall
+        shell: bash
         run: poetry install
+
+      - name: mkdocs build
+        shell: bash
+        run: poetry run poe build_docs
 
-      - name: Install Azurite
-        id: azuright
-        uses: potatoqualitee/[email protected]
-
-      - name: Generate Indexer Outputs
-        run: |
-          poetry run poe test_smoke
-          zip -jrm docsite/data/operation_dulce/dataset.zip tests/fixtures/min-csv/output/*/artifacts/*.parquet
-
-      - name: Build Jupyter Notebooks
-        run: poetry run poe convert_docsite_notebooks
-
-      - name: Build docsite
-        run: yarn build
-        working-directory: docsite
-        env:
-          DOCSITE_BASE_URL: "graphrag"
-
-      - name: List docsite files
-        run: find docsite/_site
+      - name: List Docsite Contents
+        run: find site
 
       - name: Deploy to GitHub Pages
-        uses: JamesIves/[email protected].3
+        uses: JamesIves/[email protected].4
         with:
           branch: gh-pages
-          folder: docsite/_site
-          clean: true
+          folder: site
+          clean: true
diff --git a/.github/workflows/issues-autoresolve.yml b/.github/workflows/issues-autoresolve.yml
@@ -3,6 +3,11 @@ on:
   schedule:
     - cron: "30 1 * * *"
 
+permissions:
+  actions: write
+  issues: write
+  pull-requests: write
+
 jobs:
   close-issues:
     runs-on: ubuntu-latest

diff --git a/.github/workflows/javascript-ci.yml b/.github/workflows/javascript-ci.yml
diff --git a/.github/workflows/python-ci.yml b/.github/workflows/python-ci.yml
@@ -1,9 +1,18 @@
 name: Python CI
 on:
   push:
-    branches: [main]
+    branches:
+      - "**/main" # Matches branches like feature/main
+      - "main" # Matches the main branch
   pull_request:
-    branches: [main]
+    types:
+      - opened
+      - reopened
+      - synchronize
+      - ready_for_review
+    branches:
+      - "**/main"
+      - "main"
 
 permissions:
   contents: read
@@ -19,35 +28,15 @@ env:
 
 jobs:
   python-ci:
+    # skip draft PRs
+    if: github.event.pull_request.draft == false
     strategy:
       matrix:
         python-version: ["3.10", "3.11"] # add 3.12 once gensim supports it. TODO: watch this issue - https://github.com/piskvorky/gensim/issues/3510
         os: [ubuntu-latest, windows-latest]
+      fail-fast: false # Continue running all jobs even if one fails
     env:
       DEBUG: 1
-      GRAPHRAG_LLM_TYPE: "azure_openai_chat"
-      GRAPHRAG_EMBEDDING_TYPE: "azure_openai_embedding"
-      GRAPHRAG_API_KEY: ${{ secrets.OPENAI_API_KEY }}
-      GRAPHRAG_API_BASE: ${{ secrets.GRAPHRAG_API_BASE }}
-      GRAPHRAG_API_VERSION: ${{ secrets.GRAPHRAG_API_VERSION }}
-      GRAPHRAG_LLM_DEPLOYMENT_NAME: ${{ secrets.GRAPHRAG_LLM_DEPLOYMENT_NAME }}
-      GRAPHRAG_EMBEDDING_DEPLOYMENT_NAME: ${{ secrets.GRAPHRAG_EMBEDDING_DEPLOYMENT_NAME }}
-      GRAPHRAG_CACHE_TYPE: "blob"
-      GRAPHRAG_CACHE_CONNECTION_STRING: ${{ secrets.BLOB_STORAGE_CONNECTION_STRING }}
-      GRAPHRAG_CACHE_CONTAINER_NAME: "cicache"
-      GRAPHRAG_CACHE_BASE_DIR": "cache"
-      GRAPHRAG_LLM_MODEL: gpt-3.5-turbo-16k
-      GRAPHRAG_EMBEDDING_MODEL: text-embedding-ada-002
-      # We have Windows + Linux runners in 3.10 and 3.11, so we need to divide the rate limits by 4
-      GRAPHRAG_LLM_TPM: 45_000 # 180,000 / 4
-      GRAPHRAG_LLM_RPM: 270 # 1,080 / 4
-      GRAPHRAG_EMBEDDING_TPM: 87_500 # 350,000 / 4
-      GRAPHRAG_EMBEDDING_RPM: 525 # 2,100 / 4
-      GRAPHRAG_CHUNK_SIZE: 1200
-      GRAPHRAG_CHUNK_OVERLAP: 0
-      # Azure AI Search config
-      AZURE_AI_SEARCH_URL_ENDPOINT: ${{ secrets.AZURE_AI_SEARCH_URL_ENDPOINT }}
-      AZURE_AI_SEARCH_API_KEY: ${{ secrets.AZURE_AI_SEARCH_API_KEY }}
 
     runs-on: ${{ matrix.os }}
     steps:
@@ -65,7 +54,7 @@ jobs:
               - '**/*.toml'
               - '**/*.ipynb'
               - '.github/workflows/python*.yml'
-              - 'tests/smoke/*'
+              - 'tests/**/*'
 
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v5
@@ -84,10 +73,6 @@ jobs:
           poetry run python -m pip install gensim
           poetry install
 
-      - name: Check Semversioner
-        run: |
-          poetry run semversioner check
-
       - name: Check
         run: |
           poetry run poe check
@@ -96,30 +81,6 @@ jobs:
         run: |
           poetry build
 
-      - name: Install Azurite
-        id: azuright
-        uses: potatoqualitee/[email protected]
-
       - name: Unit Test
         run: |
           poetry run poe test_unit
-
-      - name: Integration Test
-        run: |
-          poetry run poe test_integration
-
-      # - name: Smoke Test
-      #   if: steps.changes.outputs.python == 'true'
-      #   run: |
-      #     poetry run poe test_smoke
-
-      # - uses: actions/upload-artifact@v4
-      #   if: always()
-      #   with:
-      #     name: smoke-test-artifacts-${{ matrix.python-version }}-${{ matrix.poetry-version }}-${{ runner.os }}
-      #     path: tests/fixtures/*/output
-
-      # - name: E2E Test
-      #   if: steps.changes.outputs.python == 'true'
-      #   run: |
-      #     ./scripts/e2e-test.sh