Merge pull request #260 from tigergraph/dev

Merge dev to main as a release candidate for 0.9
tigergraph · Aug 12, 2024 · ac7763b · ac7763b
2 parents 81fba19 + b437299
commit ac7763b
Show file tree

Hide file tree

Showing 383 changed files with 15,000 additions and 1,871 deletions.
diff --git a/.github/workflows/build-test-ci.yaml b/.github/workflows/build-test-ci.yaml
@@ -21,11 +21,11 @@ jobs:
     - name: Set up Docker Buildx
       uses: docker/setup-buildx-action@v3
 
-    - name: Build and push Docker image
+    - name: Build Docker.tests image
       uses: docker/build-push-action@v5
       with:
-        context: .
-        file: ./Dockerfile.tests
+        context: . 
+        file: copilot/Dockerfile.tests
         push: false
         load: true
         tags: nlqs/tests:0.1
@@ -45,6 +45,7 @@ jobs:
         echo "$LLM_CONFIG_GROQ_MIXTRAL" > configs/groq_mixtral_config.json
         echo "$LLM_CONFIG_BEDROCK_CLAUDE3" > configs/bedrock_config.json
         echo "$LLM_CONFIG_HUGGINGFACE_LLAMA70B" > configs/huggingface_llama70b_config.json
+        echo "$LLM_CONFIG_WATSONX_MISTRAL_LARGE" > configs/ibm_watsonx_config.json
         echo "$MILVUS_CONFIG" > configs/milvus_config.json
       env:
         DB_CONFIG: ${{ secrets.DB_CONFIG }}
@@ -57,6 +58,7 @@ jobs:
         LLM_CONFIG_HUGGINGFACE_PHI3: ${{ secrets.LLM_CONFIG_HUGGINGFACE_PHI3 }}
         LLM_CONFIG_OPENAI_GPT4O: ${{ secrets.LLM_CONFIG_OPENAI_GPT4O }}
         LLM_CONFIG_HUGGINGFACE_LLAMA70B: ${{ secrets.LLM_CONFIG_HUGGINGFACE_LLAMA70B }}
+        LLM_CONFIG_WATSONX_MISTRAL_LARGE: ${{ secrets.LLM_CONFIG_WATSONX_MISTRAL_LARGE }}
         GCP_CREDS_CONFIG: ${{ secrets.GCP_CREDS_CONFIG }}
         LLM_TEST_EVALUATOR: ${{ secrets.LLM_TEST_EVALUATOR }}
         MILVUS_CONFIG: ${{ secrets.MILVUS_CONFIG }}
@@ -66,13 +68,13 @@ jobs:
       if: ${{ github.event_name }} == 'pull_request'
       run: |
         docker rm -f nlqs-tests || true
-        docker run -it -v $(pwd)/configs/:/code/configs -e GOOGLE_APPLICATION_CREDENTIALS=/code/configs/GCP_CREDS.json -e WANDB_API_KEY=${{ secrets.WANDB_API_KEY }} -e PR_NUMBER=${{ github.event.number }} --name nlqs-tests -d nlqs/tests:0.1
+        docker run -it -v $(pwd)/configs/:/code/app/configs -e GOOGLE_APPLICATION_CREDENTIALS=/code/app/configs/GCP_CREDS.json -e WANDB_API_KEY=${{ secrets.WANDB_API_KEY }} -e PR_NUMBER=${{ github.event.number }} --name nlqs-tests -d nlqs/tests:0.1
 
     - name: Run Docker Container for Regress
       if: ${{ github.event_name == 'schedule' }}
       run: |
         docker rm -f nlqs-tests || true
-        docker run -it -v $(pwd)/configs/:/code/configs -e GOOGLE_APPLICATION_CREDENTIALS=/code/configs/GCP_CREDS.json -e WANDB_API_KEY=${{ secrets.WANDB_API_KEY }} -e PR_NUMBER="DailyRegression" --name nlqs-tests -d nlqs/tests:0.1
+        docker run -it -v $(pwd)/configs/:/code/app/configs -e GOOGLE_APPLICATION_CREDENTIALS=/code/app/configs/GCP_CREDS.json -e WANDB_API_KEY=${{ secrets.WANDB_API_KEY }} -e PR_NUMBER="DailyRegression" --name nlqs-tests -d nlqs/tests:0.1
 
     - name: Execute PR Tests
       if: github.event_name == 'pull_request'

diff --git a/.github/workflows/cloud-build-deploy-ci.yaml b/.github/workflows/cloud-build-deploy-ci.yaml
@@ -33,12 +33,22 @@ jobs:
         echo "IMAGE=$IMAGE" >> $GITHUB_ENV
         echo "IMAGE=$IMAGE" >> $GITHUB_OUTPUT${{ needs.setup.outputs.image }} 
  
-    - name: Build and push Docker image
+    - name: Build and push Docker image CoPilot
       uses: docker/build-push-action@v5
       with:
         context: .
-        file: ./Dockerfile
+        file: ./copilot/Dockerfile
         push: true
         tags: |
            ${{ env.IMAGE }}
-           tginternal/copilot:cloud-latest
+           tginternal/copilot:cloud-latest
+ 
+    - name: Build and push Docker image ECC
+      uses: docker/build-push-action@v5
+      with:
+        context: .
+        file: ./eventual-consistency-service/Dockerfile
+        push: true
+        tags: |
+           ${{ env.IMAGE }}
+           tginternal/ecc:cloud-latest
diff --git a/.github/workflows/cloud-build-nightly.yaml b/.github/workflows/cloud-build-nightly.yaml
@@ -33,12 +33,22 @@ jobs:
         echo "IMAGE=$IMAGE" >> $GITHUB_ENV
         echo "IMAGE=$IMAGE" >> $GITHUB_OUTPUT${{ needs.setup.outputs.image }} 
  
-    - name: Build and push Docker image
+    - name: Build and push Docker image CoPilot
       uses: docker/build-push-action@v5
       with:
         context: .
-        file: ./Dockerfile
+        file: ./copilot/Dockerfile
         push: true
         tags: |
            ${{ env.IMAGE }}
-           tginternal/copilot:cloud-dev
+           tginternal/copilot:cloud-dev
+ 
+    - name: Build and push Docker image ECC
+      uses: docker/build-push-action@v5
+      with:
+        context: .
+        file: ./eventual-consistency-service/Dockerfile
+        push: true
+        tags: |
+           ${{ env.IMAGE }}
+           tginternal/ecc:cloud-dev
diff --git a/.github/workflows/onprem-build-nightly.yaml b/.github/workflows/onprem-build-nightly.yaml
@@ -33,12 +33,56 @@ jobs:
         echo "IMAGE=$IMAGE" >> $GITHUB_ENV
         echo "IMAGE=$IMAGE" >> $GITHUB_OUTPUT${{ needs.setup.outputs.image }} 
  
-    - name: Build and push Docker image
+    - name: Build and push Docker image CoPilot
       uses: docker/build-push-action@v5
       with:
         context: .
-        file: ./Dockerfile
+        file: ./copilot/Dockerfile
         push: true
         tags: |
            ${{ env.IMAGE }}
-           tigergraphml/copilot:dev
+           tigergraphml/copilot:dev
+ 
+    - name: Build and push Docker image ECC
+      uses: docker/build-push-action@v5
+      with:
+        context: .
+        file: ./eventual-consistency-service/Dockerfile
+        push: true
+        tags: |
+           ${{ env.IMAGE }}
+           tigergraphml/ecc:dev
+
+    - name: Build and push Docker image chat-history
+      uses: docker/build-push-action@v5
+      with:
+        context: chat-history/
+        file: ./chat-history/Dockerfile
+        push: true
+        tags: |
+           ${{ env.IMAGE }}
+           tigergraphml/chat-history:dev
+
+    - name: Build and push Docker image copilot-ui
+      uses: docker/build-push-action@v5
+      with:
+        context: copilot-ui/
+        file: ./copilot-ui/Dockerfile
+        push: true
+        tags: |
+           ${{ env.IMAGE }}
+           tigergraphml/copilot-ui:dev
+
+    - name: Set SSH key
+      run: |
+          mkdir -p ~/.ssh
+          echo "${{ secrets.PRIVATE_SSH_KEY }}" > ~/.ssh/id_rsa
+          chmod 600 ~/.ssh/id_rsa
+
+    - name: Update on-prem dev environment
+      run: |
+          ssh -o StrictHostKeyChecking=no ${{ secrets.DEV_USERNAME }}@${{ secrets.DEV_ADDRESS }} << 'ENDSSH'
+          cd CoPilot
+          docker compose pull
+          docker compose up -d
+          ENDSSH
diff --git a/.github/workflows/onprem-build.yaml b/.github/workflows/onprem-build.yaml
@@ -33,15 +33,25 @@ jobs:
         echo "IMAGE=$IMAGE" >> $GITHUB_ENV
         echo "IMAGE=$IMAGE" >> $GITHUB_OUTPUT${{ needs.setup.outputs.image }} 
  
-    - name: Build and push Docker image
+    - name: Build and push Docker image CoPilot
       uses: docker/build-push-action@v5
       with:
         context: .
-        file: ./Dockerfile
+        file: ./copilot/Dockerfile
         push: true
         tags: |
            ${{ env.IMAGE }}
            tigergraphml/copilot:latest
+ 
+    - name: Build and push Docker image ECC
+      uses: docker/build-push-action@v5
+      with:
+        context: .
+        file: ./eventual-consistency-service/Dockerfile
+        push: true
+        tags: |
+           ${{ env.IMAGE }}
+           tigergraphml/ecc:latest
 
     - name: Set SSH key
       run: |
@@ -55,4 +65,4 @@ jobs:
           cd CoPilot
           docker compose pull
           docker compose up -d
-          ENDSSH
+          ENDSSH
diff --git a/.github/workflows/pull-test-merge.yaml b/.github/workflows/pull-test-merge.yaml
@@ -33,12 +33,11 @@ jobs:
           python -m venv venv
           source venv/bin/activate
           python -m pip install --upgrade pip
-          pip install -r requirements.txt
+          pip install -r copilot/requirements.txt
           pip install pytest
 
       - name: Create db config
         run: |
-          source venv/bin/activate
           mkdir configs
           echo "$DB_CONFIG" > configs/db_config.json
           echo "$LLM_CONFIG_OPENAI_GPT4" > configs/llm_config.json
@@ -65,7 +64,9 @@ jobs:
       - name: Run pytest
         run: |
           source venv/bin/activate
-          ./venv/bin/python -m pytest --disable-warnings
+          cp -r copilot/tests/*test* copilot/tests/create_wandb_report.py copilot/app/ 
+          cd copilot/app
+          python -m pytest --disable-warnings
         env:
           DB_CONFIG: ${{ secrets.DB_CONFIG }}
           LLM_CONFIG:  ${{ secrets.LLM_CONFIG_OPENAI_GPT4 }}
@@ -78,4 +79,4 @@ jobs:
           LLM_TEST_EVALUATOR: ${{ secrets.LLM_TEST_EVALUATOR }}
           MILVUS_CONFIG: ${{ secrets.MILVUS_CONFIG }}
           PYTHONPATH: /opt/actions-runner/_work/CoPilot/CoPilot:/opt/actions-runner/_work/CoPilot/CoPilot/tests:/opt/actions-runner/_work/CoPilot/CoPilot/tests/app:/opt/actions-runner/_work/_tool/Python/3.11.8/x64/lib/python3.11/site-packages
-
+
diff --git a/.gitignore b/.gitignore
@@ -1,3 +1,4 @@
+.env
 __pycache__/
 *_config*.json
 tigergraph-ml-*.json
@@ -18,4 +19,25 @@ log.WARNING
 logs/*
 tmp
 .idea
-volumes
+eventual-consistency-service/app/logs/*
+eventual-consistency-service/logs/*
+eventual-consistency-service/tests/logs/*
+copilot/app/logs/*
+copilot/logs/*
+copilot/tests/logs/*
+report-service/logs/*
+report-service/app/logs/*
+volumes
+build/
+/k8s
+run.sh
+configs
+venv
+.env
+
+hld
+times*
+a.md
+gen
+mq2s
+*par_times
diff --git a/Dockerfile b/Dockerfile
diff --git a/README.md b/README.md
@@ -212,14 +212,17 @@ You can also disable the consistency_checker, which reconciles Milvus and TigerG
 ```
 
 ##### Milvus configuration
-Copy the below into `configs/milvus_config.json` and edit the `host` and `port` fields to match your Milvus configuration (keeping in mind docker configuration).  `username` and `password` can also be configured below if required by your Milvus setup.  `enabled` should always be set to "true" for now as Milvus is only the embedding store supported.
+Copy the below into `configs/milvus_config.json` and edit the `host` and `port` fields to match your Milvus configuration (keeping in mind docker configuration).  `username` and `password` can also be configured below if required by your Milvus setup.  `enabled` should always be set to "true" for now as Milvus is only the embedding store supported. `process_interval_seconds` is the number of seconds which the eventual-consistency-checker (ECC) service will be scheduled to check for new vertices in TigerGraph in order to create embeddings in Milvus. In the same way `cleanup_interval_seconds` is the number of seconds the ECC service will be scheduled to check for stale Milvus embeddings (e.g. if TigerGraph is restored from backup, or a vertex is deleted).  Batch size is the number of vertices that ECC will process in one workload; this is optional and defaults to 10.
 ```json
 {
     "host": "milvus-standalone",
     "port": 19530,
     "username": "",
     "password": "",
-    "enabled": "true"
+    "enabled": "true",
+    "process_interval_seconds": 1800,
+    "cleanup_interval_seconds": 2592000,
+    "batch_size": 10
 }
 ```
 
@@ -350,11 +353,21 @@ TigerGraph CoPilot is designed to be easily extensible. The service can be confi
 ## Testing
 A family of tests are included under the `tests` directory. If you would like to add more tests please refer to the [guide here](./docs/DeveloperGuide.md#adding-a-new-test-suite). A shell script `run_tests.sh` is also included in the folder which is the driver for running the tests. The easiest way to use this script is to execute it in the Docker Container for testing.
 
+### Testing with Pytest
+You can run testing for each service by going to the top level of the service's directory and running `python -m pytest`
+
+e.g. (from the top level)
+```sh
+cd copilot
+python -m pytest
+cd ..
+```
+
 ### Test in Docker Container
 
 First, make sure that all your LLM service provider configuration files are working properly. The configs will be mounted for the container to access. Also make sure that all the dependencies such as database and Milvus are ready. If not, you can run the included docker compose file to create those services.
 ```sh
-docker compose docker-compose.yml up -d --build
+docker compose up -d --build
 ```
 
 If you want to use Weights And Biases for logging the test results, your WandB API key needs to be set in an environment variable on the host machine.

diff --git a/app/agent/agent_generation.py b/app/agent/agent_generation.py