Merge pull request #8 from KishinNext/release-candidate/dagger

Release candidate/dagger

.github/workflows/main.yml

@@ -0,0 +1,51 @@
+name: 'ci'
+
+on:
+  push:
+    branches:
+    - main
+
+jobs:
+  dagger:
+    runs-on: ubuntu-latest
+    env:
+      PROJECT_ID: prod-datascience-393718
+    steps:
+      -
+        name: Checkout
+        uses: actions/checkout@v3
+      -
+        id: 'auth'
+        name: 'Authenticate to Google Cloud'
+        uses: 'google-github-actions/auth@v1'
+        with:
+          token_format: 'access_token'
+          credentials_json: '${{ secrets.GOOGLE_CREDENTIALS }}'
+      -
+        name: Login to Google Container Registry
+        uses: docker/login-action@v2
+        with:
+          registry: gcr.io
+          username: oauth2accesstoken
+          password: ${{ steps.auth.outputs.access_token }}
+      -
+        name: Set up Cloud SDK
+        uses: google-github-actions/setup-gcloud@v1
+        with:
+          project_id: ${{ env.PROJECT_ID }}
+      -
+        name: Setup Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.10'
+      -
+        name: Install
+        run: pip install dagger-io google-cloud-run
+
+      -
+        name: Copy development.yml on workspace
+        run: gcloud storage cp gs://prod-datascience-393718_cloudbuild/development.yaml config/.
+
+      -
+        name: Release and deploy with Dagger
+        run: python ci/main.py

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.9.16-slim-buster as base
+FROM python:3.10-slim-buster as base
 
 WORKDIR /app
 

agents/general_agent.py

@@ -138,7 +138,7 @@ def define_agent(self, template: str, tool_to_use: list) -> LLMSingleActionAgent
         llm = ChatOpenAI(
             verbose=self.verbose,
             temperature=0,
-            model_name='gpt-4',
+            model_name=config['default_model_name'],
             openai_api_key=secrets['openai_api']['token'],
             openai_organization=secrets['openai_api']['organization']
         )

app.py

@@ -37,15 +37,15 @@
 )
 
 
-@api.get('/ping')
+@api.get("/healthcheck",  status_code=status.HTTP_200_OK)
 def health_check():
     """Health check endpoint. Returns the status of the API.
 
     Returns:
         dict: The status of the API.
     """
     return {
-        'health': 'OK'
+        'heapi_healthyalth': True
     }
 
 

chains/__tests__/test_document_chain.py

@@ -15,6 +15,7 @@
     get_secrets
 )
 from utils.database import create_db_session
+import pytest
 
 config = get_config()
 secrets = get_secrets(config)
@@ -37,7 +38,7 @@ def test_generate_sql_code():
     assert result == expected
 
 
-# TODO: change the info extractor according to your database
+@pytest.mark.skipif(True, reason='should be ran only manually with AdventureWorks Database')
 def test_get_data():
     info_extractor = {
         'schemas': {
@@ -73,7 +74,7 @@ def test_process_columns_in_chunks():
     assert isinstance(result['columns'][0]['comment'], str)
 
 
-# TODO: change the info extractor according to your database
+@pytest.mark.skipif(True, reason='should be ran only manually with AdventureWorks Database')
 def test_run_sql_comment_generator():
     database = create_db_session(secrets['database']['url'])
     query = 'I need the SQL code to comment on the columns of the currency in the sales schema'

chains/__tests__/test_metabase.py

@@ -20,8 +20,8 @@
 engine = create_db_session(secrets['database']['url'])
 
 
+@pytest.mark.skipif(True, reason='should be ran only manually')  # comment this line if you want to run this test
 def test_process_metabase_queries():
-
     metabase_queries = [
         {
             "table_name": "customer",
@@ -96,6 +96,7 @@ def test_process_metabase_queries():
     assert dashboard_info["name"] == "[TEST] Dashboard for testing"
 
 
+@pytest.mark.skipif(True, reason='should be ran only manually')  # comment this line if you want to run this test
 class TestQueries:
     def test_run_graph_creator(self):
         result = run_graph_creator("""

chains/__tests__/test_sql_runner.py

@@ -6,15 +6,15 @@
     get_secrets
 )
 from utils.database import create_db_session
+import pytest
 
 config = get_config()
 secrets = get_secrets(config)
 engine = create_db_session(secrets['database']['url'])
 
 
-# TODO: Modify the test to use your own database
+@pytest.mark.skipif(True, reason='should be ran only manually with AdventureWorks Database')
 def test_general_runner():
-
     result = sql_runner('I want the total number of records of the table store of the sales schema', engine, 10)
 
     assert result['sql_code'] is not None

chains/metabase.py

@@ -180,7 +180,7 @@ def run_graph_creator(query: str, database: Engine, max_queries: int = 10) -> st
             table_info=table_data,
             table_metadata=table_metadata,
             engine_type=database.dialect.name,
-            model_name='gpt-4',
+            model_name=config['default_model_name'],
             number_of_queries=max_queries
         ).get_result()
 

ci/gcp_demo.py

@@ -0,0 +1,78 @@
+import sys
+
+import anyio
+from google.cloud import run_v2
+import os
+import dagger
+
+PROJECT = 'prod-datascience-393718'
+GCR_SERVICE_URL = f"projects/{PROJECT}/locations/us-central1/services/querycraftergg"
+GCR_PUBLISH_ADDRESS = f"gcr.io/{PROJECT}/querycrafter"
+
+root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+
+
+async def main():
+    version = '3.10'
+
+    async with dagger.Connection(dagger.Config(log_output=sys.stderr)) as client:
+        app = client.host().directory(root)  # Asegúrate de cambiar esto a la ruta real de tu código
+
+        python_container = (
+            client.container(platform=dagger.Platform("linux/amd64")).from_(f"python:{version}-slim-buster"))
+        container = (
+            python_container.with_directory("/app", app)
+            .with_workdir("/app")
+            .with_exec(["pip", "install", "poetry"])
+            .with_exec(["poetry", "config", "virtualenvs.create", "false"])
+            .with_exec(["poetry", "install", "--no-interaction", "--no-ansi"])
+            .with_exec(["pytest"])
+            .with_exposed_port(8080)
+            .with_entrypoint(["python", "-m", "app"])
+        )
+
+        # publish the container to GCR object
+        container_image = await container.publish(GCR_PUBLISH_ADDRESS)
+        print(f"Container published to {container_image}.")
+
+        gcr_client = run_v2.ServicesClient()
+
+        # define a service request
+        gcr_request = run_v2.UpdateServiceRequest(
+            service=run_v2.Service(
+                name=GCR_SERVICE_URL,
+                template=run_v2.RevisionTemplate(
+                    containers=[
+                        run_v2.Container(
+                            image=container_image,
+                            ports=[
+                                run_v2.ContainerPort(
+                                    name="http1",
+                                    container_port=8080,
+                                ),
+                            ],
+                            startup_probe=run_v2.Probe(  # Agregando el health check aquí
+                                initial_delay_seconds=30,
+                                period_seconds=5,
+                                http_get=run_v2.HTTPGetAction(
+                                    path="/healthcheck",
+                                    port=8080
+                                )
+                            ),
+                        ),
+                    ],
+                ),
+            )
+        )
+        # update service
+        gcr_operation = gcr_client.update_service(request=gcr_request)
+        print("GCP service update request sent.")
+
+        # wait for service request completion
+        response = gcr_operation.result()
+        print("GCP service update request completed.")
+
+    print(f"Deployment for image {container_image} now available at {response}.")
+
+
+anyio.run(main)

config/default.yaml

@@ -19,3 +19,4 @@ logging:
       level: INFO
       handlers:
           - console
+default_model_name: 'gpt-3.5-turbo-16k'