Updated files to communicate with database directly instead of with a…

…pp API (#31)

* Updated files to communicate with database directly instead of with app API
get_claims uses db, pipeline with temp env vars
Pipeline working with env vars
* Small pipeline fix

lab-materials/05/detect_objects.py

@@ -1,15 +1,39 @@
 import os
 import requests
 import json
+import time
+import hashlib
+
+import logging
+import boto3
+from dotenv import dotenv_values, load_dotenv
 
 from process_image import process_image
+import db_utils
+
+# Initialize logger
+logger = logging.getLogger("app")
+
+# Get config
+config = {
+    **dotenv_values(".env"),  # load shared development variables
+    **dotenv_values(".env.secret"),  # load sensitive variables
+    **os.environ,  # override loaded values with environment variables
+}
+
+db = db_utils.Database(config, logger)
+
+s3 = boto3.client(
+    's3',
+    endpoint_url=config["DB_S3_ENDPOINT_URL"],
+    aws_access_key_id=config["DB_AWS_ACCESS_KEY_ID"],
+    aws_secret_access_key=config["DB_AWS_SECRET_ACCESS_KEY"],
+    use_ssl=config["LLM_ENDPOINT"].startswith("https"),)
 
 detection_endpoint = os.environ.get("DETECTION_ENDPOINT")
-claims_endpoint = os.environ.get("CLAIMS_ENDPOINT")
 
 def download_images(claim_id):
-    claim_endpoint = claims_endpoint + f"/db/claims/{claim_id}"
-    claim_info = requests.get(claim_endpoint).json()
+    claim_info = db.get_claim_info(claim_id)
     images = claim_info["original_images"]
 
     downloaded_images = []
@@ -18,36 +42,39 @@ def download_images(claim_id):
     for image in images:
         image_name = image["image_name"]
         image_key = image["image_key"]
-        image_endpoint = claims_endpoint + f"/images/{image_key}"
 
-        response = requests.get(image_endpoint)
-
-        if response.status_code == 200:
-            content_type = response.headers['Content-Type']
-            if content_type == 'binary/octet-stream':
-                with open(f"{image_name}", 'wb') as file:
-                    file.write(response.content)
-                print(f'Saved image {image_name}.')
-            else:
-                print(f"Unexpected content type: {content_type}")
-                failed_downloads.append(image_name)
-        else:
-            print(f"Failed to retrieve image {image_name}. Status code: {response.status_code}")
-            failed_downloads.append(image_name)
+        image = s3.get_object(Bucket=config["IMAGES_BUCKET"], Key=image_key)
+        image_content = image["Body"].read()
 
+        with open(f"{image_name}", 'wb') as file:
+            file.write(image_content)
+
         downloaded_images.append(image_name)
 
     if failed_downloads:
         raise Exception(f"Not all images were downloaded. Successfully downloaded these: {downloaded_images}\nFailed to download these: {failed_downloads}")
 
     return downloaded_images
 
-def upload_images(claim_id, images):
-    upload_endpoint = claims_endpoint + f"/db/claims/{claim_id}/processed_image"
-
+def upload_image(claim_id, image):
+    filename = os.path.basename(image)
+    timestamp = int(time.time())
+    sha256_hash = hashlib.sha256(filename.encode()).hexdigest()
+    image_key = f"processed_images/{claim_id}_{timestamp}_{sha256_hash[:8]}_{filename}"
+    # Upload the image to S3
+    s3.put_object(
+        Bucket=config["IMAGES_BUCKET"],
+        Key=image_key,
+        Body=open(image, 'rb'),
+    )
+    # Save image info to DB
+    db.upload_processed_image(claim_id, filename, image_key)
+
+def upload_images(claim_id, images):    
     for image in images:
-        files = {'image': (image, open(image, 'rb'), 'image/jpeg')}
-        response = requests.post(upload_endpoint, files=files)
+        upload_image(claim_id, image)
+        # files = {'image': (image, open(image, 'rb'), 'image/jpeg')}
+        # response = requests.post(upload_endpoint, files=files)
 
 def process_images(images):
     processed_images = []

lab-materials/05/get_accident_time.py

@@ -2,25 +2,29 @@
 import os
 import json
 
+import logging
+from dotenv import dotenv_values, load_dotenv
+
 from llm_usage import infer_with_template
+import db_utils
 
-claims_endpoint = os.environ.get("CLAIMS_ENDPOINT")+"/db/claims"
+# Initialize logger
+logger = logging.getLogger("app")
 
-def upload_accident_time(claim_id, accident_time):
-    upload_endpoint = claims_endpoint + f"/{claim_id}/time"
-    data = {
-        "time": accident_time,
-    }
-    headers = {'accept': 'application/json'}
-    response = requests.post(upload_endpoint, headers=headers, params=data)
+# Get config
+config = {
+    **dotenv_values(".env"),  # load shared development variables
+    **dotenv_values(".env.secret"),  # load sensitive variables
+    **os.environ,  # override loaded values with environment variables
+}
 
-    if response.status_code!=200:
-        raise Exception(f"{response.status_code} {response.text}")
-
+db = db_utils.Database(config, logger)
+
+def upload_accident_time(claim_id, accident_time):
+    db.update_claim_time(claim_id, accident_time)
 
 def get_accident_time(claim_id = None):
-    claim_endpoint = claims_endpoint + f"/{claim_id}"
-    claim_info = requests.get(claim_endpoint).json()
+    claim_info = db.get_claim_info(claim_id)
     claim_body = claim_info["body"]
 
     with open('templates/time_template.txt') as f:

lab-materials/05/get_claims.py

@@ -2,10 +2,25 @@
 import requests
 import json
 
-claims_endpoint = os.environ.get("CLAIMS_ENDPOINT")+"/db/claims"
+import logging
+from dotenv import dotenv_values, load_dotenv
+
+import db_utils
+
+# Initialize logger
+logger = logging.getLogger("app")
+
+# Get config
+config = {
+    **dotenv_values(".env"),  # load shared development variables
+    **dotenv_values(".env.secret"),  # load sensitive variables
+    **os.environ,  # override loaded values with environment variables
+}
+
+db = db_utils.Database(config, logger)
 
 def get_unprocessed_claims():
-    claims_info = requests.get(claims_endpoint).json()
+    claims_info = db.list_claims()
 
     unprocessed_list = []
     for claim in claims_info:

lab-materials/05/get_location.py

@@ -2,25 +2,29 @@
 import os
 import json
 
+import logging
+from dotenv import dotenv_values, load_dotenv
+
 from llm_usage import infer_with_template
+import db_utils
+
+# Initialize logger
+logger = logging.getLogger("app")
 
-claims_endpoint = os.environ.get("CLAIMS_ENDPOINT")+"/db/claims"
+# Get config
+config = {
+    **dotenv_values(".env"),  # load shared development variables
+    **dotenv_values(".env.secret"),  # load sensitive variables
+    **os.environ,  # override loaded values with environment variables
+}
+
+db = db_utils.Database(config, logger)
 
 def upload_location(claim_id, location):
-    upload_endpoint = claims_endpoint + f"/{claim_id}/location"
-    data = {
-        "location": location,
-    }
-    headers = {'accept': 'application/json'}
-    response = requests.post(upload_endpoint, headers=headers, params=data)
-
-    if response.status_code!=200:
-        raise Exception(f"{response.status_code} {response.text}")
-
+    db.update_claim_location(claim_id, location) 
 
 def get_location(claim_id = None):    
-    claim_endpoint = claims_endpoint + f"/{claim_id}"
-    claim_info = requests.get(claim_endpoint).json()
+    claim_info = db.get_claim_info(claim_id)
     claim_body = claim_info["body"]
 
     with open('templates/location_template.txt') as f:

lab-materials/05/get_sentiment.py

@@ -2,25 +2,29 @@
 import os
 import json
 
+import logging
+from dotenv import dotenv_values, load_dotenv
+
 from llm_usage import infer_with_template
+import db_utils
 
-claims_endpoint = os.environ.get("CLAIMS_ENDPOINT")+"/db/claims"
+# Initialize logger
+logger = logging.getLogger("app")
 
-def upload_sentiment(claim_id, sentiment):
-    upload_endpoint = claims_endpoint + f"/{claim_id}/sentiment"
-    data = {
-        "sentiment": sentiment,
-    }
-    headers = {'accept': 'application/json'}
-    response = requests.post(upload_endpoint, headers=headers, params=data)
+# Get config
+config = {
+    **dotenv_values(".env"),  # load shared development variables
+    **dotenv_values(".env.secret"),  # load sensitive variables
+    **os.environ,  # override loaded values with environment variables
+}
 
-    if response.status_code!=200:
-        raise Exception(f"{response.status_code} {response.text}")
-
+db = db_utils.Database(config, logger)
+
+def upload_sentiment(claim_id, sentiment):
+    db.update_claim_sentiment(claim_id, sentiment)
 
 def get_claim_sentiment(claim_id = None):    
-    claim_endpoint = claims_endpoint + f"/{claim_id}"
-    claim_info = requests.get(claim_endpoint).json()
+    claim_info = db.get_claim_info(claim_id)
     claim_body = claim_info["body"]
 
     with open('templates/sentiment_template.txt') as f:

lab-materials/05/llm_usage.py

@@ -38,13 +38,4 @@ def similarity_metric(predicted_text, reference_text):
     embedding_model = HuggingFaceEmbeddings()
     evaluator = load_evaluator("embedding_distance", embeddings=embedding_model)
     distance_score = evaluator.evaluate_strings(prediction=predicted_text, reference=reference_text)
-    return 1-distance_score["score"]
-
-if __name__ == '__main__':
-    with open('example_text.txt') as f:
-        input_text = f.read()
-
-    with open('template.txt') as f:
-        template = f.read()
-
-    summarize_with_template(input_text, template)
+    return 1-distance_score["score"]