Tss 1158/related barrier merged 2

.gitignore

@@ -48,7 +48,7 @@ coverage.xml
 .hypothesis/
 /codecov.sh
 /test-reports/
-/.pytest_cache/
+*/.pytest_cache/
 
 # Translations
 *.mo
@@ -129,3 +129,5 @@ src/
 requirements-dev.txt
 /local.env
 config/settings/debug.py
+/barrier_embeddings_dict.pkl
+/barrier_similarity_scores_df.json

.profile

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+# custom initialisation tasks
+# ref - https://docs.cloudfoundry.org/devguide/deploy-apps/deploy-app.html
+
+echo "---- RUNNING release tasks (.profile) ------"
+
+echo "---- Installing Related Barrier ML Packages ------"
+/tmp/lifecycle/shell
+python -m pip install sentence-transformers==2.2.2 --no-deps
+python -m pip install torch==2.0.0 torchvision==0.15.1 --extra-index-url https://download.pytorch.org/whl/cpu
+
+echo "---- Collecting static ------"
+python manage.py collectstatic --noinput
+
+echo "---- Apply Migrations ------"
+python manage.py migrate
+
+echo "---- FINISHED release tasks (.profile) ------"

Makefile

@@ -81,7 +81,7 @@ pip-install: ## Install pip requirements inside the container.
 	@echo "$$(tput setaf 3)🙈  Installing Pip Packages  🙈$$(tput sgr 0)"
 	@docker-compose exec web poetry lock
 	@docker-compose exec web poetry export --without-hashes -f requirements.txt -o requirements.txt
-	@docker-compose exec web poetry export --dev --without-hashes -f requirements.txt -o requirements-dev.txt
+	@docker-compose exec web poetry export --with dev --without-hashes -f requirements.txt -o requirements-dev.txt
 	@docker-compose exec web pip install -r requirements-dev.txt
 	@docker-compose exec web sed -i '1i# ======\n# DO NOT EDIT - use pyproject.toml instead!\n# Generated: $(__timestamp)\n# ======' requirements.txt
 	@docker-compose exec web sed -i '1i# ======\n# DO NOT EDIT - use pyproject.toml instead!\n# Generated: $(__timestamp)\n# ======' requirements-dev.txt

api/barriers/apps.py

@@ -1,4 +1,6 @@
 from django.apps import AppConfig
+from django.conf import settings
+from django.core.cache import cache
 
 
 class BarriersConfig(AppConfig):
@@ -48,3 +50,5 @@ def ready(self):
             public_barrier_light_touch_reviews_changed,
             sender=PublicBarrierLightTouchReviews,
         )
+
+        cache.delete(settings.BARRIER_SIMILARITY_MATRIX_CACHE_KEY)

api/barriers/models.py

@@ -23,9 +23,6 @@
 from notifications_python_client.notifications import NotificationsAPIClient
 from simple_history.models import HistoricalRecords
 
-from api.barriers import validators
-from api.barriers.report_stages import REPORT_CONDITIONS, report_stage_status
-from api.barriers.utils import random_barrier_reference
 from api.collaboration import models as collaboration_models
 from api.commodities.models import Commodity
 from api.commodities.utils import format_commodity_code
@@ -55,6 +52,10 @@
     PublicBarrierStatus,
 )
 
+from . import validators
+from .report_stages import REPORT_CONDITIONS, report_stage_status
+from .utils import random_barrier_reference
+
 logger = logging.getLogger(__name__)
 
 MAX_LENGTH = settings.CHAR_FIELD_MAX_LENGTH

api/barriers/signals/handlers.py

@@ -1,7 +1,7 @@
 import logging
 
 from django.conf import settings
-from django.db.models.signals import post_save
+from django.db.models.signals import post_save, pre_save
 from django.dispatch import receiver
 from notifications_python_client.notifications import NotificationsAPIClient
 
@@ -21,6 +21,7 @@
     PublicBarrierLightTouchReviews,
 )
 from api.metadata.constants import TOP_PRIORITY_BARRIER_STATUS
+from api.related_barriers import service as related_barrier_service
 
 logger = logging.getLogger(__name__)
 
@@ -350,6 +351,24 @@ def barrier_completion_top_priority_barrier_resolved(
         )
 
 
+@receiver(pre_save, sender=Barrier)
+def barrier_update_similarity_scores(sender, instance, *args, **kwargs):
+    try:
+        current_barrier_object = sender.objects.get(pk=instance.pk)
+    except sender.DoesNotExist:
+        pass  # the barrier is new, we handle this elsewhere
+    else:
+        changed = any(
+            getattr(current_barrier_object, field) != getattr(instance, field)
+            for field in related_barrier_service.RELEVANT_BARRIER_FIELDS
+        )
+        if changed and not current_barrier_object.draft:
+            similarity_score_matrix = (
+                related_barrier_service.SimilarityScoreMatrix.retrieve_matrix()
+            )
+            similarity_score_matrix.update_matrix(instance)
+
+
 def barrier_changed_after_published(sender, instance, **kwargs):
     try:
         obj = sender.objects.get(pk=instance.pk)

api/barriers/utils.py

@@ -6,14 +6,15 @@
 CHARSET = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"
 
 
-def random_barrier_reference():
+def random_barrier_reference() -> str:
+
     """
     function to produce a random reference number for barriers
     format: B-YY-XXXX
     where YY is year and Xs are random alpha-numerics
     """
     dd = datetime.datetime.now()
     ref_code = f"B-{str(dd.year)[-2:]}-"
-    for i in range(settings.REF_CODE_LENGTH):
+    for _ in range(settings.REF_CODE_LENGTH):
         ref_code += CHARSET[randrange(0, len(CHARSET))]
     return ref_code

api/related_barriers/apps.py

@@ -0,0 +1,5 @@
+from django.apps import AppConfig
+
+
+class RelatedBarriersConfig(AppConfig):
+    name = "api.related_barriers"

api/related_barriers/management/commands/compute_related_barrier_similarity_scores.py

@@ -0,0 +1,10 @@
+from django.core.management import BaseCommand
+
+from api.related_barriers.service import SimilarityScoreMatrix
+
+
+class Command(BaseCommand):
+    help = "Compute and store all barrier similarity scores"
+
+    def handle(self, *args, **options):
+        SimilarityScoreMatrix.create_matrix()

api/related_barriers/serializers.py

@@ -0,0 +1,15 @@
+from rest_framework import serializers
+
+from api.barriers.fields import StatusField
+
+
+# TODO : standard list serialiser may suffice and the following not required base on final designs
+class BarrierRelatedListSerializer(serializers.Serializer):
+    summary = serializers.CharField(read_only=True)
+    title = serializers.CharField(read_only=True)
+    id = serializers.UUIDField(read_only=True)
+    reported_on = serializers.DateTimeField(read_only=True)
+    modified_on = serializers.DateTimeField(read_only=True)
+    status = StatusField(required=False)
+    location = serializers.CharField(read_only=True)
+    similarity = serializers.FloatField(read_only=True)

api/related_barriers/service.py

@@ -0,0 +1,195 @@
+import os
+
+import numpy as np
+import pandas as pd
+from django.conf import settings
+from django.core.cache import cache
+from django.db.models import CharField, QuerySet
+from django.db.models import Value as V
+from django.db.models.functions import Concat
+from sentence_transformers import SentenceTransformer, util
+from typing_extensions import Self
+
+from api.barriers.models import Barrier
+
+SIMILARITY_THRESHOLD = 0.19
+SIMILAR_BARRIERS_LIMIT = 5
+
+# https://www.sbert.net/docs/pretrained_models.html
+# Load the sentence transformer model
+# albert-small-v2 # 43 MB size
+# smaller model
+transformer_model = SentenceTransformer("paraphrase-MiniLM-L3-v2")
+
+RELEVANT_BARRIER_FIELDS = [
+    "title",
+    "summary",
+]
+
+
+class SimilarityScoreMatrix(pd.DataFrame):
+    similarity_score_df_path = os.path.join(
+        settings.ROOT_DIR, "barrier_similarity_scores_df.json"
+    )
+
+    @property
+    def base_class_view(self):
+        # use this to view the base class, needed for debugging in some IDEs.
+        return pd.DataFrame(self)
+
+    @property
+    def _constructor(self):
+        return SimilarityScoreMatrix
+
+    @staticmethod
+    def get_annotated_barrier_queryset(barrier_ids: list = None):
+        if barrier_ids:
+            starting_queryset = Barrier.objects.filter(id__in=barrier_ids)
+        else:
+            starting_queryset = Barrier.objects.all()
+
+        annotated_queryset = (
+            starting_queryset.exclude(draft=True)
+            .annotate(
+                barrier_corpus=Concat(
+                    "title", V(". "), "summary", output_field=CharField()
+                )
+            )
+            .values("id", "barrier_corpus")
+        )
+        if barrier_ids:
+            annotated_queryset = annotated_queryset.filter(id__in=barrier_ids)
+        return annotated_queryset
+
+    @classmethod
+    def create_matrix(cls) -> Self:
+        """Create a similarity scores matrix for all barriers.
+
+        The similarity scores matrix is a square matrix where the rows and columns are barrier ids. And the values
+        at each intersection is the similarity score between the two barriers.
+
+        The similarity score matrix is then saved to a json file and cached."""
+        barriers = cls.get_annotated_barrier_queryset()
+        barrier_ids = [str(barrier["id"]) for barrier in barriers]
+        barrier_corpuses = [barrier["barrier_corpus"] for barrier in barriers]
+        barrier_embeddings = transformer_model.encode(
+            barrier_corpuses, convert_to_tensor=True
+        )
+        cosine_scores = util.cos_sim(barrier_embeddings, barrier_embeddings)
+        new_matrix = cls(
+            cosine_scores.numpy(),
+            index=barrier_ids,
+            columns=barrier_ids,
+        )
+
+        return new_matrix.save_matrix()  # saving to JSON and caching
+
+    @classmethod
+    def retrieve_matrix(cls) -> Self:
+        """Retrieve the similarity scores matrix from the cache or from the json file if it exists."""
+        if similarity_score_json := cache.get(
+            settings.BARRIER_SIMILARITY_MATRIX_CACHE_KEY
+        ):
+            return cls(pd.read_json(similarity_score_json))  # retrieving from cache
+        # else:
+        #     if os.path.isfile(cls.similarity_score_df_path):
+        #         with open(cls.similarity_score_df_path, "r") as f:
+        #             dataframe_json = json.load(f)
+        #             cache.set(  # caching the matrix, so it can be retrieved from the cache next time
+        #                 settings.BARRIER_SIMILARITY_MATRIX_CACHE_KEY,
+        #                 dataframe_json,
+        #                 timeout=None,
+        #             )
+        #             return cls(pd.read_json(dataframe_json))
+        else:
+            return cls.create_matrix()  # creating the matrix if it doesn't exist
+
+    def update_matrix(self, barrier_object) -> Self:
+        """Given a barrier, update the similarity scores matrix column for that barrier.
+
+        Similarity scores are computed using cosine similarity between the barrier embeddings.
+        """
+        barrier_id = str(barrier_object.id)
+
+        if barrier_id not in self.columns:
+            return self.add_barrier(barrier_object)
+
+        barrier_ids = self.index.tolist()
+        print("************ barrier ids :", barrier_ids)
+        annotated_barrier_queryset = self.get_annotated_barrier_queryset(barrier_ids)
+        barrier_corpuses = [
+            barrier["barrier_corpus"] for barrier in annotated_barrier_queryset
+        ]
+        barrier_corpus = barrier_object.title + ". " + barrier_object.summary
+        this_barrier_embedding = transformer_model.encode(
+            barrier_corpus, convert_to_tensor=True
+        )
+        # TODO: We should store the barrier embeddings rather then regenerate on every update
+        # potential race/load conditions here where multiple barriers are updated at the same time
+        all_barrier_embeddings = transformer_model.encode(
+            barrier_corpuses, convert_to_tensor=True
+        )
+        # print("barrier embeddings : ", barrier_embeddings)
+        # # this_barrier_embedding = annotated_barrier_queryset.get(id=barrier_id)[
+        # #     "barrier_corpus"
+        # # ]
+        # print("++++++++ barrier embedding :", this_barrier_embedding)
+        similarity_scores = util.cos_sim(this_barrier_embedding, all_barrier_embeddings)
+        self[barrier_id] = similarity_scores.numpy()[0]
+
+        return self.save_matrix()
+
+    def save_matrix(self) -> Self:
+        """Save the similarity scores matrix to a JSON file and caching it."""
+
+        # converting to string for JSON serialisation
+        dataframe_json = self.to_json(default_handler=str)
+        cache.set(
+            settings.BARRIER_SIMILARITY_MATRIX_CACHE_KEY, dataframe_json, timeout=None
+        )
+        # with open(self.similarity_score_df_path, "w") as f:
+        #     json.dump(dataframe_json, f)
+
+        return self
+
+    def add_barrier(self, barrier_object) -> Self:
+        """Add a barrier to the similarity scores matrix.
+
+        This is made a little difficult by Pandas, so we create a new matrix with the new barrier and then return that
+        instead."""
+        barrier_id = str(barrier_object.id)
+
+        self[barrier_id] = np.nan  # creating the empty column
+        new_row = {}
+        for barrier in self:
+            new_row[barrier] = np.nan
+        new_row = pd.Series(new_row, name=barrier_id)
+
+        new_matrix = pd.concat([self, pd.DataFrame([new_row], columns=new_row.index)])
+        return new_matrix.update_matrix(barrier_object)
+
+    def retrieve_similar_barriers(
+        self,
+        barrier_object,
+        limit=SIMILAR_BARRIERS_LIMIT,
+        threshold=SIMILARITY_THRESHOLD,
+    ) -> QuerySet:
+        """Retrieve similar barriers for a given barrier object."""
+        barrier_id = str(barrier_object.id)
+
+        if barrier_id not in self.columns:
+            new_matrix = self.add_barrier(barrier_object)
+            return new_matrix.retrieve_similar_barriers(
+                barrier_object, limit, threshold
+            )
+
+        barrier_similarity_scores = (
+            self[barrier_id].sort_values(ascending=False)[:limit].drop(barrier_id)
+        )
+        barrier_similarity_scores = barrier_similarity_scores[
+            barrier_similarity_scores > threshold
+        ]
+        barrier_queryset = Barrier.objects.filter(
+            id__in=barrier_similarity_scores.index
+        )
+        return barrier_queryset

api/related_barriers/urls.py

@@ -0,0 +1,15 @@
+from django.urls import path
+from rest_framework.routers import DefaultRouter
+
+from api.related_barriers.views import related_barriers
+
+app_name = "related_barriers"
+
+router = DefaultRouter(trailing_slash=False)
+
+
+urlpatterns = router.urls + [
+    path(
+        "barriers/<uuid:pk>/related-barriers", related_barriers, name="related-barriers"
+    ),
+]