Feature/tss-1064-display-related-barriers

.gitignore

@@ -48,7 +48,7 @@ coverage.xml
 .hypothesis/
 /codecov.sh
 /test-reports/
-/.pytest_cache/
+*/.pytest_cache/
 
 # Translations
 *.mo

.profile

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+# custom initialisation tasks
+# ref - https://docs.cloudfoundry.org/devguide/deploy-apps/deploy-app.html
+
+echo "---- RUNNING release tasks (.profile) ------"
+
+echo "---- Installing Related Barrier ML Packages ------"
+/tmp/lifecycle/shell
+python -m pip install sentence-transformers==2.2.2 --no-deps
+python -m pip install torch==2.0.0 torchvision==0.15.1 --extra-index-url https://download.pytorch.org/whl/cpu
+
+echo "---- Collecting static ------"
+python manage.py collectstatic --noinput
+
+echo "---- Apply Migrations ------"
+python manage.py migrate
+
+echo "---- FINISHED release tasks (.profile) ------"

Makefile

@@ -81,7 +81,7 @@ pip-install: ## Install pip requirements inside the container.
 	@echo "$$(tput setaf 3)🙈  Installing Pip Packages  🙈$$(tput sgr 0)"
 	@docker-compose exec web poetry lock
 	@docker-compose exec web poetry export --without-hashes -f requirements.txt -o requirements.txt
-	@docker-compose exec web poetry export --dev --without-hashes -f requirements.txt -o requirements-dev.txt
+	@docker-compose exec web poetry export --with dev --without-hashes -f requirements.txt -o requirements-dev.txt
 	@docker-compose exec web pip install -r requirements-dev.txt
 	@docker-compose exec web sed -i '1i# ======\n# DO NOT EDIT - use pyproject.toml instead!\n# Generated: $(__timestamp)\n# ======' requirements.txt
 	@docker-compose exec web sed -i '1i# ======\n# DO NOT EDIT - use pyproject.toml instead!\n# Generated: $(__timestamp)\n# ======' requirements-dev.txt

api/barriers/related_barrier.py

@@ -0,0 +1,75 @@
+import numpy as np
+import pandas as pd
+from django.db.models import QuerySet
+from sentence_transformers import SentenceTransformer, util
+
+SIMILARITY_THRESHOLD = 0.19
+
+# https://www.sbert.net/docs/pretrained_models.html
+# Load the sentence transformer model
+# albert-small-v2 # 43 MB size
+# smaller model
+transformer_model = SentenceTransformer("paraphrase-MiniLM-L3-v2")
+
+
+def query_set_to_pandas_df(queryset: QuerySet) -> pd.DataFrame:
+    """
+    function to convert a django query set to a pandas dataframe
+    """
+
+    return pd.DataFrame.from_records(queryset.values())
+
+
+def __get_similar_barriers(
+    barrier_row: pd.DataFrame, barrier_id: str, df: pd.DataFrame, limit: int
+) -> pd.DataFrame:
+    """
+    function to get similar barriers based on cosine similarity
+    """
+
+    # Obtain embeddings for each processed_text
+    df["embeddings"] = df["barrier_corpus"].apply(
+        lambda x: transformer_model.encode(x, convert_to_tensor=True)
+    )
+
+    # Create a matrix of the embeddings
+    embeddings_matrix = np.vstack(df["embeddings"].to_numpy())
+
+    # Obtain embeddings for title
+    barrier_embeddings = transformer_model.encode(
+        barrier_row["barrier_corpus"].values[0]
+    )
+
+    # Calculate cosine similarity between title and all other processed_text
+    cosine_scores = util.cos_sim(barrier_embeddings, embeddings_matrix)[0]
+
+    # Add cosine scores to dataframe
+    df["similarity"] = cosine_scores
+
+    # Sort dataframe by cosine scores
+    df = df[df["similarity"] > SIMILARITY_THRESHOLD].sort_values(
+        by=["similarity"], ascending=False
+    )
+
+    # removing the barrier itself from the dataframe
+    df = df[df["id"] != barrier_id]
+
+    # trimming the dataframe according to the defined limit
+    return df.head(limit)
+
+
+def get_similar_barriers(
+    values_query_set: QuerySet, barrier_id: str, limit: int
+) -> pd.DataFrame:
+    df = query_set_to_pandas_df(values_query_set)
+
+    # Getting the barrier row from the dataframe
+    barrier_row = df[df["id"] == barrier_id].copy()
+
+    # Check if title exists
+    if barrier_row.empty:
+        raise ValueError("Barrier ID not found in data")
+
+    df = __get_similar_barriers(barrier_row, barrier_id, df, limit)
+
+    return df

api/barriers/serializers/__init__.py

@@ -1,4 +1,8 @@
-from .barriers import BarrierDetailSerializer, BarrierListSerializer  # noqa
+from .barriers import (  # noqa
+    BarrierDetailSerializer,
+    BarrierListSerializer,
+    BarrierRelatedListSerializer,
+)
 from .csv import BarrierCsvExportSerializer  # noqa
 from .data_workspace import DataWorkspaceSerializer  # noqa
 from .progress_updates import ProgressUpdateSerializer  # noqa

api/barriers/serializers/barriers.py

@@ -133,3 +133,15 @@ def get_current_valuation_assessment(instance):
             return f"{rating}"
         else:
             return None
+
+
+# TODO : standard list serialiser may suffice and the following not required base on final designs
+class BarrierRelatedListSerializer(serializers.Serializer):
+    summary = serializers.CharField(read_only=True)
+    title = serializers.CharField(read_only=True)
+    id = serializers.UUIDField(read_only=True)
+    reported_on = serializers.DateTimeField(read_only=True)
+    modified_on = serializers.DateTimeField(read_only=True)
+    status = StatusField(required=False)
+    location = serializers.CharField(read_only=True)
+    similarity = serializers.FloatField(read_only=True)

api/barriers/urls.py

@@ -24,6 +24,7 @@
     PublicBarrierActivity,
     PublicBarrierViewSet,
     barrier_count,
+    related_barriers,
 )
 
 app_name = "barriers"
@@ -175,6 +176,9 @@
         BarrierStatusChangeUnknown.as_view(),
         name="unknown-barrier",
     ),
+    path(
+        "barriers/<uuid:pk>/related-barriers", related_barriers, name="related-barriers"
+    ),
     path("counts", barrier_count, name="barrier-count"),
     path("reports", BarrierReportList.as_view(), name="list-reports"),
     path("reports/<uuid:pk>", BarrierReportDetail.as_view(), name="get-report"),

api/barriers/utils.py

@@ -6,14 +6,15 @@
 CHARSET = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"
 
 
-def random_barrier_reference():
+def random_barrier_reference() -> str:
+
     """
     function to produce a random reference number for barriers
     format: B-YY-XXXX
     where YY is year and Xs are random alpha-numerics
     """
     dd = datetime.datetime.now()
     ref_code = f"B-{str(dd.year)[-2:]}-"
-    for i in range(settings.REF_CODE_LENGTH):
+    for _ in range(settings.REF_CODE_LENGTH):
         ref_code += CHARSET[randrange(0, len(CHARSET))]
     return ref_code

api/barriers/views.py

@@ -5,7 +5,11 @@
 
 from dateutil.parser import parse
 from django.db import transaction
-from django.db.models import Case, CharField, Count, F, Value, When
+from django.db.models import Case, CharField, Count, F
+from django.db.models import Value
+from django.db.models import Value as V
+from django.db.models import When
+from django.db.models.functions import Concat
 from django.http import JsonResponse, StreamingHttpResponse
 from django.shortcuts import get_object_or_404
 from django.utils import timezone
@@ -34,6 +38,7 @@
     BarrierCsvExportSerializer,
     BarrierDetailSerializer,
     BarrierListSerializer,
+    BarrierRelatedListSerializer,
     BarrierReportSerializer,
     PublicBarrierSerializer,
 )
@@ -66,6 +71,7 @@
 
 from .models import BarrierFilterSet, BarrierProgressUpdate, PublicBarrierFilterSet
 from .public_data import public_release_to_s3
+from .related_barrier import get_similar_barriers
 from .tasks import generate_s3_and_send_email
 
 logger = logging.getLogger(__name__)
@@ -1199,3 +1205,23 @@ def update(self, request, *args, **kwargs):
             instance._prefetched_objects_cache = {}
 
         return Response(serializer.data)
+
+
+@api_view(["GET"])
+def related_barriers(request, pk) -> Response:
+    """
+    Return a list of related barriers
+    """
+    values_query_set = (
+        Barrier.objects.exclude(draft=True)
+        .annotate(
+            barrier_corpus=Concat("title", V(". "), "summary", output_field=CharField())
+        )
+        .values("id", "barrier_corpus")
+    )
+
+    related_barriers_df = get_similar_barriers(values_query_set, pk, limit=20)
+    serializer = BarrierRelatedListSerializer(
+        related_barriers_df.to_dict("records"), many=True
+    )
+    return Response(serializer.data)

docker/local/Dockerfile

@@ -28,3 +28,6 @@ RUN pip install --upgrade pip \
     && poetry export --without-hashes -f requirements.txt -o requirements.txt \
     && poetry export --dev --without-hashes -f requirements.txt -o requirements-dev.txt \
     && pip install -r requirements-dev.txt
+
+# install sentence-transformers model for similarity search
+RUN python -c 'from sentence_transformers import SentenceTransformer; model = SentenceTransformer("paraphrase-MiniLM-L3-v2"); model.save(".models")'