Merge branch 'develop' into feature/update-share-feature

.gitignore

@@ -5,3 +5,4 @@ cloud-harness/
 .vscode/
 node_modules
 secret.json
+data/

applications/visualizer/api/openapi.json

@@ -520,6 +520,20 @@
             "title": "Slicerange",
             "type": "array"
           },
+          "maxResolution": {
+            "maxItems": 2,
+            "minItems": 2,
+            "prefixItems": [
+              {
+                "type": "integer"
+              },
+              {
+                "type": "integer"
+              }
+            ],
+            "title": "Maxresolution",
+            "type": "array"
+          },
           "resourceUrl": {
             "title": "Resourceurl",
             "type": "string"
@@ -574,6 +588,7 @@
           "nbSlices",
           "tileSize",
           "sliceRange",
+          "maxResolution",
           "resourceUrl",
           "segmentationUrl",
           "segmentationSize",

applications/visualizer/api/openapi.yaml

@@ -84,6 +84,14 @@ components:
       type: object
     EMData:
       properties:
+        maxResolution:
+          maxItems: 2
+          minItems: 2
+          prefixItems:
+          - type: integer
+          - type: integer
+          title: Maxresolution
+          type: array
         maxZoom:
           title: Maxzoom
           type: integer
@@ -138,6 +146,7 @@ components:
       - nbSlices
       - tileSize
       - sliceRange
+      - maxResolution
       - resourceUrl
       - segmentationUrl
       - segmentationSize

applications/visualizer/backend/.gitignore

@@ -406,4 +406,7 @@ poetry.toml
 # LSP config files
 pyrightconfig.json
 
-# End of https://www.toptal.com/developers/gitignore/api/node,python,django
+# End of https://www.toptal.com/developers/gitignore/api/node,python,django
+
+
+static/

applications/visualizer/backend/api/api.py

@@ -1,12 +1,18 @@
+from io import StringIO
+import sys
 from collections import defaultdict
 from typing import Iterable, Optional
 
 from ninja import NinjaAPI, Router, Query, Schema
 from ninja.pagination import paginate, PageNumberPagination
+from ninja.errors import HttpError
+
 from django.shortcuts import aget_object_or_404
 from django.db.models import Q
 from django.db.models.manager import BaseManager
 from django.conf import settings
+from django.core.management import call_command
+
 
 from .utils import get_dataset_viewer_config, to_list
 
@@ -16,7 +22,9 @@
     Neuron as NeuronModel,
     Connection as ConnectionModel,
 )
+from .decorators.streaming import with_stdout_streaming
 from .services.connectivity import query_nematode_connections
+from .authenticators.basic_auth_super_user import basic_auth_superuser
 
 
 class ErrorMessage(Schema):
@@ -237,6 +245,23 @@ def get_connections(
 #     )
 
 
+## Ingestion
+
+
+@api.get("/populate_db", auth=basic_auth_superuser, tags=["ingestion"])
+@with_stdout_streaming
+def populate_db(request):
+    try:
+        print("Starting DB population...\n")
+        call_command("migrate")
+        call_command("populatedb")
+    except Exception as e:
+        raise HttpError(500)
+
+
+## Healthcheck
+
+
 @api.get("/live", tags=["healthcheck"])
 async def live(request):
     """Test if application is healthy"""

applications/visualizer/backend/api/authenticators/basic_auth_super_user.py

@@ -0,0 +1,14 @@
+from ninja.security import HttpBasicAuth
+from django.contrib.auth import authenticate as django_authenticate
+
+
+class BasicAuthSuperUser(HttpBasicAuth):
+    def authenticate(self, request, username, password):
+        # Authenticate user with Django's built-in authenticate function
+        user = django_authenticate(request, username=username, password=password)
+        if user and user.is_superuser:  # Ensure the user is a superuser
+            return user
+        return None
+
+
+basic_auth_superuser = BasicAuthSuperUser()

applications/visualizer/backend/api/decorators/streaming.py

@@ -0,0 +1,63 @@
+import asyncio
+import sys
+import threading
+from queue import Queue
+from functools import wraps
+from django.http import StreamingHttpResponse
+
+
+def with_stdout_streaming(func):
+    """
+    A decorator that:
+    - Runs the decorated function in a separate thread,
+    - Captures anything it prints to stdout,
+    - Streams that output asynchronously line-by-line as it's produced.
+    """
+
+    @wraps(func)
+    def wrapper(request, *args, **kwargs):
+        q = Queue()
+
+        def run_func():
+            # Redirect sys.stdout
+            old_stdout = sys.stdout
+
+            class QueueWriter:
+                def write(self, data):
+                    if data:
+                        q.put(data)
+
+                def flush(self):
+                    pass  # For compatibility with print
+
+            sys.stdout = QueueWriter()
+
+            try:
+                func(request, *args, **kwargs)
+            except Exception as e:
+                q.put(f"Error: {e}\n")
+            finally:
+                # Signal completion
+                q.put(None)
+                sys.stdout = old_stdout
+
+        # Run the function in a background thread
+        t = threading.Thread(target=run_func)
+        t.start()
+
+        # Async generator to yield lines from the queue
+        async def line_generator():
+            while True:
+                line = await asyncio.to_thread(q.get)
+                if line is None:  # End signal
+                    break
+                yield line
+
+        # Return a streaming response that sends data asynchronously
+        return StreamingHttpResponse(
+            line_generator(),
+            content_type="text/plain",
+            headers={"Content-Encoding": "identity"},
+        )
+
+    return wrapper

applications/visualizer/backend/api/schemas.py

@@ -27,6 +27,7 @@ class EMData(BilingualSchema):
     nb_slices: int
     tile_size: tuple[int, int]
     slice_range: tuple[int, int]
+    max_resolution: tuple[int, int] | None  # EM tiles maximum resolution
     resource_url: str
     segmentation_url: str | None
     segmentation_size: tuple[int, int] | None

applications/visualizer/backend/api/utils.py

@@ -3,6 +3,10 @@
 from .schemas import EMData
 from .models import Dataset
 
+import logging
+
+logger = logging.getLogger(__name__)
+
 
 ## Some util functions
 async def to_list(q):
@@ -15,25 +19,33 @@ async def get_dataset_viewer_config(dataset: Dataset):
         return None
     em_metadata = config.em_config
     segmentation_metadata = config.segmentation_config
-    resolution = segmentation_metadata.get("resolution")
+    segmentation_resolution = segmentation_metadata.get("resolution")
+    tiled_img_resolution = em_metadata.get("resolution")
+    if not tiled_img_resolution and not segmentation_resolution:
+        logger.warning(
+            f"There is no img resolution computed from the tiles for the dataset '{dataset.id}', there is probably missing information in your metadata file"
+        )
     return EMData(
         min_zoom=em_metadata.get("minzoom"),
         max_zoom=em_metadata.get("maxzoom"),
         nb_slices=em_metadata.get("number_slices"),
         tile_size=tuple(em_metadata.get("tile_size")),
         slice_range=tuple(em_metadata.get("slice_range")),
-        segmentation_size=tuple(resolution) if resolution else None,
+        max_resolution=tuple(tiled_img_resolution) if tiled_img_resolution else None,
+        segmentation_size=(
+            tuple(segmentation_resolution) if segmentation_resolution else None
+        ),
         resource_url=settings.DATASET_EMDATA_URL_FORMAT.format(dataset=dataset.id),
         segmentation_url=(
             settings.DATASET_EMDATA_SEGMENTATION_URL_FORMAT.format(dataset=dataset.id)
-            if resolution
+            if segmentation_resolution
             else None
         ),
         synapses_segmentation_url=(
             settings.DATASET_EMDATA_SYNAPSES_SEGMENTATION_URL_FORMAT.format(
                 dataset=dataset.id
             )
-            if resolution
+            if segmentation_resolution
             else None
         ),
     )

applications/visualizer/backend/openapi/openapi.json

@@ -520,6 +520,20 @@
             "title": "Slicerange",
             "type": "array"
           },
+          "maxResolution": {
+            "maxItems": 2,
+            "minItems": 2,
+            "prefixItems": [
+              {
+                "type": "integer"
+              },
+              {
+                "type": "integer"
+              }
+            ],
+            "title": "Maxresolution",
+            "type": "array"
+          },
           "resourceUrl": {
             "title": "Resourceurl",
             "type": "string"
@@ -574,6 +588,7 @@
           "nbSlices",
           "tileSize",
           "sliceRange",
+          "maxResolution",
           "resourceUrl",
           "segmentationUrl",
           "segmentationSize",

applications/visualizer/backend/visualizer/settings/production.py

@@ -135,14 +135,20 @@ def _pull_em_metadata(self, dataset_id):
 
     @classmethod
     def get_segmentation_metadata(cls, dataset_id):
-        file = BASE_DIR / DB_RAW_DATA_FOLDER / dataset_id / "segmentation_metadata.json"
+        file = (
+            BASE_DIR
+            / DB_RAW_DATA_FOLDER
+            / dataset_id
+            / "segmentations"
+            / "metadata.json"
+        )
         if not file.exists():
             return {}
         return json.loads(file.read_text())
 
     @classmethod
     def get_em_metadata(cls, dataset_id):
-        file = BASE_DIR / DB_RAW_DATA_FOLDER / dataset_id / "em_metadata.json"
+        file = BASE_DIR / DB_RAW_DATA_FOLDER / dataset_id / "em" / "metadata.json"
         if not file.exists():
             return {}
         return json.loads(file.read_text())

applications/visualizer/backend/visualizer/views.py

@@ -35,7 +35,7 @@ def index(request, path="", already_asked=False):
 BLACK_TILE = Image.new("RGB", (TILE_SIZE, TILE_SIZE))
 BLACK_TILE_BUFFER = io.BytesIO()
 BLACK_TILE.save(BLACK_TILE_BUFFER, format="JPEG")
-MAX_ZOOM = 6  # Should be set
+MAX_ZOOM = 5  # Should be set
 
 
 def get_tile(request, dataset, slice, x, y, zoom):