Feature/186 (#210)

* #186 feat: Add support for multiple populations in one csv
* #186 feat: Fix split cells per segment
* #186 fix: Fix get cells
* #186 feat: Update create experiment dialog
* #186 chore: Update port forward script

applications/portal/backend/api/helpers/generate_population_cells.py

@@ -4,17 +4,16 @@
 from cordmap.main import register_sections_3D
 
 from api.services.filesystem_service import move_file
-from api.utils import get_persistence_path
 
 OUTPUT_DIRECTORY = "./out"
-CELLS_CSV = "transformed_cells"
+CELLS_CSV = "cells_with_labels"
 
 
-def get_cells_filepath(data_filepath, storage_path):
+def get_cells_filepath(population_name: str, data_filepath: str, storage_path: str):
     out_dir = os.path.join(os.path.dirname(data_filepath), OUTPUT_DIRECTORY)
     register_sections_3D(data_filepath, out_dir)
     filepath = move_file(
-        os.path.join(out_dir, f"{CELLS_CSV}.csv"),
+        os.path.join(out_dir, f"{CELLS_CSV}_{population_name}.csv"),
         storage_path,
         f"{CELLS_CSV}_{str(uuid.uuid4())[:8]}.csv",
     )

applications/portal/backend/api/models/population.py

@@ -110,7 +110,7 @@ def generate_cells(self, data_filepath: str):
         self.status = PopulationStatus.RUNNING
         self.save()
         try:
-            self.cells.name = get_cells_filepath(data_filepath, self.storage_path)
+            self.cells.name = get_cells_filepath(self.name, data_filepath, self.storage_path)
         except Exception as e:
             logging.exception(e)
             self.status = PopulationStatus.ERROR

applications/portal/backend/api/serializers/experiment.py

@@ -30,7 +30,6 @@ class IntegerListField(serializers.ListField):
 
 
 class ExperimentFileUploadSerializer(serializers.Serializer):
-    population_id = serializers.IntegerField(required=False, default=None)
     key_file = serializers.FileField()
     data_file = serializers.FileField()
 

applications/portal/backend/api/services/experiment_service.py

@@ -1,6 +1,10 @@
 import os.path
+from pathlib import PosixPath
+
+from cordmap.get_all_population_keys import get_populations_from_file
+from cordmap.register.constants import population_ignore_set
+from django.conf import settings
 
-from api.helpers.exceptions import InvalidInputError
 from api.models import Experiment, Population, Tag
 from api.services.workflows_service import execute_generate_population_cells_workflow
 
@@ -27,24 +31,11 @@ def delete_tag(experiment: Experiment, tag_name: str):
     return True
 
 
-def upload_files(experiment: Experiment, data_filepath: str, population_id: int):
-    try:
-        population_name = os.path.basename(data_filepath).split("_Data")[0]
-    except Exception:
-        raise InvalidInputError
-
-    created = False
-    if population_id:
-        try:
-            population = Population.objects.get(
-                experiment_id=experiment.id, id=population_id
-            )
-        except Population.DoesNotExist:
-            raise InvalidInputError
-    else:
+def upload_files(experiment: Experiment, key_filepath: str, data_filepath: str):
+    key_path = PosixPath(os.path.join(settings.PERSISTENT_ROOT, key_filepath))
+    populations = get_populations_from_file(key_path, population_ignore_set)
+    for name in populations:
         population = Population.objects.create(
-            experiment_id=experiment.id, name=population_name
+            experiment_id=experiment.id, name=name
         )
-        created = True
-    execute_generate_population_cells_workflow(population.id, data_filepath)
-    return created
+        execute_generate_population_cells_workflow(population.id, data_filepath)

applications/portal/backend/api/services/population_service.py

@@ -31,11 +31,11 @@ def split_cells_per_segment(population):
             writer = csv.writer(file)
             file_writer_dict[s] = {"file": file, "writer": writer}
         with open(population.cells.path) as cells_file:
-            reader = csv.reader(cells_file)
+            reader = csv.DictReader(cells_file)
             for row in reader:
-                cell_depth = float(row[0])
+                cell_depth = float(row['z'])
                 subdivision = get_cell_segment(cell_depth)
-                file_writer_dict[subdivision]["writer"].writerow(row)
+                file_writer_dict[subdivision]["writer"].writerow([row['z'], row['x'], row['y']])
     except Exception:
         pass
     finally:
@@ -47,9 +47,9 @@ def get_cells(subdivision, populations):
     cells = []
     for pop in populations:
         with open(
-            pop.get_subdivision_storage_path(
-                subdivision, PopulationPersistentFiles.CSV_FILE
-            )
+                pop.get_subdivision_storage_path(
+                    subdivision, PopulationPersistentFiles.CSV_FILE
+                )
         ) as cells_file:
             reader = csv.reader(cells_file)
             for row in reader:
@@ -100,12 +100,12 @@ def generate_images(population):
 
 
 def _store_image(
-    creator: IPopulationImageCreator,
-    extension: PopulationPersistentFiles,
-    bg_atlas,
-    cells,
-    population,
-    s,
+        creator: IPopulationImageCreator,
+        extension: PopulationPersistentFiles,
+        bg_atlas,
+        cells,
+        population,
+        s,
 ):
     img = creator.create(bg_atlas=bg_atlas, subdivision=s, points=cells)
     img.save(population.get_subdivision_storage_path(s, extension))

applications/portal/backend/api/views/rest/experiment.py

@@ -22,6 +22,7 @@
 log = logging.getLogger("__name__")
 
 DATA_INDEX = 1
+KEY_INDEX = 0
 
 
 class ExperimentViewSet(viewsets.ModelViewSet):
@@ -122,15 +123,13 @@ def upload_files(self, request, **kwargs):
             validate_input_files(key_file, data_file)
         except InvalidInputError:
             return Response(status=status.HTTP_400_BAD_REQUEST)
-
-        population_id = request.FILES.get("population_id", None)
         try:
             dir_path = create_temp_dir(CORDMAP_DATA)
             filepaths = move_files([key_file, data_file], dir_path)
         except Exception as e:
             return Response(status=status.HTTP_500_INTERNAL_SERVER_ERROR)
         try:
-            created = upload_files(instance, filepaths[DATA_INDEX], population_id)
+            created = upload_files(instance, filepaths[KEY_INDEX], filepaths[DATA_INDEX])
             response_status = status.HTTP_201_CREATED if created else status.HTTP_200_OK
         except InvalidInputError:
             response_status = status.HTTP_400_BAD_REQUEST