Update cordmap to latest sow4 (#366)

* chore: Update latest cordmap (e9e12e6)

* chore: Add cordmap column swap back

applications/portal/cordmap/cordmap/entry_points/register_3D_fiducial.py

@@ -70,11 +70,11 @@ def register_3D_fiducial(
     if save:
         labeled_cells = labeled_cells.rename(columns={"x": "y", "y": "x"})  # swap x and y
         save_output(
-            output_directory,
-            labeled_cells,
-            transformed_cells,
-            save_csv=save_csv,
-        )
+                output_directory,
+                labeled_cells,
+                transformed_cells,
+                save_csv=save_csv,
+            )
 
 
 def main():

applications/portal/cordmap/cordmap/main.py

@@ -16,6 +16,7 @@
 from cordmap.data.io import load_data, save_output
 from cordmap.data.utils import filter_data_by_type, get_data_segments
 from cordmap.get_all_population_keys import get_populations_from_file
+from cordmap.napari.vis import visualise_results
 from cordmap.postprocessing.prob_map import generate_prob_map
 from cordmap.register.constants import population_ignore_set
 from cordmap.register.register_sections import (
@@ -260,11 +261,10 @@ def register_sections_3D(
     )
 
     if visualise:
-        # visualise_results(
-        #     annotation,
-        #     transformed_cells,
-        #     transformed_gm,
-        #     transformed_cord,
-        #     probability_map,
-        # )
-        pass
+        visualise_results(
+            annotation,
+            transformed_cells,
+            transformed_gm,
+            transformed_cord,
+            probability_map,
+        )

applications/portal/cordmap/cordmap/napari/points_reader.py

@@ -0,0 +1,25 @@
+import numpy as np
+from napari_plugin_engine import napari_hook_implementation
+
+
+@napari_hook_implementation(specname="napari_get_reader")
+def cordmap_load_points(path):
+
+    if not path.endswith(".npy"):
+        return None
+
+    return reader_function
+
+
+def reader_function(path):
+    return [
+        (
+            np.load(path),
+            {
+                "size": 2,
+                "n_dimensional": True,
+                "opacity": 0.8,
+            },
+            "points",
+        )
+    ]

applications/portal/cordmap/cordmap/napari/vis.py

@@ -0,0 +1,31 @@
+import napari
+
+
+def visualise_results(
+    annotation,
+    transformed_cells,
+    transformed_gm,
+    transformed_cord,
+    probability_map,
+):
+    viewer = napari.Viewer()
+    viewer.add_labels(annotation)
+    if probability_map is not None:
+        viewer.add_image(
+            probability_map,
+            opacity=0.6,
+            colormap="inferno",
+        )
+    viewer.add_points(
+        transformed_cells,
+        size=2,
+        name="Warped cells",
+    )
+    viewer.add_points(
+        transformed_gm,
+        size=2,
+        name="Warped GM boundary",
+    )
+    viewer.add_points(transformed_cord, size=2, name="Warped cord boundary")
+
+    napari.run()

applications/portal/cordmap/cordmap/register_fiducial/io.py

@@ -99,7 +99,7 @@ def correct_raw_coordinates(markers_df_slice):
     ]["x"].values
 
 
-def load_df_preprocessed(sample, z, normalise=True, scale_factor=20):
+def load_df_preprocessed(sample, z, normalise=True):
     """
     Loads single axial slice of a cord sample and applies all
     preprocessing to prepare sample image for registration.
@@ -121,8 +121,12 @@ def get_label(point):
             return np.nan
 
     def correct_single_side_offset(df_slice_z):
-        if np.count_nonzero(df_slice_z.iloc[0:9]["x"] < 0) == 3:
-            return True
+        return all(
+            df_slice_z[df_slice_z["point"].isin(["L7", "L8", "L9"])][
+                "x"
+            ].values
+            < 0
+        )
 
     df_slice_z["label"] = [get_label(point) for point in df_slice_z["point"]]
 
@@ -132,10 +136,6 @@ def correct_single_side_offset(df_slice_z):
     if normalise:
         df_slice_z = remove_padding(df_slice_z)
 
-    if scale_factor is not None:
-        df_slice_z["x"] *= scale_factor
-        df_slice_z["y"] *= scale_factor
-
     fiducial_markers = df_slice_z[df_slice_z["point"].isin(MARKER_LABELS)]
     cells = df_slice_z[~df_slice_z["point"].isin(MARKER_LABELS)]
     cell_labels = df_slice_z[~df_slice_z["point"].isin(MARKER_LABELS)][

applications/portal/cordmap/cordmap/register_fiducial/register.py

@@ -74,11 +74,19 @@ def get_atlas_z_position(z_position_sample, segment_length=1000):
     return int(atlas_pos)
 
 
+def get_scaling_factor(markers_sample, markers_atlas):
+    x_scale = markers_atlas["x"].max() / markers_sample["x"].max()
+    y_scale = markers_atlas["y"].max() / markers_sample["y"].max()
+    return np.mean([x_scale, y_scale])
+
+
 def register_fiducial_single_section(
     atlas,
     sample,
     z_position_sample=30,
     image_padding=10,
+    T1_border=-1000,
+    C1_border=8000,
 ):
     """
     Registers a single sample section to the atlas.
@@ -89,30 +97,35 @@ def register_fiducial_single_section(
     :param image_padding:
     :return:
     """
-    if z_position_sample < -1000:  # atlas does not go beyond T1
+    if z_position_sample < T1_border:  # atlas does not go beyond T1
         return None
 
-    elif z_position_sample > 8000:  # atlas does not go beyond C1
+    elif z_position_sample > C1_border:  # atlas does not go beyond C1
         return None
 
     try:
         z_position_atlas = get_atlas_z_position(z_position_sample)
         logging.info(f"z position atlas: {z_position_atlas}")
-
-        markers, cells, cells_labels = load_df_preprocessed(
+        markers_sample, cells, cells_labels = load_df_preprocessed(
             sample,
             z_position_sample,
             normalise=True,
-            scale_factor=20,
+            # scale_factor=20,
         )
+        markers_atlas = get_atlas_slice_fiducial_raw(z_position_atlas, atlas)
 
-        fixed_image = get_as_image(markers, padding=image_padding).astype(
-            np.float32
-        )
-        sample_points = np.array(markers[["x", "y"]])
-        sample_points += image_padding
+        scale_factor = get_scaling_factor(markers_sample, markers_atlas)
 
-        markers_atlas = get_atlas_slice_fiducial_raw(z_position_atlas, atlas)
+        for item in [markers_sample, cells]:
+            item["x"] *= scale_factor
+            item["y"] *= scale_factor
+
+        fixed_image = get_as_image(
+            markers_sample, padding=image_padding
+        ).astype(np.float32)
+
+        sample_points = np.array(markers_sample[["x", "y"]])
+        sample_points += image_padding
 
         atlas_pad_x = markers_atlas["x"].min()
         atlas_pad_y = markers_atlas["y"].min()

applications/portal/cordmap/doc/cordmap_data_input_guide.md

@@ -0,0 +1,115 @@
+# Cordmap data input guide
+
+## Raw data requirements
+
+### Standard cordmap registration
+
+The raw data provided by the Salk is organised as follows:
+
+Each sample is provided as two files,  the output of a custom script called Build a Brain, 
+which was also developed in house at the Salk. Both files are .csv one of these contains 
+all the data (`cardinal_subpop_Data.csv`), the other is a key (`cardinal_subpop_Key.csv`) to 
+allow selectively loading out data of particular types.
+
+This `_Key.csv` contains a list of keys that can be used as the index of the data 
+files in the "Type" column. A single type could be a cell population, a contour type
+or a segment type, for example.
+
+Each category of data is either a contour or a point. Contours are collections of 
+points that make the entire outline of particular biological features, for example 
+the outline of the grey or white matter. Each z position from a sample would therefore 
+contain a grey matter and a white matter contour. These outlines are converted to an 
+image which is then the basis of registration. The z position is used to select the 
+appropriate 2d image from the atlas to use in this registration.
+
+This `_Key.csv` file is formatted as follows:
+
+Columns are [`objectNames`, `isPoint`, `isContour`, `Type`]
+
+`objectNames`: The label of each `Type`
+`isPoint`: whether the type is a point (e.g. cell position)
+`isContour`: whether the type is a contour (e.g. grey matter outline)
+
+The `_Data.csv` file is formatted as follows:
+
+Columns are [`Type`, `Slice`, `X`, `Y`, `Z` ,`isPoint`, `isContour`, `isPoint`,`contourID`]
+
+`Type`: The index relating to a specific kind of data (e.g. `Grey` from the key file
+would be the grey matter contour used in registration, `V1 Calbindin` would be V1 calbindin 
+positive neurons). The index of these keys is stored in the `_Key.csv` and that index can be
+used to selectively load from the `Data.csv` file.
+
+`Slice`: The slice number, which indicates all the data corresponding to a single section.
+
+`X`: The x coordinate of the point represented.
+`Y`: The y coordinate of the point represented.
+`Z`: The z coordinate of the point represented.
+`isPoint` | `isContour`: Whether the row is a point or part of a contour (e.g. a cell position
+is a point but a row of type `Grey` would be the position of a point that is actually part 
+of a contour outline of the grey matter.
+`contourID`: the index of the contour.
+
+Slices are centered around 0, so slices x coordinates are expected to contain both positive and negative values 
+depending on whether the point is to the right or left of the midline. Similarly, y coordinates will contain
+positive and negative values depending on whether the point lies above or below the midpoint.
+
+Registration is computed using the grey matter and white matter outlines and therefore these are 
+fundamentally required for registration. Any population that is defined as a point (`isPoint`=1) will be treated as
+a cell population and coordinates will be transformed to standard space and saved in the output as a cell
+population.
+
+
+### Fiducial marker registration
+
+The fiducial marker registration in cordmap is fundamentally different to standard registration. Instead
+of using the grey matter contour, a small set of points are used that are selected around the white matter
+and are used to register the sample to the atlas. 
+
+![img.png](img.png)
+
+There are 9 markers in total, three on the left, three on the right and three down the midline.
+
+     e.g.    Atlas register_fiducial markers
+     --------------------------------------
+                 L9    L5     L6
+
+             L8        L4         L1
+
+                L7     L3     L2
+    ---------------------------------------
+
+Fiducial markers for all atlas sections have been annotated on a downsampled version of the
+salk_10um cord atlas and these have been interpolated to resample back to atlas resolution. 
+Fiducial markers are needed for every coronal section that is processed.
+
+
+`register_fiducial` assumes that input data will be in a single csv file that contains, for each
+coronal section:
+
+- The z position in the sample relative to C8
+- The zxy positions for each fiducial marker in the sample L1 to L9. None of these can be missing
+- The label of each of these markers (e.g. "L1")
+
+It assumes:
+- There are no missing values for the fiducial markers.
+- There are no duplicates (i.e. each section has exactly one of each fiducial marker)
+
+Other label types in the "Point" column are treated as neuronal populations and usually have the name
+`Population1`, `Population2` etc.
+
+These values should all be positive. In the originally provided data the points on the left were negative
+and offset:
+
+![img_2.png](img_2.png)
+
+Where all 3 points (L7-L9) are found to be negative, a specific correction is applied to deal with this
+edge case only. This is in `register_fiducial.io.correct_raw_coordinates`.
+
+The x and y coordinates expected for fiducial markers were originally on the order of between 0 and 20. 
+When the raw data (sample) does not conform to this, the sample will be scaled to atlas resolution during registration.
+This is done by comparing the min/max x and y of the sample and scaling this to closely match the atlas prior to 
+registering the image.
+
+This has been tested for a range of resolutions and image size but where the scale is very different it is 
+important to test and validate the registration outputs.
+

applications/portal/cordmap/cordmap/user_developer_guide.md → applications/portal/cordmap/doc/user_developer_guide.md

@@ -16,7 +16,7 @@ image which is then the basis of registration. The z position is used to select
 appropriate 2d image from the atlas to use in this registration.
 
 Data types that are of the type "point" will be treated as cell populations to be 
-transformed into atlas space. Contours will be loaded
+transformed into atlas space. Contours will be loaded using:
 
 ```python
 cordmap.data.io.load_data