Support bioformats2raw.layout

src/ome2024_ngff_challenge/resave.py

@@ -11,6 +11,7 @@
 import time
 from pathlib import Path
 
+import numpy as np
 import tensorstore as ts
 import tqdm
 import zarr
@@ -72,7 +73,9 @@ class TextBuffer(Buffer):
 
     def __init__(self, text):
         self.text = text
-        self._data = list(text)
+        if isinstance(text, str):
+            text = np.array(text.encode())
+        self._data = text
 
 
 class TSMetrics:
@@ -380,7 +383,11 @@ def write_rocrate(write_store):
     sync(write_store.set(filename, text))
 
 
-def main(ns: argparse.Namespace):
+def main(ns: argparse.Namespace) -> int:
+    """
+    Returns the number of images converted
+    """
+    converted = 0
     configs = create_configs(ns)
 
     stores = []
@@ -444,9 +451,10 @@ def main(ns: argparse.Namespace):
             ns.output_write_details,
             ns.output_script,
         )
+        converted += 1
 
     # plate...
-    elif read_root.attrs.get("plate"):
+    elif plate_attrs := read_root.attrs.get("plate"):
         ome_attrs = {"version": NGFF_VERSION}
         for key, value in read_root.attrs.items():
             # ...replaces all other versions - remove
@@ -457,7 +465,6 @@ def main(ns: argparse.Namespace):
             # dev2: everything is under 'ome' key
             write_root.attrs["ome"] = ome_attrs
 
-        plate_attrs = read_root.attrs.get("plate")
         wells = plate_attrs.get("wells")
 
         for well in tqdm.tqdm(
@@ -506,9 +513,65 @@ def main(ns: argparse.Namespace):
                     ns.output_script,
                     img_path,  # TODO: review
                 )
+                converted += 1
+    # Note: plates can *also* contain this metadata
+    elif layout := read_root.attrs.get("bioformats2raw.layout"):
+        assert layout == 3
+        ome_attrs = {"version": NGFF_VERSION}
+        for key, value in read_root.attrs.items():
+            # ...replaces all other versions - remove
+            strip_version(value)
+            ome_attrs[key] = value
+
+        if write_root is not None:  # otherwise dry run
+            # dev2: everything is under 'ome' key
+            write_root.attrs["ome"] = ome_attrs
 
+        ome_group = zarr.open_group(store=stores[0], path="OME", zarr_format=2)
+        series = ome_group.attrs.get("series", [])
+        assert series  # TODO: support implicit case where OME-XML must be read
 
-def cli(args=sys.argv):
+        for img_path in tqdm.tqdm(
+            series, position=0, desc="i", leave=False, colour="green", ncols=80
+        ):
+            # TODO: duplicated with plate section (move to class)
+            out_path = ns.output_path / img_path
+            input_path = ns.input_path / img_path
+            img_v2 = zarr.open_group(store=stores[0], path=str(img_path), zarr_format=2)
+
+            if write_root is not None:  # otherwise dry-run
+                image_group = write_root.create_group(str(img_path))
+            else:
+                image_group = None
+
+            convert_image(
+                configs,
+                img_v2,
+                input_path,
+                write_store,  # TODO: review
+                image_group,
+                out_path,
+                ns.output_overwrite,
+                ns.output_chunks,
+                ns.output_shards,
+                ns.output_read_details,
+                ns.output_write_details,
+                ns.output_script,
+                img_path,  # TODO: review
+            )
+            converted += 1
+    else:
+        LOGGER.warning(f"no convertible metadata: {read_root.attrs.keys()}")
+
+    return converted
+
+
+def cli(args=sys.argv[1:]):
+    """
+    Parses the arguments contained in `args` and passes
+    them to `main`. If no images are converted, raises
+    SystemExit. Otherwise, return the number of images.
+    """
     parser = argparse.ArgumentParser()
     parser.add_argument("--input-bucket")
     parser.add_argument("--input-endpoint")
@@ -544,8 +607,12 @@ def cli(args=sys.argv):
     ns = parser.parse_args(args)
 
     logging.basicConfig()
-    main(ns)
+
+    converted = main(ns)
+    if converted == 0:
+        raise SystemExit(1)
+    return converted
 
 
 if __name__ == "__main__":
-    cli(sys.argv)
+    cli(sys.argv[1:])

tests/data/bf2raw.fake.ini

@@ -0,0 +1,3 @@
+sizeX=8
+sizeY=8
+series=2

tests/data/bf2raw.zarr/.zattrs

@@ -0,0 +1,3 @@
+{
+  "bioformats2raw.layout" : 3
+}

tests/data/bf2raw.zarr/.zgroup

@@ -0,0 +1,3 @@
+{
+  "zarr_format" : 2
+}

tests/data/bf2raw.zarr/0/.zattrs

@@ -0,0 +1,54 @@
+{
+  "multiscales" : [ {
+    "metadata" : {
+      "method" : "loci.common.image.SimpleImageScaler",
+      "version" : "Bio-Formats 7.1.0"
+    },
+    "axes" : [ {
+      "name" : "t",
+      "type" : "time"
+    }, {
+      "name" : "c",
+      "type" : "channel"
+    }, {
+      "name" : "z",
+      "type" : "space"
+    }, {
+      "name" : "y",
+      "type" : "space"
+    }, {
+      "name" : "x",
+      "type" : "space"
+    } ],
+    "name" : "bf2raw",
+    "datasets" : [ {
+      "path" : "0",
+      "coordinateTransformations" : [ {
+        "scale" : [ 1.0, 1.0, 1.0, 1.0, 1.0 ],
+        "type" : "scale"
+      } ]
+    } ],
+    "version" : "0.4"
+  } ],
+  "omero" : {
+    "channels" : [ {
+      "color" : "808080",
+      "coefficient" : 1,
+      "active" : true,
+      "label" : "Channel 0",
+      "window" : {
+        "min" : 0.0,
+        "max" : 0.0,
+        "start" : 0.0,
+        "end" : 0.0
+      },
+      "family" : "linear",
+      "inverted" : false
+    } ],
+    "rdefs" : {
+      "defaultT" : 0,
+      "model" : "greyscale",
+      "defaultZ" : 0
+    }
+  }
+}

tests/data/bf2raw.zarr/0/.zgroup

@@ -0,0 +1,3 @@
+{
+  "zarr_format" : 2
+}

tests/data/bf2raw.zarr/0/0/.zarray

@@ -0,0 +1,17 @@
+{
+  "chunks" : [ 1, 1, 1, 8, 8 ],
+  "compressor" : {
+    "clevel" : 5,
+    "blocksize" : 0,
+    "shuffle" : 1,
+    "cname" : "lz4",
+    "id" : "blosc"
+  },
+  "dtype" : "|u1",
+  "fill_value" : 0,
+  "filters" : null,
+  "order" : "C",
+  "shape" : [ 1, 1, 1, 8, 8 ],
+  "dimension_separator" : "/",
+  "zarr_format" : 2
+}

tests/data/bf2raw.zarr/1/.zattrs

@@ -0,0 +1,54 @@
+{
+  "multiscales" : [ {
+    "metadata" : {
+      "method" : "loci.common.image.SimpleImageScaler",
+      "version" : "Bio-Formats 7.1.0"
+    },
+    "axes" : [ {
+      "name" : "t",
+      "type" : "time"
+    }, {
+      "name" : "c",
+      "type" : "channel"
+    }, {
+      "name" : "z",
+      "type" : "space"
+    }, {
+      "name" : "y",
+      "type" : "space"
+    }, {
+      "name" : "x",
+      "type" : "space"
+    } ],
+    "name" : "bf2raw 2",
+    "datasets" : [ {
+      "path" : "0",
+      "coordinateTransformations" : [ {
+        "scale" : [ 1.0, 1.0, 1.0, 1.0, 1.0 ],
+        "type" : "scale"
+      } ]
+    } ],
+    "version" : "0.4"
+  } ],
+  "omero" : {
+    "channels" : [ {
+      "color" : "808080",
+      "coefficient" : 1,
+      "active" : true,
+      "label" : "Channel 0",
+      "window" : {
+        "min" : 1.0,
+        "max" : 1.0,
+        "start" : 1.0,
+        "end" : 1.0
+      },
+      "family" : "linear",
+      "inverted" : false
+    } ],
+    "rdefs" : {
+      "defaultT" : 0,
+      "model" : "greyscale",
+      "defaultZ" : 0
+    }
+  }
+}

tests/data/bf2raw.zarr/1/.zgroup

@@ -0,0 +1,3 @@
+{
+  "zarr_format" : 2
+}

tests/data/bf2raw.zarr/1/0/.zarray

@@ -0,0 +1,17 @@
+{
+  "chunks" : [ 1, 1, 1, 8, 8 ],
+  "compressor" : {
+    "clevel" : 5,
+    "blocksize" : 0,
+    "shuffle" : 1,
+    "cname" : "lz4",
+    "id" : "blosc"
+  },
+  "dtype" : "|u1",
+  "fill_value" : 0,
+  "filters" : null,
+  "order" : "C",
+  "shape" : [ 1, 1, 1, 8, 8 ],
+  "dimension_separator" : "/",
+  "zarr_format" : 2
+}

tests/data/bf2raw.zarr/OME/.zattrs

@@ -0,0 +1,3 @@
+{
+  "series" : [ "0", "1" ]
+}

tests/data/bf2raw.zarr/OME/.zgroup

@@ -0,0 +1,3 @@
+{
+  "zarr_format" : 2
+}

tests/data/bf2raw.zarr/OME/METADATA.ome.xml

@@ -0,0 +1 @@
+<?xml version="1.0" encoding="UTF-8"?><OME xmlns="http://www.openmicroscopy.org/Schemas/OME/2016-06" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.openmicroscopy.org/Schemas/OME/2016-06 http://www.openmicroscopy.org/Schemas/OME/2016-06/ome.xsd"><Image ID="Image:0" Name="bf2raw"><Pixels BigEndian="true" DimensionOrder="XYZCT" ID="Pixels:0" Interleaved="false" SignificantBits="8" SizeC="1" SizeT="1" SizeX="8" SizeY="8" SizeZ="1" Type="uint8"><Channel ID="Channel:0:0" SamplesPerPixel="1"><LightPath/></Channel><MetadataOnly/></Pixels></Image><Image ID="Image:1" Name="bf2raw 2"><Pixels BigEndian="true" DimensionOrder="XYZCT" ID="Pixels:1" Interleaved="false" SignificantBits="8" SizeC="1" SizeT="1" SizeX="8" SizeY="8" SizeZ="1" Type="uint8"><Channel ID="Channel:1:0" SamplesPerPixel="1"><LightPath/></Channel><MetadataOnly/></Pixels></Image></OME>

tests/test_resave.py

@@ -90,18 +90,36 @@ def test_remote_simple_with_download(tmp_path):
 
 
 def test_local_2d(tmp_path):
-    resave.cli(
-        [
-            "data/2d.zarr",
-            str(tmp_path / "out.zarr"),
-        ]
+    assert (
+        resave.cli(
+            [
+                "data/2d.zarr",
+                str(tmp_path / "out.zarr"),
+            ]
+        )
+        == 1
+    )
+
+
+def test_local_bf2raw(tmp_path):
+    assert (
+        resave.cli(
+            [
+                "data/bf2raw.zarr",
+                str(tmp_path / "out.zarr"),
+            ]
+        )
+        == 2
     )
 
 
 def test_local_hcs(tmp_path):
-    resave.cli(
-        [
-            "data/hcs.zarr",
-            str(tmp_path / "out.zarr"),
-        ]
+    assert (
+        resave.cli(
+            [
+                "data/hcs.zarr",
+                str(tmp_path / "out.zarr"),
+            ]
+        )
+        == 8
     )