From 68b26bd6cdb2ffd94d46a80df6a60165e491ca1a Mon Sep 17 00:00:00 2001 From: William Moore Date: Fri, 13 Dec 2024 16:26:34 +0000 Subject: [PATCH 1/6] investigate dropping ome-zarr dependency --- napari_ome_zarr/_reader.py | 16 +-- napari_ome_zarr/ome_zarr_reader.py | 174 +++++++++++++++++++++++++++++ setup.cfg | 2 +- 3 files changed, 184 insertions(+), 8 deletions(-) create mode 100644 napari_ome_zarr/ome_zarr_reader.py diff --git a/napari_ome_zarr/_reader.py b/napari_ome_zarr/_reader.py index 295dc43..f39dfa9 100644 --- a/napari_ome_zarr/_reader.py +++ b/napari_ome_zarr/_reader.py @@ -10,7 +10,8 @@ from typing import Any, Dict, Iterator, List, Optional import numpy as np -from ome_zarr.io import parse_url +from .ome_zarr_reader import read_ome_zarr +# from ome_zarr.io import parse_url from ome_zarr.reader import Label, Node, Reader from ome_zarr.types import LayerData, PathLike, ReaderFunction from vispy.color import Colormap @@ -32,12 +33,13 @@ def napari_get_reader(path: PathLike) -> Optional[ReaderFunction]: if len(path) > 1: warnings.warn("more than one path is not currently supported") path = path[0] - zarr = parse_url(path) - if zarr: - reader = Reader(zarr) - return transform(reader()) - # Ignoring this path - return None + return read_ome_zarr(path) + # zarr = parse_url(path) + # if zarr: + # reader = Reader(zarr) + # return transform(reader()) + # # Ignoring this path + # return None def transform_properties( diff --git a/napari_ome_zarr/ome_zarr_reader.py b/napari_ome_zarr/ome_zarr_reader.py new file mode 100644 index 0000000..557b30d --- /dev/null +++ b/napari_ome_zarr/ome_zarr_reader.py @@ -0,0 +1,174 @@ + + +# zarr v3 + +import zarr +from zarr import Group +import numpy as np +import dask.array as da +from typing import List +from vispy.color import Colormap + +from typing import Any, Callable, Dict, List, Tuple, Union + +LayerData = Union[Tuple[Any], Tuple[Any, Dict], Tuple[Any, Dict, str]] + + +class Spec(): + + def __init__(self, group: Group): + self.group = group + + @staticmethod + def matches(group: Group) -> bool: + return False + + def data(self) -> List[da.core.Array] | None: + return None + + def metadata(self) -> Dict[str, Any] | None: + # napari layer metadata + return {} + + def children(self): + return [] + + def iter_nodes(self): + yield self + for child in self.children(): + for ch in child.iter_nodes(): + yield ch + + def iter_data(self): + for node in self.iter_nodes(): + data = node.data() + if data: + yield data + + @staticmethod + def get_attrs(group: Group): + if "ome" in group.attrs: + return group.attrs["ome"] + return group.attrs + + +class Multiscales(Spec): + + @staticmethod + def matches(group: Group) -> bool: + return "multiscales" in Spec.get_attrs(group) + + def children(self): + ch = [] + # test for child "labels" + try: + grp = self.group["labels"] + attrs = Spec.get_attrs(grp) + if "labels" in attrs: + for name in attrs["labels"]: + g = grp[name] + if Label.matches(g): + ch.append(Label(g)) + except KeyError: + pass + return ch + + def data(self): + attrs = Spec.get_attrs(self.group) + paths = [ds["path"] for ds in attrs["multiscales"][0]["datasets"]] + return [da.from_zarr(self.group[path]) for path in paths] + + def metadata(self): + rsp = {} + attrs = Spec.get_attrs(self.group) + axes = attrs["multiscales"][0]["axes"] + atypes = [axis["type"] for axis in axes] + if "channel" in atypes: + channel_axis = atypes.index("channel") + rsp["channel_axis"] = channel_axis + if "omero" in attrs: + colormaps = [] + for ch in attrs["omero"]["channels"]: + color = ch.get("color", None) + if color is not None: + rgb = [(int(color[i : i + 2], 16) / 255) for i in range(0, 6, 2)] + # colormap is range: black -> rgb color + colormaps.append(Colormap([[0, 0, 0], rgb])) + rsp["colormap"] = colormaps + return rsp + +class Bioformats2raw(Spec): + + @staticmethod + def matches(group: Group) -> bool: + attrs = Spec.get_attrs(group) + # Don't consider "plate" as a Bioformats2raw layout + return "bioformats2raw.layout" in attrs and "plate" not in attrs + + def children(self): + # TDOO: lookup children from series of OME/METADATA.xml + childnames = ["0"] + rv = [] + for name in childnames: + g = self.group[name] + if Multiscales.matches(g): + rv.append(Multiscales(g)) + return rv + + +class Plate(Spec): + + @staticmethod + def matches(group: Group) -> bool: + return "plate" in Spec.get_attrs(group) + + +class Label(Multiscales): + + @staticmethod + def matches(group: Group) -> bool: + # label must also be Multiscales + if not Multiscales.matches(group): + return False + return "image-label" in Spec.get_attrs(group) + + def metadata(self) -> Dict[str, Any] | None: + # override Multiscales metadata + return {} + + +def read_ome_zarr(url): + + def f(*args: Any, **kwargs: Any) -> List[LayerData]: + + results: List[LayerData] = list() + + # TODO: handle missing file + root_group = zarr.open(url) + + print("Root group", root_group.attrs.asdict()) + + if Bioformats2raw.matches(root_group): + spec = Bioformats2raw(root_group) + elif Multiscales.matches(root_group): + spec = Multiscales(root_group) + elif Plate.matches(root_group): + spec = Plate(root_group) + + if spec: + print("spec", spec) + nodes = list(spec.iter_nodes()) + print("Nodes", nodes) + for node in nodes: + node_data = node.data() + metadata = node.metadata() + # print(Spec.get_attrs(node.group)) + if Label.matches(node.group): + rv: LayerData = (node_data, metadata, "labels") + else: + rv: LayerData = (node_data, metadata) + results.append(rv) + + return results + + return f diff --git a/setup.cfg b/setup.cfg index c02f007..e48a102 100644 --- a/setup.cfg +++ b/setup.cfg @@ -33,7 +33,7 @@ python_requires = >=3.9 setup_requires = setuptools_scm # add your package requirements here install_requires = - ome-zarr>=0.3.0,!=0.8.3 + zarr==v3.0.0-beta.3 numpy vispy napari>=0.4.13 From 29fb252662e753480b6205c3f3837e68ace123ee Mon Sep 17 00:00:00 2001 From: William Moore Date: Fri, 13 Dec 2024 16:47:52 +0000 Subject: [PATCH 2/6] Handle more channel metadata --- napari_ome_zarr/ome_zarr_reader.py | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/napari_ome_zarr/ome_zarr_reader.py b/napari_ome_zarr/ome_zarr_reader.py index 557b30d..8f69ab3 100644 --- a/napari_ome_zarr/ome_zarr_reader.py +++ b/napari_ome_zarr/ome_zarr_reader.py @@ -88,13 +88,34 @@ def metadata(self): rsp["channel_axis"] = channel_axis if "omero" in attrs: colormaps = [] - for ch in attrs["omero"]["channels"]: + ch_names = [] + visibles = [] + contrast_limits = [] + + for index, ch in enumerate(attrs["omero"]["channels"]): color = ch.get("color", None) if color is not None: rgb = [(int(color[i : i + 2], 16) / 255) for i in range(0, 6, 2)] # colormap is range: black -> rgb color colormaps.append(Colormap([[0, 0, 0], rgb])) + ch_names.append(ch.get("label", str(index))) + visibles.append(ch.get("active", True)) + + window = ch.get("window", None) + if window is not None: + start = window.get("start", None) + end = window.get("end", None) + if start is None or end is None: + # Disable contrast limits settings if one is missing + contrast_limits = None + elif contrast_limits is not None: + contrast_limits.append([start, end]) + rsp["colormap"] = colormaps + rsp["name"] = ch_names + rsp["contrast_limits"] = contrast_limits + rsp["visible"] = visibles + return rsp class Bioformats2raw(Spec): From 076cc7ce590c29c3fd96edef4c74190f2249d600 Mon Sep 17 00:00:00 2001 From: William Moore Date: Mon, 16 Dec 2024 22:43:41 +0000 Subject: [PATCH 3/6] Handle no channel_axis --- napari_ome_zarr/ome_zarr_reader.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/napari_ome_zarr/ome_zarr_reader.py b/napari_ome_zarr/ome_zarr_reader.py index 8f69ab3..0c94037 100644 --- a/napari_ome_zarr/ome_zarr_reader.py +++ b/napari_ome_zarr/ome_zarr_reader.py @@ -111,10 +111,16 @@ def metadata(self): elif contrast_limits is not None: contrast_limits.append([start, end]) - rsp["colormap"] = colormaps - rsp["name"] = ch_names - rsp["contrast_limits"] = contrast_limits - rsp["visible"] = visibles + if rsp.get("channel_axis") is not None: + rsp["colormap"] = colormaps + rsp["name"] = ch_names + rsp["contrast_limits"] = contrast_limits + rsp["visible"] = visibles + else: + rsp["colormap"] = colormaps[0] + rsp["name"] = ch_names[0] + rsp["contrast_limits"] = contrast_limits[0] + rsp["visible"] = visibles[0] return rsp From 4fc57d32d629230a0824ec8e141b069a1ee95d33 Mon Sep 17 00:00:00 2001 From: William Moore Date: Mon, 6 Jan 2025 15:03:23 +0000 Subject: [PATCH 4/6] Remove ome_zarr imports and unused code from _reader.py --- napari_ome_zarr/_reader.py | 168 +------------------------------------ 1 file changed, 1 insertion(+), 167 deletions(-) diff --git a/napari_ome_zarr/_reader.py b/napari_ome_zarr/_reader.py index f39dfa9..811c115 100644 --- a/napari_ome_zarr/_reader.py +++ b/napari_ome_zarr/_reader.py @@ -4,27 +4,11 @@ """ -import logging import warnings -from importlib.metadata import version -from typing import Any, Dict, Iterator, List, Optional -import numpy as np from .ome_zarr_reader import read_ome_zarr -# from ome_zarr.io import parse_url -from ome_zarr.reader import Label, Node, Reader -from ome_zarr.types import LayerData, PathLike, ReaderFunction -from vispy.color import Colormap -LOGGER = logging.getLogger("napari_ome_zarr.reader") - -METADATA_KEYS = ("name", "visible", "contrast_limits", "colormap", "metadata") - -# major and minor versions as int -napari_version = tuple(map(int, list(version("napari").split(".")[:2]))) - - -def napari_get_reader(path: PathLike) -> Optional[ReaderFunction]: +def napari_get_reader(path): """Returns a reader for supported paths that include IDR ID. - URL of the form: https://uk1s3.embassy.ebi.ac.uk/idr/zarr/v0.1/ID.zarr/ @@ -34,153 +18,3 @@ def napari_get_reader(path: PathLike) -> Optional[ReaderFunction]: warnings.warn("more than one path is not currently supported") path = path[0] return read_ome_zarr(path) - # zarr = parse_url(path) - # if zarr: - # reader = Reader(zarr) - # return transform(reader()) - # # Ignoring this path - # return None - - -def transform_properties( - props: Optional[Dict[str, Dict]] = None -) -> Optional[Dict[str, List]]: - """ - Transform properties - - Transform a dict of {label_id : {key: value, key2: value2}} - with a key for every LABEL - into a dict of a key for every VALUE, with a list of values for each - .. code:: - - { - "index": [1381342, 1381343...] - "omero:roiId": [1381342, 1381343...], - "omero:shapeId": [1682567, 1682567...] - } - - """ - if props is None: - return None - - properties: Dict[str, List] = {} - - # First, create lists for all existing keys... - for label_id, props_dict in props.items(): - for key in props_dict.keys(): - properties[key] = [] - - keys = list(properties.keys()) - - properties["index"] = [] - for label_id, props_dict in props.items(): - properties["index"].append(label_id) - # ...in case some objects don't have all the keys - for key in keys: - properties[key].append(props_dict.get(key, None)) - return properties - - -def transform_scale( - node_metadata: Dict, metadata: Dict, channel_axis: Optional[int] -) -> None: - """ - e.g. transformation is {"scale": [0.2, 0.06, 0.06]} - Get a list of these for each level in data. Just use first? - """ - if "coordinateTransformations" in node_metadata: - level_0_transforms = node_metadata["coordinateTransformations"][0] - for transf in level_0_transforms: - if "scale" in transf: - scale = transf["scale"] - if channel_axis is not None: - scale.pop(channel_axis) - metadata["scale"] = tuple(scale) - if "translation" in transf: - translate = transf["translation"] - if channel_axis is not None: - translate.pop(channel_axis) - metadata["translate"] = tuple(translate) - - -def transform(nodes: Iterator[Node]) -> Optional[ReaderFunction]: - def f(*args: Any, **kwargs: Any) -> List[LayerData]: - results: List[LayerData] = list() - - for node in nodes: - data: List[Any] = node.data - metadata: Dict[str, Any] = {} - if data is None or len(data) < 1: - LOGGER.debug("skipping non-data %s", node) - else: - LOGGER.debug("transforming %s", node) - LOGGER.debug("node.metadata: %s", node.metadata) - - layer_type: str = "image" - channel_axis = None - try: - ch_types = [axis["type"] for axis in node.metadata["axes"]] - if "channel" in ch_types: - channel_axis = ch_types.index("channel") - except Exception: - LOGGER.error("Error reading axes: Please update ome-zarr") - raise - - transform_scale(node.metadata, metadata, channel_axis) - - if node.load(Label): - layer_type = "labels" - for x in METADATA_KEYS: - if x in node.metadata: - metadata[x] = node.metadata[x] - elif x == "colormap" and node.metadata["color"]: - # key changed 'color' -> 'colormap' in napari 0.5 - if napari_version >= (0, 5): - metadata["colormap"] = node.metadata["color"] - else: - metadata["color"] = node.metadata["color"] - if channel_axis is not None: - data = [ - np.squeeze(level, axis=channel_axis) for level in node.data - ] - else: - # Handle the removal of vispy requirement from ome-zarr-py - cms = node.metadata.get("colormap", []) - for idx, cm in enumerate(cms): - if not isinstance(cm, Colormap): - cms[idx] = Colormap(cm) - - if channel_axis is not None: - # multi-channel; Copy known metadata values - metadata["channel_axis"] = channel_axis - for x in METADATA_KEYS: - if x in node.metadata: - metadata[x] = node.metadata[x] - # overwrite 'name' if we have 'channel_names' - if "channel_names" in node.metadata: - metadata["name"] = node.metadata["channel_names"] - else: - # single channel image, so metadata just needs - # single items (not lists) - for x in METADATA_KEYS: - if x in node.metadata: - try: - metadata[x] = node.metadata[x][0] - except Exception: - pass - # overwrite 'name' if we have 'channel_names' - if "channel_names" in node.metadata: - if len(node.metadata["channel_names"]) > 0: - metadata["name"] = node.metadata["channel_names"][0] - - properties = transform_properties(node.metadata.get("properties")) - if properties is not None: - metadata["properties"] = properties - - rv: LayerData = (data, metadata, layer_type) - LOGGER.debug("Transformed: %s", rv) - results.append(rv) - - return results - - return f From e4cda75fe9b1019e7da5384837808377c5854c51 Mon Sep 17 00:00:00 2001 From: William Moore Date: Mon, 6 Jan 2025 15:06:06 +0000 Subject: [PATCH 5/6] Read OME/METADATA.xml for bioformats2raw.layout and open all series images --- napari_ome_zarr/ome_zarr_reader.py | 33 ++++++++++++++++++++++-------- 1 file changed, 25 insertions(+), 8 deletions(-) diff --git a/napari_ome_zarr/ome_zarr_reader.py b/napari_ome_zarr/ome_zarr_reader.py index 0c94037..463cf8a 100644 --- a/napari_ome_zarr/ome_zarr_reader.py +++ b/napari_ome_zarr/ome_zarr_reader.py @@ -4,12 +4,15 @@ import zarr from zarr import Group -import numpy as np +from zarr.core.sync import SyncMixin +from zarr.core.buffer import default_buffer_prototype + import dask.array as da from typing import List from vispy.color import Colormap +from xml.etree import ElementTree as ET -from typing import Any, Callable, Dict, List, Tuple, Union +from typing import Any, Dict, List, Tuple, Union LayerData = Union[Tuple[Any], Tuple[Any, Dict], Tuple[Any, Dict, str]] @@ -133,14 +136,28 @@ def matches(group: Group) -> bool: return "bioformats2raw.layout" in attrs and "plate" not in attrs def children(self): - # TDOO: lookup children from series of OME/METADATA.xml - childnames = ["0"] + # lookup children from series of OME/METADATA.xml + xml_data = SyncMixin()._sync(self.group.store.get("OME/METADATA.ome.xml", prototype=default_buffer_prototype())) + # print("xml_data", xml_data.to_bytes()) + root = ET.fromstring(xml_data.to_bytes()) rv = [] - for name in childnames: - g = self.group[name] - if Multiscales.matches(g): - rv.append(Multiscales(g)) + for child in root: + # {http://www.openmicroscopy.org/Schemas/OME/2016-06}Image + print(child.tag) + node_id = child.attrib.get("ID", "") + if child.tag.endswith("Image") and node_id.startswith("Image:"): + print("Image ID", node_id) + image_path = node_id.replace("Image:", "") + g = self.group[image_path] + if Multiscales.matches(g): + rv.append(Multiscales(g)) return rv + + # override to NOT yield self since node has no data + def iter_nodes(self): + for child in self.children(): + for ch in child.iter_nodes(): + yield ch class Plate(Spec): From 920d9cf21580d1c399cf7c519e1de055cb2011b2 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 6 Jan 2025 15:07:49 +0000 Subject: [PATCH 6/6] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .isort.cfg | 2 +- napari_ome_zarr/_reader.py | 1 + napari_ome_zarr/ome_zarr_reader.py | 46 ++++++++++++------------------ 3 files changed, 21 insertions(+), 28 deletions(-) diff --git a/.isort.cfg b/.isort.cfg index b46c472..bc7bfab 100644 --- a/.isort.cfg +++ b/.isort.cfg @@ -1,2 +1,2 @@ [settings] -known_third_party = numpy,ome_zarr,pytest,setuptools,vispy +known_third_party = dask,numpy,ome_zarr,pytest,setuptools,vispy,zarr diff --git a/napari_ome_zarr/_reader.py b/napari_ome_zarr/_reader.py index 811c115..1b30f0a 100644 --- a/napari_ome_zarr/_reader.py +++ b/napari_ome_zarr/_reader.py @@ -8,6 +8,7 @@ from .ome_zarr_reader import read_ome_zarr + def napari_get_reader(path): """Returns a reader for supported paths that include IDR ID. diff --git a/napari_ome_zarr/ome_zarr_reader.py b/napari_ome_zarr/ome_zarr_reader.py index 463cf8a..5086a3f 100644 --- a/napari_ome_zarr/ome_zarr_reader.py +++ b/napari_ome_zarr/ome_zarr_reader.py @@ -1,31 +1,26 @@ - - # zarr v3 -import zarr -from zarr import Group -from zarr.core.sync import SyncMixin -from zarr.core.buffer import default_buffer_prototype +from typing import Any, Dict, List, Tuple, Union +from xml.etree import ElementTree as ET import dask.array as da -from typing import List +import zarr from vispy.color import Colormap -from xml.etree import ElementTree as ET - -from typing import Any, Dict, List, Tuple, Union +from zarr import Group +from zarr.core.buffer import default_buffer_prototype +from zarr.core.sync import SyncMixin LayerData = Union[Tuple[Any], Tuple[Any, Dict], Tuple[Any, Dict, str]] -class Spec(): - +class Spec: def __init__(self, group: Group): self.group = group @staticmethod def matches(group: Group) -> bool: return False - + def data(self) -> List[da.core.Array] | None: return None @@ -39,8 +34,7 @@ def children(self): def iter_nodes(self): yield self for child in self.children(): - for ch in child.iter_nodes(): - yield ch + yield from child.iter_nodes() def iter_data(self): for node in self.iter_nodes(): @@ -56,7 +50,6 @@ def get_attrs(group: Group): class Multiscales(Spec): - @staticmethod def matches(group: Group) -> bool: return "multiscales" in Spec.get_attrs(group) @@ -127,8 +120,8 @@ def metadata(self): return rsp -class Bioformats2raw(Spec): +class Bioformats2raw(Spec): @staticmethod def matches(group: Group) -> bool: attrs = Spec.get_attrs(group) @@ -137,7 +130,11 @@ def matches(group: Group) -> bool: def children(self): # lookup children from series of OME/METADATA.xml - xml_data = SyncMixin()._sync(self.group.store.get("OME/METADATA.ome.xml", prototype=default_buffer_prototype())) + xml_data = SyncMixin()._sync( + self.group.store.get( + "OME/METADATA.ome.xml", prototype=default_buffer_prototype() + ) + ) # print("xml_data", xml_data.to_bytes()) root = ET.fromstring(xml_data.to_bytes()) rv = [] @@ -156,19 +153,16 @@ def children(self): # override to NOT yield self since node has no data def iter_nodes(self): for child in self.children(): - for ch in child.iter_nodes(): - yield ch - + yield from child.iter_nodes() -class Plate(Spec): +class Plate(Spec): @staticmethod def matches(group: Group) -> bool: return "plate" in Spec.get_attrs(group) class Label(Multiscales): - @staticmethod def matches(group: Group) -> bool: # label must also be Multiscales @@ -182,12 +176,10 @@ def metadata(self) -> Dict[str, Any] | None: def read_ome_zarr(url): - def f(*args: Any, **kwargs: Any) -> List[LayerData]: - results: List[LayerData] = list() - # TODO: handle missing file + # TODO: handle missing file root_group = zarr.open(url) print("Root group", root_group.attrs.asdict()) @@ -214,5 +206,5 @@ def f(*args: Any, **kwargs: Any) -> List[LayerData]: results.append(rv) return results - + return f