From 4e820d9de78e208751fc0bc8cb30c100d502acb9 Mon Sep 17 00:00:00 2001 From: Thomas Robitaille Date: Sat, 1 Feb 2025 13:50:39 +0000 Subject: [PATCH 1/5] Fix a bug that caused big-endian dask arrays to not be reprojected correctly --- reproject/interpolation/tests/test_core.py | 27 ++++++++++++++++++++++ reproject/utils.py | 2 +- 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/reproject/interpolation/tests/test_core.py b/reproject/interpolation/tests/test_core.py index 1a2d1f208..992efe4dc 100644 --- a/reproject/interpolation/tests/test_core.py +++ b/reproject/interpolation/tests/test_core.py @@ -954,3 +954,30 @@ def test_auto_block_size(): assert array_out.chunksize[0] == 350 assert footprint_out.chunksize[0] == 350 + + +def test_bigendian_dask(): + + # Regression test for an endianness issue that occurred when the input was + # passed in as (dask_array, wcs) and the dask array was big endian. + + array_in_le = da.ones((350, 250, 150), dtype=">f8") + array_in_be = da.ones((350, 250, 150), dtype=" Date: Sat, 1 Feb 2025 14:09:28 +0000 Subject: [PATCH 2/5] Pin zarr to <3 for now as it does not properly support big-endian arrays --- pyproject.toml | 2 +- tox.ini | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 509aca3d7..4fc32813a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,7 +16,7 @@ dependencies = [ "scipy>=1.9", "dask[array]>=2021.8", "cloudpickle", - "zarr", + "zarr<3", "fsspec", ] dynamic = ["version"] diff --git a/tox.ini b/tox.ini index 74ff982b3..70f5e9449 100644 --- a/tox.ini +++ b/tox.ini @@ -36,7 +36,6 @@ deps = #devdeps: asdf-astropy @ git+https://github.com/astropy/asdf-astropy.git devdeps: gwcs @ git+https://github.com/spacetelescope/gwcs.git devdeps: sunpy[map] @ git+https://github.com/sunpy/sunpy.git - devdeps: zarr<3 extras = test From d4764c2adfeab3febde7d2e6071a74bf0a49bde3 Mon Sep 17 00:00:00 2001 From: Thomas Robitaille Date: Sat, 1 Feb 2025 14:16:11 +0000 Subject: [PATCH 3/5] Cast dask array to float to avoid issues with zarr 3 --- pyproject.toml | 2 +- reproject/utils.py | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 4fc32813a..509aca3d7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,7 +16,7 @@ dependencies = [ "scipy>=1.9", "dask[array]>=2021.8", "cloudpickle", - "zarr<3", + "zarr", "fsspec", ] dynamic = ["version"] diff --git a/reproject/utils.py b/reproject/utils.py index df5bf5b78..f65303cd4 100644 --- a/reproject/utils.py +++ b/reproject/utils.py @@ -31,6 +31,11 @@ def _dask_to_numpy_memmap(dask_array, tmp_dir): if isinstance(dask_array.ravel()[0].compute(), da.Array): dask_array = dask_array.compute() + # Cast the dask array to regular float for two reasons - first, zarr 3.0.0 + # and later doesn't support big-endian arrays, and also we need to anyway + # make a native float memory mapped array below. + dask_array = dask_array.astype(float, copy=False) + # NOTE: here we use a new TemporaryDirectory context manager for the zarr # array because we can remove the temporary directory straight away after # converting the input to a Numpy memory mapped array. From 172ff1c3f4c67928c4f04dff37deaf89b24be44f Mon Sep 17 00:00:00 2001 From: Thomas Robitaille Date: Sat, 1 Feb 2025 18:09:17 +0000 Subject: [PATCH 4/5] Expand test to check for 4-byte floats --- reproject/interpolation/tests/test_core.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/reproject/interpolation/tests/test_core.py b/reproject/interpolation/tests/test_core.py index 992efe4dc..707445a9f 100644 --- a/reproject/interpolation/tests/test_core.py +++ b/reproject/interpolation/tests/test_core.py @@ -956,13 +956,14 @@ def test_auto_block_size(): assert footprint_out.chunksize[0] == 350 -def test_bigendian_dask(): +@pytest.mark.parametrize('itemsize', (4, 8)) +def test_bigendian_dask(itemsize): # Regression test for an endianness issue that occurred when the input was # passed in as (dask_array, wcs) and the dask array was big endian. - array_in_le = da.ones((350, 250, 150), dtype=">f8") - array_in_be = da.ones((350, 250, 150), dtype="f{itemsize}") + array_in_be = da.ones((350, 250, 150), dtype=f" Date: Sat, 1 Feb 2025 18:10:54 +0000 Subject: [PATCH 5/5] Fix codestyle --- reproject/interpolation/tests/test_core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reproject/interpolation/tests/test_core.py b/reproject/interpolation/tests/test_core.py index 707445a9f..63171e1e0 100644 --- a/reproject/interpolation/tests/test_core.py +++ b/reproject/interpolation/tests/test_core.py @@ -956,7 +956,7 @@ def test_auto_block_size(): assert footprint_out.chunksize[0] == 350 -@pytest.mark.parametrize('itemsize', (4, 8)) +@pytest.mark.parametrize("itemsize", (4, 8)) def test_bigendian_dask(itemsize): # Regression test for an endianness issue that occurred when the input was