diff --git a/src/zarr/api/asynchronous.py b/src/zarr/api/asynchronous.py index abd5a6acb9..699f6b542f 100644 --- a/src/zarr/api/asynchronous.py +++ b/src/zarr/api/asynchronous.py @@ -7,8 +7,15 @@ import numpy as np import numpy.typing as npt -from zarr.core.array import Array, AsyncArray -from zarr.core.common import JSON, AccessModeLiteral, ChunkCoords, MemoryOrder, ZarrFormat +from zarr.core.array import Array, AsyncArray, ChunkSpec +from zarr.core.common import ( + JSON, + AccessModeLiteral, + ChunkCoords, + MemoryOrder, + ShapeLike, + ZarrFormat, +) from zarr.core.group import AsyncGroup from zarr.core.metadata import ArrayV2Metadata, ArrayV3Metadata from zarr.store import ( @@ -561,7 +568,7 @@ async def open_group( async def create( shape: ChunkCoords, *, # Note: this is a change from v2 - chunks: ChunkCoords | None = None, # TODO: v2 allowed chunks=True + chunks: ChunkSpec | ShapeLike | None = None, # TODO: v2 allowed chunks=True dtype: npt.DTypeLike | None = None, compressor: dict[str, JSON] | None = None, # TODO: default and type change fill_value: Any = 0, # TODO: need type @@ -583,7 +590,6 @@ async def create( meta_array: Any | None = None, # TODO: need type attributes: dict[str, JSON] | None = None, # v3 only - chunk_shape: ChunkCoords | None = None, chunk_key_encoding: ( ChunkKeyEncoding | tuple[Literal["default"], Literal[".", "/"]] @@ -674,15 +680,6 @@ async def create( or _default_zarr_version() ) - if zarr_format == 2 and chunks is None: - chunks = shape - if zarr_format == 3 and chunk_shape is None: - if chunks is not None: - chunk_shape = chunks - chunks = None - else: - chunk_shape = shape - if order is not None: warnings.warn( "order is deprecated, use config `array.order` instead", @@ -729,7 +726,6 @@ async def create( filters=filters, dimension_separator=dimension_separator, zarr_format=zarr_format, - chunk_shape=chunk_shape, chunk_key_encoding=chunk_key_encoding, codecs=codecs, dimension_names=dimension_names, diff --git a/src/zarr/core/array.py b/src/zarr/core/array.py index dcd7217d7d..db567d3322 100644 --- a/src/zarr/core/array.py +++ b/src/zarr/core/array.py @@ -9,7 +9,7 @@ import numpy.typing as npt from zarr.abc.store import set_or_delete -from zarr.codecs import BytesCodec +from zarr.codecs import BytesCodec, ShardingCodec from zarr.codecs._v2 import V2Compressor, V2Filters from zarr.core.attributes import Attributes from zarr.core.buffer import BufferPrototype, NDArrayLike, NDBuffer, default_buffer_prototype @@ -71,6 +71,13 @@ # Array and AsyncArray are defined in the base ``zarr`` namespace __all__ = ["parse_array_metadata", "create_codec_pipeline"] +from typing import TypedDict + + +class ChunkSpec(TypedDict, total=False): + read_shape: tuple[int, ...] + write_shape: tuple[int, ...] + def parse_array_metadata(data: Any) -> ArrayV2Metadata | ArrayV3Metadata: if isinstance(data, ArrayV2Metadata | ArrayV3Metadata): @@ -126,8 +133,6 @@ async def create( zarr_format: ZarrFormat = 3, fill_value: Any | None = None, attributes: dict[str, JSON] | None = None, - # v3 only - chunk_shape: ChunkCoords | None = None, chunk_key_encoding: ( ChunkKeyEncoding | tuple[Literal["default"], Literal[".", "/"]] @@ -136,8 +141,8 @@ async def create( ) = None, codecs: Iterable[Codec | dict[str, JSON]] | None = None, dimension_names: Iterable[str] | None = None, + chunks: ChunkSpec | ShapeLike | None = None, # v2 only - chunks: ShapeLike | None = None, dimension_separator: Literal[".", "/"] | None = None, order: Literal["C", "F"] | None = None, filters: list[dict[str, JSON]] | None = None, @@ -150,16 +155,6 @@ async def create( shape = parse_shapelike(shape) - if chunk_shape is None: - if chunks is None: - chunk_shape = chunks = _guess_chunks(shape=shape, typesize=np.dtype(dtype).itemsize) - else: - chunks = parse_shapelike(chunks) - - chunk_shape = chunks - elif chunks is not None: - raise ValueError("Only one of chunk_shape or chunks must be provided.") - if zarr_format == 3: if dimension_separator is not None: raise ValueError( @@ -181,7 +176,7 @@ async def create( store_path, shape=shape, dtype=dtype, - chunk_shape=chunk_shape, + chunks=chunks, fill_value=fill_value, chunk_key_encoding=chunk_key_encoding, codecs=codecs, @@ -204,7 +199,7 @@ async def create( store_path, shape=shape, dtype=dtype, - chunks=chunk_shape, + chunks=chunks, dimension_separator=dimension_separator, fill_value=fill_value, order=order, @@ -214,7 +209,7 @@ async def create( exists_ok=exists_ok, ) else: - raise ValueError(f"Insupported zarr_format. Got: {zarr_format}") + raise ValueError(f"Unsupported zarr_format. Got: {zarr_format}") if data is not None: # insert user-provided data @@ -229,7 +224,7 @@ async def _create_v3( *, shape: ShapeLike, dtype: npt.DTypeLike, - chunk_shape: ChunkCoords, + chunks: ShapeLike | ChunkSpec | None = None, fill_value: Any | None = None, chunk_key_encoding: ( ChunkKeyEncoding @@ -245,8 +240,34 @@ async def _create_v3( if not exists_ok: await ensure_no_existing_node(store_path, zarr_format=3) - shape = parse_shapelike(shape) - codecs = list(codecs) if codecs is not None else [BytesCodec()] + array_shape = parse_shapelike(shape) + shard_shape: tuple[int, ...] | None = None + chunk_shape: tuple[int, ...] + + # because chunks is an optional typeddict with optional keys, it could be completely empty + # OR None, both of which result in chunks being inferred automatically + if chunks is not None and not (chunks == {}): + if isinstance(chunks, dict): + if "write_shape" in chunks: + chunk_shape = chunks["write_shape"] + if "read_shape" in chunks: + # sharding is only enabled when read_shape and write_shape are specified + # we do not special-case the condition when read_shape and write_shape are the same + shard_shape = chunks["read_shape"] + elif "read_shape" in chunks: + # if read_shape is present, but write_shape is absent, then + # set the chunk_shape to read_shape, and keep shard_shape set to `None` + chunk_shape = chunks["read_shape"] + else: + chunk_shape = parse_shapelike(chunks) + else: + # determine chunking parameters automatically + chunk_shape = _guess_chunks(array_shape, np.dtype(dtype).itemsize) + + _codecs = tuple(codecs) if codecs is not None else (BytesCodec(),) + + if shard_shape is not None: + _codecs = (ShardingCodec(chunk_shape=shard_shape, codecs=_codecs),) if fill_value is None: if dtype == np.dtype("bool"): @@ -266,12 +287,12 @@ async def _create_v3( ) metadata = ArrayV3Metadata( - shape=shape, + shape=array_shape, data_type=dtype, chunk_grid=RegularChunkGrid(chunk_shape=chunk_shape), chunk_key_encoding=chunk_key_encoding, fill_value=fill_value, - codecs=codecs, + codecs=_codecs, dimension_names=tuple(dimension_names) if dimension_names else None, attributes=attributes or {}, ) @@ -288,7 +309,7 @@ async def _create_v2( *, shape: ChunkCoords, dtype: npt.DTypeLike, - chunks: ChunkCoords, + chunks: ChunkSpec | ShapeLike | None, dimension_separator: Literal[".", "/"] | None = None, fill_value: None | int | float = None, order: Literal["C", "F"] | None = None, @@ -307,10 +328,28 @@ async def _create_v2( if dimension_separator is None: dimension_separator = "." + if chunks is None or chunks == {}: + _chunks = _guess_chunks(shape, np.dtype(dtype).itemsize) + elif isinstance(chunks, dict): + if "write_shape" in chunks: + _chunks = parse_shapelike(chunks["write_shape"]) + if "read_shape" in chunks: + if chunks["read_shape"] != chunks["write_shape"]: + msg = "Invalid chunk specification. For zarr v2, read_shape must match write_shape." + raise ValueError(msg) + elif "read_shape" in chunks: + _chunks = parse_shapelike(chunks["read_shape"]) + else: + raise ValueError( + f"Invalid chunk specification: {chunks}. Expected a dict compatible with ChunkSpec" + ) + else: + _chunks = parse_shapelike(chunks) + metadata = ArrayV2Metadata( shape=shape, dtype=np.dtype(dtype), - chunks=chunks, + chunks=_chunks, order=order, dimension_separator=dimension_separator, fill_value=0 if fill_value is None else fill_value, @@ -638,7 +677,6 @@ def create( fill_value: Any | None = None, attributes: dict[str, JSON] | None = None, # v3 only - chunk_shape: ChunkCoords | None = None, chunk_key_encoding: ( ChunkKeyEncoding | tuple[Literal["default"], Literal[".", "/"]] @@ -648,7 +686,7 @@ def create( codecs: Iterable[Codec | dict[str, JSON]] | None = None, dimension_names: Iterable[str] | None = None, # v2 only - chunks: ChunkCoords | None = None, + chunks: ChunkSpec | ShapeLike | None = None, dimension_separator: Literal[".", "/"] | None = None, order: Literal["C", "F"] | None = None, filters: list[dict[str, JSON]] | None = None, @@ -664,7 +702,6 @@ def create( zarr_format=zarr_format, attributes=attributes, fill_value=fill_value, - chunk_shape=chunk_shape, chunk_key_encoding=chunk_key_encoding, codecs=codecs, dimension_names=dimension_names, diff --git a/src/zarr/core/group.py b/src/zarr/core/group.py index 56c9c88ea5..bd60397fad 100644 --- a/src/zarr/core/group.py +++ b/src/zarr/core/group.py @@ -12,7 +12,7 @@ from zarr.abc.metadata import Metadata from zarr.abc.store import set_or_delete -from zarr.core.array import Array, AsyncArray +from zarr.core.array import Array, AsyncArray, ChunkSpec from zarr.core.attributes import Attributes from zarr.core.buffer import default_buffer_prototype from zarr.core.common import ( @@ -371,7 +371,6 @@ async def create_array( fill_value: Any | None = None, attributes: dict[str, JSON] | None = None, # v3 only - chunk_shape: ChunkCoords | None = None, chunk_key_encoding: ( ChunkKeyEncoding | tuple[Literal["default"], Literal[".", "/"]] @@ -381,7 +380,7 @@ async def create_array( codecs: Iterable[Codec | dict[str, JSON]] | None = None, dimension_names: Iterable[str] | None = None, # v2 only - chunks: ShapeLike | None = None, + chunks: ChunkSpec | ShapeLike | None = None, dimension_separator: Literal[".", "/"] | None = None, order: Literal["C", "F"] | None = None, filters: list[dict[str, JSON]] | None = None, @@ -435,7 +434,6 @@ async def create_array( self.store_path / name, shape=shape, dtype=dtype, - chunk_shape=chunk_shape, fill_value=fill_value, chunk_key_encoding=chunk_key_encoding, codecs=codecs, @@ -896,7 +894,6 @@ def create_array( fill_value: Any | None = None, attributes: dict[str, JSON] | None = None, # v3 only - chunk_shape: ChunkCoords | None = None, chunk_key_encoding: ( ChunkKeyEncoding | tuple[Literal["default"], Literal[".", "/"]] @@ -905,8 +902,7 @@ def create_array( ) = None, codecs: Iterable[Codec | dict[str, JSON]] | None = None, dimension_names: Iterable[str] | None = None, - # v2 only - chunks: ShapeLike | None = None, + chunks: ChunkSpec | ShapeLike | None = None, dimension_separator: Literal[".", "/"] | None = None, order: Literal["C", "F"] | None = None, filters: list[dict[str, JSON]] | None = None, @@ -966,7 +962,6 @@ def create_array( dtype=dtype, fill_value=fill_value, attributes=attributes, - chunk_shape=chunk_shape, chunk_key_encoding=chunk_key_encoding, codecs=codecs, dimension_names=dimension_names, @@ -1094,8 +1089,6 @@ def array( dtype: npt.DTypeLike = "float64", fill_value: Any | None = None, attributes: dict[str, JSON] | None = None, - # v3 only - chunk_shape: ChunkCoords | None = None, chunk_key_encoding: ( ChunkKeyEncoding | tuple[Literal["default"], Literal[".", "/"]] @@ -1166,7 +1159,6 @@ def array( dtype=dtype, fill_value=fill_value, attributes=attributes, - chunk_shape=chunk_shape, chunk_key_encoding=chunk_key_encoding, codecs=codecs, dimension_names=dimension_names, diff --git a/tests/v3/test_api.py b/tests/v3/test_api.py index 239dd1c3e2..82ad99a254 100644 --- a/tests/v3/test_api.py +++ b/tests/v3/test_api.py @@ -20,7 +20,7 @@ def test_create_array(memory_store: Store) -> None: assert z.shape == (100,) # create array, overwrite, specify chunk shape - z = create(shape=200, chunk_shape=20, store=store, overwrite=True) + z = create(shape=200, chunks=20, store=store, overwrite=True) assert isinstance(z, Array) assert z.shape == (200,) assert z.chunks == (20,) diff --git a/tests/v3/test_array.py b/tests/v3/test_array.py index fb726757dc..8d62cfe95e 100644 --- a/tests/v3/test_array.py +++ b/tests/v3/test_array.py @@ -4,6 +4,9 @@ import pytest from zarr import Array, Group +from zarr.codecs import BytesCodec, GzipCodec, ShardingCodec +from zarr.core.array import ChunkSpec +from zarr.core.chunk_grids import _guess_chunks from zarr.core.common import ZarrFormat from zarr.errors import ContainsArrayError, ContainsGroupError from zarr.store import LocalStore, MemoryStore @@ -64,7 +67,7 @@ def test_array_creation_existing_node( def test_array_name_properties_no_group( store: LocalStore | MemoryStore, zarr_format: ZarrFormat ) -> None: - arr = Array.create(store=store, shape=(100,), chunks=(10,), zarr_format=zarr_format, dtype="i4") + arr = Array.create(store=store, shape=(100,), zarr_format=zarr_format, dtype="i4") assert arr.path == "" assert arr.name is None assert arr.basename is None @@ -82,7 +85,7 @@ def test_array_name_properties_with_group( assert foo.basename == "foo" bar = root.create_group("bar") - spam = bar.create_array("spam", shape=(100,), chunks=(10,), dtype="i4") + spam = bar.create_array("spam", shape=(100,), dtype="i4") assert spam.path == "bar/spam" assert spam.name == "/bar/spam" @@ -107,13 +110,10 @@ def test_array_v3_fill_value_default( shape=shape, dtype=dtype_str, zarr_format=3, - chunk_shape=shape, fill_value=None, ) else: - arr = Array.create( - store=store, shape=shape, dtype=dtype_str, zarr_format=3, chunk_shape=shape - ) + arr = Array.create(store=store, shape=shape, dtype=dtype_str, zarr_format=3) assert arr.fill_value == np.dtype(dtype_str).type(default_fill_value) assert arr.fill_value.dtype == arr.dtype @@ -129,9 +129,46 @@ def test_array_v3_fill_value(store: MemoryStore, fill_value: int, dtype_str: str shape=shape, dtype=dtype_str, zarr_format=3, - chunk_shape=shape, fill_value=fill_value, ) assert arr.fill_value == np.dtype(dtype_str).type(fill_value) assert arr.fill_value.dtype == arr.dtype + + +@pytest.mark.parametrize( + "chunks", + [ + {"write_shape": (10,)}, + {"write_shape": (10,), "read_shape": (5,)}, + {"write_shape": (10,), "read_shape": (5,)}, + {"read_shape": (5,)}, + {}, + (10,), + ], +) +def test_array_chunks(chunks: ChunkSpec | tuple[int, ...] | None) -> None: + codecs = [BytesCodec(), GzipCodec()] + + arr = Array.create( + store=MemoryStore(mode="w"), shape=(100,), dtype="uint8", chunks=chunks, codecs=codecs + ) + if chunks is None or chunks == {}: + assert arr.chunks == _guess_chunks(arr.shape, arr.dtype.itemsize) + assert arr.metadata.codecs == codecs + elif isinstance(chunks, tuple): + assert arr.chunks == chunks + assert arr.metadata.codecs == codecs + elif "write_shape" in chunks: + assert arr.chunks == chunks["write_shape"] + if "read_shape" in chunks: + codecs = arr.metadata.codecs + assert isinstance(codecs[0], ShardingCodec) + assert codecs[0].chunk_shape == chunks["read_shape"] + else: + assert arr.metadata.codecs == codecs + elif "read_shape" in chunks: + assert arr.chunks == chunks["read_shape"] + assert arr.metadata.codecs == codecs + else: + raise ValueError(f"Invalid chunks: {chunks}") diff --git a/tests/v3/test_buffer.py b/tests/v3/test_buffer.py index 5a313dc1ab..0b78e64809 100644 --- a/tests/v3/test_buffer.py +++ b/tests/v3/test_buffer.py @@ -46,7 +46,6 @@ async def test_async_array_prototype() -> None: a = await AsyncArray.create( StorePath(StoreExpectingTestBuffer(mode="w")) / "test_async_array_prototype", shape=expect.shape, - chunk_shape=(5, 5), dtype=expect.dtype, fill_value=0, ) @@ -75,7 +74,6 @@ async def test_async_array_gpu_prototype() -> None: a = await AsyncArray.create( StorePath(MemoryStore(mode="w")) / "test_async_array_gpu_prototype", shape=expect.shape, - chunk_shape=(5, 5), dtype=expect.dtype, fill_value=0, ) @@ -97,7 +95,6 @@ async def test_codecs_use_of_prototype() -> None: a = await AsyncArray.create( StorePath(StoreExpectingTestBuffer(mode="w")) / "test_codecs_use_of_prototype", shape=expect.shape, - chunk_shape=(5, 5), dtype=expect.dtype, fill_value=0, codecs=[ @@ -132,7 +129,6 @@ async def test_codecs_use_of_gpu_prototype() -> None: a = await AsyncArray.create( StorePath(MemoryStore(mode="w")) / "test_codecs_use_of_gpu_prototype", shape=expect.shape, - chunk_shape=(5, 5), dtype=expect.dtype, fill_value=0, codecs=[ diff --git a/tests/v3/test_codecs/test_blosc.py b/tests/v3/test_codecs/test_blosc.py index 5de4c9fa99..f8fbf1e773 100644 --- a/tests/v3/test_codecs/test_blosc.py +++ b/tests/v3/test_codecs/test_blosc.py @@ -19,7 +19,7 @@ async def test_blosc_evolve(store: Store, dtype: str) -> None: await AsyncArray.create( spath, shape=(16, 16), - chunk_shape=(16, 16), + chunks=(16, 16), dtype=dtype, fill_value=0, codecs=[BytesCodec(), BloscCodec()], @@ -40,7 +40,7 @@ async def test_blosc_evolve(store: Store, dtype: str) -> None: await AsyncArray.create( spath2, shape=(16, 16), - chunk_shape=(16, 16), + chunks=(16, 16), dtype=dtype, fill_value=0, codecs=[ShardingCodec(chunk_shape=(16, 16), codecs=[BytesCodec(), BloscCodec()])], diff --git a/tests/v3/test_codecs/test_codecs.py b/tests/v3/test_codecs/test_codecs.py index f388fb1c1e..254b88aab6 100644 --- a/tests/v3/test_codecs/test_codecs.py +++ b/tests/v3/test_codecs/test_codecs.py @@ -92,7 +92,7 @@ async def test_order( a = await AsyncArray.create( spath, shape=data.shape, - chunk_shape=(32, 8), + chunks=(32, 8), dtype=data.dtype, fill_value=0, chunk_key_encoding=("v2", "."), @@ -154,7 +154,7 @@ def test_order_implicit( a = Array.create( spath, shape=data.shape, - chunk_shape=(16, 16), + chunks=(16, 16), dtype=data.dtype, fill_value=0, codecs=codecs_, @@ -181,7 +181,7 @@ def test_open(store: Store) -> None: a = Array.create( spath, shape=(16, 16), - chunk_shape=(16, 16), + chunks=(16, 16), dtype="int32", fill_value=0, ) @@ -228,7 +228,7 @@ def test_write_partial_chunks(store: Store) -> None: a = Array.create( spath, shape=data.shape, - chunk_shape=(20, 20), + chunks=(20, 20), dtype=data.dtype, fill_value=1, ) @@ -244,7 +244,7 @@ async def test_delete_empty_chunks(store: Store) -> None: a = await AsyncArray.create( spath, shape=data.shape, - chunk_shape=(32, 32), + chunks=(32, 32), dtype=data.dtype, fill_value=1, ) @@ -262,7 +262,7 @@ async def test_zarr_compat(store: Store) -> None: a = await AsyncArray.create( spath, shape=data.shape, - chunk_shape=(10, 10), + chunks=(10, 10), dtype=data.dtype, chunk_key_encoding=("v2", "."), fill_value=1, @@ -303,7 +303,7 @@ async def test_zarr_compat_F(store: Store) -> None: a = await AsyncArray.create( spath, shape=data.shape, - chunk_shape=(10, 10), + chunks=(10, 10), dtype=data.dtype, chunk_key_encoding=("v2", "."), fill_value=1, @@ -346,7 +346,7 @@ async def test_dimension_names(store: Store) -> None: await AsyncArray.create( spath, shape=data.shape, - chunk_shape=(16, 16), + chunks=(16, 16), dtype=data.dtype, fill_value=0, dimension_names=("x", "y"), @@ -361,7 +361,7 @@ async def test_dimension_names(store: Store) -> None: await AsyncArray.create( spath2, shape=data.shape, - chunk_shape=(16, 16), + chunks=(16, 16), dtype=data.dtype, fill_value=0, ) @@ -379,7 +379,7 @@ def test_invalid_metadata(store: Store) -> None: Array.create( spath, shape=(16, 16, 16), - chunk_shape=(16, 16), + chunks=(16, 16), dtype=np.dtype("uint8"), fill_value=0, ) @@ -388,7 +388,7 @@ def test_invalid_metadata(store: Store) -> None: Array.create( spath2, shape=(16, 16), - chunk_shape=(16, 16), + chunks=(16, 16), dtype=np.dtype("uint8"), fill_value=0, codecs=[ @@ -401,7 +401,7 @@ def test_invalid_metadata(store: Store) -> None: Array.create( spath3, shape=(16, 16), - chunk_shape=(16, 16), + chunks=(16, 16), dtype=np.dtype("uint8"), fill_value=0, codecs=[ @@ -414,7 +414,7 @@ def test_invalid_metadata(store: Store) -> None: Array.create( spath4, shape=(16, 16), - chunk_shape=(16, 16), + chunks=(16, 16), dtype=np.dtype("uint8"), fill_value=0, codecs=[ @@ -426,7 +426,7 @@ def test_invalid_metadata(store: Store) -> None: Array.create( spath5, shape=(16, 16), - chunk_shape=(16, 16), + chunks=(16, 16), dtype=np.dtype("uint8"), fill_value=0, codecs=[ @@ -438,7 +438,7 @@ def test_invalid_metadata(store: Store) -> None: Array.create( spath6, shape=(16, 16), - chunk_shape=(16, 16), + chunks=(16, 16), dtype=np.dtype("uint8"), fill_value=0, codecs=[ @@ -450,7 +450,7 @@ def test_invalid_metadata(store: Store) -> None: Array.create( spath7, shape=(16, 16), - chunk_shape=(16, 16), + chunks=(16, 16), dtype=np.dtype("uint8"), fill_value=0, codecs=[ @@ -468,7 +468,7 @@ async def test_resize(store: Store) -> None: a = await AsyncArray.create( spath, shape=data.shape, - chunk_shape=(10, 10), + chunks=(10, 10), dtype=data.dtype, chunk_key_encoding=("v2", "."), fill_value=1, diff --git a/tests/v3/test_codecs/test_endian.py b/tests/v3/test_codecs/test_endian.py index f97d95d9b7..654e5a0635 100644 --- a/tests/v3/test_codecs/test_endian.py +++ b/tests/v3/test_codecs/test_endian.py @@ -23,7 +23,7 @@ async def test_endian(store: Store, endian: Literal["big", "little"]) -> None: a = await AsyncArray.create( spath, shape=data.shape, - chunk_shape=(16, 16), + chunks=(16, 16), dtype=data.dtype, fill_value=0, chunk_key_encoding=("v2", "."), @@ -62,7 +62,7 @@ async def test_endian_write( a = await AsyncArray.create( spath, shape=data.shape, - chunk_shape=(16, 16), + chunks=(16, 16), dtype="uint16", fill_value=0, chunk_key_encoding=("v2", "."), diff --git a/tests/v3/test_codecs/test_gzip.py b/tests/v3/test_codecs/test_gzip.py index 6495f8236c..826d53e56d 100644 --- a/tests/v3/test_codecs/test_gzip.py +++ b/tests/v3/test_codecs/test_gzip.py @@ -14,7 +14,7 @@ def test_gzip(store: Store) -> None: a = Array.create( StorePath(store), shape=data.shape, - chunk_shape=(16, 16), + chunks=(16, 16), dtype=data.dtype, fill_value=0, codecs=[BytesCodec(), GzipCodec()], diff --git a/tests/v3/test_codecs/test_sharding.py b/tests/v3/test_codecs/test_sharding.py index e5f66224e9..404c77461c 100644 --- a/tests/v3/test_codecs/test_sharding.py +++ b/tests/v3/test_codecs/test_sharding.py @@ -43,7 +43,7 @@ def test_sharding( arr = Array.create( spath, shape=tuple(s + offset for s in data.shape), - chunk_shape=(64,) * data.ndim, + chunks=(64,) * data.ndim, dtype=data.dtype, fill_value=6, codecs=[ @@ -87,7 +87,7 @@ def test_sharding_partial( a = Array.create( spath, shape=tuple(a + 10 for a in data.shape), - chunk_shape=(64, 64, 64), + chunks=(64, 64, 64), dtype=data.dtype, fill_value=0, codecs=[ @@ -130,7 +130,7 @@ def test_sharding_partial_read( a = Array.create( spath, shape=tuple(a + 10 for a in data.shape), - chunk_shape=(64, 64, 64), + chunks=(64, 64, 64), dtype=data.dtype, fill_value=1, codecs=[ @@ -167,7 +167,7 @@ def test_sharding_partial_overwrite( a = Array.create( spath, shape=tuple(a + 10 for a in data.shape), - chunk_shape=(64, 64, 64), + chunks=(64, 64, 64), dtype=data.dtype, fill_value=1, codecs=[ @@ -221,7 +221,7 @@ def test_nested_sharding( a = Array.create( spath, shape=data.shape, - chunk_shape=(64, 64, 64), + chunks=(64, 64, 64), dtype=data.dtype, fill_value=0, codecs=[ @@ -249,7 +249,7 @@ def test_open_sharding(store: Store) -> None: a = Array.create( spath, shape=(16, 16), - chunk_shape=(16, 16), + chunks=(16, 16), dtype="int32", fill_value=0, codecs=[ @@ -274,7 +274,7 @@ def test_write_partial_sharded_chunks(store: Store) -> None: a = Array.create( spath, shape=(40, 40), - chunk_shape=(20, 20), + chunks=(20, 20), dtype=data.dtype, fill_value=1, codecs=[ @@ -298,7 +298,7 @@ async def test_delete_empty_shards(store: Store) -> None: a = await AsyncArray.create( spath, shape=(16, 16), - chunk_shape=(8, 16), + chunks=(8, 16), dtype="uint16", fill_value=1, codecs=[ShardingCodec(chunk_shape=(8, 8))], diff --git a/tests/v3/test_codecs/test_transpose.py b/tests/v3/test_codecs/test_transpose.py index a24c650c5b..958d69e800 100644 --- a/tests/v3/test_codecs/test_transpose.py +++ b/tests/v3/test_codecs/test_transpose.py @@ -45,7 +45,7 @@ async def test_transpose( a = await AsyncArray.create( spath, shape=data.shape, - chunk_shape=(1, 32, 8), + chunks=(1, 32, 8), dtype=data.dtype, fill_value=0, chunk_key_encoding=("v2", "."), @@ -95,7 +95,7 @@ def test_transpose_non_self_inverse(store: Store, order: list[int]) -> None: a = Array.create( spath, shape=data.shape, - chunk_shape=data.shape, + chunks=data.shape, dtype=data.dtype, fill_value=0, codecs=[TransposeCodec(order=order), BytesCodec()], @@ -116,7 +116,7 @@ def test_transpose_invalid( Array.create( spath, shape=data.shape, - chunk_shape=(1, 32, 8), + chunks=(1, 32, 8), dtype=data.dtype, fill_value=0, chunk_key_encoding=("v2", "."), diff --git a/tests/v3/test_codecs/test_zstd.py b/tests/v3/test_codecs/test_zstd.py index 0726e5944c..c5c6489836 100644 --- a/tests/v3/test_codecs/test_zstd.py +++ b/tests/v3/test_codecs/test_zstd.py @@ -15,7 +15,6 @@ def test_zstd(store: Store, checksum: bool) -> None: a = Array.create( StorePath(store, path="zstd"), shape=data.shape, - chunk_shape=(16, 16), dtype=data.dtype, fill_value=0, codecs=[BytesCodec(), ZstdCodec(level=0, checksum=checksum)], diff --git a/tests/v3/test_group.py b/tests/v3/test_group.py index 64d4cbc867..6eccf2cd00 100644 --- a/tests/v3/test_group.py +++ b/tests/v3/test_group.py @@ -94,7 +94,7 @@ def test_group_members(store: MemoryStore | LocalStore, zarr_format: ZarrFormat) subsubsubgroup = subsubgroup.create_group("subsubsubgroup") # type: ignore members_expected["subarray"] = group.create_array( - "subarray", shape=(100,), dtype="uint8", chunk_shape=(10,), exists_ok=True + "subarray", shape=(100,), dtype="uint8", exists_ok=True ) # add an extra object to the domain of the group. @@ -147,9 +147,7 @@ def test_group(store: MemoryStore | LocalStore, zarr_format: ZarrFormat) -> None # create an array from the "bar" group data = np.arange(0, 4 * 4, dtype="uint16").reshape((4, 4)) - arr = bar.create_array( - "baz", shape=data.shape, dtype=data.dtype, chunk_shape=(2, 2), exists_ok=True - ) + arr = bar.create_array("baz", shape=data.shape, dtype=data.dtype, chunks=(2, 2), exists_ok=True) arr[:] = data # check the array @@ -232,7 +230,7 @@ def test_group_getitem(store: MemoryStore | LocalStore, zarr_format: ZarrFormat) group = Group.create(store, zarr_format=zarr_format) subgroup = group.create_group(name="subgroup") - subarray = group.create_array(name="subarray", shape=(10,), chunk_shape=(10,)) + subarray = group.create_array(name="subarray", shape=(10,)) assert group["subgroup"] == subgroup assert group["subarray"] == subarray @@ -247,7 +245,7 @@ def test_group_delitem(store: MemoryStore | LocalStore, zarr_format: ZarrFormat) group = Group.create(store, zarr_format=zarr_format) subgroup = group.create_group(name="subgroup") - subarray = group.create_array(name="subarray", shape=(10,), chunk_shape=(10,)) + subarray = group.create_array(name="subarray", shape=(10,)) assert group["subgroup"] == subgroup assert group["subarray"] == subarray @@ -567,9 +565,7 @@ async def test_asyncgroup_getitem(store: LocalStore | MemoryStore, zarr_format: agroup = await AsyncGroup.create(store=store, zarr_format=zarr_format) array_name = "sub_array" - sub_array = await agroup.create_array( - name=array_name, shape=(10,), dtype="uint8", chunk_shape=(2,) - ) + sub_array = await agroup.create_array(name=array_name, shape=(10,), dtype="uint8") assert await agroup.getitem(array_name) == sub_array sub_group_path = "sub_group" @@ -585,7 +581,7 @@ async def test_asyncgroup_delitem(store: LocalStore | MemoryStore, zarr_format: agroup = await AsyncGroup.create(store=store, zarr_format=zarr_format) array_name = "sub_array" _ = await agroup.create_array( - name=array_name, shape=(10,), dtype="uint8", chunk_shape=(2,), attributes={"foo": 100} + name=array_name, shape=(10,), dtype="uint8", attributes={"foo": 100} ) await agroup.delitem(array_name) @@ -642,7 +638,7 @@ async def test_asyncgroup_create_array( shape = (10,) dtype = "uint8" - chunk_shape = (4,) + chunks = (4,) attributes = {"foo": 100} sub_node_path = "sub_array" @@ -650,7 +646,7 @@ async def test_asyncgroup_create_array( name=sub_node_path, shape=shape, dtype=dtype, - chunk_shape=chunk_shape, + chunks=chunks, attributes=attributes, ) assert isinstance(subnode, AsyncArray) @@ -661,7 +657,7 @@ async def test_asyncgroup_create_array( assert subnode.dtype == dtype # todo: fix the type annotation of array.metadata.chunk_grid so that we get some autocomplete # here. - assert subnode.metadata.chunk_grid.chunk_shape == chunk_shape + assert subnode.metadata.chunk_grid.chunk_shape == chunks assert subnode.metadata.zarr_format == zarr_format @@ -747,9 +743,7 @@ async def test_require_group(store: LocalStore | MemoryStore, zarr_format: ZarrF # test that we can get the group using require_group and overwrite=True foo_group = await root.require_group("foo", overwrite=True) - _ = await foo_group.create_array( - "bar", shape=(10,), dtype="uint8", chunk_shape=(2,), attributes={"foo": 100} - ) + _ = await foo_group.create_array("bar", shape=(10,), dtype="uint8", attributes={"foo": 100}) # test that overwriting a group w/ children fails # TODO: figure out why ensure_no_existing_node is not catching the foo.bar array diff --git a/tests/v3/test_indexing.py b/tests/v3/test_indexing.py index 0b08378726..de41b3ae52 100644 --- a/tests/v3/test_indexing.py +++ b/tests/v3/test_indexing.py @@ -42,7 +42,7 @@ def zarr_array_from_numpy_array( store=store / str(uuid4()), shape=a.shape, dtype=a.dtype, - chunk_shape=chunk_shape or a.shape, + chunks=chunk_shape or a.shape, chunk_key_encoding=("v2", "."), ) z[()] = a