diff --git a/CHANGELOG.md b/CHANGELOG.md index 3e0d8c9a..7a47a845 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- Experimental support for explicit array `bounds` and `Array.prepend` from the draft [ZEP 12](https://github.com/zarr-developers/zeps/pull/72) [#119](https://github.com/zarr-developers/zarr-java/pull/119). + ## [0.3.0] - 2026-09-09 ### Added diff --git a/USERGUIDE.md b/USERGUIDE.md index 65bf9944..183f472a 100644 --- a/USERGUIDE.md +++ b/USERGUIDE.md @@ -219,6 +219,20 @@ Array resizedArray = array.resize( true // parallel processing ); ``` +### Prepending Data (Experimental) +Zarr v3 arrays can grow before their first element without moving existing chunks, as proposed in +[ZEP 12](https://github.com/zarr-developers/zeps/pull/72). The array then stores explicit `bounds` +instead of `shape`, which readers without support for ZEP 12 refuse. +```java +// shape 6, chunks 4: c/0 = [10, 11, 12, 13], c/1 = [14, 15, _, _] +Array grown = array.prepend(ucar.ma2.Array.makeFromJavaArray(new int[]{7, 8, 9})); +// new chunk c/-1 = [_, 7, 8, 9]; c/0 and c/1 are unchanged +grown.metadata().shape; // [9] +grown.metadata().lowerBound(); // [-3] +grown.read(); // [7, 8, 9, 10, 11, 12, 13, 14, 15], indexing stays zero-based +// Create an array with explicit bounds +Array.metadataBuilder().withBounds(new long[]{-3}, new long[]{6}); +``` ### Managing Attributes ```java import dev.zarr.zarrjava.core.Attributes; diff --git a/src/main/java/dev/zarr/zarrjava/core/Array.java b/src/main/java/dev/zarr/zarrjava/core/Array.java index 67461483..4b0501e4 100644 --- a/src/main/java/dev/zarr/zarrjava/core/Array.java +++ b/src/main/java/dev/zarr/zarrjava/core/Array.java @@ -92,7 +92,9 @@ public void write(long[] offset, ucar.ma2.Array array, boolean parallel) { long[] shape = Utils.toLongArray(array.getShape()); final int[] chunkShape = metadata.chunkShape(); - Stream chunkStream = Arrays.stream(IndexingUtils.computeChunkCoords(metadata.shape, chunkShape, offset, shape)); + final long[] lowerBound = metadata.lowerBound(); + Stream chunkStream = Arrays.stream( + IndexingUtils.computeChunkCoords(metadata.shape, chunkShape, offset, shape, lowerBound)); if (parallel) { chunkStream = chunkStream.parallel(); } @@ -101,7 +103,7 @@ public void write(long[] offset, ucar.ma2.Array array, boolean parallel) { try { final IndexingUtils.ChunkProjection chunkProjection = IndexingUtils.computeProjection(chunkCoords, metadata.shape, chunkShape, offset, - shape + shape, lowerBound ); ucar.ma2.Array chunkArray; @@ -192,15 +194,11 @@ protected void cleanupChunksForResize(long[] newShape, boolean parallel) { final int[] chunkShape = metadata.chunkShape(); final int ndim = metadata.ndim(); final dev.zarr.zarrjava.core.chunkkeyencoding.ChunkKeyEncoding chunkKeyEncoding = metadata.chunkKeyEncoding(); - - // Calculate max valid chunk coordinates for the new shape - long[] newMaxChunkCoords = new long[ndim]; - for (int i = 0; i < ndim; i++) { - newMaxChunkCoords[i] = (newShape[i] + chunkShape[i] - 1) / chunkShape[i]; - } + final long[] lowerBound = metadata.lowerBound(); // Iterate over all possible chunk coordinates in the old shape - long[][] allOldChunkCoords = IndexingUtils.computeChunkCoords(metadata.shape, chunkShape); + long[][] allOldChunkCoords = IndexingUtils.computeChunkCoords(metadata.shape, chunkShape, + new long[ndim], metadata.shape, lowerBound); Stream chunkStream = Arrays.stream(allOldChunkCoords); if (parallel) { @@ -212,13 +210,14 @@ protected void cleanupChunksForResize(long[] newShape, boolean parallel) { boolean isOnBoundary = false; for (int dimIdx = 0; dimIdx < ndim; dimIdx++) { - if (chunkCoords[dimIdx] >= newMaxChunkCoords[dimIdx]) { + long newEnd = lowerBound[dimIdx] + newShape[dimIdx]; + long chunkStart = chunkCoords[dimIdx] * chunkShape[dimIdx]; + if (chunkStart >= newEnd) { isOutsideBounds = true; break; } // Check if this chunk is on the boundary (partially outside new shape) - long chunkEnd = (chunkCoords[dimIdx] + 1) * chunkShape[dimIdx]; - if (chunkEnd > newShape[dimIdx]) { + if (chunkStart + chunkShape[dimIdx] > newEnd) { isOnBoundary = true; } } @@ -251,15 +250,17 @@ protected void cleanupChunksForResize(long[] newShape, boolean parallel) { protected void trimBoundaryChunk(long[] chunkCoords, long[] newShape, int[] chunkShape) throws ZarrException { ArrayMetadata metadata = metadata(); final int ndim = metadata.ndim(); + final long[] lowerBound = metadata.lowerBound(); // Calculate the valid region within this chunk int[] validShape = new int[ndim]; boolean needsTrimming = false; for (int dimIdx = 0; dimIdx < ndim; dimIdx++) { + long newEnd = lowerBound[dimIdx] + newShape[dimIdx]; long chunkStart = chunkCoords[dimIdx] * chunkShape[dimIdx]; long chunkEnd = chunkStart + chunkShape[dimIdx]; - if (chunkEnd > newShape[dimIdx]) { - validShape[dimIdx] = (int) (newShape[dimIdx] - chunkStart); + if (chunkEnd > newEnd) { + validShape[dimIdx] = (int) (newEnd - chunkStart); needsTrimming = true; } else { validShape[dimIdx] = chunkShape[dimIdx]; @@ -356,10 +357,15 @@ public ucar.ma2.Array read(final boolean parallel) throws ZarrException { } boolean chunkIsInArray(long[] chunkCoords) { - final int[] chunkShape = metadata().chunkShape(); - for (int dimIdx = 0; dimIdx < metadata().ndim(); dimIdx++) { - if (chunkCoords[dimIdx] < 0 - || chunkCoords[dimIdx] * chunkShape[dimIdx] >= metadata().shape[dimIdx]) { + final ArrayMetadata metadata = metadata(); + final int[] chunkShape = metadata.chunkShape(); + final long[] lowerBound = metadata.lowerBound(); + for (int dimIdx = 0; dimIdx < metadata.ndim(); dimIdx++) { + final long start = lowerBound[dimIdx]; + final long end = start + metadata.shape[dimIdx]; + if (end <= start + || chunkCoords[dimIdx] < Math.floorDiv(start, chunkShape[dimIdx]) + || chunkCoords[dimIdx] > Math.floorDiv(end - 1, chunkShape[dimIdx])) { return false; } } @@ -390,8 +396,9 @@ public ucar.ma2.Array read(final long[] offset, final long[] shape, final boolea } final int[] chunkShape = metadata.chunkShape(); - if (IndexingUtils.isSingleFullChunk(offset, shape, chunkShape)) { - return readChunk(IndexingUtils.computeSingleChunkCoords(offset, chunkShape)); + final long[] lowerBound = metadata.lowerBound(); + if (IndexingUtils.isSingleFullChunk(offset, shape, chunkShape, lowerBound)) { + return readChunk(IndexingUtils.computeSingleChunkCoords(offset, chunkShape, lowerBound)); } final ucar.ma2.Array outputArray = ucar.ma2.Array.factory(metadata.dataType().getMA2DataType(), @@ -400,7 +407,8 @@ public ucar.ma2.Array read(final long[] offset, final long[] shape, final boolea if (parsedFillValue != null) { MultiArrayUtils.fill(outputArray, parsedFillValue); } - Stream chunkStream = Arrays.stream(IndexingUtils.computeChunkCoords(metadata.shape, chunkShape, offset, shape)); + Stream chunkStream = Arrays.stream( + IndexingUtils.computeChunkCoords(metadata.shape, chunkShape, offset, shape, lowerBound)); if (parallel) { chunkStream = chunkStream.parallel(); } @@ -409,7 +417,7 @@ public ucar.ma2.Array read(final long[] offset, final long[] shape, final boolea try { final IndexingUtils.ChunkProjection chunkProjection = IndexingUtils.computeProjection(chunkCoords, metadata.shape, chunkShape, offset, - shape + shape, lowerBound ); final String[] chunkKeys = metadata.chunkKeyEncoding().encodeChunkKey(chunkCoords); diff --git a/src/main/java/dev/zarr/zarrjava/core/ArrayMetadata.java b/src/main/java/dev/zarr/zarrjava/core/ArrayMetadata.java index a2fb48f3..bd65f684 100644 --- a/src/main/java/dev/zarr/zarrjava/core/ArrayMetadata.java +++ b/src/main/java/dev/zarr/zarrjava/core/ArrayMetadata.java @@ -138,6 +138,14 @@ public int ndim() { return shape.length; } + /** + * The coordinate of the array's first element along each dimension. Indices passed to reads and + * writes stay zero-based; index {@code i} is coordinate {@code lowerBound()[dim] + i}. + */ + public long[] lowerBound() { + return new long[ndim()]; + } + public abstract int[] chunkShape(); public abstract DataType dataType(); diff --git a/src/main/java/dev/zarr/zarrjava/utils/IndexingUtils.java b/src/main/java/dev/zarr/zarrjava/utils/IndexingUtils.java index 404d3145..49c76c3b 100644 --- a/src/main/java/dev/zarr/zarrjava/utils/IndexingUtils.java +++ b/src/main/java/dev/zarr/zarrjava/utils/IndexingUtils.java @@ -15,13 +15,26 @@ public static long[][] computeChunkCoords(int[] arrayShape, int[] chunkShape) { public static long[][] computeChunkCoords(long[] arrayShape, int[] chunkShape, long[] selOffset, long[] selShape) { + return computeChunkCoords(arrayShape, chunkShape, selOffset, selShape, + new long[arrayShape.length]); + } + + /** + * Computes the coordinates of all chunks that intersect a selection. + * + * @param lowerBound the coordinate of the array's first element along each dimension; selections + * are relative to it, chunk coordinates are not + */ + public static long[][] computeChunkCoords(long[] arrayShape, int[] chunkShape, long[] selOffset, + long[] selShape, long[] lowerBound) { final int ndim = arrayShape.length; long[] start = new long[ndim]; long[] end = new long[ndim]; long numChunks = 1; for (int dimIdx = 0; dimIdx < ndim; dimIdx++) { - final int staIdx = (int) (selOffset[dimIdx] / chunkShape[dimIdx]); - final int endIdx = (int) ((selOffset[dimIdx] + selShape[dimIdx] - 1) / chunkShape[dimIdx]); + final long selStart = lowerBound[dimIdx] + selOffset[dimIdx]; + final long staIdx = Math.floorDiv(selStart, chunkShape[dimIdx]); + final long endIdx = Math.floorDiv(selStart + selShape[dimIdx] - 1, chunkShape[dimIdx]); numChunks *= (endIdx - staIdx + 1); start[dimIdx] = staIdx; end[dimIdx] = endIdx; @@ -66,6 +79,21 @@ public static ChunkProjection computeProjection( final long[] chunkCoords, final long[] arrayShape, final int[] chunkShape, final long[] selOffset, final long[] selShape + ) { + return computeProjection(chunkCoords, arrayShape, chunkShape, selOffset, selShape, + new long[chunkCoords.length]); + } + + /** + * Computes how a chunk maps onto a selection. + * + * @param lowerBound the coordinate of the array's first element along each dimension; selections + * are relative to it, chunk coordinates are not + */ + public static ChunkProjection computeProjection( + final long[] chunkCoords, final long[] arrayShape, + final int[] chunkShape, final long[] selOffset, + final long[] selShape, final long[] lowerBound ) { final int ndim = chunkCoords.length; final int[] chunkOffset = new int[ndim]; @@ -74,9 +102,9 @@ public static ChunkProjection computeProjection( for (int dimIdx = 0; dimIdx < chunkCoords.length; dimIdx++) { // compute offsets for chunk within overall array - final long dimOffset = (long) chunkShape[dimIdx] * chunkCoords[dimIdx]; + final long dimOffset = (long) chunkShape[dimIdx] * chunkCoords[dimIdx] - lowerBound[dimIdx]; final long dimLimit = Math.min(arrayShape[dimIdx], - (chunkCoords[dimIdx] + 1) * (long) chunkShape[dimIdx]); + (chunkCoords[dimIdx] + 1) * (long) chunkShape[dimIdx] - lowerBound[dimIdx]); if (selOffset[dimIdx] < dimOffset) { // selection starts before current chunk @@ -136,6 +164,11 @@ public static boolean isFullChunk(final int[] selOffset, final int[] selShape, public static boolean isSingleFullChunk(final long[] selOffset, final long[] selShape, final int[] chunkShape) { + return isSingleFullChunk(selOffset, selShape, chunkShape, new long[selOffset.length]); + } + + public static boolean isSingleFullChunk(final long[] selOffset, final long[] selShape, + final int[] chunkShape, final long[] lowerBound) { if (selOffset.length != selShape.length) { throw new IllegalArgumentException("'selOffset' and 'selShape' need to have the same rank."); } @@ -144,7 +177,8 @@ public static boolean isSingleFullChunk(final long[] selOffset, final long[] sel "'selOffset' and 'chunkShape' need to have the same rank."); } for (int dimIdx = 0; dimIdx < selOffset.length; dimIdx++) { - if (selOffset[dimIdx] % chunkShape[dimIdx] != 0 || selShape[dimIdx] != chunkShape[dimIdx]) { + if (Math.floorMod(lowerBound[dimIdx] + selOffset[dimIdx], chunkShape[dimIdx]) != 0 + || selShape[dimIdx] != chunkShape[dimIdx]) { return false; } } @@ -152,13 +186,18 @@ public static boolean isSingleFullChunk(final long[] selOffset, final long[] sel } public static long[] computeSingleChunkCoords(final long[] selOffset, final int[] chunkShape) { + return computeSingleChunkCoords(selOffset, chunkShape, new long[selOffset.length]); + } + + public static long[] computeSingleChunkCoords(final long[] selOffset, final int[] chunkShape, + final long[] lowerBound) { if (selOffset.length != chunkShape.length) { throw new IllegalArgumentException( "'selOffset' and 'chunkShape' need to have the same rank."); } final long[] chunkCoords = new long[selOffset.length]; for (int dimIdx = 0; dimIdx < selOffset.length; dimIdx++) { - chunkCoords[dimIdx] = (selOffset[dimIdx] / chunkShape[dimIdx]); + chunkCoords[dimIdx] = Math.floorDiv(lowerBound[dimIdx] + selOffset[dimIdx], chunkShape[dimIdx]); } return chunkCoords; } diff --git a/src/main/java/dev/zarr/zarrjava/v3/Array.java b/src/main/java/dev/zarr/zarrjava/v3/Array.java index 6b30e297..c9867b52 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/Array.java +++ b/src/main/java/dev/zarr/zarrjava/v3/Array.java @@ -257,6 +257,55 @@ public Array resize(long[] newShape, boolean resizeMetadataOnly, boolean paralle return writeMetadata(newArrayMetadata); } + /** + * Adds data before the first element along the first dimension. See {@link #prepend(ucar.ma2.Array, int)}. + * + * @param data the data to add + * @throws ZarrException if the new metadata is invalid or the write fails + * @throws IOException throws IOException if the new metadata cannot be serialized + */ + public Array prepend(ucar.ma2.Array data) throws ZarrException, IOException { + return prepend(data, 0); + } + + /** + * Adds data before the first element along a dimension. The lower bound of the array moves down + * by the length of {@code data} along {@code axis}; existing chunks keep their keys and are not + * rewritten, except for the one holding the old first element if it is only partially filled. + * Indexing stays zero-based, so {@code data} starts at index 0 afterwards. + * This method returns a new instance of the Zarr array class and the old instance becomes invalid. + * + * @param data the data to add, with the array's shape along every other dimension + * @param axis the dimension to grow + * @throws ZarrException if the new metadata is invalid or the write fails + * @throws IOException throws IOException if the new metadata cannot be serialized + */ + public Array prepend(ucar.ma2.Array data, int axis) throws ZarrException, IOException { + final int ndim = metadata.ndim(); + if (axis < 0 || axis >= ndim) { + throw new IllegalArgumentException("'axis' needs to be in [0, " + ndim + ")."); + } + if (data.getRank() != ndim) { + throw new IllegalArgumentException("'data' needs to have rank '" + ndim + "'."); + } + int[] dataShape = data.getShape(); + for (int i = 0; i < ndim; i++) { + if (i != axis && dataShape[i] != metadata.shape[i]) { + throw new IllegalArgumentException( + "'data' needs to match the array's shape along dimension " + i + "."); + } + } + + long[] inclusiveMin = metadata.lowerBound(); + long[] exclusiveMax = metadata.bounds().exclusiveMax; + inclusiveMin[axis] -= dataShape[axis]; + Array newArray = writeMetadata(ArrayMetadataBuilder.fromArrayMetadata(metadata) + .withBounds(inclusiveMin, exclusiveMax) + .build()); + newArray.write(new long[ndim], data); + return newArray; + } + /** * Sets the attributes of the Zarr array. It overwrites and removes any existing attributes. This diff --git a/src/main/java/dev/zarr/zarrjava/v3/ArrayMetadata.java b/src/main/java/dev/zarr/zarrjava/v3/ArrayMetadata.java index cb880cf0..7ef8bb78 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/ArrayMetadata.java +++ b/src/main/java/dev/zarr/zarrjava/v3/ArrayMetadata.java @@ -61,6 +61,9 @@ public final class ArrayMetadata extends dev.zarr.zarrjava.core.ArrayMetadata { @JsonIgnore public CoreArrayMetadata coreArrayMetadata; + @JsonIgnore + private final long[] lowerBound; + public ArrayMetadata( long[] shape, DataType dataType, ChunkGrid chunkGrid, ChunkKeyEncoding chunkKeyEncoding, Object fillValue, @@ -108,11 +111,35 @@ public ArrayMetadata( ); } + public ArrayMetadata( + int zarrFormat, + String nodeType, + long[] shape, + DataType dataType, + ChunkGrid chunkGrid, + ChunkKeyEncoding chunkKeyEncoding, + Object fillValue, + @Nonnull Codec[] codecs, + @Nullable String[] dimensionNames, + @Nullable Attributes attributes, + @Nullable Map[] storageTransformers, + @Nullable Map extraFields + ) throws ZarrException { + this(zarrFormat, nodeType, shape, null, dataType, chunkGrid, chunkKeyEncoding, fillValue, codecs, + dimensionNames, attributes, storageTransformers, extraFields + ); + } + + /** + * Creates array metadata from either a {@code shape} or explicit {@code bounds}; exactly one of + * them must be given. + */ @JsonCreator(mode = JsonCreator.Mode.PROPERTIES) public ArrayMetadata( @JsonProperty(value = "zarr_format", required = true) int zarrFormat, @JsonProperty(value = "node_type", required = true) String nodeType, - @JsonProperty(value = "shape", required = true) long[] shape, + @Nullable @JsonProperty(value = "shape") long[] shape, + @Nullable @JsonProperty(value = "bounds") Bounds bounds, @JsonProperty(value = "data_type", required = true) DataType dataType, @JsonProperty(value = "chunk_grid", required = true) ChunkGrid chunkGrid, @JsonProperty(value = "chunk_key_encoding", required = true) ChunkKeyEncoding chunkKeyEncoding, @@ -123,7 +150,8 @@ public ArrayMetadata( @Nullable @JsonProperty(value = "storage_transformers") Map[] storageTransformers, @Nullable @JsonAnySetter Map extraFields ) throws ZarrException { - super(shape, fillValue, dataType); + super(derivedShape(shape, bounds), fillValue, dataType); + this.lowerBound = bounds != null ? bounds.inclusiveMin.clone() : new long[this.shape.length]; this.extraFields = ExtraFields.validatedArrayFields(extraFields); if (zarrFormat != this.zarrFormat) { throw new ZarrException( @@ -139,8 +167,8 @@ public ArrayMetadata( } if (chunkGrid instanceof RegularChunkGrid) { int[] chunkShape = ((RegularChunkGrid) chunkGrid).configuration.chunkShape; - if (shape.length != chunkShape.length) { - throw new ZarrException("Shape (ndim=" + shape.length + ") and chunk shape (ndim=" + + if (this.shape.length != chunkShape.length) { + throw new ZarrException("Shape (ndim=" + this.shape.length + ") and chunk shape (ndim=" + chunkShape.length + ") need to have the same number of dimensions."); } @@ -186,6 +214,68 @@ public Map extraFields() { return extraFields; } + private static long[] derivedShape(@Nullable long[] shape, @Nullable Bounds bounds) + throws ZarrException { + if ((shape == null) == (bounds == null)) { + throw new ZarrException("Array metadata needs to contain exactly one of 'shape' and 'bounds'."); + } + if (shape != null) { + return shape; + } + if (bounds.inclusiveMin == null || bounds.exclusiveMax == null) { + throw new ZarrException("'bounds' needs to contain 'inclusive_min' and 'exclusive_max'."); + } + if (bounds.inclusiveMin.length != bounds.exclusiveMax.length) { + throw new ZarrException("'inclusive_min' and 'exclusive_max' need to have the same length."); + } + long[] derived = new long[bounds.inclusiveMin.length]; + for (int i = 0; i < derived.length; i++) { + if (bounds.inclusiveMin[i] > bounds.exclusiveMax[i]) { + throw new ZarrException("'inclusive_min' needs to be less than or equal to 'exclusive_max'."); + } + try { + derived[i] = Math.subtractExact(bounds.exclusiveMax[i], bounds.inclusiveMin[i]); + } catch (ArithmeticException e) { + throw new ZarrException("The length of dimension " + i + " exceeds the int64 range."); + } + } + return derived; + } + + @Override + public long[] lowerBound() { + return lowerBound.clone(); + } + + /** + * The coordinate range of the array: the first and one past the last coordinate along each + * dimension. + */ + @JsonIgnore + public Bounds bounds() { + long[] exclusiveMax = new long[ndim()]; + for (int i = 0; i < exclusiveMax.length; i++) { + exclusiveMax[i] = lowerBound[i] + shape[i]; + } + return new Bounds(lowerBound.clone(), exclusiveMax); + } + + private boolean hasZeroLowerBound() { + return Arrays.stream(lowerBound).allMatch(b -> b == 0); + } + + @Nullable + @JsonProperty("shape") + private long[] jsonShape() { + return hasZeroLowerBound() ? shape : null; + } + + @Nullable + @JsonProperty("bounds") + private Bounds jsonBounds() { + return hasZeroLowerBound() ? null : bounds(); + } + public static Optional getShardingIndexedCodec(Codec[] codecs) { return Arrays.stream(codecs).filter(codec -> codec instanceof ShardingIndexedCodec).findFirst(); } @@ -230,5 +320,24 @@ public int chunkByteLength() { return coreArrayMetadata.chunkByteLength(); } + /** + * A half-open coordinate range per dimension. + */ + public static final class Bounds { + @JsonProperty("inclusive_min") + public final long[] inclusiveMin; + @JsonProperty("exclusive_max") + public final long[] exclusiveMax; + + @JsonCreator + public Bounds( + @JsonProperty(value = "inclusive_min", required = true) long[] inclusiveMin, + @JsonProperty(value = "exclusive_max", required = true) long[] exclusiveMax + ) { + this.inclusiveMin = inclusiveMin; + this.exclusiveMax = exclusiveMax; + } + } + } diff --git a/src/main/java/dev/zarr/zarrjava/v3/ArrayMetadataBuilder.java b/src/main/java/dev/zarr/zarrjava/v3/ArrayMetadataBuilder.java index fd6b5c37..3adcacc1 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/ArrayMetadataBuilder.java +++ b/src/main/java/dev/zarr/zarrjava/v3/ArrayMetadataBuilder.java @@ -14,6 +14,7 @@ import dev.zarr.zarrjava.v3.codec.CodecBuilder; import dev.zarr.zarrjava.v3.codec.core.BytesCodec; +import java.util.Arrays; import java.util.HashMap; import java.util.Map; import java.util.function.Function; @@ -21,6 +22,7 @@ public class ArrayMetadataBuilder { long[] shape = null; + long[] lowerBound = null; DataType dataType = null; ChunkGrid chunkGrid = null; ChunkKeyEncoding chunkKeyEncoding = @@ -43,6 +45,7 @@ protected static ArrayMetadataBuilder fromArrayMetadata(ArrayMetadata arrayMetad protected static ArrayMetadataBuilder fromArrayMetadata(ArrayMetadata arrayMetadata, boolean withAttributes) { ArrayMetadataBuilder builder = new ArrayMetadataBuilder(); builder.shape = arrayMetadata.shape; + builder.lowerBound = arrayMetadata.lowerBound(); builder.dataType = arrayMetadata.dataType; builder.chunkGrid = arrayMetadata.chunkGrid; builder.chunkKeyEncoding = arrayMetadata.chunkKeyEncoding; @@ -57,11 +60,32 @@ protected static ArrayMetadataBuilder fromArrayMetadata(ArrayMetadata arrayMetad return builder; } + /** + * Sets the shape of the array. A lower bound set earlier, e.g. through {@link #withBounds}, is + * kept. + */ public ArrayMetadataBuilder withShape(long... shape) { this.shape = shape; return this; } + /** + * Sets the coordinate range of the array: the first and one past the last coordinate along each + * dimension. The lower bound may be negative. + */ + public ArrayMetadataBuilder withBounds(long[] inclusiveMin, long[] exclusiveMax) throws ZarrException { + if (inclusiveMin.length != exclusiveMax.length) { + throw new ZarrException("'inclusiveMin' and 'exclusiveMax' need to have the same length."); + } + long[] shape = new long[inclusiveMin.length]; + for (int i = 0; i < shape.length; i++) { + shape[i] = exclusiveMax[i] - inclusiveMin[i]; + } + this.shape = shape; + this.lowerBound = inclusiveMin; + return this; + } + public ArrayMetadataBuilder withDataType(DataType dataType) { this.dataType = dataType; return this; @@ -182,11 +206,21 @@ public ArrayMetadata build() throws ZarrException { chunkGrid = new RegularChunkGrid(new RegularChunkGrid.Configuration(defaultChunks)); } - return new ArrayMetadata(shape, dataType, chunkGrid, chunkKeyEncoding, fillValue, codecs, - dimensionNames, - attributes, - storageTransformers, - extraFields + ArrayMetadata.Bounds bounds = null; + if (lowerBound != null && Arrays.stream(lowerBound).anyMatch(b -> b != 0)) { + if (lowerBound.length != shape.length) { + throw new ZarrException("The lower bound and the shape need to have the same rank."); + } + long[] exclusiveMax = new long[shape.length]; + for (int i = 0; i < shape.length; i++) { + exclusiveMax[i] = lowerBound[i] + shape[i]; + } + bounds = new ArrayMetadata.Bounds(lowerBound, exclusiveMax); + } + + return new ArrayMetadata(ArrayMetadata.ZARR_FORMAT, ArrayMetadata.NODE_TYPE, + bounds == null ? shape : null, bounds, dataType, chunkGrid, chunkKeyEncoding, fillValue, + codecs, dimensionNames, attributes, storageTransformers, extraFields ); } } diff --git a/src/main/java/dev/zarr/zarrjava/v3/ExtraFields.java b/src/main/java/dev/zarr/zarrjava/v3/ExtraFields.java index 6b353381..69ec2505 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/ExtraFields.java +++ b/src/main/java/dev/zarr/zarrjava/v3/ExtraFields.java @@ -31,7 +31,7 @@ final class ExtraFields { * {@code zarr.json} array document is an extra field. */ private static final Set ARRAY_METADATA_KEYS = unmodifiableSetOf( - "zarr_format", "node_type", "shape", "data_type", "chunk_grid", "chunk_key_encoding", + "zarr_format", "node_type", "shape", "bounds", "data_type", "chunk_grid", "chunk_key_encoding", "fill_value", "codecs", "attributes", "dimension_names", "storage_transformers" ); diff --git a/src/test/java/dev/zarr/zarrjava/ArrayBoundsTest.java b/src/test/java/dev/zarr/zarrjava/ArrayBoundsTest.java new file mode 100644 index 00000000..9b596d3b --- /dev/null +++ b/src/test/java/dev/zarr/zarrjava/ArrayBoundsTest.java @@ -0,0 +1,257 @@ +package dev.zarr.zarrjava; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import dev.zarr.zarrjava.store.FilesystemStore; +import dev.zarr.zarrjava.store.StoreHandle; +import dev.zarr.zarrjava.utils.MultiArrayUtils; +import dev.zarr.zarrjava.v3.Array; +import dev.zarr.zarrjava.v3.DataType; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Random; + +import static dev.zarr.zarrjava.core.Node.ZARR_JSON; + +public class ArrayBoundsTest extends ZarrTest { + + static final ObjectMapper MAPPER = new ObjectMapper(); + + static Array createExample(Path path) throws Exception { + Array array = Array.create(new FilesystemStore(path).resolve(), + Array.metadataBuilder() + .withShape(6) + .withDataType(DataType.INT32) + .withChunkShape(4) + .withFillValue(0) + .build()); + array.write(ucar.ma2.Array.makeFromJavaArray(new int[]{10, 11, 12, 13, 14, 15})); + return array; + } + + static JsonNode readZarrJson(Path path) throws IOException { + return MAPPER.readTree(path.resolve(ZARR_JSON).toFile()); + } + + @Test + public void testPrependKeepsExistingChunks() throws Exception { + Path path = TESTOUTPUT.resolve("bounds_prepend"); + Array array = createExample(path); + byte[] chunk0 = Files.readAllBytes(path.resolve("c/0")); + byte[] chunk1 = Files.readAllBytes(path.resolve("c/1")); + + array = array.prepend(ucar.ma2.Array.makeFromJavaArray(new int[]{7, 8, 9})); + + Assertions.assertArrayEquals(new long[]{9}, array.metadata().shape); + Assertions.assertArrayEquals(new long[]{-3}, array.metadata().lowerBound()); + Assertions.assertArrayEquals(new int[]{7, 8, 9, 10, 11, 12, 13, 14, 15}, + (int[]) array.read().get1DJavaArray(ucar.ma2.DataType.INT)); + + Assertions.assertArrayEquals(chunk0, Files.readAllBytes(path.resolve("c/0"))); + Assertions.assertArrayEquals(chunk1, Files.readAllBytes(path.resolve("c/1"))); + Assertions.assertArrayEquals(new int[]{0, 7, 8, 9}, + (int[]) array.readChunk(new long[]{-1}).get1DJavaArray(ucar.ma2.DataType.INT)); + + JsonNode zarrJson = readZarrJson(path); + Assertions.assertFalse(zarrJson.has("shape")); + Assertions.assertEquals("{\"inclusive_min\":[-3],\"exclusive_max\":[6]}", + zarrJson.get("bounds").toString()); + + Array reopened = Array.open(path); + Assertions.assertArrayEquals(new long[]{-3}, reopened.metadata().lowerBound()); + Assertions.assertArrayEquals(new int[]{8, 9, 10}, + (int[]) reopened.read(new long[]{1}, new long[]{3}).get1DJavaArray(ucar.ma2.DataType.INT)); + } + + @Test + public void testPrependAcrossSeveralChunksAndAxes() throws Exception { + Path path = TESTOUTPUT.resolve("bounds_prepend_2d"); + Array array = Array.create(new FilesystemStore(path).resolve(), + Array.metadataBuilder() + .withShape(4, 3) + .withDataType(DataType.INT32) + .withChunkShape(2, 2) + .withFillValue(-1) + .build()); + ucar.ma2.Array original = arange(new int[]{4, 3}, 0); + array.write(original); + + ucar.ma2.Array rows = arange(new int[]{5, 3}, 100); + array = array.prepend(rows, 0); + ucar.ma2.Array cols = arange(new int[]{9, 1}, 200); + array = array.prepend(cols, 1); + + Assertions.assertArrayEquals(new long[]{-5, -1}, array.metadata().lowerBound()); + Assertions.assertArrayEquals(new long[]{9, 4}, array.metadata().shape); + ucar.ma2.Array result = array.read(); + for (int i = 0; i < 9; i++) { + Assertions.assertEquals(200 + i, result.getInt(result.getIndex().set(i, 0))); + for (int j = 0; j < 3; j++) { + int expected = i < 5 ? 100 + i * 3 + j : (i - 5) * 3 + j; + Assertions.assertEquals(expected, result.getInt(result.getIndex().set(i, j + 1))); + } + } + Assertions.assertTrue(Files.exists(path.resolve("c/-3/-1"))); + Assertions.assertTrue(Files.exists(path.resolve("c/1/1"))); + } + + @ParameterizedTest + @ValueSource(longs = {-9, -7, -4, -3, -1, 0, 2, 5}) + public void testReadWriteMatchesReference(long lowerBound) throws Exception { + StoreHandle store = new FilesystemStore(TESTOUTPUT).resolve("bounds_reference_" + lowerBound); + Array array = Array.create(store, + Array.metadataBuilder() + .withBounds(new long[]{lowerBound, -lowerBound}, new long[]{lowerBound + 11, -lowerBound + 7}) + .withDataType(DataType.INT32) + .withChunkShape(4, 3) + .withFillValue(0) + .build()); + ucar.ma2.Array reference = ucar.ma2.Array.factory(ucar.ma2.DataType.INT, new int[]{11, 7}); + Random random = new Random(lowerBound); + for (int n = 0; n < 20; n++) { + int[] offset = {random.nextInt(11), random.nextInt(7)}; + int[] shape = {1 + random.nextInt(11 - offset[0]), 1 + random.nextInt(7 - offset[1])}; + ucar.ma2.Array data = arange(shape, 1 + n * 100); + array.write(new long[]{offset[0], offset[1]}, data); + MultiArrayUtils.copyRegion(data, new int[2], reference, offset, shape); + + int[] readOffset = {random.nextInt(11), random.nextInt(7)}; + int[] readShape = {1 + random.nextInt(11 - readOffset[0]), 1 + random.nextInt(7 - readOffset[1])}; + Assertions.assertTrue(MultiArrayUtils.allValuesEqual( + reference.sectionNoReduce(readOffset, readShape, null), + array.read(new long[]{readOffset[0], readOffset[1]}, new long[]{readShape[0], readShape[1]}))); + } + Assertions.assertTrue(MultiArrayUtils.allValuesEqual(reference, Array.open(store).read())); + } + + @Test + public void testPrependWithSharding() throws Exception { + Path path = TESTOUTPUT.resolve("bounds_sharding"); + Array array = Array.create(new FilesystemStore(path).resolve(), + Array.metadataBuilder() + .withShape(8, 8) + .withDataType(DataType.INT32) + .withChunkShape(4, 4) + .withFillValue(0) + .withCodecs(c -> c.withSharding(new int[]{2, 2})) + .build()); + array.write(arange(new int[]{8, 8}, 1)); + byte[] shard = Files.readAllBytes(path.resolve("c/0/0")); + + array = array.prepend(arange(new int[]{3, 8}, 1000), 0); + + Assertions.assertArrayEquals(shard, Files.readAllBytes(path.resolve("c/0/0"))); + ucar.ma2.Array result = Array.open(path).read(); + Assertions.assertEquals(1000, result.getInt(result.getIndex().set(0, 0))); + Assertions.assertEquals(1, result.getInt(result.getIndex().set(3, 0))); + Assertions.assertEquals(64, result.getInt(result.getIndex().set(10, 7))); + } + + @Test + public void testResizeKeepsLowerBound() throws Exception { + Path path = TESTOUTPUT.resolve("bounds_resize"); + Array array = createExample(path).prepend(ucar.ma2.Array.makeFromJavaArray(new int[]{7, 8, 9})); + + array = array.resize(new long[]{5}, false); + Assertions.assertArrayEquals(new long[]{-3}, array.metadata().lowerBound()); + Assertions.assertFalse(Files.exists(path.resolve("c/1"))); + Assertions.assertArrayEquals(new int[]{7, 8, 9, 10, 11}, + (int[]) array.read().get1DJavaArray(ucar.ma2.DataType.INT)); + + array = array.resize(new long[]{9}); + Assertions.assertArrayEquals(new int[]{7, 8, 9, 10, 11, 0, 0, 0, 0}, + (int[]) array.read().get1DJavaArray(ucar.ma2.DataType.INT)); + } + + @Test + public void testZeroLowerBoundIsWrittenAsShape() throws Exception { + Path path = TESTOUTPUT.resolve("bounds_zero"); + writeZarrJson(path, "\"bounds\": {\"inclusive_min\": [0, 0], \"exclusive_max\": [6, 7]}"); + Array array = Array.open(path); + Assertions.assertArrayEquals(new long[]{6, 7}, array.metadata().shape); + + array.setAttributes(new dev.zarr.zarrjava.core.Attributes()); + JsonNode zarrJson = readZarrJson(path); + Assertions.assertEquals("[6,7]", zarrJson.get("shape").toString()); + Assertions.assertFalse(zarrJson.has("bounds")); + } + + @Test + public void testReadExampleFromZep() throws Exception { + Path path = TESTOUTPUT.resolve("bounds_zep_example"); + writeZarrJson(path, "\"bounds\": {\"inclusive_min\": [-3, -2], \"exclusive_max\": [6, 7]}"); + Array array = Array.open(path); + Assertions.assertArrayEquals(new long[]{9, 9}, array.metadata().shape); + + array.write(arange(new int[]{9, 9}, 1)); + for (String key : new String[]{"c/-1/-1", "c/-1/0", "c/-1/1", "c/0/-1", "c/0/0", "c/0/1", "c/1/-1", "c/1/0", "c/1/1"}) { + Assertions.assertTrue(Files.exists(path.resolve(key)), key); + } + // the first element is at position 1 of chunk -1 along dim 0 and position 3 of chunk -1 along dim 1 + ucar.ma2.Array chunk = array.readChunk(new long[]{-1, -1}); + Assertions.assertEquals(1, chunk.getInt(chunk.getIndex().set(1, 3))); + Assertions.assertEquals(2, chunk.getInt(chunk.getIndex().set(1, 4))); + Assertions.assertEquals(0, chunk.getInt(chunk.getIndex().set(0, 0))); + } + + @Test + public void testInvalidMetadata() throws Exception { + String bounds = "\"bounds\": {\"inclusive_min\": [-3], \"exclusive_max\": [6]}"; + writeZarrJson(TESTOUTPUT.resolve("bounds_valid"), bounds, "[4]"); + Assertions.assertArrayEquals(new long[]{9}, Array.open(TESTOUTPUT.resolve("bounds_valid")).metadata().shape); + assertOpenFails("bounds_and_shape", bounds + ", \"shape\": [9]"); + assertOpenFails("bounds_none", null); + assertOpenFails("bounds_inverted", "\"bounds\": {\"inclusive_min\": [6], \"exclusive_max\": [-3]}"); + assertOpenFails("bounds_length", "\"bounds\": {\"inclusive_min\": [0, 0], \"exclusive_max\": [6]}"); + assertOpenFails("bounds_missing_max", "\"bounds\": {\"inclusive_min\": [0]}"); + } + + @Test + public void testOriginIsRejected() throws Exception { + Path path = TESTOUTPUT.resolve("bounds_origin"); + Files.createDirectories(path); + Files.write(path.resolve(ZARR_JSON), ("{\"zarr_format\": 3, \"node_type\": \"array\", " + + "\"bounds\": {\"inclusive_min\": [-3], \"exclusive_max\": [6]}, \"data_type\": \"int32\", " + + "\"chunk_grid\": {\"name\": \"regular\", \"configuration\": {\"chunk_shape\": [4], \"origin\": [1]}}, " + + "\"chunk_key_encoding\": {\"name\": \"default\"}, " + + "\"codecs\": [{\"name\": \"bytes\", \"configuration\": {\"endian\": \"little\"}}], " + + "\"fill_value\": 0}").getBytes(StandardCharsets.UTF_8)); + Assertions.assertThrows(Exception.class, () -> Array.open(path)); + } + + void assertOpenFails(String name, String extent) throws IOException { + Path path = TESTOUTPUT.resolve(name); + writeZarrJson(path, extent, "[4]"); + Assertions.assertThrows(Exception.class, () -> Array.open(path)); + } + + static void writeZarrJson(Path path, String extent) throws IOException { + writeZarrJson(path, extent, "[4, 5]"); + } + + static void writeZarrJson(Path path, String extent, String chunkShape) throws IOException { + Files.createDirectories(path); + Files.write(path.resolve(ZARR_JSON), ("{\"zarr_format\": 3, \"node_type\": \"array\", " + + (extent == null ? "" : extent + ", ") + + "\"data_type\": \"int32\", " + + "\"chunk_grid\": {\"name\": \"regular\", \"configuration\": {\"chunk_shape\": " + chunkShape + "}}, " + + "\"chunk_key_encoding\": {\"name\": \"default\"}, " + + "\"codecs\": [{\"name\": \"bytes\", \"configuration\": {\"endian\": \"little\"}}], " + + "\"fill_value\": 0}").getBytes(StandardCharsets.UTF_8)); + } + + static ucar.ma2.Array arange(int[] shape, int start) { + ucar.ma2.Array array = ucar.ma2.Array.factory(ucar.ma2.DataType.INT, shape); + for (int i = 0; i < array.getSize(); i++) { + array.setInt(i, start + i); + } + return array; + } +}