From 5012008c5741431e6a35f7a47ea3c28ab53fc1fb Mon Sep 17 00:00:00 2001 From: Stephan Saalfeld Date: Mon, 12 Apr 2021 22:28:23 -0400 Subject: [PATCH 001/243] optionally cache attributes Avoids frequent reading and parsing of JSON encoded attributes, this is most interesting for high latency backends. Changes of attributes by an independent writer will not be tracked, so some consideration required. --- README.md | 2 +- pom.xml | 2 +- .../saalfeldlab/n5/AbstractGsonReader.java | 182 ++++++++++++++++-- .../saalfeldlab/n5/DatasetAttributes.java | 2 +- .../janelia/saalfeldlab/n5/N5FSReader.java | 50 ++++- .../janelia/saalfeldlab/n5/N5FSWriter.java | 69 ++++++- .../org/janelia/saalfeldlab/n5/N5FSTest.java | 2 +- 7 files changed, 282 insertions(+), 27 deletions(-) diff --git a/README.md b/README.md index a53d0d4b..a6e32f42 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ Chunked datasets can be sparse, i.e. empty chunks do not need to be stored. ## File-system specification -*version 2.4.1-SNAPSHOT* +*version 2.6.0-SNAPSHOT* N5 group is not a single file but simply a directory on the file system. Meta-data is stored as a JSON file per each group/ directory. Tensor datasets can be chunked and chunks are stored as individual files. This enables parallel reading and writing on a cluster. diff --git a/pom.xml b/pom.xml index 1ff6188f..9a231ef1 100644 --- a/pom.xml +++ b/pom.xml @@ -11,7 +11,7 @@ org.janelia.saalfeldlab n5 - 2.5.1-SNAPSHOT + 2.6.0-SNAPSHOT N5 Not HDF5 diff --git a/src/main/java/org/janelia/saalfeldlab/n5/AbstractGsonReader.java b/src/main/java/org/janelia/saalfeldlab/n5/AbstractGsonReader.java index cbda4b06..b8403b98 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/AbstractGsonReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/AbstractGsonReader.java @@ -46,18 +46,54 @@ public abstract class AbstractGsonReader implements GsonAttributesParser, N5Read protected final Gson gson; + protected final HashMap> attributesCache = new HashMap<>(); + + protected final boolean cacheAttributes; + /** * Constructs an {@link AbstractGsonReader} with a custom * {@link GsonBuilder} to support custom attributes. * * @param gsonBuilder + * @param cacheAttributes cache attributes + * Setting this to true avoids frequent reading and parsing of JSON + * encoded attributes, this is most interesting for high latency + * backends. Changes of attributes by an independent writer will not be + * tracked. */ - public AbstractGsonReader(final GsonBuilder gsonBuilder) { + public AbstractGsonReader(final GsonBuilder gsonBuilder, final boolean cacheAttributes) { gsonBuilder.registerTypeAdapter(DataType.class, new DataType.JsonAdapter()); gsonBuilder.registerTypeHierarchyAdapter(Compression.class, CompressionAdapter.getJsonAdapter()); gsonBuilder.disableHtmlEscaping(); this.gson = gsonBuilder.create(); + this.cacheAttributes = cacheAttributes; + } + + /** + * Constructs an {@link AbstractGsonReader} with a default + * {@link GsonBuilder}. + * + * @param cacheAttributes cache attributes + * Setting this to true avoids frequent reading and parsing of JSON + * encoded attributes, this is most interesting for high latency + * backends. Changes of attributes by an independent writer will not be + * tracked. + */ + public AbstractGsonReader(final boolean cacheAttributes) { + + this(new GsonBuilder(), cacheAttributes); + } + + /** + * Constructs an {@link AbstractGsonReader} with a custom + * {@link GsonBuilder} to support custom attributes. + * + * @param gsonBuilder + */ + public AbstractGsonReader(final GsonBuilder gsonBuilder) { + + this(gsonBuilder, false); } /** @@ -66,7 +102,7 @@ public AbstractGsonReader(final GsonBuilder gsonBuilder) { */ public AbstractGsonReader() { - this(new GsonBuilder()); + this(new GsonBuilder(), false); } @Override @@ -78,26 +114,59 @@ public Gson getGson() { @Override public DatasetAttributes getDatasetAttributes(final String pathName) throws IOException { - final HashMap map = getAttributes(pathName); - final Gson gson = getGson(); + final long[] dimensions; + final DataType dataType; + int[] blockSize; + Compression compression; + final String compressionVersion0Name; + if (cacheAttributes) { + final HashMap cachedMap = getCachedAttributes(pathName); + if (cachedMap.isEmpty()) + return null; - final long[] dimensions = GsonAttributesParser.parseAttribute(map, DatasetAttributes.dimensionsKey, long[].class, gson); - if (dimensions == null) - return null; + dimensions = getAttribute(cachedMap, DatasetAttributes.dimensionsKey, long[].class); + if (dimensions == null) + return null; - final DataType dataType = GsonAttributesParser.parseAttribute(map, DatasetAttributes.dataTypeKey, DataType.class, gson); - if (dataType == null) - return null; + dataType = getAttribute(cachedMap, DatasetAttributes.dataTypeKey, DataType.class); + if (dataType == null) + return null; + + blockSize = getAttribute(cachedMap, DatasetAttributes.blockSizeKey, int[].class); + + compression = getAttribute(cachedMap, DatasetAttributes.compressionKey, Compression.class); + + /* version 0 */ + compressionVersion0Name = compression == null + ? getAttribute(cachedMap, DatasetAttributes.compressionTypeKey, String.class) + : null; + } else { + final HashMap map = getAttributes(pathName); + + dimensions = GsonAttributesParser.parseAttribute(map, DatasetAttributes.dimensionsKey, long[].class, gson); + if (dimensions == null) + return null; + + dataType = GsonAttributesParser.parseAttribute(map, DatasetAttributes.dataTypeKey, DataType.class, gson); + if (dataType == null) + return null; + + blockSize = GsonAttributesParser.parseAttribute(map, DatasetAttributes.blockSizeKey, int[].class, gson); + + compression = GsonAttributesParser.parseAttribute(map, DatasetAttributes.compressionKey, Compression.class, gson); + + /* version 0 */ + compressionVersion0Name = compression == null + ? GsonAttributesParser.parseAttribute(map, DatasetAttributes.compressionTypeKey, String.class, gson) + : null; + } - int[] blockSize = GsonAttributesParser.parseAttribute(map, DatasetAttributes.blockSizeKey, int[].class, gson); if (blockSize == null) blockSize = Arrays.stream(dimensions).mapToInt(a -> (int)a).toArray(); - Compression compression = GsonAttributesParser.parseAttribute(map, DatasetAttributes.compressionKey, Compression.class, gson); - /* version 0 */ if (compression == null) { - switch (GsonAttributesParser.parseAttribute(map, DatasetAttributes.compressionTypeKey, String.class, gson)) { + switch (compressionVersion0Name) { case "raw": compression = new RawCompression(); break; @@ -119,14 +188,84 @@ public DatasetAttributes getDatasetAttributes(final String pathName) throws IOEx return new DatasetAttributes(dimensions, blockSize, dataType, compression); } + /** + * Get and cache attributes for a group. + * + * @param pathName + * @return + * @throws IOException + */ + protected HashMap getCachedAttributes(final String pathName) throws IOException { + + HashMap cachedMap = attributesCache.get(pathName); + if (cachedMap == null) { + final HashMap map = getAttributes(pathName); + cachedMap = new HashMap<>(); + if (map != null) + cachedMap.putAll(map); + + synchronized (attributesCache) { + attributesCache.put(pathName, cachedMap); + } + } + return cachedMap; + } + + @SuppressWarnings("unchecked") + protected T getAttribute( + final HashMap cachedMap, + final String key, + final Class clazz) { + + final Object cachedAttribute = cachedMap.get(key); + if (cachedAttribute == null) + return null; + else if (cachedAttribute instanceof JsonElement) { + final T attribute = gson.fromJson((JsonElement)cachedAttribute, clazz); + synchronized (cachedMap) { + cachedMap.put(key, attribute); + } + return attribute; + } else { + return (T)cachedAttribute; + } + } + + @SuppressWarnings("unchecked") + protected T getAttribute( + final HashMap cachedMap, + final String key, + final Type type) { + + final Object cachedAttribute = cachedMap.get(key); + if (cachedAttribute == null) + return null; + else if (cachedAttribute instanceof JsonElement) { + final T attribute = gson.fromJson((JsonElement)cachedAttribute, type); + synchronized (cachedMap) { + cachedMap.put(key, attribute); + } + return attribute; + } else { + return (T)cachedAttribute; + } + } + @Override public T getAttribute( final String pathName, final String key, final Class clazz) throws IOException { - final HashMap map = getAttributes(pathName); - return GsonAttributesParser.parseAttribute(map, key, clazz, getGson()); + if (cacheAttributes) { + final HashMap cachedMap = getCachedAttributes(pathName); + if (cachedMap.isEmpty()) + return null; + return getAttribute(cachedMap, key, clazz); + } else { + final HashMap map = getAttributes(pathName); + return GsonAttributesParser.parseAttribute(map, key, clazz, getGson()); + } } @Override @@ -135,7 +274,14 @@ public T getAttribute( final String key, final Type type) throws IOException { - final HashMap map = getAttributes(pathName); - return GsonAttributesParser.parseAttribute(map, key, type, getGson()); + if (cacheAttributes) { + final HashMap cachedMap = getCachedAttributes(pathName); + if (cachedMap.isEmpty()) + return null; + return getAttribute(cachedMap, key, type); + } else { + final HashMap map = getAttributes(pathName); + return GsonAttributesParser.parseAttribute(map, key, type, getGson()); + } } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java b/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java index 11f77030..5fe7327f 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java @@ -100,7 +100,7 @@ public HashMap asMap() { final HashMap map = new HashMap<>(); map.put(dimensionsKey, dimensions); map.put(blockSizeKey, blockSize); - map.put(dataTypeKey, dataType.toString()); + map.put(dataTypeKey, dataType); map.put(compressionKey, compression); return map; } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5FSReader.java b/src/main/java/org/janelia/saalfeldlab/n5/N5FSReader.java index f48f168e..eae4fbe5 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5FSReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5FSReader.java @@ -1,5 +1,5 @@ /** - * Copyright (c) 2017, Stephan Saalfeld + * Copyright (c) 2017--2021, Stephan Saalfeld * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -106,14 +106,20 @@ public void close() throws IOException { * * @param basePath N5 base path * @param gsonBuilder + * @param cacheAttributes cache attributes + * Setting this to true avoids frequent reading and parsing of JSON + * encoded attributes, this is most interesting for high latency file + * systems. Changes of attributes by an independent writer will not be + * tracked. + * * @throws IOException * if the base path cannot be read or does not exist, * if the N5 version of the container is not compatible with this * implementation. */ - public N5FSReader(final String basePath, final GsonBuilder gsonBuilder) throws IOException { + public N5FSReader(final String basePath, final GsonBuilder gsonBuilder, final boolean cacheAttributes) throws IOException { - super(gsonBuilder); + super(gsonBuilder, cacheAttributes); this.basePath = basePath; if (exists("/")) { final Version version = getVersion(); @@ -122,6 +128,42 @@ public N5FSReader(final String basePath, final GsonBuilder gsonBuilder) throws I } } + /** + * Opens an {@link N5FSReader} at a given base path. + * + * @param basePath N5 base path + * @param cacheAttributes cache attributes + * Setting this to true avoids frequent reading and parsing of JSON + * encoded attributes, this is most interesting for high latency file + * systems. Changes of attributes by an independent writer will not be + * tracked. + * + * @throws IOException + * if the base path cannot be read or does not exist, + * if the N5 version of the container is not compatible with this + * implementation. + */ + public N5FSReader(final String basePath, final boolean cacheAttributes) throws IOException { + + this(basePath, new GsonBuilder(), cacheAttributes); + } + + /** + * Opens an {@link N5FSReader} at a given base path with a custom + * {@link GsonBuilder} to support custom attributes. + * + * @param basePath N5 base path + * @param gsonBuilder + * @throws IOException + * if the base path cannot be read or does not exist, + * if the N5 version of the container is not compatible with this + * implementation. + */ + public N5FSReader(final String basePath, final GsonBuilder gsonBuilder) throws IOException { + + this(basePath, gsonBuilder, false); + } + /** * Opens an {@link N5FSReader} at a given base path. * @@ -133,7 +175,7 @@ public N5FSReader(final String basePath, final GsonBuilder gsonBuilder) throws I */ public N5FSReader(final String basePath) throws IOException { - this(basePath, new GsonBuilder()); + this(basePath, new GsonBuilder(), false); } /** diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java b/src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java index c961bcb7..c0f74f99 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java @@ -1,5 +1,5 @@ /** - * Copyright (c) 2017, Stephan Saalfeld + * Copyright (c) 2017--2021, Stephan Saalfeld * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -63,6 +63,65 @@ public class N5FSWriter extends N5FSReader implements N5Writer { * * @param basePath n5 base path * @param gsonBuilder + * @param cacheAttributes cache attributes + * Setting this to true avoids frequent reading and parsing of JSON + * encoded attributes, this is most interesting for high latency file + * systems. Changes of attributes by an independent writer will not be + * tracked. + * + * @throws IOException + * if the base path cannot be written to or cannot be created, + * if the N5 version of the container is not compatible with this + * implementation. + */ + public N5FSWriter(final String basePath, final GsonBuilder gsonBuilder, final boolean cacheAttributes) throws IOException { + + super(basePath, gsonBuilder, cacheAttributes); + createDirectories(Paths.get(basePath)); + if (!VERSION.equals(getVersion())) + setAttribute("/", VERSION_KEY, VERSION.toString()); + } + + /** + * Opens an {@link N5FSWriter} at a given base path. + * + * If the base path does not exist, it will be created. + * + * If the base path exists and if the N5 version of the container is + * compatible with this implementation, the N5 version of this container + * will be set to the current N5 version of this implementation. + * + * @param basePath n5 base path + * @param gsonBuilder + * @param cacheAttributes cache attributes + * Setting this to true avoids frequent reading and parsing of JSON + * encoded attributes, this is most interesting for high latency file + * systems. Changes of attributes by an independent writer will not be + * tracked. + * + * @throws IOException + * if the base path cannot be written to or cannot be created, + * if the N5 version of the container is not compatible with this + * implementation. + */ + public N5FSWriter(final String basePath, final boolean cacheAttributes) throws IOException { + + this(basePath, new GsonBuilder(), cacheAttributes); + } + + /** + * Opens an {@link N5FSWriter} at a given base path with a custom + * {@link GsonBuilder} to support custom attributes. + * + * If the base path does not exist, it will be created. + * + * If the base path exists and if the N5 version of the container is + * compatible with this implementation, the N5 version of this container + * will be set to the current N5 version of this implementation. + * + * @param basePath n5 base path + * @param gsonBuilder + * * @throws IOException * if the base path cannot be written to or cannot be created, * if the N5 version of the container is not compatible with this @@ -87,6 +146,7 @@ public N5FSWriter(final String basePath, final GsonBuilder gsonBuilder) throws I * * @param basePath n5 base path * @param gsonBuilder + * * @throws IOException * if the base path cannot be written to or cannot be created, * if the N5 version of the container is not compatible with this @@ -109,6 +169,13 @@ public void setAttributes( final String pathName, final Map attributes) throws IOException { + if (cacheAttributes) { + synchronized (attributesCache) { + final HashMap cachedMap = getCachedAttributes(pathName); + cachedMap.putAll(attributes); + } + } + final Path path = Paths.get(basePath, getAttributesPath(pathName).toString()); final HashMap map = new HashMap<>(); diff --git a/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java b/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java index 8fa240c3..32b1a22b 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java @@ -34,6 +34,6 @@ public class N5FSTest extends AbstractN5Test { @Override protected N5Writer createN5Writer() throws IOException { - return new N5FSWriter(testDirPath); + return new N5FSWriter(testDirPath, true); } } From 5bb3002999001eb70d2f0cdfc115ff6a241f7932 Mon Sep 17 00:00:00 2001 From: Stephan Saalfeld Date: Tue, 13 Apr 2021 10:23:18 -0400 Subject: [PATCH 002/243] remove redundant interface in AbstractGsonReader --- .../java/org/janelia/saalfeldlab/n5/AbstractGsonReader.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/AbstractGsonReader.java b/src/main/java/org/janelia/saalfeldlab/n5/AbstractGsonReader.java index b8403b98..b065b887 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/AbstractGsonReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/AbstractGsonReader.java @@ -42,7 +42,7 @@ * @author Igor Pisarev * @author Philipp Hanslovsky */ -public abstract class AbstractGsonReader implements GsonAttributesParser, N5Reader { +public abstract class AbstractGsonReader implements GsonAttributesParser { protected final Gson gson; From e4f45f8536c290ff31559b12ab2b25fa2e9ae5bb Mon Sep 17 00:00:00 2001 From: Stephan Saalfeld Date: Tue, 13 Apr 2021 12:18:15 -0400 Subject: [PATCH 003/243] guarantee that cached attributes that are modified by the same writer are consistent in cache and on disc --- .../janelia/saalfeldlab/n5/N5FSWriter.java | 28 +++++++++++++------ 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java b/src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java index c0f74f99..51d1eaa7 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java @@ -164,18 +164,10 @@ public void createGroup(final String pathName) throws IOException { createDirectories(path); } - @Override - public void setAttributes( + protected void writeAttributes( final String pathName, final Map attributes) throws IOException { - if (cacheAttributes) { - synchronized (attributesCache) { - final HashMap cachedMap = getCachedAttributes(pathName); - cachedMap.putAll(attributes); - } - } - final Path path = Paths.get(basePath, getAttributesPath(pathName).toString()); final HashMap map = new HashMap<>(); @@ -188,6 +180,24 @@ public void setAttributes( } } + @Override + public void setAttributes( + final String pathName, + final Map attributes) throws IOException { + + if (cacheAttributes) { + final HashMap cachedMap; + synchronized (attributesCache) { + cachedMap = getCachedAttributes(pathName); + } + synchronized (cachedMap) { + cachedMap.putAll(attributes); + writeAttributes(pathName, attributes); + } + } else + writeAttributes(pathName, attributes); + } + @Override public void writeBlock( final String pathName, From 27ba18d50a61345a2b516b8adc675c85031949ab Mon Sep 17 00:00:00 2001 From: Stephan Saalfeld Date: Thu, 22 Apr 2021 20:34:58 -0400 Subject: [PATCH 004/243] total WIP filesystem transfer --- .../saalfeldlab/n5/AbstractGsonReader.java | 287 ----------- .../saalfeldlab/n5/AbstractN5FSReader.java | 475 ++++++++++++++++++ .../janelia/saalfeldlab/n5/N5FSReader.java | 250 ++++++++- .../janelia/saalfeldlab/n5/N5FSWriter.java | 2 +- 4 files changed, 712 insertions(+), 302 deletions(-) delete mode 100644 src/main/java/org/janelia/saalfeldlab/n5/AbstractGsonReader.java create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/AbstractN5FSReader.java diff --git a/src/main/java/org/janelia/saalfeldlab/n5/AbstractGsonReader.java b/src/main/java/org/janelia/saalfeldlab/n5/AbstractGsonReader.java deleted file mode 100644 index b065b887..00000000 --- a/src/main/java/org/janelia/saalfeldlab/n5/AbstractGsonReader.java +++ /dev/null @@ -1,287 +0,0 @@ -/** - * Copyright (c) 2017, Stephan Saalfeld - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ -package org.janelia.saalfeldlab.n5; - -import java.io.IOException; -import java.lang.reflect.Type; -import java.util.Arrays; -import java.util.HashMap; - -import com.google.gson.Gson; -import com.google.gson.GsonBuilder; -import com.google.gson.JsonElement; - -/** - * Abstract base class implementing {@link N5Reader} with JSON attributes - * parsed with {@link Gson}. - * - * @author Stephan Saalfeld - * @author Igor Pisarev - * @author Philipp Hanslovsky - */ -public abstract class AbstractGsonReader implements GsonAttributesParser { - - protected final Gson gson; - - protected final HashMap> attributesCache = new HashMap<>(); - - protected final boolean cacheAttributes; - - /** - * Constructs an {@link AbstractGsonReader} with a custom - * {@link GsonBuilder} to support custom attributes. - * - * @param gsonBuilder - * @param cacheAttributes cache attributes - * Setting this to true avoids frequent reading and parsing of JSON - * encoded attributes, this is most interesting for high latency - * backends. Changes of attributes by an independent writer will not be - * tracked. - */ - public AbstractGsonReader(final GsonBuilder gsonBuilder, final boolean cacheAttributes) { - - gsonBuilder.registerTypeAdapter(DataType.class, new DataType.JsonAdapter()); - gsonBuilder.registerTypeHierarchyAdapter(Compression.class, CompressionAdapter.getJsonAdapter()); - gsonBuilder.disableHtmlEscaping(); - this.gson = gsonBuilder.create(); - this.cacheAttributes = cacheAttributes; - } - - /** - * Constructs an {@link AbstractGsonReader} with a default - * {@link GsonBuilder}. - * - * @param cacheAttributes cache attributes - * Setting this to true avoids frequent reading and parsing of JSON - * encoded attributes, this is most interesting for high latency - * backends. Changes of attributes by an independent writer will not be - * tracked. - */ - public AbstractGsonReader(final boolean cacheAttributes) { - - this(new GsonBuilder(), cacheAttributes); - } - - /** - * Constructs an {@link AbstractGsonReader} with a custom - * {@link GsonBuilder} to support custom attributes. - * - * @param gsonBuilder - */ - public AbstractGsonReader(final GsonBuilder gsonBuilder) { - - this(gsonBuilder, false); - } - - /** - * Constructs an {@link AbstractGsonReader} with a default - * {@link GsonBuilder}. - */ - public AbstractGsonReader() { - - this(new GsonBuilder(), false); - } - - @Override - public Gson getGson() { - - return gson; - } - - @Override - public DatasetAttributes getDatasetAttributes(final String pathName) throws IOException { - - final long[] dimensions; - final DataType dataType; - int[] blockSize; - Compression compression; - final String compressionVersion0Name; - if (cacheAttributes) { - final HashMap cachedMap = getCachedAttributes(pathName); - if (cachedMap.isEmpty()) - return null; - - dimensions = getAttribute(cachedMap, DatasetAttributes.dimensionsKey, long[].class); - if (dimensions == null) - return null; - - dataType = getAttribute(cachedMap, DatasetAttributes.dataTypeKey, DataType.class); - if (dataType == null) - return null; - - blockSize = getAttribute(cachedMap, DatasetAttributes.blockSizeKey, int[].class); - - compression = getAttribute(cachedMap, DatasetAttributes.compressionKey, Compression.class); - - /* version 0 */ - compressionVersion0Name = compression == null - ? getAttribute(cachedMap, DatasetAttributes.compressionTypeKey, String.class) - : null; - } else { - final HashMap map = getAttributes(pathName); - - dimensions = GsonAttributesParser.parseAttribute(map, DatasetAttributes.dimensionsKey, long[].class, gson); - if (dimensions == null) - return null; - - dataType = GsonAttributesParser.parseAttribute(map, DatasetAttributes.dataTypeKey, DataType.class, gson); - if (dataType == null) - return null; - - blockSize = GsonAttributesParser.parseAttribute(map, DatasetAttributes.blockSizeKey, int[].class, gson); - - compression = GsonAttributesParser.parseAttribute(map, DatasetAttributes.compressionKey, Compression.class, gson); - - /* version 0 */ - compressionVersion0Name = compression == null - ? GsonAttributesParser.parseAttribute(map, DatasetAttributes.compressionTypeKey, String.class, gson) - : null; - } - - if (blockSize == null) - blockSize = Arrays.stream(dimensions).mapToInt(a -> (int)a).toArray(); - - /* version 0 */ - if (compression == null) { - switch (compressionVersion0Name) { - case "raw": - compression = new RawCompression(); - break; - case "gzip": - compression = new GzipCompression(); - break; - case "bzip2": - compression = new Bzip2Compression(); - break; - case "lz4": - compression = new Lz4Compression(); - break; - case "xz": - compression = new XzCompression(); - break; - } - } - - return new DatasetAttributes(dimensions, blockSize, dataType, compression); - } - - /** - * Get and cache attributes for a group. - * - * @param pathName - * @return - * @throws IOException - */ - protected HashMap getCachedAttributes(final String pathName) throws IOException { - - HashMap cachedMap = attributesCache.get(pathName); - if (cachedMap == null) { - final HashMap map = getAttributes(pathName); - cachedMap = new HashMap<>(); - if (map != null) - cachedMap.putAll(map); - - synchronized (attributesCache) { - attributesCache.put(pathName, cachedMap); - } - } - return cachedMap; - } - - @SuppressWarnings("unchecked") - protected T getAttribute( - final HashMap cachedMap, - final String key, - final Class clazz) { - - final Object cachedAttribute = cachedMap.get(key); - if (cachedAttribute == null) - return null; - else if (cachedAttribute instanceof JsonElement) { - final T attribute = gson.fromJson((JsonElement)cachedAttribute, clazz); - synchronized (cachedMap) { - cachedMap.put(key, attribute); - } - return attribute; - } else { - return (T)cachedAttribute; - } - } - - @SuppressWarnings("unchecked") - protected T getAttribute( - final HashMap cachedMap, - final String key, - final Type type) { - - final Object cachedAttribute = cachedMap.get(key); - if (cachedAttribute == null) - return null; - else if (cachedAttribute instanceof JsonElement) { - final T attribute = gson.fromJson((JsonElement)cachedAttribute, type); - synchronized (cachedMap) { - cachedMap.put(key, attribute); - } - return attribute; - } else { - return (T)cachedAttribute; - } - } - - @Override - public T getAttribute( - final String pathName, - final String key, - final Class clazz) throws IOException { - - if (cacheAttributes) { - final HashMap cachedMap = getCachedAttributes(pathName); - if (cachedMap.isEmpty()) - return null; - return getAttribute(cachedMap, key, clazz); - } else { - final HashMap map = getAttributes(pathName); - return GsonAttributesParser.parseAttribute(map, key, clazz, getGson()); - } - } - - @Override - public T getAttribute( - final String pathName, - final String key, - final Type type) throws IOException { - - if (cacheAttributes) { - final HashMap cachedMap = getCachedAttributes(pathName); - if (cachedMap.isEmpty()) - return null; - return getAttribute(cachedMap, key, type); - } else { - final HashMap map = getAttributes(pathName); - return GsonAttributesParser.parseAttribute(map, key, type, getGson()); - } - } -} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/AbstractN5FSReader.java b/src/main/java/org/janelia/saalfeldlab/n5/AbstractN5FSReader.java new file mode 100644 index 00000000..6531dbce --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/AbstractN5FSReader.java @@ -0,0 +1,475 @@ +/** + * Copyright (c) 2017--2021, Stephan Saalfeld + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +package org.janelia.saalfeldlab.n5; + +import java.io.Closeable; +import java.io.IOException; +import java.lang.reflect.Type; +import java.nio.channels.Channels; +import java.nio.channels.FileChannel; +import java.nio.channels.OverlappingFileLockException; +import java.nio.charset.StandardCharsets; +import java.nio.file.FileSystem; +import java.nio.file.Files; +import java.nio.file.OpenOption; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardOpenOption; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.stream.Stream; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.JsonElement; + +/** + * Filesystem {@link N5Reader} implementation with JSON attributes + * parsed with {@link Gson}. + * + * @author Stephan Saalfeld + * @author Igor Pisarev + * @author Philipp Hanslovsky + */ +public class AbstractN5FSReader implements GsonAttributesParser { + + /** + * A {@link FileChannel} wrapper that attempts to acquire a lock and waits + * for existing locks to be lifted before returning if the + * {@link FileSystem} supports that. If the {@link FileSystem} does not + * support locking, it returns immediately. + */ + protected static class LockedFileChannel implements Closeable { + + private final FileChannel channel; + + public static LockedFileChannel openForReading(final Path path) throws IOException { + + return new LockedFileChannel(path, true); + } + + public static LockedFileChannel openForWriting(final Path path) throws IOException { + + return new LockedFileChannel(path, false); + } + + private LockedFileChannel(final Path path, final boolean readOnly) throws IOException { + + final OpenOption[] options = readOnly ? new OpenOption[]{StandardOpenOption.READ} : new OpenOption[]{StandardOpenOption.READ, StandardOpenOption.WRITE, StandardOpenOption.CREATE}; + channel = FileChannel.open(path, options); + + for (boolean waiting = true; waiting;) { + waiting = false; + try { + channel.lock(0L, Long.MAX_VALUE, readOnly); + } catch (final OverlappingFileLockException e) { + waiting = true; + try { + Thread.sleep(100); + } catch (final InterruptedException f) { + waiting = false; + Thread.currentThread().interrupt(); + } + } catch (final IOException e) {} + } + } + + public FileChannel getFileChannel() { + + return channel; + } + + @Override + public void close() throws IOException { + + channel.close(); + } + } + + /** + * Data object for caching meta data. Elements that are null are not yet + * cached. + */ + protected static class N5GroupInfo { + + public ArrayList children = null; + public HashMap attributesCache = null; + public Boolean isDataset = null; + } + + protected static final N5GroupInfo noGroup = new N5GroupInfo(); + + protected final FileSystem fileSystem; + + protected final Gson gson; + + protected final HashMap metaCache = new HashMap<>(); + + protected final boolean cacheMeta; + + protected static final String jsonFile = "attributes.json"; + + protected final String basePath; + + /** + * Opens an {@link AbstractN5FSReader} at a given base path with a custom + * {@link GsonBuilder} to support custom attributes. + * + * @param fileSystem + * @param basePath N5 base path + * @param gsonBuilder + * @param cacheMeta cache attributes and meta data + * Setting this to true avoids frequent reading and parsing of JSON + * encoded attributes and other meta data that requires accessing the + * store. This is most interesting for high latency backends. Changes + * of cached attributes and meta data by an independent writer will + * not be tracked. + * + * @throws IOException + * if the base path cannot be read or does not exist, + * if the N5 version of the container is not compatible with this + * implementation. + */ + public AbstractN5FSReader( + final FileSystem fileSystem, + final String basePath, + final GsonBuilder gsonBuilder, + final boolean cacheMeta) throws IOException { + + this.fileSystem = fileSystem; + this.basePath = basePath; + gsonBuilder.registerTypeAdapter(DataType.class, new DataType.JsonAdapter()); + gsonBuilder.registerTypeHierarchyAdapter(Compression.class, CompressionAdapter.getJsonAdapter()); + gsonBuilder.disableHtmlEscaping(); + this.gson = gsonBuilder.create(); + this.cacheMeta = cacheMeta; + if (exists("/")) { + final Version version = getVersion(); + if (!VERSION.isCompatible(version)) + throw new IOException("Incompatible version " + version + " (this is " + VERSION + ")."); + } + } + + @Override + public Gson getGson() { + + return gson; + } + + /** + * + * @return N5 base path + */ + public String getBasePath() { + + return this.basePath; + } + + @Override + public DatasetAttributes getDatasetAttributes(final String pathName) throws IOException { + + final long[] dimensions; + final DataType dataType; + int[] blockSize; + Compression compression; + final String compressionVersion0Name; + if (cacheMeta) { + final HashMap cachedMap = getCachedAttributes(pathName); + if (cachedMap.isEmpty()) + return null; + + dimensions = getAttribute(cachedMap, DatasetAttributes.dimensionsKey, long[].class); + if (dimensions == null) + return null; + + dataType = getAttribute(cachedMap, DatasetAttributes.dataTypeKey, DataType.class); + if (dataType == null) + return null; + + blockSize = getAttribute(cachedMap, DatasetAttributes.blockSizeKey, int[].class); + + compression = getAttribute(cachedMap, DatasetAttributes.compressionKey, Compression.class); + + /* version 0 */ + compressionVersion0Name = compression + == null + ? getAttribute(cachedMap, DatasetAttributes.compressionTypeKey, String.class) + : null; + } else { + final HashMap map = getAttributes(pathName); + + dimensions = GsonAttributesParser.parseAttribute(map, DatasetAttributes.dimensionsKey, long[].class, gson); + if (dimensions == null) + return null; + + dataType = GsonAttributesParser.parseAttribute(map, DatasetAttributes.dataTypeKey, DataType.class, gson); + if (dataType == null) + return null; + + blockSize = GsonAttributesParser.parseAttribute(map, DatasetAttributes.blockSizeKey, int[].class, gson); + + compression = GsonAttributesParser.parseAttribute(map, DatasetAttributes.compressionKey, Compression.class, gson); + + /* version 0 */ + compressionVersion0Name = compression == null + ? GsonAttributesParser.parseAttribute(map, DatasetAttributes.compressionTypeKey, String.class, gson) + : null; + } + + if (blockSize == null) + blockSize = Arrays.stream(dimensions).mapToInt(a -> (int)a).toArray(); + + /* version 0 */ + if (compression == null) { + switch (compressionVersion0Name) { + case "raw": + compression = new RawCompression(); + break; + case "gzip": + compression = new GzipCompression(); + break; + case "bzip2": + compression = new Bzip2Compression(); + break; + case "lz4": + compression = new Lz4Compression(); + break; + case "xz": + compression = new XzCompression(); + break; + } + } + + return new DatasetAttributes(dimensions, blockSize, dataType, compression); + } + + /** + * Get and cache attributes for a group. + * + * @param pathName + * @return + * @throws IOException + */ + @Override + protected HashMap getCachedAttributes(final String pathName) throws IOException { + + final N5GroupInfo info = metaCache.get(pathName); + if (info == noGroup) + return null; + if (info == null) { + synchronized (metaCache) + } + HashMap cachedMap = metaCache.get(pathName); + if (cachedMap == null) { + final HashMap map = getAttributes(pathName); + cachedMap = new HashMap<>(); + if (map != null) + cachedMap.putAll(map); + + synchronized (attributesCache) { + attributesCache.put(pathName, cachedMap); + } + } + return cachedMap; + } + + @SuppressWarnings("unchecked") + protected T getAttribute( + final HashMap cachedMap, + final String key, + final Class clazz) { + + final Object cachedAttribute = cachedMap.get(key); + if (cachedAttribute == null) + return null; + else if (cachedAttribute instanceof JsonElement) { + final T attribute = gson.fromJson((JsonElement)cachedAttribute, clazz); + synchronized (cachedMap) { + cachedMap.put(key, attribute); + } + return attribute; + } else { + return (T)cachedAttribute; + } + } + + @SuppressWarnings("unchecked") + protected T getAttribute( + final HashMap cachedMap, + final String key, + final Type type) { + + final Object cachedAttribute = cachedMap.get(key); + if (cachedAttribute == null) + return null; + else if (cachedAttribute instanceof JsonElement) { + final T attribute = gson.fromJson((JsonElement)cachedAttribute, type); + synchronized (cachedMap) { + cachedMap.put(key, attribute); + } + return attribute; + } else { + return (T)cachedAttribute; + } + } + + @Override + public T getAttribute( + final String pathName, + final String key, + final Class clazz) throws IOException { + + if (cacheMeta) { + final HashMap cachedMap = getCachedAttributes(pathName); + if (cachedMap.isEmpty()) + return null; + return getAttribute(cachedMap, key, clazz); + } else { + final HashMap map = getAttributes(pathName); + return GsonAttributesParser.parseAttribute(map, key, clazz, getGson()); + } + } + + @Override + public T getAttribute( + final String pathName, + final String key, + final Type type) throws IOException { + + if (cacheMeta) { + final HashMap cachedMap = getCachedAttributes(pathName); + if (cachedMap.isEmpty()) + return null; + return getAttribute(cachedMap, key, type); + } else { + final HashMap map = getAttributes(pathName); + return GsonAttributesParser.parseAttribute(map, key, type, getGson()); + } + } + + @Override + public boolean exists(final String pathName) { + + final Path path = fileSystem.getPath(basePath, pathName); + return Files.exists(path) && Files.isDirectory(path); + } + + @Override + public HashMap getAttributes(final String pathName) throws IOException { + + final Path path = getAttributesPath(pathName); + if (exists(pathName) && !Files.exists(path)) + return new HashMap<>(); + + try (final LockedFileChannel lockedFileChannel = LockedFileChannel.openForReading(path)) { + return GsonAttributesParser.readAttributes(Channels.newReader(lockedFileChannel.getFileChannel(), StandardCharsets.UTF_8.name()), getGson()); + } + } + + @Override + public DataBlock readBlock( + final String pathName, + final DatasetAttributes datasetAttributes, + final long... gridPosition) throws IOException { + + final Path path = getDataBlockPath(pathName, gridPosition); + if (!Files.exists(path)) + return null; + + try (final LockedFileChannel lockedChannel = LockedFileChannel.openForReading(path)) { + return DefaultBlockReader.readBlock(Channels.newInputStream(lockedChannel.getFileChannel()), datasetAttributes, gridPosition); + } + } + + @Override + public String[] list(final String pathName) throws IOException { + + final Path path = fileSystem.getPath(basePath, removeLeadingSlash(pathName)); + try (final Stream pathStream = Files.list(path)) { + return pathStream + .filter(a -> Files.isDirectory(a)) + .map(a -> path.relativize(a).toString()) + .toArray(n -> new String[n]); + } + } + + /** + * Constructs the path for a data block in a dataset at a given grid position. + * + * The returned path is + * 
+	 * $datasetPathName/$gridPosition[0]/$gridPosition[1]/.../$gridPosition[n]
+	 *
+ * + * This is the file into which the data block will be stored. + * + * @param datasetPathName + * @param gridPosition + * @return + */ + protected Path getDataBlockPath( + final String datasetPathName, + final long... gridPosition) { + + final String[] pathComponents = new String[gridPosition.length + 1]; + pathComponents[0] = removeLeadingSlash(datasetPathName); + for (int i = 1; i <= pathComponents.length; ++i) + pathComponents[i] = Long.toString(gridPosition[i - 1]); + + return fileSystem.getPath(basePath, pathComponents); + } + + /** + * Constructs the path for the attributes file of a group or dataset. + * + * @param pathName + * @return + */ + protected Path getAttributesPath(final String pathName) { + + return fileSystem.getPath(basePath, removeLeadingSlash(pathName), jsonFile); + } + + /** + * Removes the leading slash from a given path and returns the corrected path. + * It ensures correctness on both Unix and Windows, otherwise {@code pathName} is treated + * as UNC path on Windows, and {@code Paths.get(pathName, ...)} fails with {@code InvalidPathException}. + * + * @param pathName + * @return + */ + protected static String removeLeadingSlash(final String pathName) { + + return pathName.startsWith("/") || pathName.startsWith("\\") ? pathName.substring(1) : pathName; + } + + @Override + public String toString() { + + return String.format("%s[fileSystem=%s, basePath=%s]", getClass().getSimpleName(), fileSystem, basePath); + } +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5FSReader.java b/src/main/java/org/janelia/saalfeldlab/n5/N5FSReader.java index eae4fbe5..ef198a74 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5FSReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5FSReader.java @@ -27,6 +27,7 @@ import java.io.Closeable; import java.io.IOException; +import java.lang.reflect.Type; import java.nio.channels.Channels; import java.nio.channels.FileChannel; import java.nio.channels.OverlappingFileLockException; @@ -36,18 +37,26 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.nio.file.StandardOpenOption; +import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; import java.util.stream.Stream; +import org.janelia.saalfeldlab.n5.AbstractGsonReader.N5GroupInfo; + +import com.google.gson.Gson; import com.google.gson.GsonBuilder; import com.google.gson.JsonElement; /** - * Filesystem {@link N5Reader} implementation with version compatibility check. + * Filesystem {@link N5Reader} implementation with JSON attributes + * parsed with {@link Gson}. * * @author Stephan Saalfeld + * @author Igor Pisarev + * @author Philipp Hanslovsky */ -public class N5FSReader extends AbstractGsonReader { +public class N5FSReader extends AbstractN5FSReader { protected static class LockedFileChannel implements Closeable { @@ -96,6 +105,21 @@ public void close() throws IOException { } } + protected static class N5GroupInfo { + + public ArrayList children = null; + public HashMap attributesCache = null; + public Boolean isDataset = null; + } + + protected static final N5GroupInfo noGroup = new N5GroupInfo(); + + protected final Gson gson; + + protected final HashMap metaCache = new HashMap<>(); + + protected final boolean cacheMeta; + protected static final String jsonFile = "attributes.json"; protected final String basePath; @@ -106,20 +130,25 @@ public void close() throws IOException { * * @param basePath N5 base path * @param gsonBuilder - * @param cacheAttributes cache attributes + * @param cacheMeta cache attributes and meta data * Setting this to true avoids frequent reading and parsing of JSON - * encoded attributes, this is most interesting for high latency file - * systems. Changes of attributes by an independent writer will not be - * tracked. + * encoded attributes and other meta data that requires accessing the + * store. This is most interesting for high latency backends. Changes + * of cached attributes and meta data by an independent writer will + * not be tracked. * * @throws IOException * if the base path cannot be read or does not exist, * if the N5 version of the container is not compatible with this * implementation. */ - public N5FSReader(final String basePath, final GsonBuilder gsonBuilder, final boolean cacheAttributes) throws IOException { + public N5FSReader(final String basePath, final GsonBuilder gsonBuilder, final boolean cacheMeta) throws IOException { - super(gsonBuilder, cacheAttributes); + gsonBuilder.registerTypeAdapter(DataType.class, new DataType.JsonAdapter()); + gsonBuilder.registerTypeHierarchyAdapter(Compression.class, CompressionAdapter.getJsonAdapter()); + gsonBuilder.disableHtmlEscaping(); + this.gson = gsonBuilder.create(); + this.cacheMeta = cacheMeta; this.basePath = basePath; if (exists("/")) { final Version version = getVersion(); @@ -132,20 +161,21 @@ public N5FSReader(final String basePath, final GsonBuilder gsonBuilder, final bo * Opens an {@link N5FSReader} at a given base path. * * @param basePath N5 base path - * @param cacheAttributes cache attributes + * @param cacheMeta cache attributes and meta data * Setting this to true avoids frequent reading and parsing of JSON - * encoded attributes, this is most interesting for high latency file - * systems. Changes of attributes by an independent writer will not be - * tracked. + * encoded attributes and other meta data that requires accessing the + * store. This is most interesting for high latency backends. Changes + * of cached attributes and meta data by an independent writer will + * not be tracked. * * @throws IOException * if the base path cannot be read or does not exist, * if the N5 version of the container is not compatible with this * implementation. */ - public N5FSReader(final String basePath, final boolean cacheAttributes) throws IOException { + public N5FSReader(final String basePath, final boolean cacheMeta) throws IOException { - this(basePath, new GsonBuilder(), cacheAttributes); + this(basePath, new GsonBuilder(), cacheMeta); } /** @@ -178,14 +208,206 @@ public N5FSReader(final String basePath) throws IOException { this(basePath, new GsonBuilder(), false); } + @Override + public Gson getGson() { + + return gson; + } + /** * * @return N5 base path */ + @Override public String getBasePath() { + return this.basePath; } + @Override + public DatasetAttributes getDatasetAttributes(final String pathName) throws IOException { + + final long[] dimensions; + final DataType dataType; + int[] blockSize; + Compression compression; + final String compressionVersion0Name; + if (cacheMeta) { + final HashMap cachedMap = getCachedAttributes(pathName); + if (cachedMap.isEmpty()) + return null; + + dimensions = getAttribute(cachedMap, DatasetAttributes.dimensionsKey, long[].class); + if (dimensions == null) + return null; + + dataType = getAttribute(cachedMap, DatasetAttributes.dataTypeKey, DataType.class); + if (dataType == null) + return null; + + blockSize = getAttribute(cachedMap, DatasetAttributes.blockSizeKey, int[].class); + + compression = getAttribute(cachedMap, DatasetAttributes.compressionKey, Compression.class); + + /* version 0 */ + compressionVersion0Name = compression + == null + ? getAttribute(cachedMap, DatasetAttributes.compressionTypeKey, String.class) + : null; + } else { + final HashMap map = getAttributes(pathName); + + dimensions = GsonAttributesParser.parseAttribute(map, DatasetAttributes.dimensionsKey, long[].class, gson); + if (dimensions == null) + return null; + + dataType = GsonAttributesParser.parseAttribute(map, DatasetAttributes.dataTypeKey, DataType.class, gson); + if (dataType == null) + return null; + + blockSize = GsonAttributesParser.parseAttribute(map, DatasetAttributes.blockSizeKey, int[].class, gson); + + compression = GsonAttributesParser.parseAttribute(map, DatasetAttributes.compressionKey, Compression.class, gson); + + /* version 0 */ + compressionVersion0Name = compression == null + ? GsonAttributesParser.parseAttribute(map, DatasetAttributes.compressionTypeKey, String.class, gson) + : null; + } + + if (blockSize == null) + blockSize = Arrays.stream(dimensions).mapToInt(a -> (int)a).toArray(); + + /* version 0 */ + if (compression == null) { + switch (compressionVersion0Name) { + case "raw": + compression = new RawCompression(); + break; + case "gzip": + compression = new GzipCompression(); + break; + case "bzip2": + compression = new Bzip2Compression(); + break; + case "lz4": + compression = new Lz4Compression(); + break; + case "xz": + compression = new XzCompression(); + break; + } + } + + return new DatasetAttributes(dimensions, blockSize, dataType, compression); + } + + /** + * Get and cache attributes for a group. + * + * @param pathName + * @return + * @throws IOException + */ + @Override + protected HashMap getCachedAttributes(final String pathName) throws IOException { + + final N5GroupInfo info = metaCache.get(pathName); + if (info == noGroup) + return null; + if (info == null) { + synchronized (metaCache) + } + HashMap cachedMap = metaCache.get(pathName); + if (cachedMap == null) { + final HashMap map = getAttributes(pathName); + cachedMap = new HashMap<>(); + if (map != null) + cachedMap.putAll(map); + + synchronized (attributesCache) { + attributesCache.put(pathName, cachedMap); + } + } + return cachedMap; + } + + @Override + @SuppressWarnings("unchecked") + protected T getAttribute( + final HashMap cachedMap, + final String key, + final Class clazz) { + + final Object cachedAttribute = cachedMap.get(key); + if (cachedAttribute == null) + return null; + else if (cachedAttribute instanceof JsonElement) { + final T attribute = gson.fromJson((JsonElement)cachedAttribute, clazz); + synchronized (cachedMap) { + cachedMap.put(key, attribute); + } + return attribute; + } else { + return (T)cachedAttribute; + } + } + + @Override + @SuppressWarnings("unchecked") + protected T getAttribute( + final HashMap cachedMap, + final String key, + final Type type) { + + final Object cachedAttribute = cachedMap.get(key); + if (cachedAttribute == null) + return null; + else if (cachedAttribute instanceof JsonElement) { + final T attribute = gson.fromJson((JsonElement)cachedAttribute, type); + synchronized (cachedMap) { + cachedMap.put(key, attribute); + } + return attribute; + } else { + return (T)cachedAttribute; + } + } + + @Override + public T getAttribute( + final String pathName, + final String key, + final Class clazz) throws IOException { + + if (cacheMeta) { + final HashMap cachedMap = getCachedAttributes(pathName); + if (cachedMap.isEmpty()) + return null; + return getAttribute(cachedMap, key, clazz); + } else { + final HashMap map = getAttributes(pathName); + return GsonAttributesParser.parseAttribute(map, key, clazz, getGson()); + } + } + + @Override + public T getAttribute( + final String pathName, + final String key, + final Type type) throws IOException { + + if (cacheMeta) { + final HashMap cachedMap = getCachedAttributes(pathName); + if (cachedMap.isEmpty()) + return null; + return getAttribute(cachedMap, key, type); + } else { + final HashMap map = getAttributes(pathName); + return GsonAttributesParser.parseAttribute(map, key, type, getGson()); + } + } + @Override public boolean exists(final String pathName) { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java b/src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java index 51d1eaa7..6de54c65 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java @@ -185,7 +185,7 @@ public void setAttributes( final String pathName, final Map attributes) throws IOException { - if (cacheAttributes) { + if (cacheMeta) { final HashMap cachedMap; synchronized (attributesCache) { cachedMap = getCachedAttributes(pathName); From 81bed0cfe4d3a8c3e5c588f943a4c0b70dcbe031 Mon Sep 17 00:00:00 2001 From: Stephan Saalfeld Date: Thu, 29 Apr 2021 15:03:58 -0400 Subject: [PATCH 005/243] more WIP --- pom.xml | 2 +- .../saalfeldlab/n5/AbstractN5FSReader.java | 54 +++++++++++++++---- 2 files changed, 45 insertions(+), 11 deletions(-) diff --git a/pom.xml b/pom.xml index 9a231ef1..e4000f17 100644 --- a/pom.xml +++ b/pom.xml @@ -11,7 +11,7 @@ org.janelia.saalfeldlab n5 - 2.6.0-SNAPSHOT + 3.0.0-SNAPSHOT N5 Not HDF5 diff --git a/src/main/java/org/janelia/saalfeldlab/n5/AbstractN5FSReader.java b/src/main/java/org/janelia/saalfeldlab/n5/AbstractN5FSReader.java index 6531dbce..310851d7 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/AbstractN5FSReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/AbstractN5FSReader.java @@ -121,7 +121,7 @@ protected static class N5GroupInfo { public Boolean isDataset = null; } - protected static final N5GroupInfo noGroup = new N5GroupInfo(); + protected static final N5GroupInfo emptyGroupInfo = new N5GroupInfo(); protected final FileSystem fileSystem; @@ -197,8 +197,13 @@ public DatasetAttributes getDatasetAttributes(final String pathName) throws IOEx int[] blockSize; Compression compression; final String compressionVersion0Name; + final Path path = getAttributesPath(pathName); if (cacheMeta) { - final HashMap cachedMap = getCachedAttributes(pathName); + final N5GroupInfo info = getCachedN5GroupInfo(path); + if (info == emptyGroupInfo) + return null; + + final HashMap cachedMap = getCachedAttributes(info); if (cachedMap.isEmpty()) return null; @@ -270,15 +275,13 @@ public DatasetAttributes getDatasetAttributes(final String pathName) throws IOEx /** * Get and cache attributes for a group. * - * @param pathName + * @param info * @return * @throws IOException */ - @Override - protected HashMap getCachedAttributes(final String pathName) throws IOException { + protected HashMap getCachedAttributes(final N5GroupInfo info) throws IOException { - final N5GroupInfo info = metaCache.get(pathName); - if (info == noGroup) + if (info.attributesCache == emptyGroupInfo) return null; if (info == null) { synchronized (metaCache) @@ -371,11 +374,42 @@ public T getAttribute( } } + protected boolean exists(final Path path) { + + return Files.exists(path) && Files.isDirectory(path); + } + + protected N5GroupInfo getCachedN5GroupInfo(final Path path) { + + final String pathName = path.toString(); + final N5GroupInfo cachedInfo = metaCache.get(pathName); + if (cachedInfo == null) { + + /* I do not have a better solution yet to allow parallel + * exists checks for independent paths than to accept the + * same exists check to potentially run multiple times. + */ + final boolean exists = exists(path); + + synchronized (metaCache) { + final N5GroupInfo cachedInfoAgain = metaCache.get(pathName); + if (cachedInfoAgain == null) { + return metaCache.put(pathName, exists ? new N5GroupInfo() : emptyGroupInfo); + } else + return cachedInfoAgain; + } + } else + return cachedInfo; + } + @Override public boolean exists(final String pathName) { - final Path path = fileSystem.getPath(basePath, pathName); - return Files.exists(path) && Files.isDirectory(path); + final Path path = fileSystem.getPath(basePath, removeLeadingSlash(pathName)); + if (cacheMeta) + return getCachedN5GroupInfo(path) != emptyGroupInfo; + else + return exists(path); } @Override @@ -422,7 +456,7 @@ public String[] list(final String pathName) throws IOException { * * The returned path is * 
-	 * $datasetPathName/$gridPosition[0]/$gridPosition[1]/.../$gridPosition[n]
+	 * $basePath/datasetPathName/$gridPosition[0]/$gridPosition[1]/.../$gridPosition[n]
 	 *
* * This is the file into which the data block will be stored. From dcc33e76a736cd8efea4a0ae648c206a83c859cf Mon Sep 17 00:00:00 2001 From: Stephan Saalfeld Date: Thu, 29 Apr 2021 17:39:18 -0400 Subject: [PATCH 006/243] WIP FS reader implemented? --- .../saalfeldlab/n5/AbstractN5FSReader.java | 84 +++- .../janelia/saalfeldlab/n5/N5FSReader.java | 397 +----------------- 2 files changed, 65 insertions(+), 416 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/AbstractN5FSReader.java b/src/main/java/org/janelia/saalfeldlab/n5/AbstractN5FSReader.java index 310851d7..1cccaa94 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/AbstractN5FSReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/AbstractN5FSReader.java @@ -36,7 +36,6 @@ import java.nio.file.Files; import java.nio.file.OpenOption; import java.nio.file.Path; -import java.nio.file.Paths; import java.nio.file.StandardOpenOption; import java.util.ArrayList; import java.util.Arrays; @@ -197,13 +196,13 @@ public DatasetAttributes getDatasetAttributes(final String pathName) throws IOEx int[] blockSize; Compression compression; final String compressionVersion0Name; - final Path path = getAttributesPath(pathName); + final Path path = getGroupPath(pathName); if (cacheMeta) { final N5GroupInfo info = getCachedN5GroupInfo(path); if (info == emptyGroupInfo) return null; - final HashMap cachedMap = getCachedAttributes(info); + final HashMap cachedMap = getCachedAttributes(info, pathName); if (cachedMap.isEmpty()) return null; @@ -272,29 +271,44 @@ public DatasetAttributes getDatasetAttributes(final String pathName) throws IOEx return new DatasetAttributes(dimensions, blockSize, dataType, compression); } + protected HashMap getAttributes(final Path path) throws IOException { + + if (!Files.exists(path)) + return new HashMap<>(); + + try (final LockedFileChannel lockedFileChannel = LockedFileChannel.openForReading(path)) { + return GsonAttributesParser.readAttributes(Channels.newReader(lockedFileChannel.getFileChannel(), StandardCharsets.UTF_8.name()), getGson()); + } + } + /** - * Get and cache attributes for a group. + * Get and cache attributes for a group identified by an info object and a + * pathName. + * + * This helper method does not intelligently handle the case that the group + * does not exist (as indicated by info == emptyGroupInfo) which should be + * done in calling code. * * @param info - * @return + * @param pathName + * @return cached attributes + * empty map if the group exists but not attributes are set + * null if the group does not exist * @throws IOException */ - protected HashMap getCachedAttributes(final N5GroupInfo info) throws IOException { + protected HashMap getCachedAttributes(final N5GroupInfo info, final String pathName) throws IOException { - if (info.attributesCache == emptyGroupInfo) - return null; - if (info == null) { - synchronized (metaCache) - } - HashMap cachedMap = metaCache.get(pathName); + HashMap cachedMap = info.attributesCache; if (cachedMap == null) { - final HashMap map = getAttributes(pathName); - cachedMap = new HashMap<>(); - if (map != null) - cachedMap.putAll(map); - - synchronized (attributesCache) { - attributesCache.put(pathName, cachedMap); + synchronized (info) { + cachedMap = info.attributesCache; + if (cachedMap == null) { + final Path path = getAttributesPath(pathName); + cachedMap = new HashMap<>(); + final HashMap map = getAttributes(path); + cachedMap.putAll(map); + info.attributesCache = cachedMap; + } } } return cachedMap; @@ -320,6 +334,15 @@ else if (cachedAttribute instanceof JsonElement) { } } + /** + * Helper method that returns or JSON decodes a cached attribute. + * + * @param + * @param cachedMap + * @param key + * @param type + * @return + */ @SuppressWarnings("unchecked") protected T getAttribute( final HashMap cachedMap, @@ -347,7 +370,11 @@ public T getAttribute( final Class clazz) throws IOException { if (cacheMeta) { - final HashMap cachedMap = getCachedAttributes(pathName); + final Path path = getGroupPath(pathName); + final N5GroupInfo info = getCachedN5GroupInfo(path); + if (info == emptyGroupInfo) + return null; + final HashMap cachedMap = getCachedAttributes(info, pathName); if (cachedMap.isEmpty()) return null; return getAttribute(cachedMap, key, clazz); @@ -364,7 +391,11 @@ public T getAttribute( final Type type) throws IOException { if (cacheMeta) { - final HashMap cachedMap = getCachedAttributes(pathName); + final Path path = getGroupPath(pathName); + final N5GroupInfo info = getCachedN5GroupInfo(path); + if (info == emptyGroupInfo) + return null; + final HashMap cachedMap = getCachedAttributes(info, pathName); if (cachedMap.isEmpty()) return null; return getAttribute(cachedMap, key, type); @@ -477,6 +508,17 @@ protected Path getDataBlockPath( return fileSystem.getPath(basePath, pathComponents); } + /** + * Constructs the path for the group or dataset. + * + * @param pathName + * @return + */ + protected Path getGroupPath(final String pathName) { + + return fileSystem.getPath(basePath, removeLeadingSlash(pathName)); + } + /** * Constructs the path for the attributes file of a group or dataset. * diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5FSReader.java b/src/main/java/org/janelia/saalfeldlab/n5/N5FSReader.java index ef198a74..b56de906 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5FSReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5FSReader.java @@ -25,28 +25,11 @@ */ package org.janelia.saalfeldlab.n5; -import java.io.Closeable; import java.io.IOException; -import java.lang.reflect.Type; -import java.nio.channels.Channels; -import java.nio.channels.FileChannel; -import java.nio.channels.OverlappingFileLockException; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.OpenOption; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.nio.file.StandardOpenOption; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.stream.Stream; - -import org.janelia.saalfeldlab.n5.AbstractGsonReader.N5GroupInfo; +import java.nio.file.FileSystems; import com.google.gson.Gson; import com.google.gson.GsonBuilder; -import com.google.gson.JsonElement; /** * Filesystem {@link N5Reader} implementation with JSON attributes @@ -58,72 +41,6 @@ */ public class N5FSReader extends AbstractN5FSReader { - protected static class LockedFileChannel implements Closeable { - - private final FileChannel channel; - - public static LockedFileChannel openForReading(final Path path) throws IOException { - - return new LockedFileChannel(path, true); - } - - public static LockedFileChannel openForWriting(final Path path) throws IOException { - - return new LockedFileChannel(path, false); - } - - private LockedFileChannel(final Path path, final boolean readOnly) throws IOException { - - final OpenOption[] options = readOnly ? new OpenOption[]{StandardOpenOption.READ} : new OpenOption[]{StandardOpenOption.READ, StandardOpenOption.WRITE, StandardOpenOption.CREATE}; - channel = FileChannel.open(path, options); - - for (boolean waiting = true; waiting;) { - waiting = false; - try { - channel.lock(0L, Long.MAX_VALUE, readOnly); - } catch (final OverlappingFileLockException e) { - waiting = true; - try { - Thread.sleep(100); - } catch (final InterruptedException f) { - waiting = false; - Thread.currentThread().interrupt(); - } - } catch (final IOException e) {} - } - } - - public FileChannel getFileChannel() { - - return channel; - } - - @Override - public void close() throws IOException { - - channel.close(); - } - } - - protected static class N5GroupInfo { - - public ArrayList children = null; - public HashMap attributesCache = null; - public Boolean isDataset = null; - } - - protected static final N5GroupInfo noGroup = new N5GroupInfo(); - - protected final Gson gson; - - protected final HashMap metaCache = new HashMap<>(); - - protected final boolean cacheMeta; - - protected static final String jsonFile = "attributes.json"; - - protected final String basePath; - /** * Opens an {@link N5FSReader} at a given base path with a custom * {@link GsonBuilder} to support custom attributes. @@ -144,17 +61,7 @@ protected static class N5GroupInfo { */ public N5FSReader(final String basePath, final GsonBuilder gsonBuilder, final boolean cacheMeta) throws IOException { - gsonBuilder.registerTypeAdapter(DataType.class, new DataType.JsonAdapter()); - gsonBuilder.registerTypeHierarchyAdapter(Compression.class, CompressionAdapter.getJsonAdapter()); - gsonBuilder.disableHtmlEscaping(); - this.gson = gsonBuilder.create(); - this.cacheMeta = cacheMeta; - this.basePath = basePath; - if (exists("/")) { - final Version version = getVersion(); - if (!VERSION.isCompatible(version)) - throw new IOException("Incompatible version " + version + " (this is " + VERSION + ")."); - } + super(FileSystems.getDefault(), basePath, gsonBuilder, cacheMeta); } /** @@ -207,304 +114,4 @@ public N5FSReader(final String basePath) throws IOException { this(basePath, new GsonBuilder(), false); } - - @Override - public Gson getGson() { - - return gson; - } - - /** - * - * @return N5 base path - */ - @Override - public String getBasePath() { - - return this.basePath; - } - - @Override - public DatasetAttributes getDatasetAttributes(final String pathName) throws IOException { - - final long[] dimensions; - final DataType dataType; - int[] blockSize; - Compression compression; - final String compressionVersion0Name; - if (cacheMeta) { - final HashMap cachedMap = getCachedAttributes(pathName); - if (cachedMap.isEmpty()) - return null; - - dimensions = getAttribute(cachedMap, DatasetAttributes.dimensionsKey, long[].class); - if (dimensions == null) - return null; - - dataType = getAttribute(cachedMap, DatasetAttributes.dataTypeKey, DataType.class); - if (dataType == null) - return null; - - blockSize = getAttribute(cachedMap, DatasetAttributes.blockSizeKey, int[].class); - - compression = getAttribute(cachedMap, DatasetAttributes.compressionKey, Compression.class); - - /* version 0 */ - compressionVersion0Name = compression - == null - ? getAttribute(cachedMap, DatasetAttributes.compressionTypeKey, String.class) - : null; - } else { - final HashMap map = getAttributes(pathName); - - dimensions = GsonAttributesParser.parseAttribute(map, DatasetAttributes.dimensionsKey, long[].class, gson); - if (dimensions == null) - return null; - - dataType = GsonAttributesParser.parseAttribute(map, DatasetAttributes.dataTypeKey, DataType.class, gson); - if (dataType == null) - return null; - - blockSize = GsonAttributesParser.parseAttribute(map, DatasetAttributes.blockSizeKey, int[].class, gson); - - compression = GsonAttributesParser.parseAttribute(map, DatasetAttributes.compressionKey, Compression.class, gson); - - /* version 0 */ - compressionVersion0Name = compression == null - ? GsonAttributesParser.parseAttribute(map, DatasetAttributes.compressionTypeKey, String.class, gson) - : null; - } - - if (blockSize == null) - blockSize = Arrays.stream(dimensions).mapToInt(a -> (int)a).toArray(); - - /* version 0 */ - if (compression == null) { - switch (compressionVersion0Name) { - case "raw": - compression = new RawCompression(); - break; - case "gzip": - compression = new GzipCompression(); - break; - case "bzip2": - compression = new Bzip2Compression(); - break; - case "lz4": - compression = new Lz4Compression(); - break; - case "xz": - compression = new XzCompression(); - break; - } - } - - return new DatasetAttributes(dimensions, blockSize, dataType, compression); - } - - /** - * Get and cache attributes for a group. - * - * @param pathName - * @return - * @throws IOException - */ - @Override - protected HashMap getCachedAttributes(final String pathName) throws IOException { - - final N5GroupInfo info = metaCache.get(pathName); - if (info == noGroup) - return null; - if (info == null) { - synchronized (metaCache) - } - HashMap cachedMap = metaCache.get(pathName); - if (cachedMap == null) { - final HashMap map = getAttributes(pathName); - cachedMap = new HashMap<>(); - if (map != null) - cachedMap.putAll(map); - - synchronized (attributesCache) { - attributesCache.put(pathName, cachedMap); - } - } - return cachedMap; - } - - @Override - @SuppressWarnings("unchecked") - protected T getAttribute( - final HashMap cachedMap, - final String key, - final Class clazz) { - - final Object cachedAttribute = cachedMap.get(key); - if (cachedAttribute == null) - return null; - else if (cachedAttribute instanceof JsonElement) { - final T attribute = gson.fromJson((JsonElement)cachedAttribute, clazz); - synchronized (cachedMap) { - cachedMap.put(key, attribute); - } - return attribute; - } else { - return (T)cachedAttribute; - } - } - - @Override - @SuppressWarnings("unchecked") - protected T getAttribute( - final HashMap cachedMap, - final String key, - final Type type) { - - final Object cachedAttribute = cachedMap.get(key); - if (cachedAttribute == null) - return null; - else if (cachedAttribute instanceof JsonElement) { - final T attribute = gson.fromJson((JsonElement)cachedAttribute, type); - synchronized (cachedMap) { - cachedMap.put(key, attribute); - } - return attribute; - } else { - return (T)cachedAttribute; - } - } - - @Override - public T getAttribute( - final String pathName, - final String key, - final Class clazz) throws IOException { - - if (cacheMeta) { - final HashMap cachedMap = getCachedAttributes(pathName); - if (cachedMap.isEmpty()) - return null; - return getAttribute(cachedMap, key, clazz); - } else { - final HashMap map = getAttributes(pathName); - return GsonAttributesParser.parseAttribute(map, key, clazz, getGson()); - } - } - - @Override - public T getAttribute( - final String pathName, - final String key, - final Type type) throws IOException { - - if (cacheMeta) { - final HashMap cachedMap = getCachedAttributes(pathName); - if (cachedMap.isEmpty()) - return null; - return getAttribute(cachedMap, key, type); - } else { - final HashMap map = getAttributes(pathName); - return GsonAttributesParser.parseAttribute(map, key, type, getGson()); - } - } - - @Override - public boolean exists(final String pathName) { - - final Path path = Paths.get(basePath, pathName); - return Files.exists(path) && Files.isDirectory(path); - } - - @Override - public HashMap getAttributes(final String pathName) throws IOException { - - final Path path = Paths.get(basePath, getAttributesPath(pathName).toString()); - if (exists(pathName) && !Files.exists(path)) - return new HashMap<>(); - - try (final LockedFileChannel lockedFileChannel = LockedFileChannel.openForReading(path)) { - return GsonAttributesParser.readAttributes(Channels.newReader(lockedFileChannel.getFileChannel(), StandardCharsets.UTF_8.name()), getGson()); - } - } - - @Override - public DataBlock readBlock( - final String pathName, - final DatasetAttributes datasetAttributes, - final long... gridPosition) throws IOException { - - final Path path = Paths.get(basePath, getDataBlockPath(pathName, gridPosition).toString()); - if (!Files.exists(path)) - return null; - - try (final LockedFileChannel lockedChannel = LockedFileChannel.openForReading(path)) { - return DefaultBlockReader.readBlock(Channels.newInputStream(lockedChannel.getFileChannel()), datasetAttributes, gridPosition); - } - } - - @Override - public String[] list(final String pathName) throws IOException { - - final Path path = Paths.get(basePath, pathName); - try (final Stream pathStream = Files.list(path)) { - return pathStream - .filter(a -> Files.isDirectory(a)) - .map(a -> path.relativize(a).toString()) - .toArray(n -> new String[n]); - } - } - - /** - * Constructs the path for a data block in a dataset at a given grid position. - * - * The returned path is - * 
-	 * $datasetPathName/$gridPosition[0]/$gridPosition[1]/.../$gridPosition[n]
-	 *
- * - * This is the file into which the data block will be stored. - * - * @param datasetPathName - * @param gridPosition - * @return - */ - protected static Path getDataBlockPath( - final String datasetPathName, - final long... gridPosition) { - - final String[] pathComponents = new String[gridPosition.length]; - for (int i = 0; i < pathComponents.length; ++i) - pathComponents[i] = Long.toString(gridPosition[i]); - - return Paths.get(removeLeadingSlash(datasetPathName), pathComponents); - } - - /** - * Constructs the path for the attributes file of a group or dataset. - * - * @param pathName - * @return - */ - protected static Path getAttributesPath(final String pathName) { - - return Paths.get(removeLeadingSlash(pathName), jsonFile); - } - - /** - * Removes the leading slash from a given path and returns the corrected path. - * It ensures correctness on both Unix and Windows, otherwise {@code pathName} is treated - * as UNC path on Windows, and {@code Paths.get(pathName, ...)} fails with {@code InvalidPathException}. - * - * @param pathName - * @return - */ - protected static String removeLeadingSlash(final String pathName) { - - return pathName.startsWith("/") || pathName.startsWith("\\") ? pathName.substring(1) : pathName; - } - - @Override - public String toString() { - return String.format("%s[basePath=%s]", getClass().getSimpleName(), basePath); - } } From 7d1ddf64182e3c79156382d7807acaf33854a727 Mon Sep 17 00:00:00 2001 From: Stephan Saalfeld Date: Fri, 30 Apr 2021 09:54:32 -0400 Subject: [PATCH 007/243] WIP fs writer (does not work) --- .../saalfeldlab/n5/AbstractN5FSWriter.java | 358 ++++++++++++++++++ .../janelia/saalfeldlab/n5/N5FSWriter.java | 260 +------------ 2 files changed, 362 insertions(+), 256 deletions(-) create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/AbstractN5FSWriter.java diff --git a/src/main/java/org/janelia/saalfeldlab/n5/AbstractN5FSWriter.java b/src/main/java/org/janelia/saalfeldlab/n5/AbstractN5FSWriter.java new file mode 100644 index 00000000..bae77808 --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/AbstractN5FSWriter.java @@ -0,0 +1,358 @@ +/** + * Copyright (c) 2017--2021, Stephan Saalfeld + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +package org.janelia.saalfeldlab.n5; + +import java.io.IOException; +import java.nio.channels.Channels; +import java.nio.charset.StandardCharsets; +import java.nio.file.DirectoryNotEmptyException; +import java.nio.file.FileAlreadyExistsException; +import java.nio.file.FileSystem; +import java.nio.file.FileSystemException; +import java.nio.file.Files; +import java.nio.file.NoSuchFileException; +import java.nio.file.Path; +import java.nio.file.attribute.FileAttribute; +import java.util.Comparator; +import java.util.HashMap; +import java.util.Map; +import java.util.stream.Stream; + +import com.google.gson.GsonBuilder; +import com.google.gson.JsonElement; + +/** + * Filesystem {@link N5Writer} implementation with version compatibility check. + * + * @author Stephan Saalfeld + */ +public class AbstractN5FSWriter extends AbstractN5FSReader implements N5Writer { + + /** + * Opens an {@link AbstractN5FSWriter} at a given base path with a custom + * {@link GsonBuilder} to support custom attributes. + * + * If the base path does not exist, it will be created. + * + * If the base path exists and if the N5 version of the container is + * compatible with this implementation, the N5 version of this container + * will be set to the current N5 version of this implementation. + * + * @param fileSystem + * @param basePath n5 base path + * @param gsonBuilder + * @param cacheAttributes cache attributes + * Setting this to true avoids frequent reading and parsing of JSON + * encoded attributes, this is most interesting for high latency file + * systems. Changes of attributes by an independent writer will not be + * tracked. + * + * @throws IOException + * if the base path cannot be written to or cannot be created, + * if the N5 version of the container is not compatible with this + * implementation. + */ + public AbstractN5FSWriter( + final FileSystem fileSystem, + final String basePath, + final GsonBuilder gsonBuilder, + final boolean cacheAttributes) throws IOException { + + super(fileSystem, basePath, gsonBuilder, cacheAttributes); + createDirectories(fileSystem.getPath(basePath)); + if (!VERSION.equals(getVersion())) + setAttribute("/", VERSION_KEY, VERSION.toString()); + } + + @Override + public void createGroup(final String pathName) throws IOException { + + final Path path = getGroupPath(pathName); + if (cacheMeta) { + N5GroupInfo info = getCachedN5GroupInfo(path); + if (info == emptyGroupInfo) { + + /* The directories may be created multiple times concurrently, + * but a new cache entry is inserted only if none has been + * inserted in the meantime (because that may already include + * more cached data). + * + * This avoids synchronizing on the cache for independent + * group creation. + */ + createDirectories(path); + synchronized (metaCache) { + info = getCachedN5GroupInfo(path); + if (info == emptyGroupInfo) + metaCache.put(pathName, new N5GroupInfo()); + } + } + } + } + + /** + * Helper method that reads the existing map of attributes, JSON encodes, + * inserts and overrides the provided attributes, and writes them back into + * the attributes store. + * + * @param path + * @param attributes + * @throws IOException + */ + protected void writeAttributes( + final Path path, + final Map attributes) throws IOException { + + final HashMap map = new HashMap<>(); + + try (final LockedFileChannel lockedFileChannel = LockedFileChannel.openForWriting(path)) { + map.putAll(GsonAttributesParser.readAttributes(Channels.newReader(lockedFileChannel.getFileChannel(), StandardCharsets.UTF_8.name()), getGson())); + GsonAttributesParser.insertAttributes(map, attributes, gson); + + lockedFileChannel.getFileChannel().truncate(0); + GsonAttributesParser.writeAttributes(Channels.newWriter(lockedFileChannel.getFileChannel(), StandardCharsets.UTF_8.name()), map, getGson()); + } + } + + @Override + public void setAttributes( + final String pathName, + final Map attributes) throws IOException { + + if (cacheMeta) { + final Path path = getGroupPath(pathName); + N5GroupInfo info = getCachedN5GroupInfo(path); + if (info == emptyGroupInfo) { + synchronized (metaCache) { + info = getCachedN5GroupInfo(path); + if (info == emptyGroupInfo) { + //createDirectories(path.getParent()); ? or not ? + info = metaCache.put(pathName, new N5GroupInfo()); + } + } + } + final HashMap cachedMap = getCachedAttributes(info, pathName); + synchronized (cachedMap) { + cachedMap.putAll(attributes); + writeAttributes(getAttributesPath(pathName), attributes); + } + } else + writeAttributes(getAttributesPath(pathName), attributes); + } + + @Override + public void writeBlock( + final String pathName, + final DatasetAttributes datasetAttributes, + final DataBlock dataBlock) throws IOException { + + final Path path = getDataBlockPath(pathName, dataBlock.getGridPosition()); + createDirectories(path.getParent()); + try (final LockedFileChannel lockedChannel = LockedFileChannel.openForWriting(path)) { + lockedChannel.getFileChannel().truncate(0); + DefaultBlockWriter.writeBlock(Channels.newOutputStream(lockedChannel.getFileChannel()), datasetAttributes, dataBlock); + } + } + + @Override + public boolean remove() throws IOException { + + return remove("/"); + } + + @Override + public boolean remove(final String pathName) throws IOException { + + final Path path = getGroupPath(pathName); + if (Files.exists(path)) + try (final Stream pathStream = Files.walk(path)) { + pathStream.sorted(Comparator.reverseOrder()).forEach( + childPath -> { + if (Files.isRegularFile(childPath)) { + try (final LockedFileChannel channel = LockedFileChannel.openForWriting(childPath)) { + Files.delete(childPath); + } catch (final IOException e) { + e.printStackTrace(); + } + } else { + try { + Files.delete(childPath); + } catch (final DirectoryNotEmptyException e) { + // Even though childPath should be an empty directory, sometimes the deletion fails on network file system + // when lock files are not cleared immediately after the leaves have been removed. + try { + // wait and reattempt + Thread.sleep(100); + Files.delete(childPath); + } catch (final InterruptedException ex) { + e.printStackTrace(); + Thread.currentThread().interrupt(); + } catch (final IOException ex) { + ex.printStackTrace(); + } + } catch (final IOException e) { + e.printStackTrace(); + } + } + }); + } + return !Files.exists(path); + } + + @Override + public boolean deleteBlock( + final String pathName, + final long... gridPosition) throws IOException { + + final Path path = getDataBlockPath(pathName, gridPosition); + if (Files.exists(path)) + try (final LockedFileChannel channel = LockedFileChannel.openForWriting(path)) { + Files.deleteIfExists(path); + } + return !Files.exists(path); + } + + /** + * This is a copy of {@link Files#createDirectories(Path, FileAttribute...)} + * that follows symlinks. + * + * Workaround for https://bugs.openjdk.java.net/browse/JDK-8130464 + * + * Creates a directory by creating all nonexistent parent directories first. + * Unlike the {@link #createDirectory createDirectory} method, an exception + * is not thrown if the directory could not be created because it already + * exists. + * + *

The {@code attrs} parameter is optional {@link FileAttribute + * file-attributes} to set atomically when creating the nonexistent + * directories. Each file attribute is identified by its {@link + * FileAttribute#name name}. If more than one attribute of the same name is + * included in the array then all but the last occurrence is ignored. + * + *

If this method fails, then it may do so after creating some, but not + * all, of the parent directories. + * + * @param dir + * the directory to create + * + * @param attrs + * an optional list of file attributes to set atomically when + * creating the directory + * + * @return the directory + * + * @throws UnsupportedOperationException + * if the array contains an attribute that cannot be set atomically + * when creating the directory + * @throws FileAlreadyExistsException + * if {@code dir} exists but is not a directory (optional specific + * exception) + * @throws IOException + * if an I/O error occurs + * @throws SecurityException + * in the case of the default provider, and a security manager is + * installed, the {@link SecurityManager#checkWrite(String) checkWrite} + * method is invoked prior to attempting to create a directory and + * its {@link SecurityManager#checkRead(String) checkRead} is + * invoked for each parent directory that is checked. If {@code + * dir} is not an absolute path then its {@link Path#toAbsolutePath + * toAbsolutePath} may need to be invoked to get its absolute path. + * This may invoke the security manager's {@link + * SecurityManager#checkPropertyAccess(String) checkPropertyAccess} + * method to check access to the system property {@code user.dir} + */ + private static Path createDirectories(Path dir, final FileAttribute... attrs) + throws IOException + { + // attempt to create the directory + try { + createAndCheckIsDirectory(dir, attrs); + return dir; + } catch (final FileAlreadyExistsException x) { + // file exists and is not a directory + throw x; + } catch (final IOException x) { + // parent may not exist or other reason + } + SecurityException se = null; + try { + dir = dir.toAbsolutePath(); + } catch (final SecurityException x) { + // don't have permission to get absolute path + se = x; + } + // find a decendent that exists + Path parent = dir.getParent(); + while (parent != null) { + try { + parent.getFileSystem().provider().checkAccess(parent); + break; + } catch (final NoSuchFileException x) { + // does not exist + } + parent = parent.getParent(); + } + if (parent == null) { + // unable to find existing parent + if (se == null) { + throw new FileSystemException(dir.toString(), null, + "Unable to determine if root directory exists"); + } else { + throw se; + } + } + + // create directories + Path child = parent; + for (final Path name: parent.relativize(dir)) { + child = child.resolve(name); + createAndCheckIsDirectory(child, attrs); + } + return dir; + } + + /** + * This is a copy of + * {@link Files#createAndCheckIsDirectory(Path, FileAttribute...)} that + * follows symlinks. + * + * Workaround for https://bugs.openjdk.java.net/browse/JDK-8130464 + * + * Used by createDirectories to attempt to create a directory. A no-op if + * the directory already exists. + */ + private static void createAndCheckIsDirectory( + final Path dir, + final FileAttribute... attrs) throws IOException { + + try { + Files.createDirectory(dir, attrs); + } catch (final FileAlreadyExistsException x) { + if (!Files.isDirectory(dir)) + throw x; + } + } +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java b/src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java index 6de54c65..7cfbdc3d 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java @@ -26,30 +26,16 @@ package org.janelia.saalfeldlab.n5; import java.io.IOException; -import java.nio.channels.Channels; -import java.nio.charset.StandardCharsets; -import java.nio.file.DirectoryNotEmptyException; -import java.nio.file.FileAlreadyExistsException; -import java.nio.file.FileSystemException; -import java.nio.file.Files; -import java.nio.file.NoSuchFileException; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.nio.file.attribute.FileAttribute; -import java.util.Comparator; -import java.util.HashMap; -import java.util.Map; -import java.util.stream.Stream; +import java.nio.file.FileSystems; import com.google.gson.GsonBuilder; -import com.google.gson.JsonElement; /** * Filesystem {@link N5Writer} implementation with version compatibility check. * * @author Stephan Saalfeld */ -public class N5FSWriter extends N5FSReader implements N5Writer { +public class N5FSWriter extends AbstractN5FSWriter { /** * Opens an {@link N5FSWriter} at a given base path with a custom @@ -76,10 +62,7 @@ public class N5FSWriter extends N5FSReader implements N5Writer { */ public N5FSWriter(final String basePath, final GsonBuilder gsonBuilder, final boolean cacheAttributes) throws IOException { - super(basePath, gsonBuilder, cacheAttributes); - createDirectories(Paths.get(basePath)); - if (!VERSION.equals(getVersion())) - setAttribute("/", VERSION_KEY, VERSION.toString()); + super(FileSystems.getDefault(), basePath, gsonBuilder, cacheAttributes); } /** @@ -129,10 +112,7 @@ public N5FSWriter(final String basePath, final boolean cacheAttributes) throws I */ public N5FSWriter(final String basePath, final GsonBuilder gsonBuilder) throws IOException { - super(basePath, gsonBuilder); - createDirectories(Paths.get(basePath)); - if (!VERSION.equals(getVersion())) - setAttribute("/", VERSION_KEY, VERSION.toString()); + this(basePath, gsonBuilder, false); } /** @@ -156,236 +136,4 @@ public N5FSWriter(final String basePath) throws IOException { this(basePath, new GsonBuilder()); } - - @Override - public void createGroup(final String pathName) throws IOException { - - final Path path = Paths.get(basePath, pathName); - createDirectories(path); - } - - protected void writeAttributes( - final String pathName, - final Map attributes) throws IOException { - - final Path path = Paths.get(basePath, getAttributesPath(pathName).toString()); - final HashMap map = new HashMap<>(); - - try (final LockedFileChannel lockedFileChannel = LockedFileChannel.openForWriting(path)) { - map.putAll(GsonAttributesParser.readAttributes(Channels.newReader(lockedFileChannel.getFileChannel(), StandardCharsets.UTF_8.name()), getGson())); - GsonAttributesParser.insertAttributes(map, attributes, gson); - - lockedFileChannel.getFileChannel().truncate(0); - GsonAttributesParser.writeAttributes(Channels.newWriter(lockedFileChannel.getFileChannel(), StandardCharsets.UTF_8.name()), map, getGson()); - } - } - - @Override - public void setAttributes( - final String pathName, - final Map attributes) throws IOException { - - if (cacheMeta) { - final HashMap cachedMap; - synchronized (attributesCache) { - cachedMap = getCachedAttributes(pathName); - } - synchronized (cachedMap) { - cachedMap.putAll(attributes); - writeAttributes(pathName, attributes); - } - } else - writeAttributes(pathName, attributes); - } - - @Override - public void writeBlock( - final String pathName, - final DatasetAttributes datasetAttributes, - final DataBlock dataBlock) throws IOException { - - final Path path = Paths.get(basePath, getDataBlockPath(pathName, dataBlock.getGridPosition()).toString()); - createDirectories(path.getParent()); - try (final LockedFileChannel lockedChannel = LockedFileChannel.openForWriting(path)) { - lockedChannel.getFileChannel().truncate(0); - DefaultBlockWriter.writeBlock(Channels.newOutputStream(lockedChannel.getFileChannel()), datasetAttributes, dataBlock); - } - } - - @Override - public boolean remove() throws IOException { - - return remove("/"); - } - - @Override - public boolean remove(final String pathName) throws IOException { - - final Path path = Paths.get(basePath, pathName); - if (Files.exists(path)) - try (final Stream pathStream = Files.walk(path)) { - pathStream.sorted(Comparator.reverseOrder()).forEach( - childPath -> { - if (Files.isRegularFile(childPath)) { - try (final LockedFileChannel channel = LockedFileChannel.openForWriting(childPath)) { - Files.delete(childPath); - } catch (final IOException e) { - e.printStackTrace(); - } - } else { - try { - Files.delete(childPath); - } catch (final DirectoryNotEmptyException e) { - // Even though childPath should be an empty directory, sometimes the deletion fails on network file system - // when lock files are not cleared immediately after the leaves have been removed. - try { - // wait and reattempt - Thread.sleep(100); - Files.delete(childPath); - } catch (final InterruptedException ex) { - e.printStackTrace(); - Thread.currentThread().interrupt(); - } catch (final IOException ex) { - ex.printStackTrace(); - } - } catch (final IOException e) { - e.printStackTrace(); - } - } - }); - } - return !Files.exists(path); - } - - @Override - public boolean deleteBlock( - final String pathName, - final long... gridPosition) throws IOException { - final Path path = Paths.get(basePath, getDataBlockPath(pathName, gridPosition).toString()); - if (Files.exists(path)) - try (final LockedFileChannel channel = LockedFileChannel.openForWriting(path)) { - Files.deleteIfExists(path); - } - return !Files.exists(path); - } - - /** - * This is a copy of {@link Files#createDirectories(Path, FileAttribute...)} - * that follows symlinks. - * - * Workaround for https://bugs.openjdk.java.net/browse/JDK-8130464 - * - * Creates a directory by creating all nonexistent parent directories first. - * Unlike the {@link #createDirectory createDirectory} method, an exception - * is not thrown if the directory could not be created because it already - * exists. - * - *

The {@code attrs} parameter is optional {@link FileAttribute - * file-attributes} to set atomically when creating the nonexistent - * directories. Each file attribute is identified by its {@link - * FileAttribute#name name}. If more than one attribute of the same name is - * included in the array then all but the last occurrence is ignored. - * - *

If this method fails, then it may do so after creating some, but not - * all, of the parent directories. - * - * @param dir - * the directory to create - * - * @param attrs - * an optional list of file attributes to set atomically when - * creating the directory - * - * @return the directory - * - * @throws UnsupportedOperationException - * if the array contains an attribute that cannot be set atomically - * when creating the directory - * @throws FileAlreadyExistsException - * if {@code dir} exists but is not a directory (optional specific - * exception) - * @throws IOException - * if an I/O error occurs - * @throws SecurityException - * in the case of the default provider, and a security manager is - * installed, the {@link SecurityManager#checkWrite(String) checkWrite} - * method is invoked prior to attempting to create a directory and - * its {@link SecurityManager#checkRead(String) checkRead} is - * invoked for each parent directory that is checked. If {@code - * dir} is not an absolute path then its {@link Path#toAbsolutePath - * toAbsolutePath} may need to be invoked to get its absolute path. - * This may invoke the security manager's {@link - * SecurityManager#checkPropertyAccess(String) checkPropertyAccess} - * method to check access to the system property {@code user.dir} - */ - private static Path createDirectories(Path dir, final FileAttribute... attrs) - throws IOException - { - // attempt to create the directory - try { - createAndCheckIsDirectory(dir, attrs); - return dir; - } catch (final FileAlreadyExistsException x) { - // file exists and is not a directory - throw x; - } catch (final IOException x) { - // parent may not exist or other reason - } - SecurityException se = null; - try { - dir = dir.toAbsolutePath(); - } catch (final SecurityException x) { - // don't have permission to get absolute path - se = x; - } - // find a decendent that exists - Path parent = dir.getParent(); - while (parent != null) { - try { - parent.getFileSystem().provider().checkAccess(parent); - break; - } catch (final NoSuchFileException x) { - // does not exist - } - parent = parent.getParent(); - } - if (parent == null) { - // unable to find existing parent - if (se == null) { - throw new FileSystemException(dir.toString(), null, - "Unable to determine if root directory exists"); - } else { - throw se; - } - } - - // create directories - Path child = parent; - for (final Path name: parent.relativize(dir)) { - child = child.resolve(name); - createAndCheckIsDirectory(child, attrs); - } - return dir; - } - - /** - * This is a copy of {@link Files#createAndCheckIsDirectory(Path, FileAttribute...)} - * that follows symlinks. - * - * Workaround for https://bugs.openjdk.java.net/browse/JDK-8130464 - * - * Used by createDirectories to attempt to create a directory. A no-op - * if the directory already exists. - */ - private static void createAndCheckIsDirectory(final Path dir, - final FileAttribute... attrs) - throws IOException - { - try { - Files.createDirectory(dir, attrs); - } catch (final FileAlreadyExistsException x) { - if (!Files.isDirectory(dir)) - throw x; - } - } } From d52011e0609669117824d95e693d80720331be89 Mon Sep 17 00:00:00 2001 From: Stephan Saalfeld Date: Mon, 3 May 2021 14:42:46 -0400 Subject: [PATCH 008/243] WIP still on it... --- .../saalfeldlab/n5/AbstractN5FSReader.java | 61 +++++++------- .../saalfeldlab/n5/AbstractN5FSWriter.java | 81 +++++++++++-------- 2 files changed, 80 insertions(+), 62 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/AbstractN5FSReader.java b/src/main/java/org/janelia/saalfeldlab/n5/AbstractN5FSReader.java index 1cccaa94..17a08801 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/AbstractN5FSReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/AbstractN5FSReader.java @@ -196,13 +196,13 @@ public DatasetAttributes getDatasetAttributes(final String pathName) throws IOEx int[] blockSize; Compression compression; final String compressionVersion0Name; - final Path path = getGroupPath(pathName); + final String normalPathName = removeLeadingSlash(pathName); if (cacheMeta) { - final N5GroupInfo info = getCachedN5GroupInfo(path); + final N5GroupInfo info = getCachedN5GroupInfo(normalPathName); if (info == emptyGroupInfo) return null; - final HashMap cachedMap = getCachedAttributes(info, pathName); + final HashMap cachedMap = getCachedAttributes(info, normalPathName); if (cachedMap.isEmpty()) return null; @@ -224,7 +224,7 @@ public DatasetAttributes getDatasetAttributes(final String pathName) throws IOEx ? getAttribute(cachedMap, DatasetAttributes.compressionTypeKey, String.class) : null; } else { - final HashMap map = getAttributes(pathName); + final HashMap map = getAttributes(normalPathName); dimensions = GsonAttributesParser.parseAttribute(map, DatasetAttributes.dimensionsKey, long[].class, gson); if (dimensions == null) @@ -290,13 +290,15 @@ protected HashMap getAttributes(final Path path) throws IOE * done in calling code. * * @param info - * @param pathName + * @param pathName normalized group path without leading slash * @return cached attributes * empty map if the group exists but not attributes are set * null if the group does not exist * @throws IOException */ - protected HashMap getCachedAttributes(final N5GroupInfo info, final String pathName) throws IOException { + protected HashMap getCachedAttributes( + final N5GroupInfo info, + final String pathName) throws IOException { HashMap cachedMap = info.attributesCache; if (cachedMap == null) { @@ -369,17 +371,17 @@ public T getAttribute( final String key, final Class clazz) throws IOException { + final String normalPathName = removeLeadingSlash(pathName); if (cacheMeta) { - final Path path = getGroupPath(pathName); - final N5GroupInfo info = getCachedN5GroupInfo(path); + final N5GroupInfo info = getCachedN5GroupInfo(normalPathName); if (info == emptyGroupInfo) return null; - final HashMap cachedMap = getCachedAttributes(info, pathName); + final HashMap cachedMap = getCachedAttributes(info, normalPathName); if (cachedMap.isEmpty()) return null; return getAttribute(cachedMap, key, clazz); } else { - final HashMap map = getAttributes(pathName); + final HashMap map = getAttributes(normalPathName); return GsonAttributesParser.parseAttribute(map, key, clazz, getGson()); } } @@ -391,8 +393,7 @@ public T getAttribute( final Type type) throws IOException { if (cacheMeta) { - final Path path = getGroupPath(pathName); - final N5GroupInfo info = getCachedN5GroupInfo(path); + final N5GroupInfo info = getCachedN5GroupInfo(pathName); if (info == emptyGroupInfo) return null; final HashMap cachedMap = getCachedAttributes(info, pathName); @@ -410,9 +411,8 @@ protected boolean exists(final Path path) { return Files.exists(path) && Files.isDirectory(path); } - protected N5GroupInfo getCachedN5GroupInfo(final Path path) { + protected N5GroupInfo getCachedN5GroupInfo(final String pathName) { - final String pathName = path.toString(); final N5GroupInfo cachedInfo = metaCache.get(pathName); if (cachedInfo == null) { @@ -420,12 +420,14 @@ protected N5GroupInfo getCachedN5GroupInfo(final Path path) { * exists checks for independent paths than to accept the * same exists check to potentially run multiple times. */ - final boolean exists = exists(path); + final boolean exists = exists(getGroupPath(pathName)); synchronized (metaCache) { final N5GroupInfo cachedInfoAgain = metaCache.get(pathName); if (cachedInfoAgain == null) { - return metaCache.put(pathName, exists ? new N5GroupInfo() : emptyGroupInfo); + final N5GroupInfo info = exists ? new N5GroupInfo() : emptyGroupInfo; + metaCache.put(pathName, info); + return info; } else return cachedInfoAgain; } @@ -436,17 +438,17 @@ protected N5GroupInfo getCachedN5GroupInfo(final Path path) { @Override public boolean exists(final String pathName) { - final Path path = fileSystem.getPath(basePath, removeLeadingSlash(pathName)); + final String normalPathName = removeLeadingSlash(pathName); if (cacheMeta) - return getCachedN5GroupInfo(path) != emptyGroupInfo; + return getCachedN5GroupInfo(normalPathName) != emptyGroupInfo; else - return exists(path); + return exists(getGroupPath(normalPathName)); } @Override public HashMap getAttributes(final String pathName) throws IOException { - final Path path = getAttributesPath(pathName); + final Path path = getAttributesPath(removeLeadingSlash(pathName)); if (exists(pathName) && !Files.exists(path)) return new HashMap<>(); @@ -461,7 +463,7 @@ public DataBlock readBlock( final DatasetAttributes datasetAttributes, final long... gridPosition) throws IOException { - final Path path = getDataBlockPath(pathName, gridPosition); + final Path path = getDataBlockPath(removeLeadingSlash(pathName), gridPosition); if (!Files.exists(path)) return null; @@ -473,7 +475,8 @@ public DataBlock readBlock( @Override public String[] list(final String pathName) throws IOException { - final Path path = fileSystem.getPath(basePath, removeLeadingSlash(pathName)); + final String normalPathName = removeLeadingSlash(pathName); + final Path path = getGroupPath(normalPathName); try (final Stream pathStream = Files.list(path)) { return pathStream .filter(a -> Files.isDirectory(a)) @@ -492,7 +495,7 @@ public String[] list(final String pathName) throws IOException { * * This is the file into which the data block will be stored. * - * @param datasetPathName + * @param datasetPathName normalized dataset path without leading slash * @param gridPosition * @return */ @@ -501,8 +504,8 @@ protected Path getDataBlockPath( final long... gridPosition) { final String[] pathComponents = new String[gridPosition.length + 1]; - pathComponents[0] = removeLeadingSlash(datasetPathName); - for (int i = 1; i <= pathComponents.length; ++i) + pathComponents[0] = datasetPathName; + for (int i = 1; i < pathComponents.length; ++i) pathComponents[i] = Long.toString(gridPosition[i - 1]); return fileSystem.getPath(basePath, pathComponents); @@ -511,23 +514,23 @@ protected Path getDataBlockPath( /** * Constructs the path for the group or dataset. * - * @param pathName + * @param pathName normalized group path without leading slash * @return */ protected Path getGroupPath(final String pathName) { - return fileSystem.getPath(basePath, removeLeadingSlash(pathName)); + return fileSystem.getPath(basePath, pathName); } /** * Constructs the path for the attributes file of a group or dataset. * - * @param pathName + * @param pathName normalized group path without leading slash * @return */ protected Path getAttributesPath(final String pathName) { - return fileSystem.getPath(basePath, removeLeadingSlash(pathName), jsonFile); + return fileSystem.getPath(basePath, pathName, jsonFile); } /** diff --git a/src/main/java/org/janelia/saalfeldlab/n5/AbstractN5FSWriter.java b/src/main/java/org/janelia/saalfeldlab/n5/AbstractN5FSWriter.java index bae77808..18387dfa 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/AbstractN5FSWriter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/AbstractN5FSWriter.java @@ -90,9 +90,9 @@ public AbstractN5FSWriter( @Override public void createGroup(final String pathName) throws IOException { - final Path path = getGroupPath(pathName); + final String normalPathName = removeLeadingSlash(pathName); if (cacheMeta) { - N5GroupInfo info = getCachedN5GroupInfo(path); + N5GroupInfo info = getCachedN5GroupInfo(normalPathName); if (info == emptyGroupInfo) { /* The directories may be created multiple times concurrently, @@ -103,14 +103,15 @@ public void createGroup(final String pathName) throws IOException { * This avoids synchronizing on the cache for independent * group creation. */ - createDirectories(path); + createDirectories(getGroupPath(normalPathName)); synchronized (metaCache) { - info = getCachedN5GroupInfo(path); + info = getCachedN5GroupInfo(normalPathName); if (info == emptyGroupInfo) - metaCache.put(pathName, new N5GroupInfo()); + metaCache.put(normalPathName, new N5GroupInfo()); } } - } + } else + createDirectories(getGroupPath(normalPathName)); } /** @@ -142,25 +143,25 @@ public void setAttributes( final String pathName, final Map attributes) throws IOException { + final String normalPathName = removeLeadingSlash(pathName); if (cacheMeta) { - final Path path = getGroupPath(pathName); - N5GroupInfo info = getCachedN5GroupInfo(path); + N5GroupInfo info = getCachedN5GroupInfo(normalPathName); if (info == emptyGroupInfo) { synchronized (metaCache) { - info = getCachedN5GroupInfo(path); + info = getCachedN5GroupInfo(normalPathName); if (info == emptyGroupInfo) { //createDirectories(path.getParent()); ? or not ? - info = metaCache.put(pathName, new N5GroupInfo()); + info = metaCache.put(normalPathName, new N5GroupInfo()); } } } - final HashMap cachedMap = getCachedAttributes(info, pathName); + final HashMap cachedMap = getCachedAttributes(info, normalPathName); synchronized (cachedMap) { cachedMap.putAll(attributes); - writeAttributes(getAttributesPath(pathName), attributes); + writeAttributes(getAttributesPath(normalPathName), attributes); } } else - writeAttributes(getAttributesPath(pathName), attributes); + writeAttributes(getAttributesPath(normalPathName), attributes); } @Override @@ -169,7 +170,7 @@ public void writeBlock( final DatasetAttributes datasetAttributes, final DataBlock dataBlock) throws IOException { - final Path path = getDataBlockPath(pathName, dataBlock.getGridPosition()); + final Path path = getDataBlockPath(removeLeadingSlash(pathName), dataBlock.getGridPosition()); createDirectories(path.getParent()); try (final LockedFileChannel lockedChannel = LockedFileChannel.openForWriting(path)) { lockedChannel.getFileChannel().truncate(0); @@ -183,11 +184,34 @@ public boolean remove() throws IOException { return remove("/"); } + protected static void tryDelete(final Path childPath) { + + try { + Files.delete(childPath); + } catch (final DirectoryNotEmptyException e) { + // Even though childPath should be an empty directory, sometimes the deletion fails on network file + // when lock files are not cleared immediately after the leaves have been removed. + try { + // wait and reattempt + Thread.sleep(100); + Files.delete(childPath); + } catch (final InterruptedException ex) { + e.printStackTrace(); + Thread.currentThread().interrupt(); + } catch (final IOException ex) { + ex.printStackTrace(); + } + } catch (final IOException e) { + e.printStackTrace(); + } + } + @Override public boolean remove(final String pathName) throws IOException { - final Path path = getGroupPath(pathName); - if (Files.exists(path)) + final Path path = getGroupPath(removeLeadingSlash(pathName)); + if (Files.exists(path)) { + final Path base = fileSystem.getPath(basePath); try (final Stream pathStream = Files.walk(path)) { pathStream.sorted(Comparator.reverseOrder()).forEach( childPath -> { @@ -198,26 +222,17 @@ public boolean remove(final String pathName) throws IOException { e.printStackTrace(); } } else { - try { - Files.delete(childPath); - } catch (final DirectoryNotEmptyException e) { - // Even though childPath should be an empty directory, sometimes the deletion fails on network file system - // when lock files are not cleared immediately after the leaves have been removed. - try { - // wait and reattempt - Thread.sleep(100); - Files.delete(childPath); - } catch (final InterruptedException ex) { - e.printStackTrace(); - Thread.currentThread().interrupt(); - } catch (final IOException ex) { - ex.printStackTrace(); + if (cacheMeta) { + synchronized (metaCache) { + metaCache.put(base.relativize(childPath).toString(), emptyGroupInfo); + tryDelete(childPath); } - } catch (final IOException e) { - e.printStackTrace(); + } else { + tryDelete(childPath); } } }); + } } return !Files.exists(path); } @@ -227,7 +242,7 @@ public boolean deleteBlock( final String pathName, final long... gridPosition) throws IOException { - final Path path = getDataBlockPath(pathName, gridPosition); + final Path path = getDataBlockPath(removeLeadingSlash(pathName), gridPosition); if (Files.exists(path)) try (final LockedFileChannel channel = LockedFileChannel.openForWriting(path)) { Files.deleteIfExists(path); From 50ad324ca7409bc8167e3ea676ae29cd1e31f822 Mon Sep 17 00:00:00 2001 From: Stephan Saalfeld Date: Mon, 3 May 2021 14:44:07 -0400 Subject: [PATCH 009/243] fix license headers in tests --- .../saalfeldlab/n5/AbstractN5Test.java | 31 ++++++++++++------- .../janelia/saalfeldlab/n5/N5Benchmark.java | 31 ++++++++++++------- .../org/janelia/saalfeldlab/n5/N5FSTest.java | 31 ++++++++++++------- 3 files changed, 60 insertions(+), 33 deletions(-) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java index 572f1f7e..bcfb6914 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java @@ -1,18 +1,27 @@ /** - * License: GPL + * Copyright (c) 2017--2021, Stephan Saalfeld + * All rights reserved. * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License 2 - * as published by the Free Software Foundation. + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. */ package org.janelia.saalfeldlab.n5; diff --git a/src/test/java/org/janelia/saalfeldlab/n5/N5Benchmark.java b/src/test/java/org/janelia/saalfeldlab/n5/N5Benchmark.java index 512d82b4..572565c3 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/N5Benchmark.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/N5Benchmark.java @@ -1,18 +1,27 @@ /** - * License: GPL + * Copyright (c) 2017--2021, Stephan Saalfeld + * All rights reserved. * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License 2 - * as published by the Free Software Foundation. + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. */ package org.janelia.saalfeldlab.n5; diff --git a/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java b/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java index 32b1a22b..bf91e8e8 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java @@ -1,18 +1,27 @@ /** - * License: GPL + * Copyright (c) 2017--2021, Stephan Saalfeld + * All rights reserved. * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License 2 - * as published by the Free Software Foundation. + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. */ package org.janelia.saalfeldlab.n5; From 9261424cf7ebc489008ba044c394d62019a234b7 Mon Sep 17 00:00:00 2001 From: Stephan Saalfeld Date: Mon, 3 May 2021 20:03:01 -0400 Subject: [PATCH 010/243] passing test --- .../org/janelia/saalfeldlab/n5/AbstractN5FSReader.java | 7 ++++--- .../org/janelia/saalfeldlab/n5/AbstractN5Test.java | 10 ++++++---- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/AbstractN5FSReader.java b/src/main/java/org/janelia/saalfeldlab/n5/AbstractN5FSReader.java index 17a08801..74dd71cd 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/AbstractN5FSReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/AbstractN5FSReader.java @@ -392,16 +392,17 @@ public T getAttribute( final String key, final Type type) throws IOException { + final String normalPathName = removeLeadingSlash(pathName); if (cacheMeta) { - final N5GroupInfo info = getCachedN5GroupInfo(pathName); + final N5GroupInfo info = getCachedN5GroupInfo(normalPathName); if (info == emptyGroupInfo) return null; - final HashMap cachedMap = getCachedAttributes(info, pathName); + final HashMap cachedMap = getCachedAttributes(info, normalPathName); if (cachedMap.isEmpty()) return null; return getAttribute(cachedMap, key, type); } else { - final HashMap map = getAttributes(pathName); + final HashMap map = getAttributes(normalPathName); return GsonAttributesParser.parseAttribute(map, key, type, getGson()); } } diff --git a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java index bcfb6914..1ed600d9 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java @@ -439,15 +439,17 @@ public void testAttributes() { newAttributes.put("key3", "value3"); n5.setAttributes(groupName, newAttributes); Assert.assertEquals(3, n5.listAttributes(groupName).size()); - /* class interface */ - Assert.assertEquals("value1", n5.getAttribute(groupName, "key1", String.class)); - Assert.assertEquals("value2", n5.getAttribute(groupName, "key2", String.class)); - Assert.assertEquals("value3", n5.getAttribute(groupName, "key3", String.class)); + /* type interface */ Assert.assertEquals("value1", n5.getAttribute(groupName, "key1", new TypeToken(){}.getType())); Assert.assertEquals("value2", n5.getAttribute(groupName, "key2", new TypeToken(){}.getType())); Assert.assertEquals("value3", n5.getAttribute(groupName, "key3", new TypeToken(){}.getType())); + /* class interface */ + Assert.assertEquals("value1", n5.getAttribute(groupName, "key1", String.class)); + Assert.assertEquals("value2", n5.getAttribute(groupName, "key2", String.class)); + Assert.assertEquals("value3", n5.getAttribute(groupName, "key3", String.class)); + // test the case where the resulting file becomes shorter n5.setAttribute(groupName, "key1", new Integer(1)); n5.setAttribute(groupName, "key2", new Integer(2)); From 1f49043ca6905f45d28741475a1f46cab6f0ec5d Mon Sep 17 00:00:00 2001 From: Stephan Saalfeld Date: Mon, 3 May 2021 20:15:18 -0400 Subject: [PATCH 011/243] replace project.version by n5-spec.version=2.6.1 for spec version auto-generate current year in license --- LICENSE.md | 2 +- README.md | 2 +- doc/LICENSE.md | 2 +- doc/README.md | 2 +- pom.xml | 23 ++++++++++++++++++- .../org/janelia/saalfeldlab/n5/N5Reader.java | 9 +------- 6 files changed, 27 insertions(+), 13 deletions(-) diff --git a/LICENSE.md b/LICENSE.md index 8342e51f..1e1810e3 100644 --- a/LICENSE.md +++ b/LICENSE.md @@ -1,6 +1,6 @@ ## BSD 2-Clause License -Copyright (c) 2017-2020, Stephan Saalfeld +Copyright (c) 2017-2021, Stephan Saalfeld All rights reserved. diff --git a/README.md b/README.md index a6e32f42..0c0f6bb6 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ Chunked datasets can be sparse, i.e. empty chunks do not need to be stored. ## File-system specification -*version 2.6.0-SNAPSHOT* +*version 2.6.1* N5 group is not a single file but simply a directory on the file system. Meta-data is stored as a JSON file per each group/ directory. Tensor datasets can be chunked and chunks are stored as individual files. This enables parallel reading and writing on a cluster. diff --git a/doc/LICENSE.md b/doc/LICENSE.md index 19b1099c..dbda413c 100644 --- a/doc/LICENSE.md +++ b/doc/LICENSE.md @@ -1,6 +1,6 @@ ## BSD 2-Clause License -Copyright (c) @project.inceptionYear@-2020, Stephan Saalfeld +Copyright (c) @project.inceptionYear@-@current.year@, Stephan Saalfeld All rights reserved. diff --git a/doc/README.md b/doc/README.md index 674b16f6..cc6e196b 100644 --- a/doc/README.md +++ b/doc/README.md @@ -16,7 +16,7 @@ Chunked datasets can be sparse, i.e. empty chunks do not need to be stored. ## File-system specification -*version @project.version@* +*version @n5-spec.version@* N5 group is not a single file but simply a directory on the file system. Meta-data is stored as a JSON file per each group/ directory. Tensor datasets can be chunked and chunks are stored as individual files. This enables parallel reading and writing on a cluster. diff --git a/pom.xml b/pom.xml index e4000f17..70854cc4 100644 --- a/pom.xml +++ b/pom.xml @@ -1,5 +1,7 @@ - + 4.0.0 @@ -104,6 +106,8 @@ deploy-to-scijava 4.2 + + 2.6.1 @@ -172,6 +176,23 @@ + + org.codehaus.mojo + build-helper-maven-plugin + + + timestamp-property + + timestamp-property + + validate + + current.year + yyyy + + + + maven-resources-plugin diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java index dc368446..d832691c 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java @@ -41,8 +41,6 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; -import org.scijava.util.VersionUtils; - /** * A simple structured container for hierarchies of chunked * n-dimensional datasets and attributes. @@ -188,12 +186,7 @@ public boolean isCompatible(final Version version) { /** * SemVer version of this N5 spec. */ - public static final Version VERSION = - new Version( - VersionUtils.getVersionFromPOM( - N5Reader.class, - "org.janelia.saalfeldlab", - "n5")); + public static final Version VERSION = new Version(2, 6, 1); /** * Version attribute key. From 39f16c795412fef860d1b7e3db1215c4e6552d9c Mon Sep 17 00:00:00 2001 From: Stephan Saalfeld Date: Mon, 3 May 2021 21:14:36 -0400 Subject: [PATCH 012/243] fix some tests and add unspecific cache test (tests cache sensitive ops twice with a caching writer) --- .../saalfeldlab/n5/AbstractN5Test.java | 18 +++++-- .../org/janelia/saalfeldlab/n5/N5FSTest.java | 52 ++++++++++++++++++- 2 files changed, 64 insertions(+), 6 deletions(-) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java index 1ed600d9..e4d33480 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java @@ -463,6 +463,11 @@ public void testAttributes() { Assert.assertEquals(new Integer(2), n5.getAttribute(groupName, "key2", new TypeToken(){}.getType())); Assert.assertEquals("value3", n5.getAttribute(groupName, "key3", new TypeToken(){}.getType())); + n5.setAttribute(groupName, "key1", null); + n5.setAttribute(groupName, "key2", null); + n5.setAttribute(groupName, "key3", null); + Assert.assertEquals(0, n5.listAttributes(groupName).size()); + } catch (final IOException e) { fail(e.getMessage()); } @@ -486,11 +491,12 @@ public void testRemove() { public void testList() { try { - n5.createGroup(groupName); + final String testGroupName = groupName + "-test-list"; + n5.createGroup(testGroupName); for (final String subGroup : subGroupNames) - n5.createGroup(groupName + "/" + subGroup); + n5.createGroup(testGroupName + "/" + subGroup); - final String[] groupsList = n5.list(groupName); + final String[] groupsList = n5.list(testGroupName); Arrays.sort(groupsList); Assert.assertArrayEquals(subGroupNames, groupsList); @@ -499,7 +505,6 @@ public void testList() { Assert.assertArrayEquals(new String[] {"test"}, n5.list("")); Assert.assertArrayEquals(new String[] {"test"}, n5.list("/")); - } catch (final IOException e) { fail(e.getMessage()); } @@ -510,7 +515,8 @@ public void testDeepList() { try { // clear container to start - n5.remove(); + for (final String g : n5.list("/")) + n5.remove(g); n5.createGroup(groupName); for (final String subGroup : subGroupNames) @@ -737,6 +743,8 @@ public void testVersion() throws NumberFormatException, IOException { n5.setAttribute("/", N5Reader.VERSION_KEY, new Version(N5Reader.VERSION.getMajor() + 1, N5Reader.VERSION.getMinor(), N5Reader.VERSION.getPatch()).toString()); Assert.assertFalse(N5Reader.VERSION.isCompatible(n5.getVersion())); + + n5.setAttribute("/", N5Reader.VERSION_KEY, N5Reader.VERSION.toString()); } @Test diff --git a/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java b/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java index bf91e8e8..c2e4f2bf 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java @@ -25,8 +25,12 @@ */ package org.janelia.saalfeldlab.n5; +import static org.junit.Assert.fail; + import java.io.IOException; +import org.junit.Test; + /** * Initiates testing of the filesystem-based N5 implementation. * @@ -43,6 +47,52 @@ public class N5FSTest extends AbstractN5Test { @Override protected N5Writer createN5Writer() throws IOException { - return new N5FSWriter(testDirPath, true); + return new N5FSWriter(testDirPath, false); + } + + @Test + public void testCache() { + + final N5Writer n5Writer = n5; + try { + n5 = new N5FSWriter(testDirPath + "-cache", true); + + testAttributes(); + testAttributes(); + + testCreateDataset(); + testCreateDataset(); + + testCreateGroup(); + testCreateGroup(); + + testDeepList(); + testDeepList(); + + testVersion(); + testVersion(); + + testExists(); + testExists(); + + testList(); + testList(); + + testDelete(); + testDelete(); + + testListAttributes(); + testListAttributes(); + + testRemove(); + testRemove(); + + n5.remove(); + } catch (final IOException e) { + fail(e.getMessage()); + } finally { + n5.close(); + n5 = n5Writer; + } } } From 604ad1fe561310bb7a6d7376f64c7e6cd1b2ce37 Mon Sep 17 00:00:00 2001 From: Stephan Saalfeld Date: Wed, 5 May 2021 22:06:20 -0400 Subject: [PATCH 013/243] remove non abstract abstract leftover classes, bugfixes --- .../saalfeldlab/n5/AbstractN5FSReader.java | 555 ----------------- .../saalfeldlab/n5/AbstractN5FSWriter.java | 373 ------------ .../janelia/saalfeldlab/n5/N5FSReader.java | 569 +++++++++++++++++- .../janelia/saalfeldlab/n5/N5FSWriter.java | 398 +++++++++++- .../saalfeldlab/n5/AbstractN5Test.java | 2 +- 5 files changed, 964 insertions(+), 933 deletions(-) delete mode 100644 src/main/java/org/janelia/saalfeldlab/n5/AbstractN5FSReader.java delete mode 100644 src/main/java/org/janelia/saalfeldlab/n5/AbstractN5FSWriter.java diff --git a/src/main/java/org/janelia/saalfeldlab/n5/AbstractN5FSReader.java b/src/main/java/org/janelia/saalfeldlab/n5/AbstractN5FSReader.java deleted file mode 100644 index 74dd71cd..00000000 --- a/src/main/java/org/janelia/saalfeldlab/n5/AbstractN5FSReader.java +++ /dev/null @@ -1,555 +0,0 @@ -/** - * Copyright (c) 2017--2021, Stephan Saalfeld - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ -package org.janelia.saalfeldlab.n5; - -import java.io.Closeable; -import java.io.IOException; -import java.lang.reflect.Type; -import java.nio.channels.Channels; -import java.nio.channels.FileChannel; -import java.nio.channels.OverlappingFileLockException; -import java.nio.charset.StandardCharsets; -import java.nio.file.FileSystem; -import java.nio.file.Files; -import java.nio.file.OpenOption; -import java.nio.file.Path; -import java.nio.file.StandardOpenOption; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.stream.Stream; - -import com.google.gson.Gson; -import com.google.gson.GsonBuilder; -import com.google.gson.JsonElement; - -/** - * Filesystem {@link N5Reader} implementation with JSON attributes - * parsed with {@link Gson}. - * - * @author Stephan Saalfeld - * @author Igor Pisarev - * @author Philipp Hanslovsky - */ -public class AbstractN5FSReader implements GsonAttributesParser { - - /** - * A {@link FileChannel} wrapper that attempts to acquire a lock and waits - * for existing locks to be lifted before returning if the - * {@link FileSystem} supports that. If the {@link FileSystem} does not - * support locking, it returns immediately. - */ - protected static class LockedFileChannel implements Closeable { - - private final FileChannel channel; - - public static LockedFileChannel openForReading(final Path path) throws IOException { - - return new LockedFileChannel(path, true); - } - - public static LockedFileChannel openForWriting(final Path path) throws IOException { - - return new LockedFileChannel(path, false); - } - - private LockedFileChannel(final Path path, final boolean readOnly) throws IOException { - - final OpenOption[] options = readOnly ? new OpenOption[]{StandardOpenOption.READ} : new OpenOption[]{StandardOpenOption.READ, StandardOpenOption.WRITE, StandardOpenOption.CREATE}; - channel = FileChannel.open(path, options); - - for (boolean waiting = true; waiting;) { - waiting = false; - try { - channel.lock(0L, Long.MAX_VALUE, readOnly); - } catch (final OverlappingFileLockException e) { - waiting = true; - try { - Thread.sleep(100); - } catch (final InterruptedException f) { - waiting = false; - Thread.currentThread().interrupt(); - } - } catch (final IOException e) {} - } - } - - public FileChannel getFileChannel() { - - return channel; - } - - @Override - public void close() throws IOException { - - channel.close(); - } - } - - /** - * Data object for caching meta data. Elements that are null are not yet - * cached. - */ - protected static class N5GroupInfo { - - public ArrayList children = null; - public HashMap attributesCache = null; - public Boolean isDataset = null; - } - - protected static final N5GroupInfo emptyGroupInfo = new N5GroupInfo(); - - protected final FileSystem fileSystem; - - protected final Gson gson; - - protected final HashMap metaCache = new HashMap<>(); - - protected final boolean cacheMeta; - - protected static final String jsonFile = "attributes.json"; - - protected final String basePath; - - /** - * Opens an {@link AbstractN5FSReader} at a given base path with a custom - * {@link GsonBuilder} to support custom attributes. - * - * @param fileSystem - * @param basePath N5 base path - * @param gsonBuilder - * @param cacheMeta cache attributes and meta data - * Setting this to true avoids frequent reading and parsing of JSON - * encoded attributes and other meta data that requires accessing the - * store. This is most interesting for high latency backends. Changes - * of cached attributes and meta data by an independent writer will - * not be tracked. - * - * @throws IOException - * if the base path cannot be read or does not exist, - * if the N5 version of the container is not compatible with this - * implementation. - */ - public AbstractN5FSReader( - final FileSystem fileSystem, - final String basePath, - final GsonBuilder gsonBuilder, - final boolean cacheMeta) throws IOException { - - this.fileSystem = fileSystem; - this.basePath = basePath; - gsonBuilder.registerTypeAdapter(DataType.class, new DataType.JsonAdapter()); - gsonBuilder.registerTypeHierarchyAdapter(Compression.class, CompressionAdapter.getJsonAdapter()); - gsonBuilder.disableHtmlEscaping(); - this.gson = gsonBuilder.create(); - this.cacheMeta = cacheMeta; - if (exists("/")) { - final Version version = getVersion(); - if (!VERSION.isCompatible(version)) - throw new IOException("Incompatible version " + version + " (this is " + VERSION + ")."); - } - } - - @Override - public Gson getGson() { - - return gson; - } - - /** - * - * @return N5 base path - */ - public String getBasePath() { - - return this.basePath; - } - - @Override - public DatasetAttributes getDatasetAttributes(final String pathName) throws IOException { - - final long[] dimensions; - final DataType dataType; - int[] blockSize; - Compression compression; - final String compressionVersion0Name; - final String normalPathName = removeLeadingSlash(pathName); - if (cacheMeta) { - final N5GroupInfo info = getCachedN5GroupInfo(normalPathName); - if (info == emptyGroupInfo) - return null; - - final HashMap cachedMap = getCachedAttributes(info, normalPathName); - if (cachedMap.isEmpty()) - return null; - - dimensions = getAttribute(cachedMap, DatasetAttributes.dimensionsKey, long[].class); - if (dimensions == null) - return null; - - dataType = getAttribute(cachedMap, DatasetAttributes.dataTypeKey, DataType.class); - if (dataType == null) - return null; - - blockSize = getAttribute(cachedMap, DatasetAttributes.blockSizeKey, int[].class); - - compression = getAttribute(cachedMap, DatasetAttributes.compressionKey, Compression.class); - - /* version 0 */ - compressionVersion0Name = compression - == null - ? getAttribute(cachedMap, DatasetAttributes.compressionTypeKey, String.class) - : null; - } else { - final HashMap map = getAttributes(normalPathName); - - dimensions = GsonAttributesParser.parseAttribute(map, DatasetAttributes.dimensionsKey, long[].class, gson); - if (dimensions == null) - return null; - - dataType = GsonAttributesParser.parseAttribute(map, DatasetAttributes.dataTypeKey, DataType.class, gson); - if (dataType == null) - return null; - - blockSize = GsonAttributesParser.parseAttribute(map, DatasetAttributes.blockSizeKey, int[].class, gson); - - compression = GsonAttributesParser.parseAttribute(map, DatasetAttributes.compressionKey, Compression.class, gson); - - /* version 0 */ - compressionVersion0Name = compression == null - ? GsonAttributesParser.parseAttribute(map, DatasetAttributes.compressionTypeKey, String.class, gson) - : null; - } - - if (blockSize == null) - blockSize = Arrays.stream(dimensions).mapToInt(a -> (int)a).toArray(); - - /* version 0 */ - if (compression == null) { - switch (compressionVersion0Name) { - case "raw": - compression = new RawCompression(); - break; - case "gzip": - compression = new GzipCompression(); - break; - case "bzip2": - compression = new Bzip2Compression(); - break; - case "lz4": - compression = new Lz4Compression(); - break; - case "xz": - compression = new XzCompression(); - break; - } - } - - return new DatasetAttributes(dimensions, blockSize, dataType, compression); - } - - protected HashMap getAttributes(final Path path) throws IOException { - - if (!Files.exists(path)) - return new HashMap<>(); - - try (final LockedFileChannel lockedFileChannel = LockedFileChannel.openForReading(path)) { - return GsonAttributesParser.readAttributes(Channels.newReader(lockedFileChannel.getFileChannel(), StandardCharsets.UTF_8.name()), getGson()); - } - } - - /** - * Get and cache attributes for a group identified by an info object and a - * pathName. - * - * This helper method does not intelligently handle the case that the group - * does not exist (as indicated by info == emptyGroupInfo) which should be - * done in calling code. - * - * @param info - * @param pathName normalized group path without leading slash - * @return cached attributes - * empty map if the group exists but not attributes are set - * null if the group does not exist - * @throws IOException - */ - protected HashMap getCachedAttributes( - final N5GroupInfo info, - final String pathName) throws IOException { - - HashMap cachedMap = info.attributesCache; - if (cachedMap == null) { - synchronized (info) { - cachedMap = info.attributesCache; - if (cachedMap == null) { - final Path path = getAttributesPath(pathName); - cachedMap = new HashMap<>(); - final HashMap map = getAttributes(path); - cachedMap.putAll(map); - info.attributesCache = cachedMap; - } - } - } - return cachedMap; - } - - @SuppressWarnings("unchecked") - protected T getAttribute( - final HashMap cachedMap, - final String key, - final Class clazz) { - - final Object cachedAttribute = cachedMap.get(key); - if (cachedAttribute == null) - return null; - else if (cachedAttribute instanceof JsonElement) { - final T attribute = gson.fromJson((JsonElement)cachedAttribute, clazz); - synchronized (cachedMap) { - cachedMap.put(key, attribute); - } - return attribute; - } else { - return (T)cachedAttribute; - } - } - - /** - * Helper method that returns or JSON decodes a cached attribute. - * - * @param - * @param cachedMap - * @param key - * @param type - * @return - */ - @SuppressWarnings("unchecked") - protected T getAttribute( - final HashMap cachedMap, - final String key, - final Type type) { - - final Object cachedAttribute = cachedMap.get(key); - if (cachedAttribute == null) - return null; - else if (cachedAttribute instanceof JsonElement) { - final T attribute = gson.fromJson((JsonElement)cachedAttribute, type); - synchronized (cachedMap) { - cachedMap.put(key, attribute); - } - return attribute; - } else { - return (T)cachedAttribute; - } - } - - @Override - public T getAttribute( - final String pathName, - final String key, - final Class clazz) throws IOException { - - final String normalPathName = removeLeadingSlash(pathName); - if (cacheMeta) { - final N5GroupInfo info = getCachedN5GroupInfo(normalPathName); - if (info == emptyGroupInfo) - return null; - final HashMap cachedMap = getCachedAttributes(info, normalPathName); - if (cachedMap.isEmpty()) - return null; - return getAttribute(cachedMap, key, clazz); - } else { - final HashMap map = getAttributes(normalPathName); - return GsonAttributesParser.parseAttribute(map, key, clazz, getGson()); - } - } - - @Override - public T getAttribute( - final String pathName, - final String key, - final Type type) throws IOException { - - final String normalPathName = removeLeadingSlash(pathName); - if (cacheMeta) { - final N5GroupInfo info = getCachedN5GroupInfo(normalPathName); - if (info == emptyGroupInfo) - return null; - final HashMap cachedMap = getCachedAttributes(info, normalPathName); - if (cachedMap.isEmpty()) - return null; - return getAttribute(cachedMap, key, type); - } else { - final HashMap map = getAttributes(normalPathName); - return GsonAttributesParser.parseAttribute(map, key, type, getGson()); - } - } - - protected boolean exists(final Path path) { - - return Files.exists(path) && Files.isDirectory(path); - } - - protected N5GroupInfo getCachedN5GroupInfo(final String pathName) { - - final N5GroupInfo cachedInfo = metaCache.get(pathName); - if (cachedInfo == null) { - - /* I do not have a better solution yet to allow parallel - * exists checks for independent paths than to accept the - * same exists check to potentially run multiple times. - */ - final boolean exists = exists(getGroupPath(pathName)); - - synchronized (metaCache) { - final N5GroupInfo cachedInfoAgain = metaCache.get(pathName); - if (cachedInfoAgain == null) { - final N5GroupInfo info = exists ? new N5GroupInfo() : emptyGroupInfo; - metaCache.put(pathName, info); - return info; - } else - return cachedInfoAgain; - } - } else - return cachedInfo; - } - - @Override - public boolean exists(final String pathName) { - - final String normalPathName = removeLeadingSlash(pathName); - if (cacheMeta) - return getCachedN5GroupInfo(normalPathName) != emptyGroupInfo; - else - return exists(getGroupPath(normalPathName)); - } - - @Override - public HashMap getAttributes(final String pathName) throws IOException { - - final Path path = getAttributesPath(removeLeadingSlash(pathName)); - if (exists(pathName) && !Files.exists(path)) - return new HashMap<>(); - - try (final LockedFileChannel lockedFileChannel = LockedFileChannel.openForReading(path)) { - return GsonAttributesParser.readAttributes(Channels.newReader(lockedFileChannel.getFileChannel(), StandardCharsets.UTF_8.name()), getGson()); - } - } - - @Override - public DataBlock readBlock( - final String pathName, - final DatasetAttributes datasetAttributes, - final long... gridPosition) throws IOException { - - final Path path = getDataBlockPath(removeLeadingSlash(pathName), gridPosition); - if (!Files.exists(path)) - return null; - - try (final LockedFileChannel lockedChannel = LockedFileChannel.openForReading(path)) { - return DefaultBlockReader.readBlock(Channels.newInputStream(lockedChannel.getFileChannel()), datasetAttributes, gridPosition); - } - } - - @Override - public String[] list(final String pathName) throws IOException { - - final String normalPathName = removeLeadingSlash(pathName); - final Path path = getGroupPath(normalPathName); - try (final Stream pathStream = Files.list(path)) { - return pathStream - .filter(a -> Files.isDirectory(a)) - .map(a -> path.relativize(a).toString()) - .toArray(n -> new String[n]); - } - } - - /** - * Constructs the path for a data block in a dataset at a given grid position. - * - * The returned path is - * 

-	 * $basePath/datasetPathName/$gridPosition[0]/$gridPosition[1]/.../$gridPosition[n]
-	 *
- * - * This is the file into which the data block will be stored. - * - * @param datasetPathName normalized dataset path without leading slash - * @param gridPosition - * @return - */ - protected Path getDataBlockPath( - final String datasetPathName, - final long... gridPosition) { - - final String[] pathComponents = new String[gridPosition.length + 1]; - pathComponents[0] = datasetPathName; - for (int i = 1; i < pathComponents.length; ++i) - pathComponents[i] = Long.toString(gridPosition[i - 1]); - - return fileSystem.getPath(basePath, pathComponents); - } - - /** - * Constructs the path for the group or dataset. - * - * @param pathName normalized group path without leading slash - * @return - */ - protected Path getGroupPath(final String pathName) { - - return fileSystem.getPath(basePath, pathName); - } - - /** - * Constructs the path for the attributes file of a group or dataset. - * - * @param pathName normalized group path without leading slash - * @return - */ - protected Path getAttributesPath(final String pathName) { - - return fileSystem.getPath(basePath, pathName, jsonFile); - } - - /** - * Removes the leading slash from a given path and returns the corrected path. - * It ensures correctness on both Unix and Windows, otherwise {@code pathName} is treated - * as UNC path on Windows, and {@code Paths.get(pathName, ...)} fails with {@code InvalidPathException}. - * - * @param pathName - * @return - */ - protected static String removeLeadingSlash(final String pathName) { - - return pathName.startsWith("/") || pathName.startsWith("\\") ? pathName.substring(1) : pathName; - } - - @Override - public String toString() { - - return String.format("%s[fileSystem=%s, basePath=%s]", getClass().getSimpleName(), fileSystem, basePath); - } -} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/AbstractN5FSWriter.java b/src/main/java/org/janelia/saalfeldlab/n5/AbstractN5FSWriter.java deleted file mode 100644 index 18387dfa..00000000 --- a/src/main/java/org/janelia/saalfeldlab/n5/AbstractN5FSWriter.java +++ /dev/null @@ -1,373 +0,0 @@ -/** - * Copyright (c) 2017--2021, Stephan Saalfeld - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ -package org.janelia.saalfeldlab.n5; - -import java.io.IOException; -import java.nio.channels.Channels; -import java.nio.charset.StandardCharsets; -import java.nio.file.DirectoryNotEmptyException; -import java.nio.file.FileAlreadyExistsException; -import java.nio.file.FileSystem; -import java.nio.file.FileSystemException; -import java.nio.file.Files; -import java.nio.file.NoSuchFileException; -import java.nio.file.Path; -import java.nio.file.attribute.FileAttribute; -import java.util.Comparator; -import java.util.HashMap; -import java.util.Map; -import java.util.stream.Stream; - -import com.google.gson.GsonBuilder; -import com.google.gson.JsonElement; - -/** - * Filesystem {@link N5Writer} implementation with version compatibility check. - * - * @author Stephan Saalfeld - */ -public class AbstractN5FSWriter extends AbstractN5FSReader implements N5Writer { - - /** - * Opens an {@link AbstractN5FSWriter} at a given base path with a custom - * {@link GsonBuilder} to support custom attributes. - * - * If the base path does not exist, it will be created. - * - * If the base path exists and if the N5 version of the container is - * compatible with this implementation, the N5 version of this container - * will be set to the current N5 version of this implementation. - * - * @param fileSystem - * @param basePath n5 base path - * @param gsonBuilder - * @param cacheAttributes cache attributes - * Setting this to true avoids frequent reading and parsing of JSON - * encoded attributes, this is most interesting for high latency file - * systems. Changes of attributes by an independent writer will not be - * tracked. - * - * @throws IOException - * if the base path cannot be written to or cannot be created, - * if the N5 version of the container is not compatible with this - * implementation. - */ - public AbstractN5FSWriter( - final FileSystem fileSystem, - final String basePath, - final GsonBuilder gsonBuilder, - final boolean cacheAttributes) throws IOException { - - super(fileSystem, basePath, gsonBuilder, cacheAttributes); - createDirectories(fileSystem.getPath(basePath)); - if (!VERSION.equals(getVersion())) - setAttribute("/", VERSION_KEY, VERSION.toString()); - } - - @Override - public void createGroup(final String pathName) throws IOException { - - final String normalPathName = removeLeadingSlash(pathName); - if (cacheMeta) { - N5GroupInfo info = getCachedN5GroupInfo(normalPathName); - if (info == emptyGroupInfo) { - - /* The directories may be created multiple times concurrently, - * but a new cache entry is inserted only if none has been - * inserted in the meantime (because that may already include - * more cached data). - * - * This avoids synchronizing on the cache for independent - * group creation. - */ - createDirectories(getGroupPath(normalPathName)); - synchronized (metaCache) { - info = getCachedN5GroupInfo(normalPathName); - if (info == emptyGroupInfo) - metaCache.put(normalPathName, new N5GroupInfo()); - } - } - } else - createDirectories(getGroupPath(normalPathName)); - } - - /** - * Helper method that reads the existing map of attributes, JSON encodes, - * inserts and overrides the provided attributes, and writes them back into - * the attributes store. - * - * @param path - * @param attributes - * @throws IOException - */ - protected void writeAttributes( - final Path path, - final Map attributes) throws IOException { - - final HashMap map = new HashMap<>(); - - try (final LockedFileChannel lockedFileChannel = LockedFileChannel.openForWriting(path)) { - map.putAll(GsonAttributesParser.readAttributes(Channels.newReader(lockedFileChannel.getFileChannel(), StandardCharsets.UTF_8.name()), getGson())); - GsonAttributesParser.insertAttributes(map, attributes, gson); - - lockedFileChannel.getFileChannel().truncate(0); - GsonAttributesParser.writeAttributes(Channels.newWriter(lockedFileChannel.getFileChannel(), StandardCharsets.UTF_8.name()), map, getGson()); - } - } - - @Override - public void setAttributes( - final String pathName, - final Map attributes) throws IOException { - - final String normalPathName = removeLeadingSlash(pathName); - if (cacheMeta) { - N5GroupInfo info = getCachedN5GroupInfo(normalPathName); - if (info == emptyGroupInfo) { - synchronized (metaCache) { - info = getCachedN5GroupInfo(normalPathName); - if (info == emptyGroupInfo) { - //createDirectories(path.getParent()); ? or not ? - info = metaCache.put(normalPathName, new N5GroupInfo()); - } - } - } - final HashMap cachedMap = getCachedAttributes(info, normalPathName); - synchronized (cachedMap) { - cachedMap.putAll(attributes); - writeAttributes(getAttributesPath(normalPathName), attributes); - } - } else - writeAttributes(getAttributesPath(normalPathName), attributes); - } - - @Override - public void writeBlock( - final String pathName, - final DatasetAttributes datasetAttributes, - final DataBlock dataBlock) throws IOException { - - final Path path = getDataBlockPath(removeLeadingSlash(pathName), dataBlock.getGridPosition()); - createDirectories(path.getParent()); - try (final LockedFileChannel lockedChannel = LockedFileChannel.openForWriting(path)) { - lockedChannel.getFileChannel().truncate(0); - DefaultBlockWriter.writeBlock(Channels.newOutputStream(lockedChannel.getFileChannel()), datasetAttributes, dataBlock); - } - } - - @Override - public boolean remove() throws IOException { - - return remove("/"); - } - - protected static void tryDelete(final Path childPath) { - - try { - Files.delete(childPath); - } catch (final DirectoryNotEmptyException e) { - // Even though childPath should be an empty directory, sometimes the deletion fails on network file - // when lock files are not cleared immediately after the leaves have been removed. - try { - // wait and reattempt - Thread.sleep(100); - Files.delete(childPath); - } catch (final InterruptedException ex) { - e.printStackTrace(); - Thread.currentThread().interrupt(); - } catch (final IOException ex) { - ex.printStackTrace(); - } - } catch (final IOException e) { - e.printStackTrace(); - } - } - - @Override - public boolean remove(final String pathName) throws IOException { - - final Path path = getGroupPath(removeLeadingSlash(pathName)); - if (Files.exists(path)) { - final Path base = fileSystem.getPath(basePath); - try (final Stream pathStream = Files.walk(path)) { - pathStream.sorted(Comparator.reverseOrder()).forEach( - childPath -> { - if (Files.isRegularFile(childPath)) { - try (final LockedFileChannel channel = LockedFileChannel.openForWriting(childPath)) { - Files.delete(childPath); - } catch (final IOException e) { - e.printStackTrace(); - } - } else { - if (cacheMeta) { - synchronized (metaCache) { - metaCache.put(base.relativize(childPath).toString(), emptyGroupInfo); - tryDelete(childPath); - } - } else { - tryDelete(childPath); - } - } - }); - } - } - return !Files.exists(path); - } - - @Override - public boolean deleteBlock( - final String pathName, - final long... gridPosition) throws IOException { - - final Path path = getDataBlockPath(removeLeadingSlash(pathName), gridPosition); - if (Files.exists(path)) - try (final LockedFileChannel channel = LockedFileChannel.openForWriting(path)) { - Files.deleteIfExists(path); - } - return !Files.exists(path); - } - - /** - * This is a copy of {@link Files#createDirectories(Path, FileAttribute...)} - * that follows symlinks. - * - * Workaround for https://bugs.openjdk.java.net/browse/JDK-8130464 - * - * Creates a directory by creating all nonexistent parent directories first. - * Unlike the {@link #createDirectory createDirectory} method, an exception - * is not thrown if the directory could not be created because it already - * exists. - * - *

The {@code attrs} parameter is optional {@link FileAttribute - * file-attributes} to set atomically when creating the nonexistent - * directories. Each file attribute is identified by its {@link - * FileAttribute#name name}. If more than one attribute of the same name is - * included in the array then all but the last occurrence is ignored. - * - *

If this method fails, then it may do so after creating some, but not - * all, of the parent directories. - * - * @param dir - * the directory to create - * - * @param attrs - * an optional list of file attributes to set atomically when - * creating the directory - * - * @return the directory - * - * @throws UnsupportedOperationException - * if the array contains an attribute that cannot be set atomically - * when creating the directory - * @throws FileAlreadyExistsException - * if {@code dir} exists but is not a directory (optional specific - * exception) - * @throws IOException - * if an I/O error occurs - * @throws SecurityException - * in the case of the default provider, and a security manager is - * installed, the {@link SecurityManager#checkWrite(String) checkWrite} - * method is invoked prior to attempting to create a directory and - * its {@link SecurityManager#checkRead(String) checkRead} is - * invoked for each parent directory that is checked. If {@code - * dir} is not an absolute path then its {@link Path#toAbsolutePath - * toAbsolutePath} may need to be invoked to get its absolute path. - * This may invoke the security manager's {@link - * SecurityManager#checkPropertyAccess(String) checkPropertyAccess} - * method to check access to the system property {@code user.dir} - */ - private static Path createDirectories(Path dir, final FileAttribute... attrs) - throws IOException - { - // attempt to create the directory - try { - createAndCheckIsDirectory(dir, attrs); - return dir; - } catch (final FileAlreadyExistsException x) { - // file exists and is not a directory - throw x; - } catch (final IOException x) { - // parent may not exist or other reason - } - SecurityException se = null; - try { - dir = dir.toAbsolutePath(); - } catch (final SecurityException x) { - // don't have permission to get absolute path - se = x; - } - // find a decendent that exists - Path parent = dir.getParent(); - while (parent != null) { - try { - parent.getFileSystem().provider().checkAccess(parent); - break; - } catch (final NoSuchFileException x) { - // does not exist - } - parent = parent.getParent(); - } - if (parent == null) { - // unable to find existing parent - if (se == null) { - throw new FileSystemException(dir.toString(), null, - "Unable to determine if root directory exists"); - } else { - throw se; - } - } - - // create directories - Path child = parent; - for (final Path name: parent.relativize(dir)) { - child = child.resolve(name); - createAndCheckIsDirectory(child, attrs); - } - return dir; - } - - /** - * This is a copy of - * {@link Files#createAndCheckIsDirectory(Path, FileAttribute...)} that - * follows symlinks. - * - * Workaround for https://bugs.openjdk.java.net/browse/JDK-8130464 - * - * Used by createDirectories to attempt to create a directory. A no-op if - * the directory already exists. - */ - private static void createAndCheckIsDirectory( - final Path dir, - final FileAttribute... attrs) throws IOException { - - try { - Files.createDirectory(dir, attrs); - } catch (final FileAlreadyExistsException x) { - if (!Files.isDirectory(dir)) - throw x; - } - } -} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5FSReader.java b/src/main/java/org/janelia/saalfeldlab/n5/N5FSReader.java index b56de906..41978399 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5FSReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5FSReader.java @@ -25,11 +25,27 @@ */ package org.janelia.saalfeldlab.n5; +import java.io.Closeable; import java.io.IOException; +import java.lang.reflect.Type; +import java.nio.channels.Channels; +import java.nio.channels.FileChannel; +import java.nio.channels.OverlappingFileLockException; +import java.nio.charset.StandardCharsets; +import java.nio.file.FileSystem; import java.nio.file.FileSystems; +import java.nio.file.Files; +import java.nio.file.OpenOption; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.stream.Stream; import com.google.gson.Gson; import com.google.gson.GsonBuilder; +import com.google.gson.JsonElement; /** * Filesystem {@link N5Reader} implementation with JSON attributes @@ -39,7 +55,124 @@ * @author Igor Pisarev * @author Philipp Hanslovsky */ -public class N5FSReader extends AbstractN5FSReader { +public class N5FSReader implements GsonAttributesParser { + + /** + * A {@link FileChannel} wrapper that attempts to acquire a lock and waits + * for existing locks to be lifted before returning if the + * {@link FileSystem} supports that. If the {@link FileSystem} does not + * support locking, it returns immediately. + */ + protected static class LockedFileChannel implements Closeable { + + private final FileChannel channel; + + public static LockedFileChannel openForReading(final Path path) throws IOException { + + return new LockedFileChannel(path, true); + } + + public static LockedFileChannel openForWriting(final Path path) throws IOException { + + return new LockedFileChannel(path, false); + } + + private LockedFileChannel(final Path path, final boolean readOnly) throws IOException { + + final OpenOption[] options = readOnly ? new OpenOption[]{StandardOpenOption.READ} : new OpenOption[]{StandardOpenOption.READ, StandardOpenOption.WRITE, StandardOpenOption.CREATE}; + channel = FileChannel.open(path, options); + + for (boolean waiting = true; waiting;) { + waiting = false; + try { + channel.lock(0L, Long.MAX_VALUE, readOnly); + } catch (final OverlappingFileLockException e) { + waiting = true; + try { + Thread.sleep(100); + } catch (final InterruptedException f) { + waiting = false; + Thread.currentThread().interrupt(); + } + } catch (final IOException e) {} + } + } + + public FileChannel getFileChannel() { + + return channel; + } + + @Override + public void close() throws IOException { + + channel.close(); + } + } + + /** + * Data object for caching meta data. Elements that are null are not yet + * cached. + */ + protected static class N5GroupInfo { + + public ArrayList children = null; + public HashMap attributesCache = null; + public Boolean isDataset = null; + } + + protected static final N5GroupInfo emptyGroupInfo = new N5GroupInfo(); + + protected final FileSystem fileSystem; + + protected final Gson gson; + + protected final HashMap metaCache = new HashMap<>(); + + protected final boolean cacheMeta; + + protected static final String jsonFile = "attributes.json"; + + protected final String basePath; + + /** + * Opens an {@link N5FSReader} at a given base path with a custom + * {@link GsonBuilder} to support custom attributes. + * + * @param fileSystem + * @param basePath N5 base path + * @param gsonBuilder + * @param cacheMeta cache attributes and meta data + * Setting this to true avoids frequent reading and parsing of JSON + * encoded attributes and other meta data that requires accessing the + * store. This is most interesting for high latency backends. Changes + * of cached attributes and meta data by an independent writer will + * not be tracked. + * + * @throws IOException + * if the base path cannot be read or does not exist, + * if the N5 version of the container is not compatible with this + * implementation. + */ + public N5FSReader( + final FileSystem fileSystem, + final String basePath, + final GsonBuilder gsonBuilder, + final boolean cacheMeta) throws IOException { + + this.fileSystem = fileSystem; + this.basePath = basePath; + gsonBuilder.registerTypeAdapter(DataType.class, new DataType.JsonAdapter()); + gsonBuilder.registerTypeHierarchyAdapter(Compression.class, CompressionAdapter.getJsonAdapter()); + gsonBuilder.disableHtmlEscaping(); + this.gson = gsonBuilder.create(); + this.cacheMeta = cacheMeta; + if (exists("/")) { + final Version version = getVersion(); + if (!VERSION.isCompatible(version)) + throw new IOException("Incompatible version " + version + " (this is " + VERSION + ")."); + } + } /** * Opens an {@link N5FSReader} at a given base path with a custom @@ -61,7 +194,7 @@ public class N5FSReader extends AbstractN5FSReader { */ public N5FSReader(final String basePath, final GsonBuilder gsonBuilder, final boolean cacheMeta) throws IOException { - super(FileSystems.getDefault(), basePath, gsonBuilder, cacheMeta); + this(FileSystems.getDefault(), basePath, gsonBuilder, cacheMeta); } /** @@ -114,4 +247,436 @@ public N5FSReader(final String basePath) throws IOException { this(basePath, new GsonBuilder(), false); } + + @Override + public Gson getGson() { + + return gson; + } + + /** + * + * @return N5 base path + */ + public String getBasePath() { + + return this.basePath; + } + + @Override + public DatasetAttributes getDatasetAttributes(final String pathName) throws IOException { + + final long[] dimensions; + final DataType dataType; + int[] blockSize; + Compression compression; + final String compressionVersion0Name; + final String normalPathName = removeLeadingSlash(pathName); + if (cacheMeta) { + final N5GroupInfo info = getCachedN5GroupInfo(normalPathName); + if (info == emptyGroupInfo) + return null; + + final HashMap cachedMap; + if (info.isDataset == null) { + + synchronized (info) { + + cachedMap = getCachedAttributes(info, normalPathName); + if (cachedMap.isEmpty()) { + info.isDataset = false; + return null; + } + + dimensions = getAttribute(cachedMap, DatasetAttributes.dimensionsKey, long[].class); + if (dimensions == null) { + info.isDataset = false; + return null; + } + + dataType = getAttribute(cachedMap, DatasetAttributes.dataTypeKey, DataType.class); + if (dataType == null) { + info.isDataset = false; + return null; + } + + info.isDataset = true; + } + } else if (!info.isDataset) { + return null; + } else { + + cachedMap = getCachedAttributes(info, normalPathName); + dimensions = getAttribute(cachedMap, DatasetAttributes.dimensionsKey, long[].class); + dataType = getAttribute(cachedMap, DatasetAttributes.dataTypeKey, DataType.class); + } + + blockSize = getAttribute(cachedMap, DatasetAttributes.blockSizeKey, int[].class); + + compression = getAttribute(cachedMap, DatasetAttributes.compressionKey, Compression.class); + + /* version 0 */ + compressionVersion0Name = compression + == null + ? getAttribute(cachedMap, DatasetAttributes.compressionTypeKey, String.class) + : null; + + + } else { + final HashMap map = getAttributes(normalPathName); + + dimensions = GsonAttributesParser.parseAttribute(map, DatasetAttributes.dimensionsKey, long[].class, gson); + if (dimensions == null) + return null; + + dataType = GsonAttributesParser.parseAttribute(map, DatasetAttributes.dataTypeKey, DataType.class, gson); + if (dataType == null) + return null; + + blockSize = GsonAttributesParser.parseAttribute(map, DatasetAttributes.blockSizeKey, int[].class, gson); + + compression = GsonAttributesParser.parseAttribute(map, DatasetAttributes.compressionKey, Compression.class, gson); + + /* version 0 */ + compressionVersion0Name = compression == null + ? GsonAttributesParser.parseAttribute(map, DatasetAttributes.compressionTypeKey, String.class, gson) + : null; + } + + if (blockSize == null) + blockSize = Arrays.stream(dimensions).mapToInt(a -> (int)a).toArray(); + + /* version 0 */ + if (compression == null) { + switch (compressionVersion0Name) { + case "raw": + compression = new RawCompression(); + break; + case "gzip": + compression = new GzipCompression(); + break; + case "bzip2": + compression = new Bzip2Compression(); + break; + case "lz4": + compression = new Lz4Compression(); + break; + case "xz": + compression = new XzCompression(); + break; + } + } + + return new DatasetAttributes(dimensions, blockSize, dataType, compression); + } + + protected HashMap getAttributes(final Path path) throws IOException { + + if (!Files.exists(path)) + return new HashMap<>(); + + try (final LockedFileChannel lockedFileChannel = LockedFileChannel.openForReading(path)) { + return GsonAttributesParser.readAttributes(Channels.newReader(lockedFileChannel.getFileChannel(), StandardCharsets.UTF_8.name()), getGson()); + } + } + + /** + * Get and cache attributes for a group identified by an info object and a + * pathName. + * + * This helper method does not intelligently handle the case that the group + * does not exist (as indicated by info == emptyGroupInfo) which should be + * done in calling code. + * + * @param info + * @param pathName normalized group path without leading slash + * @return cached attributes + * empty map if the group exists but not attributes are set + * null if the group does not exist + * @throws IOException + */ + protected HashMap getCachedAttributes( + final N5GroupInfo info, + final String pathName) throws IOException { + + HashMap cachedMap = info.attributesCache; + if (cachedMap == null) { + synchronized (info) { + cachedMap = info.attributesCache; + if (cachedMap == null) { + final Path path = getAttributesPath(pathName); + cachedMap = new HashMap<>(); + final HashMap map = getAttributes(path); + cachedMap.putAll(map); + info.attributesCache = cachedMap; + } + } + } + return cachedMap; + } + + @SuppressWarnings("unchecked") + protected T getAttribute( + final HashMap cachedMap, + final String key, + final Class clazz) { + + final Object cachedAttribute = cachedMap.get(key); + if (cachedAttribute == null) + return null; + else if (cachedAttribute instanceof JsonElement) { + final T attribute = gson.fromJson((JsonElement)cachedAttribute, clazz); + synchronized (cachedMap) { + cachedMap.put(key, attribute); + } + return attribute; + } else { + return (T)cachedAttribute; + } + } + + /** + * Helper method that returns or JSON decodes a cached attribute. + * + * @param + * @param cachedMap + * @param key + * @param type + * @return + */ + @SuppressWarnings("unchecked") + protected T getAttribute( + final HashMap cachedMap, + final String key, + final Type type) { + + final Object cachedAttribute = cachedMap.get(key); + if (cachedAttribute == null) + return null; + else if (cachedAttribute instanceof JsonElement) { + final T attribute = gson.fromJson((JsonElement)cachedAttribute, type); + synchronized (cachedMap) { + cachedMap.put(key, attribute); + } + return attribute; + } else { + return (T)cachedAttribute; + } + } + + @Override + public T getAttribute( + final String pathName, + final String key, + final Class clazz) throws IOException { + + final String normalPathName = removeLeadingSlash(pathName); + if (cacheMeta) { + final N5GroupInfo info = getCachedN5GroupInfo(normalPathName); + if (info == emptyGroupInfo) + return null; + final HashMap cachedMap = getCachedAttributes(info, normalPathName); + if (cachedMap.isEmpty()) + return null; + return getAttribute(cachedMap, key, clazz); + } else { + final HashMap map = getAttributes(normalPathName); + return GsonAttributesParser.parseAttribute(map, key, clazz, getGson()); + } + } + + @Override + public T getAttribute( + final String pathName, + final String key, + final Type type) throws IOException { + + final String normalPathName = removeLeadingSlash(pathName); + if (cacheMeta) { + final N5GroupInfo info = getCachedN5GroupInfo(normalPathName); + if (info == emptyGroupInfo) + return null; + final HashMap cachedMap = getCachedAttributes(info, normalPathName); + if (cachedMap.isEmpty()) + return null; + return getAttribute(cachedMap, key, type); + } else { + final HashMap map = getAttributes(normalPathName); + return GsonAttributesParser.parseAttribute(map, key, type, getGson()); + } + } + + protected boolean exists(final Path path) { + + return Files.exists(path) && Files.isDirectory(path); + } + + /** + * Get an existing cached N5 group info object or create it. + * + * @param pathName normalized group path without leading slash + * @return + */ + protected N5GroupInfo getCachedN5GroupInfo(final String pathName) { + + final N5GroupInfo cachedInfo = metaCache.get(pathName); + if (cachedInfo == null) { + + /* I do not have a better solution yet to allow parallel + * exists checks for independent paths than to accept the + * same exists check to potentially run multiple times. + */ + final boolean exists = exists(getGroupPath(pathName)); + + synchronized (metaCache) { + final N5GroupInfo cachedInfoAgain = metaCache.get(pathName); + if (cachedInfoAgain == null) { + final N5GroupInfo info = exists ? new N5GroupInfo() : emptyGroupInfo; + metaCache.put(pathName, info); + return info; + } else + return cachedInfoAgain; + } + } else + return cachedInfo; + } + + @Override + public boolean exists(final String pathName) { + + final String normalPathName = removeLeadingSlash(pathName); + if (cacheMeta) + return getCachedN5GroupInfo(normalPathName) != emptyGroupInfo; + else + return exists(getGroupPath(normalPathName)); + } + + @Override + public boolean datasetExists(final String pathName) throws IOException { + + if (cacheMeta) { + final String normalPathName = removeLeadingSlash(pathName); + final N5GroupInfo info = getCachedN5GroupInfo(normalPathName); + if (info == emptyGroupInfo) + return false; + if (info.isDataset == null) { + synchronized (info) { + if (info.isDataset == null ) { + + } + else + return info.isDataset; + } + } else + return info.isDataset; + } + return exists(pathName) && getDatasetAttributes(pathName) != null; + } + + @Override + public HashMap getAttributes(final String pathName) throws IOException { + + final Path path = getAttributesPath(removeLeadingSlash(pathName)); + if (exists(pathName) && !Files.exists(path)) + return new HashMap<>(); + + try (final LockedFileChannel lockedFileChannel = LockedFileChannel.openForReading(path)) { + return GsonAttributesParser.readAttributes(Channels.newReader(lockedFileChannel.getFileChannel(), StandardCharsets.UTF_8.name()), getGson()); + } + } + + @Override + public DataBlock readBlock( + final String pathName, + final DatasetAttributes datasetAttributes, + final long... gridPosition) throws IOException { + + final Path path = getDataBlockPath(removeLeadingSlash(pathName), gridPosition); + if (!Files.exists(path)) + return null; + + try (final LockedFileChannel lockedChannel = LockedFileChannel.openForReading(path)) { + return DefaultBlockReader.readBlock(Channels.newInputStream(lockedChannel.getFileChannel()), datasetAttributes, gridPosition); + } + } + + @Override + public String[] list(final String pathName) throws IOException { + + final String normalPathName = removeLeadingSlash(pathName); + final Path path = getGroupPath(normalPathName); + try (final Stream pathStream = Files.list(path)) { + return pathStream + .filter(a -> Files.isDirectory(a)) + .map(a -> path.relativize(a).toString()) + .toArray(n -> new String[n]); + } + } + + /** + * Constructs the path for a data block in a dataset at a given grid position. + * + * The returned path is + * 

+	 * $basePath/datasetPathName/$gridPosition[0]/$gridPosition[1]/.../$gridPosition[n]
+	 *
+ * + * This is the file into which the data block will be stored. + * + * @param datasetPathName normalized dataset path without leading slash + * @param gridPosition + * @return + */ + protected Path getDataBlockPath( + final String datasetPathName, + final long... gridPosition) { + + final String[] pathComponents = new String[gridPosition.length + 1]; + pathComponents[0] = datasetPathName; + for (int i = 1; i < pathComponents.length; ++i) + pathComponents[i] = Long.toString(gridPosition[i - 1]); + + return fileSystem.getPath(basePath, pathComponents); + } + + /** + * Constructs the path for the group or dataset. + * + * @param pathName normalized group path without leading slash + * @return + */ + protected Path getGroupPath(final String pathName) { + + return fileSystem.getPath(basePath, pathName); + } + + /** + * Constructs the path for the attributes file of a group or dataset. + * + * @param pathName normalized group path without leading slash + * @return + */ + protected Path getAttributesPath(final String pathName) { + + return fileSystem.getPath(basePath, pathName, jsonFile); + } + + /** + * Removes the leading slash from a given path and returns the corrected path. + * It ensures correctness on both Unix and Windows, otherwise {@code pathName} is treated + * as UNC path on Windows, and {@code Paths.get(pathName, ...)} fails with {@code InvalidPathException}. + * + * @param pathName + * @return + */ + protected static String removeLeadingSlash(final String pathName) { + + return pathName.startsWith("/") || pathName.startsWith("\\") ? pathName.substring(1) : pathName; + } + + @Override + public String toString() { + + return String.format("%s[fileSystem=%s, basePath=%s]", getClass().getSimpleName(), fileSystem, basePath); + } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java b/src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java index 7cfbdc3d..400b0470 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java @@ -26,16 +26,67 @@ package org.janelia.saalfeldlab.n5; import java.io.IOException; +import java.nio.channels.Channels; +import java.nio.charset.StandardCharsets; +import java.nio.file.DirectoryNotEmptyException; +import java.nio.file.FileAlreadyExistsException; +import java.nio.file.FileSystem; +import java.nio.file.FileSystemException; import java.nio.file.FileSystems; +import java.nio.file.Files; +import java.nio.file.NoSuchFileException; +import java.nio.file.Path; +import java.nio.file.attribute.FileAttribute; +import java.util.Comparator; +import java.util.HashMap; +import java.util.Map; +import java.util.stream.Stream; import com.google.gson.GsonBuilder; +import com.google.gson.JsonElement; /** * Filesystem {@link N5Writer} implementation with version compatibility check. * * @author Stephan Saalfeld */ -public class N5FSWriter extends AbstractN5FSWriter { +public class N5FSWriter extends N5FSReader implements N5Writer { + + /** + * Opens an {@link N5FSWriter} at a given base path with a custom + * {@link GsonBuilder} to support custom attributes. + * + * If the base path does not exist, it will be created. + * + * If the base path exists and if the N5 version of the container is + * compatible with this implementation, the N5 version of this container + * will be set to the current N5 version of this implementation. + * + * @param fileSystem + * @param basePath n5 base path + * @param gsonBuilder + * @param cacheAttributes cache attributes + * Setting this to true avoids frequent reading and parsing of JSON + * encoded attributes, this is most interesting for high latency file + * systems. Changes of attributes by an independent writer will not be + * tracked. + * + * @throws IOException + * if the base path cannot be written to or cannot be created, + * if the N5 version of the container is not compatible with this + * implementation. + */ + public N5FSWriter( + final FileSystem fileSystem, + final String basePath, + final GsonBuilder gsonBuilder, + final boolean cacheAttributes) throws IOException { + + super(fileSystem, basePath, gsonBuilder, cacheAttributes); + createDirectories(fileSystem.getPath(basePath)); + if (!VERSION.equals(getVersion())) + setAttribute("/", VERSION_KEY, VERSION.toString()); + } /** * Opens an {@link N5FSWriter} at a given base path with a custom @@ -62,7 +113,7 @@ public class N5FSWriter extends AbstractN5FSWriter { */ public N5FSWriter(final String basePath, final GsonBuilder gsonBuilder, final boolean cacheAttributes) throws IOException { - super(FileSystems.getDefault(), basePath, gsonBuilder, cacheAttributes); + this(FileSystems.getDefault(), basePath, gsonBuilder, cacheAttributes); } /** @@ -136,4 +187,347 @@ public N5FSWriter(final String basePath) throws IOException { this(basePath, new GsonBuilder()); } + + /** + * Helper method to create and cache a group. + * + * @param pathName normalized group path without leading slash + * @return + * @throws IOException + */ + protected N5GroupInfo createCachedGroup(final String pathName) throws IOException { + + N5GroupInfo info = getCachedN5GroupInfo(pathName); + if (info == emptyGroupInfo) { + + /* The directories may be created multiple times concurrently, + * but a new cache entry is inserted only if none has been + * inserted in the meantime (because that may already include + * more cached data). + * + * This avoids synchronizing on the cache for independent + * group creation. + */ + createDirectories(getGroupPath(pathName)); + synchronized (metaCache) { + info = getCachedN5GroupInfo(pathName); + if (info == emptyGroupInfo) { + info = new N5GroupInfo(); + metaCache.put(pathName, new N5GroupInfo()); + } + } + } + return info; + } + + @Override + public void createGroup(final String pathName) throws IOException { + + final String normalPathName = removeLeadingSlash(pathName); + if (cacheMeta) { + final N5GroupInfo info = createCachedGroup(normalPathName); + synchronized (info) { + if (info.isDataset == null) + info.isDataset = false; + } + } else + createDirectories(getGroupPath(normalPathName)); + } + + @Override + public void createDataset( + final String pathName, + final DatasetAttributes datasetAttributes) throws IOException { + + final String normalPathName = removeLeadingSlash(pathName); + if (cacheMeta) { + final N5GroupInfo info = createCachedGroup(normalPathName); + synchronized (info) { + setDatasetAttributes(normalPathName, datasetAttributes); + info.isDataset = true; + } + } else { + createGroup(pathName); + setDatasetAttributes(normalPathName, datasetAttributes); + } + } + + /** + * Helper method that reads the existing map of attributes, JSON encodes, + * inserts and overrides the provided attributes, and writes them back into + * the attributes store. + * + * @param path + * @param attributes + * @throws IOException + */ + protected void writeAttributes( + final Path path, + final Map attributes) throws IOException { + + final HashMap map = new HashMap<>(); + + try (final LockedFileChannel lockedFileChannel = LockedFileChannel.openForWriting(path)) { + map.putAll(GsonAttributesParser.readAttributes(Channels.newReader(lockedFileChannel.getFileChannel(), StandardCharsets.UTF_8.name()), getGson())); + GsonAttributesParser.insertAttributes(map, attributes, gson); + + lockedFileChannel.getFileChannel().truncate(0); + GsonAttributesParser.writeAttributes(Channels.newWriter(lockedFileChannel.getFileChannel(), StandardCharsets.UTF_8.name()), map, getGson()); + } + } + + protected static boolean hasCachedDatasetAttributes(final Map cachedAttributes) { + + return cachedAttributes.keySet().contains(DatasetAttributes.dimensionsKey) && cachedAttributes.keySet().contains(DatasetAttributes.dataTypeKey); + } + + + /** + * Helper method to cache and write attributes. + * + * @param pathName normalized group path without leading slash + * @param attributes + * @param isDataset + * @return + * @throws IOException + */ + protected N5GroupInfo setCachedAttributes( + final String pathName, + final Map attributes) throws IOException { + + N5GroupInfo info = getCachedN5GroupInfo(pathName); + if (info == emptyGroupInfo) { + synchronized (metaCache) { + info = getCachedN5GroupInfo(pathName); + if (info == emptyGroupInfo) + throw new IOException("N5 group '" + pathName + "' does not exist. Cannot set attributes."); + } + } + final HashMap cachedMap = getCachedAttributes(info, pathName); + synchronized (info) { + cachedMap.putAll(attributes); + writeAttributes(getAttributesPath(pathName), attributes); + info.isDataset = hasCachedDatasetAttributes(cachedMap); + } + return info; + } + + @Override + public void setAttributes( + final String pathName, + final Map attributes) throws IOException { + + final String normalPathName = removeLeadingSlash(pathName); + if (cacheMeta) + setCachedAttributes(normalPathName, attributes); + else + writeAttributes(getAttributesPath(normalPathName), attributes); + } + + @Override + public void writeBlock( + final String pathName, + final DatasetAttributes datasetAttributes, + final DataBlock dataBlock) throws IOException { + + final Path path = getDataBlockPath(removeLeadingSlash(pathName), dataBlock.getGridPosition()); + createDirectories(path.getParent()); + try (final LockedFileChannel lockedChannel = LockedFileChannel.openForWriting(path)) { + lockedChannel.getFileChannel().truncate(0); + DefaultBlockWriter.writeBlock(Channels.newOutputStream(lockedChannel.getFileChannel()), datasetAttributes, dataBlock); + } + } + + @Override + public boolean remove() throws IOException { + + return remove("/"); + } + + protected static void tryDelete(final Path childPath) { + + try { + Files.delete(childPath); + } catch (final DirectoryNotEmptyException e) { + // Even though childPath should be an empty directory, sometimes the deletion fails on network file + // when lock files are not cleared immediately after the leaves have been removed. + try { + // wait and reattempt + Thread.sleep(100); + Files.delete(childPath); + } catch (final InterruptedException ex) { + e.printStackTrace(); + Thread.currentThread().interrupt(); + } catch (final IOException ex) { + ex.printStackTrace(); + } + } catch (final IOException e) { + e.printStackTrace(); + } + } + + @Override + public boolean remove(final String pathName) throws IOException { + + final Path path = getGroupPath(removeLeadingSlash(pathName)); + if (Files.exists(path)) { + final Path base = fileSystem.getPath(basePath); + try (final Stream pathStream = Files.walk(path)) { + pathStream.sorted(Comparator.reverseOrder()).forEach( + childPath -> { + if (Files.isRegularFile(childPath)) { + try (final LockedFileChannel channel = LockedFileChannel.openForWriting(childPath)) { + Files.delete(childPath); + } catch (final IOException e) { + e.printStackTrace(); + } + } else { + if (cacheMeta) { + synchronized (metaCache) { + metaCache.put(base.relativize(childPath).toString(), emptyGroupInfo); + tryDelete(childPath); + } + } else { + tryDelete(childPath); + } + } + }); + } + } + return !Files.exists(path); + } + + @Override + public boolean deleteBlock( + final String pathName, + final long... gridPosition) throws IOException { + + final Path path = getDataBlockPath(removeLeadingSlash(pathName), gridPosition); + if (Files.exists(path)) + try (final LockedFileChannel channel = LockedFileChannel.openForWriting(path)) { + Files.deleteIfExists(path); + } + return !Files.exists(path); + } + + /** + * This is a copy of {@link Files#createDirectories(Path, FileAttribute...)} + * that follows symlinks. + * + * Workaround for https://bugs.openjdk.java.net/browse/JDK-8130464 + * + * Creates a directory by creating all nonexistent parent directories first. + * Unlike the {@link #createDirectory createDirectory} method, an exception + * is not thrown if the directory could not be created because it already + * exists. + * + *

The {@code attrs} parameter is optional {@link FileAttribute + * file-attributes} to set atomically when creating the nonexistent + * directories. Each file attribute is identified by its {@link + * FileAttribute#name name}. If more than one attribute of the same name is + * included in the array then all but the last occurrence is ignored. + * + *

If this method fails, then it may do so after creating some, but not + * all, of the parent directories. + * + * @param dir + * the directory to create + * + * @param attrs + * an optional list of file attributes to set atomically when + * creating the directory + * + * @return the directory + * + * @throws UnsupportedOperationException + * if the array contains an attribute that cannot be set atomically + * when creating the directory + * @throws FileAlreadyExistsException + * if {@code dir} exists but is not a directory (optional specific + * exception) + * @throws IOException + * if an I/O error occurs + * @throws SecurityException + * in the case of the default provider, and a security manager is + * installed, the {@link SecurityManager#checkWrite(String) checkWrite} + * method is invoked prior to attempting to create a directory and + * its {@link SecurityManager#checkRead(String) checkRead} is + * invoked for each parent directory that is checked. If {@code + * dir} is not an absolute path then its {@link Path#toAbsolutePath + * toAbsolutePath} may need to be invoked to get its absolute path. + * This may invoke the security manager's {@link + * SecurityManager#checkPropertyAccess(String) checkPropertyAccess} + * method to check access to the system property {@code user.dir} + */ + private static Path createDirectories(Path dir, final FileAttribute... attrs) + throws IOException + { + // attempt to create the directory + try { + createAndCheckIsDirectory(dir, attrs); + return dir; + } catch (final FileAlreadyExistsException x) { + // file exists and is not a directory + throw x; + } catch (final IOException x) { + // parent may not exist or other reason + } + SecurityException se = null; + try { + dir = dir.toAbsolutePath(); + } catch (final SecurityException x) { + // don't have permission to get absolute path + se = x; + } + // find a decendent that exists + Path parent = dir.getParent(); + while (parent != null) { + try { + parent.getFileSystem().provider().checkAccess(parent); + break; + } catch (final NoSuchFileException x) { + // does not exist + } + parent = parent.getParent(); + } + if (parent == null) { + // unable to find existing parent + if (se == null) { + throw new FileSystemException(dir.toString(), null, + "Unable to determine if root directory exists"); + } else { + throw se; + } + } + + // create directories + Path child = parent; + for (final Path name: parent.relativize(dir)) { + child = child.resolve(name); + createAndCheckIsDirectory(child, attrs); + } + return dir; + } + + /** + * This is a copy of + * {@link Files#createAndCheckIsDirectory(Path, FileAttribute...)} that + * follows symlinks. + * + * Workaround for https://bugs.openjdk.java.net/browse/JDK-8130464 + * + * Used by createDirectories to attempt to create a directory. A no-op if + * the directory already exists. + */ + private static void createAndCheckIsDirectory( + final Path dir, + final FileAttribute... attrs) throws IOException { + + try { + Files.createDirectory(dir, attrs); + } catch (final FileAlreadyExistsException x) { + if (!Files.isDirectory(dir)) + throw x; + } + } } diff --git a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java index e4d33480..67265777 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java @@ -122,7 +122,7 @@ public void setUpOnce() throws IOException { public static void rampDownAfterClass() throws IOException { if (n5 != null) { - Assert.assertTrue(n5.remove()); +// Assert.assertTrue(n5.remove()); n5 = null; } } From e015bb7e92dc5909c9b7c5579309827e40b404a0 Mon Sep 17 00:00:00 2001 From: Stephan Saalfeld Date: Thu, 6 May 2021 12:54:37 -0400 Subject: [PATCH 014/243] list caching TODO implement updates in create and remove methods --- .../janelia/saalfeldlab/n5/N5FSReader.java | 40 ++++++++++++++++--- 1 file changed, 34 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5FSReader.java b/src/main/java/org/janelia/saalfeldlab/n5/N5FSReader.java index 41978399..a788a7b9 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5FSReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5FSReader.java @@ -38,9 +38,10 @@ import java.nio.file.OpenOption; import java.nio.file.Path; import java.nio.file.StandardOpenOption; -import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; +import java.util.HashSet; +import java.util.Set; import java.util.stream.Stream; import com.google.gson.Gson; @@ -116,7 +117,7 @@ public void close() throws IOException { */ protected static class N5GroupInfo { - public ArrayList children = null; + public HashSet children = null; public HashMap attributesCache = null; public Boolean isDataset = null; } @@ -600,11 +601,9 @@ public DataBlock readBlock( } } - @Override - public String[] list(final String pathName) throws IOException { + protected String[] normalList(final String pathName) throws IOException { - final String normalPathName = removeLeadingSlash(pathName); - final Path path = getGroupPath(normalPathName); + final Path path = getGroupPath(pathName); try (final Stream pathStream = Files.list(path)) { return pathStream .filter(a -> Files.isDirectory(a)) @@ -613,6 +612,35 @@ public String[] list(final String pathName) throws IOException { } } + @Override + public String[] list(final String pathName) throws IOException { + + if (cacheMeta) { + final N5GroupInfo info = getCachedN5GroupInfo(pathName); + if (info == emptyGroupInfo) + throw new IOException("Group '" + pathName +"' does not exist."); + else { + Set children = info.children; + final String[] list; + if (children == null) { + synchronized (info) { + children = info.children; + if (children == null) { + list = normalList(removeLeadingSlash(pathName)); + info.children = new HashSet<>(Arrays.asList(list)); + } else + list = children.toArray(new String[children.size()]); + } + } else + list = children.toArray(new String[children.size()]); + + return list; + } + } else { + return normalList(removeLeadingSlash(pathName)); + } + } + /** * Constructs the path for a data block in a dataset at a given grid position. * From f01d5e5a4de2ca52297ac07c78b625345b998a80 Mon Sep 17 00:00:00 2001 From: Stephan Saalfeld Date: Thu, 6 May 2021 20:21:24 -0400 Subject: [PATCH 015/243] insert root cache entry at writer creation ... TODO shouldn't readers do that too? --- src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java | 5 +++++ src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java | 4 ++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java b/src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java index 400b0470..eb750f6c 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java @@ -84,6 +84,11 @@ public N5FSWriter( super(fileSystem, basePath, gsonBuilder, cacheAttributes); createDirectories(fileSystem.getPath(basePath)); + if (cacheAttributes) { + final N5GroupInfo info = new N5GroupInfo(); + info.isDataset = false; + metaCache.put("", info); + } if (!VERSION.equals(getVersion())) setAttribute("/", VERSION_KEY, VERSION.toString()); } diff --git a/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java b/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java index c2e4f2bf..93c27d13 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java @@ -66,8 +66,8 @@ public void testCache() { testCreateGroup(); testCreateGroup(); - testDeepList(); - testDeepList(); +// testDeepList(); +// testDeepList(); testVersion(); testVersion(); From 25bb2e5e6662f93754c1ae9881977f20c531667f Mon Sep 17 00:00:00 2001 From: Stephan Saalfeld Date: Mon, 10 May 2021 15:08:02 -0400 Subject: [PATCH 016/243] run writer constructor through createGroup("/") --- src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java b/src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java index eb750f6c..29341a11 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java @@ -83,12 +83,7 @@ public N5FSWriter( final boolean cacheAttributes) throws IOException { super(fileSystem, basePath, gsonBuilder, cacheAttributes); - createDirectories(fileSystem.getPath(basePath)); - if (cacheAttributes) { - final N5GroupInfo info = new N5GroupInfo(); - info.isDataset = false; - metaCache.put("", info); - } + createGroup("/"); if (!VERSION.equals(getVersion())) setAttribute("/", VERSION_KEY, VERSION.toString()); } From 274cad0886dbf3e14680b66841a011ca3cc2efb9 Mon Sep 17 00:00:00 2001 From: Stephan Saalfeld Date: Mon, 10 May 2021 15:08:46 -0400 Subject: [PATCH 017/243] move trivial remove() implementation into interface default --- src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java | 6 ------ src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java | 5 ++++- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java b/src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java index 29341a11..065cb6c6 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java @@ -338,12 +338,6 @@ public void writeBlock( } } - @Override - public boolean remove() throws IOException { - - return remove("/"); - } - protected static void tryDelete(final Path childPath) { try { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java index c87a563a..bea3b1b1 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java @@ -115,7 +115,10 @@ public default void setDatasetAttributes( * @return true if removal was successful, false otherwise * @throws IOException */ - public boolean remove() throws IOException; + public default boolean remove() throws IOException { + + return remove("/"); + } /** * Creates a dataset. This does not create any data but the path and From 8a53377a5c4fc7073655235a47b99ded874ff742 Mon Sep 17 00:00:00 2001 From: Stephan Saalfeld Date: Mon, 10 May 2021 15:11:48 -0400 Subject: [PATCH 018/243] add children to parent cache when creating a group --- .../java/org/janelia/saalfeldlab/n5/N5FSWriter.java | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java b/src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java index 065cb6c6..6844e176 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java @@ -39,6 +39,7 @@ import java.nio.file.attribute.FileAttribute; import java.util.Comparator; import java.util.HashMap; +import java.util.HashSet; import java.util.Map; import java.util.stream.Stream; @@ -216,6 +217,18 @@ protected N5GroupInfo createCachedGroup(final String pathName) throws IOExceptio metaCache.put(pathName, new N5GroupInfo()); } } + if (!pathName.equals("")) { // not root + final Path parent = getGroupPath(pathName).getParent(); + final N5GroupInfo parentInfo = getCachedN5GroupInfo( + fileSystem.getPath(basePath).relativize(parent).toString()); // group must exist + final HashSet children = parentInfo.children; + if (children != null) { + synchronized (children) { + children.add( + parent.relativize(fileSystem.getPath(pathName)).toString()); + } + } + } } return info; } From 24e663ef3014ce6d097d34b03d2ed91c3af37ded Mon Sep 17 00:00:00 2001 From: Stephan Saalfeld Date: Mon, 10 May 2021 21:10:49 -0400 Subject: [PATCH 019/243] update parent cache on remove TODO fails deepList test for a mix of test and implementation issues --- .../janelia/saalfeldlab/n5/N5FSReader.java | 30 +++++++++------ .../janelia/saalfeldlab/n5/N5FSWriter.java | 37 ++++++++++++++----- .../saalfeldlab/n5/AbstractN5Test.java | 4 +- .../org/janelia/saalfeldlab/n5/N5FSTest.java | 4 +- 4 files changed, 50 insertions(+), 25 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5FSReader.java b/src/main/java/org/janelia/saalfeldlab/n5/N5FSReader.java index a788a7b9..e67f736d 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5FSReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5FSReader.java @@ -272,7 +272,7 @@ public DatasetAttributes getDatasetAttributes(final String pathName) throws IOEx int[] blockSize; Compression compression; final String compressionVersion0Name; - final String normalPathName = removeLeadingSlash(pathName); + final String normalPathName = normalize(pathName); if (cacheMeta) { final N5GroupInfo info = getCachedN5GroupInfo(normalPathName); if (info == emptyGroupInfo) @@ -471,7 +471,7 @@ public T getAttribute( final String key, final Class clazz) throws IOException { - final String normalPathName = removeLeadingSlash(pathName); + final String normalPathName = normalize(pathName); if (cacheMeta) { final N5GroupInfo info = getCachedN5GroupInfo(normalPathName); if (info == emptyGroupInfo) @@ -492,7 +492,7 @@ public T getAttribute( final String key, final Type type) throws IOException { - final String normalPathName = removeLeadingSlash(pathName); + final String normalPathName = normalize(pathName); if (cacheMeta) { final N5GroupInfo info = getCachedN5GroupInfo(normalPathName); if (info == emptyGroupInfo) @@ -545,7 +545,7 @@ protected N5GroupInfo getCachedN5GroupInfo(final String pathName) { @Override public boolean exists(final String pathName) { - final String normalPathName = removeLeadingSlash(pathName); + final String normalPathName = normalize(pathName); if (cacheMeta) return getCachedN5GroupInfo(normalPathName) != emptyGroupInfo; else @@ -556,7 +556,7 @@ public boolean exists(final String pathName) { public boolean datasetExists(final String pathName) throws IOException { if (cacheMeta) { - final String normalPathName = removeLeadingSlash(pathName); + final String normalPathName = normalize(pathName); final N5GroupInfo info = getCachedN5GroupInfo(normalPathName); if (info == emptyGroupInfo) return false; @@ -577,7 +577,7 @@ public boolean datasetExists(final String pathName) throws IOException { @Override public HashMap getAttributes(final String pathName) throws IOException { - final Path path = getAttributesPath(removeLeadingSlash(pathName)); + final Path path = getAttributesPath(normalize(pathName)); if (exists(pathName) && !Files.exists(path)) return new HashMap<>(); @@ -592,7 +592,7 @@ public DataBlock readBlock( final DatasetAttributes datasetAttributes, final long... gridPosition) throws IOException { - final Path path = getDataBlockPath(removeLeadingSlash(pathName), gridPosition); + final Path path = getDataBlockPath(normalize(pathName), gridPosition); if (!Files.exists(path)) return null; @@ -626,7 +626,7 @@ public String[] list(final String pathName) throws IOException { synchronized (info) { children = info.children; if (children == null) { - list = normalList(removeLeadingSlash(pathName)); + list = normalList(normalize(pathName)); info.children = new HashSet<>(Arrays.asList(list)); } else list = children.toArray(new String[children.size()]); @@ -637,7 +637,7 @@ public String[] list(final String pathName) throws IOException { return list; } } else { - return normalList(removeLeadingSlash(pathName)); + return normalList(normalize(pathName)); } } @@ -697,9 +697,15 @@ protected Path getAttributesPath(final String pathName) { * @param pathName * @return */ - protected static String removeLeadingSlash(final String pathName) { - - return pathName.startsWith("/") || pathName.startsWith("\\") ? pathName.substring(1) : pathName; + protected String normalize(final String pathName) { + + return fileSystem.getPath(basePath) + .relativize( + fileSystem.getPath( + basePath, + pathName.startsWith("/") || pathName.startsWith("\\") ? pathName.substring(1) : pathName) + .normalize()) + .toString(); } @Override diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java b/src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java index 6844e176..13b098b6 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java @@ -236,7 +236,7 @@ protected N5GroupInfo createCachedGroup(final String pathName) throws IOExceptio @Override public void createGroup(final String pathName) throws IOException { - final String normalPathName = removeLeadingSlash(pathName); + final String normalPathName = normalize(pathName); if (cacheMeta) { final N5GroupInfo info = createCachedGroup(normalPathName); synchronized (info) { @@ -252,7 +252,7 @@ public void createDataset( final String pathName, final DatasetAttributes datasetAttributes) throws IOException { - final String normalPathName = removeLeadingSlash(pathName); + final String normalPathName = normalize(pathName); if (cacheMeta) { final N5GroupInfo info = createCachedGroup(normalPathName); synchronized (info) { @@ -330,7 +330,7 @@ public void setAttributes( final String pathName, final Map attributes) throws IOException { - final String normalPathName = removeLeadingSlash(pathName); + final String normalPathName = normalize(pathName); if (cacheMeta) setCachedAttributes(normalPathName, attributes); else @@ -343,7 +343,7 @@ public void writeBlock( final DatasetAttributes datasetAttributes, final DataBlock dataBlock) throws IOException { - final Path path = getDataBlockPath(removeLeadingSlash(pathName), dataBlock.getGridPosition()); + final Path path = getDataBlockPath(normalize(pathName), dataBlock.getGridPosition()); createDirectories(path.getParent()); try (final LockedFileChannel lockedChannel = LockedFileChannel.openForWriting(path)) { lockedChannel.getFileChannel().truncate(0); @@ -376,8 +376,10 @@ protected static void tryDelete(final Path childPath) { @Override public boolean remove(final String pathName) throws IOException { - final Path path = getGroupPath(removeLeadingSlash(pathName)); - if (Files.exists(path)) { + final String normalPathName = normalize(pathName); + final Path path = getGroupPath(normalPathName); + boolean exists = Files.exists(path); + if (exists) { final Path base = fileSystem.getPath(basePath); try (final Stream pathStream = Files.walk(path)) { pathStream.sorted(Comparator.reverseOrder()).forEach( @@ -399,9 +401,26 @@ public boolean remove(final String pathName) throws IOException { } } }); - } } - return !Files.exists(path); + if (cacheMeta) { + if (!normalPathName.equals("")) { // not root + final Path parent = getGroupPath(normalPathName).getParent(); + final N5GroupInfo parentInfo = getCachedN5GroupInfo( + fileSystem.getPath(basePath).relativize(parent).toString()); // group must exist + final HashSet children = parentInfo.children; + if (children != null) { + synchronized (children) { + exists = Files.exists(path); + if (exists) + children.remove( + parent.relativize(fileSystem.getPath(normalPathName)).toString()); + } + } + } + } else + exists = Files.exists(path); + } + return !exists; } @Override @@ -409,7 +428,7 @@ public boolean deleteBlock( final String pathName, final long... gridPosition) throws IOException { - final Path path = getDataBlockPath(removeLeadingSlash(pathName), gridPosition); + final Path path = getDataBlockPath(normalize(pathName), gridPosition); if (Files.exists(path)) try (final LockedFileChannel channel = LockedFileChannel.openForWriting(path)) { Files.deleteIfExists(path); diff --git a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java index 67265777..cad8aa83 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java @@ -122,7 +122,7 @@ public void setUpOnce() throws IOException { public static void rampDownAfterClass() throws IOException { if (n5 != null) { -// Assert.assertTrue(n5.remove()); + Assert.assertTrue(n5.remove()); n5 = null; } } @@ -542,7 +542,7 @@ public void testDeepList() { final List datasetList2 = Arrays.asList(n5.deepList("")); for (final String subGroup : subGroupNames) Assert.assertTrue("deepList contents", datasetList2.contains(groupName.replaceFirst("/", "") + "/" + subGroup)); - Assert.assertTrue("deepList contents", datasetList2.contains(datasetName.replaceFirst("/", ""))); +// Assert.assertTrue("deepList contents", datasetList2.contains(datasetName.replaceFirst("/", ""))); Assert.assertFalse("deepList stops at datasets", datasetList2.contains(datasetName + "/0")); final String prefix = "/test"; diff --git a/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java b/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java index 93c27d13..c2e4f2bf 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java @@ -66,8 +66,8 @@ public void testCache() { testCreateGroup(); testCreateGroup(); -// testDeepList(); -// testDeepList(); + testDeepList(); + testDeepList(); testVersion(); testVersion(); From 2158733236a530b6a8d41d81e103dec6cd7616b6 Mon Sep 17 00:00:00 2001 From: Stephan Saalfeld Date: Wed, 12 May 2021 17:29:51 -0400 Subject: [PATCH 020/243] all tests passing with and without cache --- .../janelia/saalfeldlab/n5/N5FSReader.java | 50 ++++++++-------- .../janelia/saalfeldlab/n5/N5FSWriter.java | 57 +++++++++++-------- .../saalfeldlab/n5/AbstractN5Test.java | 6 +- 3 files changed, 66 insertions(+), 47 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5FSReader.java b/src/main/java/org/janelia/saalfeldlab/n5/N5FSReader.java index e67f736d..efd14e28 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5FSReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5FSReader.java @@ -515,31 +515,29 @@ protected boolean exists(final Path path) { /** * Get an existing cached N5 group info object or create it. * - * @param pathName normalized group path without leading slash + * @param normalPathName normalized group path without leading slash * @return */ - protected N5GroupInfo getCachedN5GroupInfo(final String pathName) { + protected N5GroupInfo getCachedN5GroupInfo(final String normalPathName) { - final N5GroupInfo cachedInfo = metaCache.get(pathName); - if (cachedInfo == null) { + N5GroupInfo info = metaCache.get(normalPathName); + if (info == null) { /* I do not have a better solution yet to allow parallel * exists checks for independent paths than to accept the * same exists check to potentially run multiple times. */ - final boolean exists = exists(getGroupPath(pathName)); + final boolean exists = exists(getGroupPath(normalPathName)); synchronized (metaCache) { - final N5GroupInfo cachedInfoAgain = metaCache.get(pathName); - if (cachedInfoAgain == null) { - final N5GroupInfo info = exists ? new N5GroupInfo() : emptyGroupInfo; - metaCache.put(pathName, info); - return info; - } else - return cachedInfoAgain; + info = metaCache.get(normalPathName); + if (info == null) { + info = exists ? new N5GroupInfo() : emptyGroupInfo; + metaCache.put(normalPathName, info); + } } - } else - return cachedInfo; + } + return info; } @Override @@ -601,9 +599,15 @@ public DataBlock readBlock( } } - protected String[] normalList(final String pathName) throws IOException { + /** + * + * @param normalPathName normalized path name + * @return + * @throws IOException + */ + protected String[] normalList(final String normalPathName) throws IOException { - final Path path = getGroupPath(pathName); + final Path path = getGroupPath(normalPathName); try (final Stream pathStream = Files.list(path)) { return pathStream .filter(a -> Files.isDirectory(a)) @@ -616,7 +620,7 @@ protected String[] normalList(final String pathName) throws IOException { public String[] list(final String pathName) throws IOException { if (cacheMeta) { - final N5GroupInfo info = getCachedN5GroupInfo(pathName); + final N5GroupInfo info = getCachedN5GroupInfo(normalize(pathName)); if (info == emptyGroupInfo) throw new IOException("Group '" + pathName +"' does not exist."); else { @@ -651,16 +655,16 @@ public String[] list(final String pathName) throws IOException { * * This is the file into which the data block will be stored. * - * @param datasetPathName normalized dataset path without leading slash + * @param normalDatasetPathName normalized dataset path without leading slash * @param gridPosition * @return */ protected Path getDataBlockPath( - final String datasetPathName, + final String normalDatasetPathName, final long... gridPosition) { final String[] pathComponents = new String[gridPosition.length + 1]; - pathComponents[0] = datasetPathName; + pathComponents[0] = normalDatasetPathName; for (int i = 1; i < pathComponents.length; ++i) pathComponents[i] = Long.toString(gridPosition[i - 1]); @@ -681,12 +685,12 @@ protected Path getGroupPath(final String pathName) { /** * Constructs the path for the attributes file of a group or dataset. * - * @param pathName normalized group path without leading slash + * @param normalPathName normalized group path without leading slash * @return */ - protected Path getAttributesPath(final String pathName) { + protected Path getAttributesPath(final String normalPathName) { - return fileSystem.getPath(basePath, pathName, jsonFile); + return fileSystem.getPath(basePath, normalPathName, jsonFile); } /** diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java b/src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java index 13b098b6..57bce3bc 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java @@ -192,13 +192,13 @@ public N5FSWriter(final String basePath) throws IOException { /** * Helper method to create and cache a group. * - * @param pathName normalized group path without leading slash + * @param normalPathName normalized group path without leading slash * @return * @throws IOException */ - protected N5GroupInfo createCachedGroup(final String pathName) throws IOException { + protected N5GroupInfo createCachedGroup(final String normalPathName) throws IOException { - N5GroupInfo info = getCachedN5GroupInfo(pathName); + N5GroupInfo info = getCachedN5GroupInfo(normalPathName); if (info == emptyGroupInfo) { /* The directories may be created multiple times concurrently, @@ -209,24 +209,29 @@ protected N5GroupInfo createCachedGroup(final String pathName) throws IOExceptio * This avoids synchronizing on the cache for independent * group creation. */ - createDirectories(getGroupPath(pathName)); + createDirectories(getGroupPath(normalPathName)); synchronized (metaCache) { - info = getCachedN5GroupInfo(pathName); + info = getCachedN5GroupInfo(normalPathName); if (info == emptyGroupInfo) { info = new N5GroupInfo(); - metaCache.put(pathName, new N5GroupInfo()); + metaCache.put(normalPathName, info); } - } - if (!pathName.equals("")) { // not root - final Path parent = getGroupPath(pathName).getParent(); - final N5GroupInfo parentInfo = getCachedN5GroupInfo( - fileSystem.getPath(basePath).relativize(parent).toString()); // group must exist - final HashSet children = parentInfo.children; - if (children != null) { - synchronized (children) { - children.add( - parent.relativize(fileSystem.getPath(pathName)).toString()); + for (String childPathName = normalPathName; !childPathName.equals("");) { + final Path parent = getGroupPath(childPathName).getParent(); + final String parentPathName = fileSystem.getPath(basePath).relativize(parent).toString(); + N5GroupInfo parentInfo = getCachedN5GroupInfo(parentPathName); + if (parentInfo == emptyGroupInfo) { + parentInfo = new N5GroupInfo(); + metaCache.put(parentPathName, parentInfo); + } + final HashSet children = parentInfo.children; + if (children != null) { + synchronized (children) { + children.add( + parent.relativize(getGroupPath(childPathName)).toString()); + } } + childPathName = parentPathName; } } } @@ -289,6 +294,12 @@ protected void writeAttributes( } } + /** + * Check for attributes that are required for a group to be a dataset. + * + * @param cachedAttributes + * @return + */ protected static boolean hasCachedDatasetAttributes(final Map cachedAttributes) { return cachedAttributes.keySet().contains(DatasetAttributes.dimensionsKey) && cachedAttributes.keySet().contains(DatasetAttributes.dataTypeKey); @@ -298,28 +309,28 @@ protected static boolean hasCachedDatasetAttributes(final Map ca /** * Helper method to cache and write attributes. * - * @param pathName normalized group path without leading slash + * @param normalPathName normalized group path without leading slash * @param attributes * @param isDataset * @return * @throws IOException */ protected N5GroupInfo setCachedAttributes( - final String pathName, + final String normalPathName, final Map attributes) throws IOException { - N5GroupInfo info = getCachedN5GroupInfo(pathName); + N5GroupInfo info = getCachedN5GroupInfo(normalPathName); if (info == emptyGroupInfo) { synchronized (metaCache) { - info = getCachedN5GroupInfo(pathName); + info = getCachedN5GroupInfo(normalPathName); if (info == emptyGroupInfo) - throw new IOException("N5 group '" + pathName + "' does not exist. Cannot set attributes."); + throw new IOException("N5 group '" + normalPathName + "' does not exist. Cannot set attributes."); } } - final HashMap cachedMap = getCachedAttributes(info, pathName); + final HashMap cachedMap = getCachedAttributes(info, normalPathName); synchronized (info) { cachedMap.putAll(attributes); - writeAttributes(getAttributesPath(pathName), attributes); + writeAttributes(getAttributesPath(normalPathName), attributes); info.isDataset = hasCachedDatasetAttributes(cachedMap); } return info; diff --git a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java index cad8aa83..f75516b2 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java @@ -535,14 +535,17 @@ public void testDeepList() { n5.writeBlock(datasetName, datasetAttributes, dataBlock); final List datasetList = Arrays.asList(n5.deepList("/")); + final N5Writer n5Writer = n5; + System.out.println(datasetList); for (final String subGroup : subGroupNames) Assert.assertTrue("deepList contents", datasetList.contains(groupName.replaceFirst("/", "") + "/" + subGroup)); + Assert.assertTrue("deepList contents", datasetList.contains(datasetName.replaceFirst("/", ""))); Assert.assertFalse("deepList stops at datasets", datasetList.contains(datasetName + "/0")); final List datasetList2 = Arrays.asList(n5.deepList("")); for (final String subGroup : subGroupNames) Assert.assertTrue("deepList contents", datasetList2.contains(groupName.replaceFirst("/", "") + "/" + subGroup)); -// Assert.assertTrue("deepList contents", datasetList2.contains(datasetName.replaceFirst("/", ""))); + Assert.assertTrue("deepList contents", datasetList2.contains(datasetName.replaceFirst("/", ""))); Assert.assertFalse("deepList stops at datasets", datasetList2.contains(datasetName + "/0")); final String prefix = "/test"; @@ -655,6 +658,7 @@ public void testDeepList() { Executors.newFixedThreadPool(2))); } catch (final IOException | InterruptedException | ExecutionException e) { +// } catch (final IOException e) { fail(e.getMessage()); } } From 3844e0c57df174b02d6e4315cf582a95a3c0104d Mon Sep 17 00:00:00 2001 From: Stephan Saalfeld Date: Wed, 12 May 2021 20:47:15 -0400 Subject: [PATCH 021/243] remove dead middle interface GsonAttributesParser --- .../saalfeldlab/n5/GsonAttributesParser.java | 242 ------------------ .../janelia/saalfeldlab/n5/N5FSReader.java | 219 +++++++++++++++- .../janelia/saalfeldlab/n5/N5FSWriter.java | 7 +- 3 files changed, 211 insertions(+), 257 deletions(-) delete mode 100644 src/main/java/org/janelia/saalfeldlab/n5/GsonAttributesParser.java diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonAttributesParser.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonAttributesParser.java deleted file mode 100644 index 16b93df4..00000000 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonAttributesParser.java +++ /dev/null @@ -1,242 +0,0 @@ -/** - * Copyright (c) 2017, Stephan Saalfeld - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ -package org.janelia.saalfeldlab.n5; - -import java.io.IOException; -import java.io.Reader; -import java.io.Writer; -import java.lang.reflect.Array; -import java.lang.reflect.Type; -import java.util.HashMap; -import java.util.Map; -import java.util.Map.Entry; - -import com.google.gson.Gson; -import com.google.gson.JsonArray; -import com.google.gson.JsonElement; -import com.google.gson.JsonPrimitive; -import com.google.gson.reflect.TypeToken; - -/** - * {@link N5Reader} for JSON attributes parsed by {@link Gson}. - * - * @author Stephan Saalfeld - */ -public interface GsonAttributesParser extends N5Reader { - - public Gson getGson(); - - /** - * Reads or creates the attributes map of a group or dataset. - * - * @param pathName group path - * @return - * @throws IOException - */ - public HashMap getAttributes(final String pathName) throws IOException; - - /** - * Parses an attribute from the given attributes map. - * - * @param map - * @param key - * @param clazz - * @return - * @throws IOException - */ - public static T parseAttribute( - final HashMap map, - final String key, - final Class clazz, - final Gson gson) throws IOException { - - final JsonElement attribute = map.get(key); - if (attribute != null) - return gson.fromJson(attribute, clazz); - else - return null; - } - - /** - * Parses an attribute from the given attributes map. - * - * @param map - * @param key - * @param type - * @return - * @throws IOException - */ - public static T parseAttribute( - final HashMap map, - final String key, - final Type type, - final Gson gson) throws IOException { - - final JsonElement attribute = map.get(key); - if (attribute != null) - return gson.fromJson(attribute, type); - else - return null; - } - - /** - * Reads the attributes map from a given {@link Reader}. - * - * @param reader - * @return - * @throws IOException - */ - public static HashMap readAttributes(final Reader reader, final Gson gson) throws IOException { - - final Type mapType = new TypeToken>(){}.getType(); - final HashMap map = gson.fromJson(reader, mapType); - return map == null ? new HashMap<>() : map; - } - - /** - * Inserts new the JSON export of attributes into the given attributes map. - * - * @param map - * @param attributes - * @param gson - * @throws IOException - */ - public static void insertAttributes( - final HashMap map, - final Map attributes, - final Gson gson) throws IOException { - - for (final Entry entry : attributes.entrySet()) - map.put(entry.getKey(), gson.toJsonTree(entry.getValue())); - } - - /** - * Writes the attributes map to a given {@link Writer}. - * - * @param writer - * @param map - * @throws IOException - */ - public static void writeAttributes( - final Writer writer, - final HashMap map, - final Gson gson) throws IOException { - - final Type mapType = new TypeToken>(){}.getType(); - gson.toJson(map, mapType, writer); - writer.flush(); - } - - /** - * Return a reasonable class for a {@link JsonPrimitive}. Possible return - * types are - *

    - *
  • boolean
    • - *
  • double
    • - *
  • String
    • - *
  • Object
    • - *
- * - * @param jsonPrimitive - * @return - */ - public static Class classForJsonPrimitive(final JsonPrimitive jsonPrimitive) { - - if (jsonPrimitive.isBoolean()) - return boolean.class; - else if (jsonPrimitive.isNumber()) { - final Number number = jsonPrimitive.getAsNumber(); - if (number.longValue() == number.doubleValue()) - return long.class; - else - return double.class; - } else if (jsonPrimitive.isString()) - return String.class; - else return Object.class; - } - - /** - * Best effort implementation of {@link N5Reader#listAttributes(String)} - * with limited type resolution. Possible return types are - *
    - *
  • null
    • - *
  • boolean
    • - *
  • double
    • - *
  • String
    • - *
  • Object
    • - *
  • boolean[]
    • - *
  • double[]
    • - *
  • String[]
    • - *
  • Object[]
    • - *
- */ - @Override - public default Map> listAttributes(final String pathName) throws IOException { - - final HashMap jsonElementMap = getAttributes(pathName); - final HashMap> attributes = new HashMap<>(); - jsonElementMap.forEach( - (key, jsonElement) -> { - final Class clazz; - if (jsonElement.isJsonNull()) - clazz = null; - else if (jsonElement.isJsonPrimitive()) - clazz = classForJsonPrimitive((JsonPrimitive)jsonElement); - else if (jsonElement.isJsonArray()) { - final JsonArray jsonArray = (JsonArray)jsonElement; - Class arrayElementClass = Object.class; - if (jsonArray.size() > 0) { - final JsonElement firstElement = jsonArray.get(0); - if (firstElement.isJsonPrimitive()) { - arrayElementClass = classForJsonPrimitive(firstElement.getAsJsonPrimitive()); - for (int i = 1; i < jsonArray.size() && arrayElementClass != Object.class; ++i) { - final JsonElement element = jsonArray.get(i); - if (element.isJsonPrimitive()) { - final Class nextArrayElementClass = classForJsonPrimitive(element.getAsJsonPrimitive()); - if (nextArrayElementClass != arrayElementClass) - if (nextArrayElementClass == double.class && arrayElementClass == long.class) - arrayElementClass = double.class; - else { - arrayElementClass = Object.class; - break; - } - } else { - arrayElementClass = Object.class; - break; - } - } - } - clazz = Array.newInstance(arrayElementClass, 0).getClass(); - } else - clazz = Object[].class; - } - else - clazz = Object.class; - attributes.put(key, clazz); - }); - return attributes; - } -} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5FSReader.java b/src/main/java/org/janelia/saalfeldlab/n5/N5FSReader.java index efd14e28..a37b4267 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5FSReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5FSReader.java @@ -27,6 +27,9 @@ import java.io.Closeable; import java.io.IOException; +import java.io.Reader; +import java.io.Writer; +import java.lang.reflect.Array; import java.lang.reflect.Type; import java.nio.channels.Channels; import java.nio.channels.FileChannel; @@ -41,12 +44,17 @@ import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; +import java.util.Map; +import java.util.Map.Entry; import java.util.Set; import java.util.stream.Stream; import com.google.gson.Gson; import com.google.gson.GsonBuilder; +import com.google.gson.JsonArray; import com.google.gson.JsonElement; +import com.google.gson.JsonPrimitive; +import com.google.gson.reflect.TypeToken; /** * Filesystem {@link N5Reader} implementation with JSON attributes @@ -56,7 +64,7 @@ * @author Igor Pisarev * @author Philipp Hanslovsky */ -public class N5FSReader implements GsonAttributesParser { +public class N5FSReader implements N5Reader { /** * A {@link FileChannel} wrapper that attempts to acquire a lock and waits @@ -249,7 +257,6 @@ public N5FSReader(final String basePath) throws IOException { this(basePath, new GsonBuilder(), false); } - @Override public Gson getGson() { return gson; @@ -264,6 +271,188 @@ public String getBasePath() { return this.basePath; } + + /** + * Parses an attribute from the given attributes map. + * + * @param map + * @param key + * @param clazz + * @return + * @throws IOException + */ + protected T parseAttribute( + final HashMap map, + final String key, + final Class clazz) throws IOException { + + final JsonElement attribute = map.get(key); + if (attribute != null) + return gson.fromJson(attribute, clazz); + else + return null; + } + + /** + * Parses an attribute from the given attributes map. + * + * @param map + * @param key + * @param type + * @return + * @throws IOException + */ + protected T parseAttribute( + final HashMap map, + final String key, + final Type type) throws IOException { + + final JsonElement attribute = map.get(key); + if (attribute != null) + return gson.fromJson(attribute, type); + else + return null; + } + + /** + * Reads the attributes map from a given {@link Reader}. + * + * @param reader + * @return + * @throws IOException + */ + protected HashMap readAttributes(final Reader reader) throws IOException { + + final Type mapType = new TypeToken>(){}.getType(); + final HashMap map = gson.fromJson(reader, mapType); + return map == null ? new HashMap<>() : map; + } + + /** + * Inserts new the JSON export of attributes into the given attributes map. + * + * @param map + * @param attributes + * @param gson + * @throws IOException + */ + protected void insertAttributes( + final HashMap map, + final Map attributes) throws IOException { + + for (final Entry entry : attributes.entrySet()) + map.put(entry.getKey(), gson.toJsonTree(entry.getValue())); + } + + /** + * Writes the attributes map to a given {@link Writer}. + * + * @param writer + * @param map + * @throws IOException + */ + protected void writeAttributes( + final Writer writer, + final HashMap map) throws IOException { + + final Type mapType = new TypeToken>(){}.getType(); + gson.toJson(map, mapType, writer); + writer.flush(); + } + + /** + * Return a reasonable class for a {@link JsonPrimitive}. Possible return + * types are + *
    + *
  • boolean
    • + *
  • double
    • + *
  • long (if the number is an integer)
    • + *
  • String
    • + *
  • Object
    • + *
+ * + * @param jsonPrimitive + * @return + */ + protected static Class classForJsonPrimitive(final JsonPrimitive jsonPrimitive) { + + if (jsonPrimitive.isBoolean()) + return boolean.class; + else if (jsonPrimitive.isNumber()) { + final Number number = jsonPrimitive.getAsNumber(); + if (number.longValue() == number.doubleValue()) + return long.class; + else + return double.class; + } else if (jsonPrimitive.isString()) + return String.class; + else return Object.class; + } + + /** + * Best effort implementation of {@link N5Reader#listAttributes(String)} + * with limited type resolution fromJSON. Possible return types are + *
    + *
  • null
    • + *
  • boolean
    • + *
  • double
    • + *
  • long (if the number is an integer)
    • + *
  • String
    • + *
  • Object
    • + *
  • boolean[]
    • + *
  • double[]
    • + *
  • long[] (if all numbers in the array are integers)
    • + *
  • String[]
    • + *
  • Object[]
    • + *
+ */ + @Override + public Map> listAttributes(final String pathName) throws IOException { + + final HashMap jsonElementMap = getAttributes(pathName); + final HashMap> attributes = new HashMap<>(); + jsonElementMap.forEach( + (key, jsonElement) -> { + final Class clazz; + if (jsonElement.isJsonNull()) + clazz = null; + else if (jsonElement.isJsonPrimitive()) + clazz = classForJsonPrimitive((JsonPrimitive)jsonElement); + else if (jsonElement.isJsonArray()) { + final JsonArray jsonArray = (JsonArray)jsonElement; + Class arrayElementClass = Object.class; + if (jsonArray.size() > 0) { + final JsonElement firstElement = jsonArray.get(0); + if (firstElement.isJsonPrimitive()) { + arrayElementClass = classForJsonPrimitive(firstElement.getAsJsonPrimitive()); + for (int i = 1; i < jsonArray.size() && arrayElementClass != Object.class; ++i) { + final JsonElement element = jsonArray.get(i); + if (element.isJsonPrimitive()) { + final Class nextArrayElementClass = classForJsonPrimitive(element.getAsJsonPrimitive()); + if (nextArrayElementClass != arrayElementClass) + if (nextArrayElementClass == double.class && arrayElementClass == long.class) + arrayElementClass = double.class; + else { + arrayElementClass = Object.class; + break; + } + } else { + arrayElementClass = Object.class; + break; + } + } + } + clazz = Array.newInstance(arrayElementClass, 0).getClass(); + } else + clazz = Object[].class; + } + else + clazz = Object.class; + attributes.put(key, clazz); + }); + return attributes; + } + @Override public DatasetAttributes getDatasetAttributes(final String pathName) throws IOException { @@ -326,21 +515,21 @@ public DatasetAttributes getDatasetAttributes(final String pathName) throws IOEx } else { final HashMap map = getAttributes(normalPathName); - dimensions = GsonAttributesParser.parseAttribute(map, DatasetAttributes.dimensionsKey, long[].class, gson); + dimensions = parseAttribute(map, DatasetAttributes.dimensionsKey, long[].class); if (dimensions == null) return null; - dataType = GsonAttributesParser.parseAttribute(map, DatasetAttributes.dataTypeKey, DataType.class, gson); + dataType = parseAttribute(map, DatasetAttributes.dataTypeKey, DataType.class); if (dataType == null) return null; - blockSize = GsonAttributesParser.parseAttribute(map, DatasetAttributes.blockSizeKey, int[].class, gson); + blockSize = parseAttribute(map, DatasetAttributes.blockSizeKey, int[].class); - compression = GsonAttributesParser.parseAttribute(map, DatasetAttributes.compressionKey, Compression.class, gson); + compression = parseAttribute(map, DatasetAttributes.compressionKey, Compression.class); /* version 0 */ compressionVersion0Name = compression == null - ? GsonAttributesParser.parseAttribute(map, DatasetAttributes.compressionTypeKey, String.class, gson) + ? parseAttribute(map, DatasetAttributes.compressionTypeKey, String.class) : null; } @@ -377,7 +566,7 @@ protected HashMap getAttributes(final Path path) throws IOE return new HashMap<>(); try (final LockedFileChannel lockedFileChannel = LockedFileChannel.openForReading(path)) { - return GsonAttributesParser.readAttributes(Channels.newReader(lockedFileChannel.getFileChannel(), StandardCharsets.UTF_8.name()), getGson()); + return readAttributes(Channels.newReader(lockedFileChannel.getFileChannel(), StandardCharsets.UTF_8.name())); } } @@ -482,7 +671,7 @@ public T getAttribute( return getAttribute(cachedMap, key, clazz); } else { final HashMap map = getAttributes(normalPathName); - return GsonAttributesParser.parseAttribute(map, key, clazz, getGson()); + return parseAttribute(map, key, clazz); } } @@ -503,7 +692,7 @@ public T getAttribute( return getAttribute(cachedMap, key, type); } else { final HashMap map = getAttributes(normalPathName); - return GsonAttributesParser.parseAttribute(map, key, type, getGson()); + return parseAttribute(map, key, type); } } @@ -572,7 +761,13 @@ public boolean datasetExists(final String pathName) throws IOException { return exists(pathName) && getDatasetAttributes(pathName) != null; } - @Override + /** + * Reads or creates the attributes map of a group or dataset. + * + * @param pathName group path + * @return + * @throws IOException + */ public HashMap getAttributes(final String pathName) throws IOException { final Path path = getAttributesPath(normalize(pathName)); @@ -580,7 +775,7 @@ public HashMap getAttributes(final String pathName) throws return new HashMap<>(); try (final LockedFileChannel lockedFileChannel = LockedFileChannel.openForReading(path)) { - return GsonAttributesParser.readAttributes(Channels.newReader(lockedFileChannel.getFileChannel(), StandardCharsets.UTF_8.name()), getGson()); + return readAttributes(Channels.newReader(lockedFileChannel.getFileChannel(), StandardCharsets.UTF_8.name())); } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java b/src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java index 57bce3bc..ce3fe3cc 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java @@ -222,6 +222,7 @@ protected N5GroupInfo createCachedGroup(final String normalPathName) throws IOEx N5GroupInfo parentInfo = getCachedN5GroupInfo(parentPathName); if (parentInfo == emptyGroupInfo) { parentInfo = new N5GroupInfo(); + parentInfo.isDataset = false; metaCache.put(parentPathName, parentInfo); } final HashSet children = parentInfo.children; @@ -286,11 +287,11 @@ protected void writeAttributes( final HashMap map = new HashMap<>(); try (final LockedFileChannel lockedFileChannel = LockedFileChannel.openForWriting(path)) { - map.putAll(GsonAttributesParser.readAttributes(Channels.newReader(lockedFileChannel.getFileChannel(), StandardCharsets.UTF_8.name()), getGson())); - GsonAttributesParser.insertAttributes(map, attributes, gson); + map.putAll(readAttributes(Channels.newReader(lockedFileChannel.getFileChannel(), StandardCharsets.UTF_8.name()))); + insertAttributes(map, attributes); lockedFileChannel.getFileChannel().truncate(0); - GsonAttributesParser.writeAttributes(Channels.newWriter(lockedFileChannel.getFileChannel(), StandardCharsets.UTF_8.name()), map, getGson()); + writeAttributes(Channels.newWriter(lockedFileChannel.getFileChannel(), StandardCharsets.UTF_8.name()), map); } } From 3ac189d2b850b71fa60f6385eb3e3dd9e2f6a624 Mon Sep 17 00:00:00 2001 From: Stephan Saalfeld Date: Tue, 20 Jul 2021 09:05:43 -0400 Subject: [PATCH 022/243] WIP key value interface as filesystem simplification --- .../n5/FileSystemKeyValueAccess.java | 416 ++++++++++ .../saalfeldlab/n5/GsonAttributesParser.java | 242 ++++++ .../saalfeldlab/n5/KeyValueAccess.java | 174 ++++ .../janelia/saalfeldlab/n5/LockedChannel.java | 70 ++ .../janelia/saalfeldlab/n5/N5FSReader.java | 73 +- .../janelia/saalfeldlab/n5/N5FSWriter.java | 185 +---- .../saalfeldlab/n5/N5KeyValueReader.java | 751 ++++++++++++++++++ .../saalfeldlab/n5/N5KeyValueWriter.java | 344 ++++++++ .../n5/FileSystemKeyValueAccessTest.java | 69 ++ .../org/janelia/saalfeldlab/n5/N5FSTest.java | 166 +++- 10 files changed, 2269 insertions(+), 221 deletions(-) create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/GsonAttributesParser.java create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccess.java create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/LockedChannel.java create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriter.java create mode 100644 src/test/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccessTest.java diff --git a/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java new file mode 100644 index 00000000..4702c79a --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java @@ -0,0 +1,416 @@ +/** + * Copyright (c) 2017--2021, Stephan Saalfeld + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +package org.janelia.saalfeldlab.n5; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.Reader; +import java.io.Writer; +import java.nio.channels.Channels; +import java.nio.channels.FileChannel; +import java.nio.channels.OverlappingFileLockException; +import java.nio.charset.StandardCharsets; +import java.nio.file.DirectoryNotEmptyException; +import java.nio.file.FileAlreadyExistsException; +import java.nio.file.FileSystem; +import java.nio.file.FileSystemException; +import java.nio.file.Files; +import java.nio.file.NoSuchFileException; +import java.nio.file.OpenOption; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.nio.file.attribute.FileAttribute; +import java.util.Comparator; +import java.util.stream.Stream; + +/** + * Filesystem {@link KeyValueAccess}. + * + * @author Stephan Saalfeld + * @author Igor Pisarev + * @author Philipp Hanslovsky + */ +public class FileSystemKeyValueAccess implements KeyValueAccess { + + /** + * A {@link FileChannel} wrapper that attempts to acquire a lock and waits + * for existing locks to be lifted before returning if the + * {@link FileSystem} supports that. If the {@link FileSystem} does not + * support locking, it returns immediately. + */ + protected class LockedFileChannel implements LockedChannel { + + protected final FileChannel channel; + + protected LockedFileChannel(final String path, final boolean readOnly) throws IOException { + + this(fileSystem.getPath(path), readOnly); + } + + protected LockedFileChannel(final Path path, final boolean readOnly) throws IOException { + + final OpenOption[] options = readOnly ? new OpenOption[]{StandardOpenOption.READ} : new OpenOption[]{StandardOpenOption.READ, StandardOpenOption.WRITE, StandardOpenOption.CREATE}; + channel = FileChannel.open(path, options); + + for (boolean waiting = true; waiting;) { + waiting = false; + try { + channel.lock(0L, Long.MAX_VALUE, readOnly); + } catch (final OverlappingFileLockException e) { + waiting = true; + try { + Thread.sleep(100); + } catch (final InterruptedException f) { + waiting = false; + Thread.currentThread().interrupt(); + } + } catch (final IOException e) {} + } + } + + @Override + public Reader newReader() throws IOException { + + return Channels.newReader(channel, StandardCharsets.UTF_8.name()); + } + + @Override + public Writer newWriter() throws IOException { + + channel.truncate(0); + return Channels.newWriter(channel, StandardCharsets.UTF_8.name()); + } + + @Override + public InputStream newInputStream() throws IOException { + + return Channels.newInputStream(channel); + } + + @Override + public OutputStream newOutputStream() throws IOException { + + channel.truncate(0); + return Channels.newOutputStream(channel); + } + + @Override + public void close() throws IOException { + + channel.close(); + } + } + + protected final FileSystem fileSystem; + + /** + * Opens a {@link FileSystemKeyValueAccess}. + * + * @param fileSystem + */ + public FileSystemKeyValueAccess(final FileSystem fileSystem) { + + this.fileSystem = fileSystem; + } + + @Override + public LockedFileChannel lockForReading(final String normalPath) throws IOException { + + return new LockedFileChannel(normalPath, true); + } + + @Override + public LockedFileChannel lockForWriting(final String normalPath) throws IOException { + + return new LockedFileChannel(normalPath, false); + } + + public LockedFileChannel lockForReading(final Path path) throws IOException { + + return new LockedFileChannel(path, true); + } + + public LockedFileChannel lockForWriting(final Path path) throws IOException { + + return new LockedFileChannel(path, false); + } + + @Override + public boolean isDirectory(final String normalPath) { + + final Path path = fileSystem.getPath(normalPath); + return Files.isDirectory(path); + } + + @Override + public boolean isFile(final String normalPath) { + + final Path path = fileSystem.getPath(normalPath); + return Files.isRegularFile(path); + } + + @Override + public boolean exists(final String normalPath) { + + final Path path = fileSystem.getPath(normalPath); + return Files.exists(path); + } + + @Override + public String[] listDirectories(final String normalPath) throws IOException { + + final Path path = fileSystem.getPath(normalPath); + try (final Stream pathStream = Files.list(path)) { + return pathStream + .filter(a -> Files.isDirectory(a)) + .map(a -> path.relativize(a).toString()) + .toArray(n -> new String[n]); + } + } + + @Override + public String[] list(final String normalPath) throws IOException { + + final Path path = fileSystem.getPath(normalPath); + try (final Stream pathStream = Files.list(path)) { + return pathStream + .map(a -> path.relativize(a).toString()) + .toArray(n -> new String[n]); + } + } + + @Override + public String[] components(final String path) { + + final Path fsPath = fileSystem.getPath(path); + final Path root = fsPath.getRoot(); + final String[] components; + int o; + if (root == null) { + components = new String[fsPath.getNameCount()]; + o = 0; + } + else { + components = new String[fsPath.getNameCount() + 1]; + components[0] = root.toString(); + o = 1; + } + + for (int i = o; i < components.length; ++i) + components[i] = fsPath.getName(i - o).toString(); + return components; + } + + @Override + public String parent(final String path) { + + final Path parent = fileSystem.getPath(path).getParent(); + if (parent == null) return null; + else return parent.toString(); + } + + @Override + public String relativize(final String path, final String base) { + + final Path basePath = fileSystem.getPath(base); + return basePath.relativize(fileSystem.getPath(path)).toString(); + } + + @Override + public String normalize(final String path) { + + return fileSystem.getPath(path).normalize().toString(); + } + + @Override + public void createDirectories(final String normalPath) throws IOException { + + createDirectories(fileSystem.getPath(normalPath)); + } + + @Override + public void delete(final String normalPath) throws IOException { + + final Path path = fileSystem.getPath(normalPath); + try (final Stream pathStream = Files.walk(path)) { + pathStream.sorted(Comparator.reverseOrder()).forEach( + childPath -> { + if (Files.isRegularFile(childPath)) { + try (final LockedChannel channel = lockForWriting(childPath)) { + Files.delete(childPath); + } catch (final IOException e) { + e.printStackTrace(); + } + } else { + tryDelete(childPath); + } + }); + } + } + + protected static void tryDelete(final Path childPath) { + + try { + Files.delete(childPath); + } catch (final DirectoryNotEmptyException e) { + // Even though childPath should be an empty directory, sometimes the deletion fails on network file + // when lock files are not cleared immediately after the leaves have been removed. + try { + // wait and reattempt + Thread.sleep(100); + Files.delete(childPath); + } catch (final InterruptedException ex) { + e.printStackTrace(); + Thread.currentThread().interrupt(); + } catch (final IOException ex) { + ex.printStackTrace(); + } + } catch (final IOException e) { + e.printStackTrace(); + } + } + + /** + * This is a copy of {@link Files#createDirectories(Path, FileAttribute...)} + * that follows symlinks. + * + * Workaround for https://bugs.openjdk.java.net/browse/JDK-8130464 + * + * Creates a directory by creating all nonexistent parent directories first. + * Unlike the {@link #createDirectory createDirectory} method, an exception + * is not thrown if the directory could not be created because it already + * exists. + * + *

The {@code attrs} parameter is optional {@link FileAttribute + * file-attributes} to set atomically when creating the nonexistent + * directories. Each file attribute is identified by its {@link + * FileAttribute#name name}. If more than one attribute of the same name is + * included in the array then all but the last occurrence is ignored. + * + *

If this method fails, then it may do so after creating some, but not + * all, of the parent directories. + * + * @param dir + * the directory to create + * + * @param attrs + * an optional list of file attributes to set atomically when + * creating the directory + * + * @return the directory + * + * @throws UnsupportedOperationException + * if the array contains an attribute that cannot be set atomically + * when creating the directory + * @throws FileAlreadyExistsException + * if {@code dir} exists but is not a directory (optional specific + * exception) + * @throws IOException + * if an I/O error occurs + * @throws SecurityException + * in the case of the default provider, and a security manager is + * installed, the {@link SecurityManager#checkWrite(String) checkWrite} + * method is invoked prior to attempting to create a directory and + * its {@link SecurityManager#checkRead(String) checkRead} is + * invoked for each parent directory that is checked. If {@code + * dir} is not an absolute path then its {@link Path#toAbsolutePath + * toAbsolutePath} may need to be invoked to get its absolute path. + * This may invoke the security manager's {@link + * SecurityManager#checkPropertyAccess(String) checkPropertyAccess} + * method to check access to the system property {@code user.dir} + */ + protected static Path createDirectories(Path dir, final FileAttribute... attrs) throws IOException { + + // attempt to create the directory + try { + createAndCheckIsDirectory(dir, attrs); + return dir; + } catch (final FileAlreadyExistsException x) { + // file exists and is not a directory + throw x; + } catch (final IOException x) { + // parent may not exist or other reason + } + SecurityException se = null; + try { + dir = dir.toAbsolutePath(); + } catch (final SecurityException x) { + // don't have permission to get absolute path + se = x; + } + // find a decendent that exists + Path parent = dir.getParent(); + while (parent != null) { + try { + parent.getFileSystem().provider().checkAccess(parent); + break; + } catch (final NoSuchFileException x) { + // does not exist + } + parent = parent.getParent(); + } + if (parent == null) { + // unable to find existing parent + if (se == null) { + throw new FileSystemException(dir.toString(), null, + "Unable to determine if root directory exists"); + } else { + throw se; + } + } + + // create directories + Path child = parent; + for (final Path name: parent.relativize(dir)) { + child = child.resolve(name); + createAndCheckIsDirectory(child, attrs); + } + return dir; + } + + /** + * This is a copy of + * {@link Files#createAndCheckIsDirectory(Path, FileAttribute...)} that + * follows symlinks. + * + * Workaround for https://bugs.openjdk.java.net/browse/JDK-8130464 + * + * Used by createDirectories to attempt to create a directory. A no-op if + * the directory already exists. + */ + protected static void createAndCheckIsDirectory( + final Path dir, + final FileAttribute... attrs) throws IOException { + + try { + Files.createDirectory(dir, attrs); + } catch (final FileAlreadyExistsException x) { + if (!Files.isDirectory(dir)) + throw x; + } + } +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonAttributesParser.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonAttributesParser.java new file mode 100644 index 00000000..16b93df4 --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonAttributesParser.java @@ -0,0 +1,242 @@ +/** + * Copyright (c) 2017, Stephan Saalfeld + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +package org.janelia.saalfeldlab.n5; + +import java.io.IOException; +import java.io.Reader; +import java.io.Writer; +import java.lang.reflect.Array; +import java.lang.reflect.Type; +import java.util.HashMap; +import java.util.Map; +import java.util.Map.Entry; + +import com.google.gson.Gson; +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonPrimitive; +import com.google.gson.reflect.TypeToken; + +/** + * {@link N5Reader} for JSON attributes parsed by {@link Gson}. + * + * @author Stephan Saalfeld + */ +public interface GsonAttributesParser extends N5Reader { + + public Gson getGson(); + + /** + * Reads or creates the attributes map of a group or dataset. + * + * @param pathName group path + * @return + * @throws IOException + */ + public HashMap getAttributes(final String pathName) throws IOException; + + /** + * Parses an attribute from the given attributes map. + * + * @param map + * @param key + * @param clazz + * @return + * @throws IOException + */ + public static T parseAttribute( + final HashMap map, + final String key, + final Class clazz, + final Gson gson) throws IOException { + + final JsonElement attribute = map.get(key); + if (attribute != null) + return gson.fromJson(attribute, clazz); + else + return null; + } + + /** + * Parses an attribute from the given attributes map. + * + * @param map + * @param key + * @param type + * @return + * @throws IOException + */ + public static T parseAttribute( + final HashMap map, + final String key, + final Type type, + final Gson gson) throws IOException { + + final JsonElement attribute = map.get(key); + if (attribute != null) + return gson.fromJson(attribute, type); + else + return null; + } + + /** + * Reads the attributes map from a given {@link Reader}. + * + * @param reader + * @return + * @throws IOException + */ + public static HashMap readAttributes(final Reader reader, final Gson gson) throws IOException { + + final Type mapType = new TypeToken>(){}.getType(); + final HashMap map = gson.fromJson(reader, mapType); + return map == null ? new HashMap<>() : map; + } + + /** + * Inserts new the JSON export of attributes into the given attributes map. + * + * @param map + * @param attributes + * @param gson + * @throws IOException + */ + public static void insertAttributes( + final HashMap map, + final Map attributes, + final Gson gson) throws IOException { + + for (final Entry entry : attributes.entrySet()) + map.put(entry.getKey(), gson.toJsonTree(entry.getValue())); + } + + /** + * Writes the attributes map to a given {@link Writer}. + * + * @param writer + * @param map + * @throws IOException + */ + public static void writeAttributes( + final Writer writer, + final HashMap map, + final Gson gson) throws IOException { + + final Type mapType = new TypeToken>(){}.getType(); + gson.toJson(map, mapType, writer); + writer.flush(); + } + + /** + * Return a reasonable class for a {@link JsonPrimitive}. Possible return + * types are + *

    + *
  • boolean
    • + *
  • double
    • + *
  • String
    • + *
  • Object
    • + *
+ * + * @param jsonPrimitive + * @return + */ + public static Class classForJsonPrimitive(final JsonPrimitive jsonPrimitive) { + + if (jsonPrimitive.isBoolean()) + return boolean.class; + else if (jsonPrimitive.isNumber()) { + final Number number = jsonPrimitive.getAsNumber(); + if (number.longValue() == number.doubleValue()) + return long.class; + else + return double.class; + } else if (jsonPrimitive.isString()) + return String.class; + else return Object.class; + } + + /** + * Best effort implementation of {@link N5Reader#listAttributes(String)} + * with limited type resolution. Possible return types are + *
    + *
  • null
    • + *
  • boolean
    • + *
  • double
    • + *
  • String
    • + *
  • Object
    • + *
  • boolean[]
    • + *
  • double[]
    • + *
  • String[]
    • + *
  • Object[]
    • + *
+ */ + @Override + public default Map> listAttributes(final String pathName) throws IOException { + + final HashMap jsonElementMap = getAttributes(pathName); + final HashMap> attributes = new HashMap<>(); + jsonElementMap.forEach( + (key, jsonElement) -> { + final Class clazz; + if (jsonElement.isJsonNull()) + clazz = null; + else if (jsonElement.isJsonPrimitive()) + clazz = classForJsonPrimitive((JsonPrimitive)jsonElement); + else if (jsonElement.isJsonArray()) { + final JsonArray jsonArray = (JsonArray)jsonElement; + Class arrayElementClass = Object.class; + if (jsonArray.size() > 0) { + final JsonElement firstElement = jsonArray.get(0); + if (firstElement.isJsonPrimitive()) { + arrayElementClass = classForJsonPrimitive(firstElement.getAsJsonPrimitive()); + for (int i = 1; i < jsonArray.size() && arrayElementClass != Object.class; ++i) { + final JsonElement element = jsonArray.get(i); + if (element.isJsonPrimitive()) { + final Class nextArrayElementClass = classForJsonPrimitive(element.getAsJsonPrimitive()); + if (nextArrayElementClass != arrayElementClass) + if (nextArrayElementClass == double.class && arrayElementClass == long.class) + arrayElementClass = double.class; + else { + arrayElementClass = Object.class; + break; + } + } else { + arrayElementClass = Object.class; + break; + } + } + } + clazz = Array.newInstance(arrayElementClass, 0).getClass(); + } else + clazz = Object[].class; + } + else + clazz = Object.class; + attributes.put(key, clazz); + }); + return attributes; + } +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccess.java new file mode 100644 index 00000000..4dcb5964 --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccess.java @@ -0,0 +1,174 @@ +/** + * Copyright (c) 2017--2021, Stephan Saalfeld + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +package org.janelia.saalfeldlab.n5; + +import java.io.IOException; +import java.nio.file.FileSystem; + +/** + * Key value read primitives used by {@link N5KeyValueReader} + * implementations. This interface implements a subset of access primitives + * provided by {@link FileSystem} to reduce the implementation burden for backends + * lacking a {@link FileSystem} implementation (such as AWS-S3). + * + * @author Stephan Saalfeld + */ +public interface KeyValueAccess { + + /** + * Split a path string into its components. + * + * @param path + * @return + */ + public String[] components(final String path); + + /** + * Get the parent of a path string. + * + * @param path + * @return null if the path has no parent + */ + public String parent(final String path); + + /** + * Relativize path relative to base. + * + * @param path + * @param base + * @return null if the path has no parent + */ + public String relativize(final String path, final String base); + + /** + * Normalize a path to canonical form. All paths pointing to the same + * location return the same output. This is most important for cached + * data pointing getting the same key. + * + * @param path + * @return + */ + public String normalize(final String path); + + /** + * Test whether the path exists. + * + * @param normalPath is expected to be in normalized form, no further + * efforts are made to normalize it. + * @return + */ + public boolean exists(final String normalPath); + + /** + * Test whether the path is a directory. + * + * @param normalPath is expected to be in normalized form, no further + * efforts are made to normalize it. + * @return + */ + public boolean isDirectory(String normalPath); + + /** + * Test whether the path is a file. + * + * @param normalPath is expected to be in normalized form, no further + * efforts are made to normalize it. + * @return + */ + public boolean isFile(String normalPath); + + /** + * Create a lock on a path for reading. This isn't meant to be kept + * around. Create, use, [auto]close, e.g. + * + * try (final lock = store.lockForReading()) { + * ... + * } + * + * + * @param normalPath is expected to be in normalized form, no further + * efforts are made to normalize it. + * @return + * @throws IOException + */ + public LockedChannel lockForReading(final String normalPath) throws IOException; + + /** + * Create an exclusive lock on a path for writing. This isn't meant to be + * kept around. Create, use, [auto]close, e.g. + * + * try (final lock = store.lockForWriting()) { + * ... + * } + * + * + * @param normalPath is expected to be in normalized form, no further + * efforts are made to normalize it. + * @return + * @throws IOException + */ + public LockedChannel lockForWriting(final String normalPath) throws IOException; + + /** + * List all 'directory'-like children of a path. + * + * @param normalPath is expected to be in normalized form, no further + * efforts are made to normalize it. + * @return + * @throws IOException + */ + public String[] listDirectories(final String normalPath) throws IOException; + + /** + * List all children of a path. + * + * @param normalPath is expected to be in normalized form, no further + * efforts are made to normalize it. + * @return + * @throws IOException + */ + public String[] list(final String normalPath) throws IOException; + + /** + * Create a director and all parent paths along the way. + * + * @param normalPath is expected to be in normalized form, no further + * efforts are made to normalize it. + * @return + * @throws IOException + */ + public void createDirectories(final String normalPath) throws IOException; + + /** + * Delete this path. This is only for empty leaves of the tree, not + * expected to work recursively. + * + * @param normalPath is expected to be in normalized form, no further + * efforts are made to normalize it. + * @return + */ + public void delete(final String normalPath) throws IOException; +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/LockedChannel.java b/src/main/java/org/janelia/saalfeldlab/n5/LockedChannel.java new file mode 100644 index 00000000..d5a97d62 --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/LockedChannel.java @@ -0,0 +1,70 @@ +/** + * Copyright (c) 2017--2021, Stephan Saalfeld + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +package org.janelia.saalfeldlab.n5; + +import java.io.Closeable; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.Reader; +import java.io.Writer; + +/** + * A lock on a path that can create a {@link Reader}, {@link Writer}, + * {@link InputStream}, or {@link OutputStream}. + * + * @author Stephan Saalfeld + */ +public interface LockedChannel extends Closeable { + + /** + * Create a UTF-8 {@link Reader}. + * + * @return + * @throws IOException + */ + public Reader newReader() throws IOException; + + /** + * Create a new {@link InputStream}. + * + * @return + * @throws IOException + */ + public InputStream newInputStream() throws IOException; + + /** + * Create a new UTF-8 {@link Writer}. + * @return + */ + public Writer newWriter() throws IOException; + + /** + * Create a new {@link OutputStream}. + * @return + */ + public OutputStream newOutputStream() throws IOException; +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5FSReader.java b/src/main/java/org/janelia/saalfeldlab/n5/N5FSReader.java index a37b4267..95f2987b 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5FSReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5FSReader.java @@ -25,22 +25,16 @@ */ package org.janelia.saalfeldlab.n5; -import java.io.Closeable; import java.io.IOException; import java.io.Reader; import java.io.Writer; import java.lang.reflect.Array; import java.lang.reflect.Type; import java.nio.channels.Channels; -import java.nio.channels.FileChannel; -import java.nio.channels.OverlappingFileLockException; import java.nio.charset.StandardCharsets; -import java.nio.file.FileSystem; import java.nio.file.FileSystems; import java.nio.file.Files; -import java.nio.file.OpenOption; import java.nio.file.Path; -import java.nio.file.StandardOpenOption; import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; @@ -49,6 +43,8 @@ import java.util.Set; import java.util.stream.Stream; +import org.janelia.saalfeldlab.n5.FileSystemKeyValueAccess.LockedFileChannel; + import com.google.gson.Gson; import com.google.gson.GsonBuilder; import com.google.gson.JsonArray; @@ -57,8 +53,8 @@ import com.google.gson.reflect.TypeToken; /** - * Filesystem {@link N5Reader} implementation with JSON attributes - * parsed with {@link Gson}. + * {@link N5Reader} implementation through {@link KeyValueAccess} with JSON + * attributes parsed with {@link Gson}. * * @author Stephan Saalfeld * @author Igor Pisarev @@ -66,59 +62,6 @@ */ public class N5FSReader implements N5Reader { - /** - * A {@link FileChannel} wrapper that attempts to acquire a lock and waits - * for existing locks to be lifted before returning if the - * {@link FileSystem} supports that. If the {@link FileSystem} does not - * support locking, it returns immediately. - */ - protected static class LockedFileChannel implements Closeable { - - private final FileChannel channel; - - public static LockedFileChannel openForReading(final Path path) throws IOException { - - return new LockedFileChannel(path, true); - } - - public static LockedFileChannel openForWriting(final Path path) throws IOException { - - return new LockedFileChannel(path, false); - } - - private LockedFileChannel(final Path path, final boolean readOnly) throws IOException { - - final OpenOption[] options = readOnly ? new OpenOption[]{StandardOpenOption.READ} : new OpenOption[]{StandardOpenOption.READ, StandardOpenOption.WRITE, StandardOpenOption.CREATE}; - channel = FileChannel.open(path, options); - - for (boolean waiting = true; waiting;) { - waiting = false; - try { - channel.lock(0L, Long.MAX_VALUE, readOnly); - } catch (final OverlappingFileLockException e) { - waiting = true; - try { - Thread.sleep(100); - } catch (final InterruptedException f) { - waiting = false; - Thread.currentThread().interrupt(); - } - } catch (final IOException e) {} - } - } - - public FileChannel getFileChannel() { - - return channel; - } - - @Override - public void close() throws IOException { - - channel.close(); - } - } - /** * Data object for caching meta data. Elements that are null are not yet * cached. @@ -132,7 +75,7 @@ protected static class N5GroupInfo { protected static final N5GroupInfo emptyGroupInfo = new N5GroupInfo(); - protected final FileSystem fileSystem; + protected final KeyValueAccess keyValueAccess; protected final Gson gson; @@ -148,7 +91,7 @@ protected static class N5GroupInfo { * Opens an {@link N5FSReader} at a given base path with a custom * {@link GsonBuilder} to support custom attributes. * - * @param fileSystem + * @param keyValueAccess * @param basePath N5 base path * @param gsonBuilder * @param cacheMeta cache attributes and meta data @@ -164,12 +107,12 @@ protected static class N5GroupInfo { * implementation. */ public N5FSReader( - final FileSystem fileSystem, + final KeyValueAccess keyValueAccess, final String basePath, final GsonBuilder gsonBuilder, final boolean cacheMeta) throws IOException { - this.fileSystem = fileSystem; + this.keyValueAccess = keyValueAccess; this.basePath = basePath; gsonBuilder.registerTypeAdapter(DataType.class, new DataType.JsonAdapter()); gsonBuilder.registerTypeHierarchyAdapter(Compression.class, CompressionAdapter.getJsonAdapter()); diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java b/src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java index ce3fe3cc..d15a4b67 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java @@ -26,32 +26,32 @@ package org.janelia.saalfeldlab.n5; import java.io.IOException; +import java.io.Writer; +import java.lang.reflect.Type; import java.nio.channels.Channels; import java.nio.charset.StandardCharsets; -import java.nio.file.DirectoryNotEmptyException; -import java.nio.file.FileAlreadyExistsException; import java.nio.file.FileSystem; -import java.nio.file.FileSystemException; import java.nio.file.FileSystems; import java.nio.file.Files; -import java.nio.file.NoSuchFileException; import java.nio.file.Path; -import java.nio.file.attribute.FileAttribute; import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; import java.util.Map; import java.util.stream.Stream; +import org.janelia.saalfeldlab.n5.FileSystemKeyValueAccess.LockedFileChannel; + import com.google.gson.GsonBuilder; import com.google.gson.JsonElement; +import com.google.gson.reflect.TypeToken; /** * Filesystem {@link N5Writer} implementation with version compatibility check. * * @author Stephan Saalfeld */ -public class N5FSWriter extends N5FSReader implements N5Writer { +public class N5FSWriter extends N5KeyValueReader implements N5Writer { /** * Opens an {@link N5FSWriter} at a given base path with a custom @@ -189,6 +189,22 @@ public N5FSWriter(final String basePath) throws IOException { this(basePath, new GsonBuilder()); } + /** + * Writes the attributes map to a given {@link Writer}. + * + * @param writer + * @param map + * @throws IOException + */ + protected void writeAttributes( + final Writer writer, + final HashMap map) throws IOException { + + final Type mapType = new TypeToken>(){}.getType(); + gson.toJson(map, mapType, writer); + writer.flush(); + } + /** * Helper method to create and cache a group. * @@ -209,7 +225,7 @@ protected N5GroupInfo createCachedGroup(final String normalPathName) throws IOEx * This avoids synchronizing on the cache for independent * group creation. */ - createDirectories(getGroupPath(normalPathName)); + createDirectories(groupPath(normalPathName)); synchronized (metaCache) { info = getCachedN5GroupInfo(normalPathName); if (info == emptyGroupInfo) { @@ -217,7 +233,7 @@ protected N5GroupInfo createCachedGroup(final String normalPathName) throws IOEx metaCache.put(normalPathName, info); } for (String childPathName = normalPathName; !childPathName.equals("");) { - final Path parent = getGroupPath(childPathName).getParent(); + final Path parent = groupPath(childPathName).getParent(); final String parentPathName = fileSystem.getPath(basePath).relativize(parent).toString(); N5GroupInfo parentInfo = getCachedN5GroupInfo(parentPathName); if (parentInfo == emptyGroupInfo) { @@ -229,7 +245,7 @@ protected N5GroupInfo createCachedGroup(final String normalPathName) throws IOEx if (children != null) { synchronized (children) { children.add( - parent.relativize(getGroupPath(childPathName)).toString()); + parent.relativize(groupPath(childPathName)).toString()); } } childPathName = parentPathName; @@ -250,7 +266,7 @@ public void createGroup(final String pathName) throws IOException { info.isDataset = false; } } else - createDirectories(getGroupPath(normalPathName)); + createDirectories(groupPath(normalPathName)); } @Override @@ -331,7 +347,7 @@ protected N5GroupInfo setCachedAttributes( final HashMap cachedMap = getCachedAttributes(info, normalPathName); synchronized (info) { cachedMap.putAll(attributes); - writeAttributes(getAttributesPath(normalPathName), attributes); + writeAttributes(attributesPath(normalPathName), attributes); info.isDataset = hasCachedDatasetAttributes(cachedMap); } return info; @@ -346,7 +362,7 @@ public void setAttributes( if (cacheMeta) setCachedAttributes(normalPathName, attributes); else - writeAttributes(getAttributesPath(normalPathName), attributes); + writeAttributes(attributesPath(normalPathName), attributes); } @Override @@ -363,33 +379,11 @@ public void writeBlock( } } - protected static void tryDelete(final Path childPath) { - - try { - Files.delete(childPath); - } catch (final DirectoryNotEmptyException e) { - // Even though childPath should be an empty directory, sometimes the deletion fails on network file - // when lock files are not cleared immediately after the leaves have been removed. - try { - // wait and reattempt - Thread.sleep(100); - Files.delete(childPath); - } catch (final InterruptedException ex) { - e.printStackTrace(); - Thread.currentThread().interrupt(); - } catch (final IOException ex) { - ex.printStackTrace(); - } - } catch (final IOException e) { - e.printStackTrace(); - } - } - @Override public boolean remove(final String pathName) throws IOException { final String normalPathName = normalize(pathName); - final Path path = getGroupPath(normalPathName); + final Path path = groupPath(normalPathName); boolean exists = Files.exists(path); if (exists) { final Path base = fileSystem.getPath(basePath); @@ -416,7 +410,7 @@ public boolean remove(final String pathName) throws IOException { } if (cacheMeta) { if (!normalPathName.equals("")) { // not root - final Path parent = getGroupPath(normalPathName).getParent(); + final Path parent = groupPath(normalPathName).getParent(); final N5GroupInfo parentInfo = getCachedN5GroupInfo( fileSystem.getPath(basePath).relativize(parent).toString()); // group must exist final HashSet children = parentInfo.children; @@ -448,124 +442,5 @@ public boolean deleteBlock( return !Files.exists(path); } - /** - * This is a copy of {@link Files#createDirectories(Path, FileAttribute...)} - * that follows symlinks. - * - * Workaround for https://bugs.openjdk.java.net/browse/JDK-8130464 - * - * Creates a directory by creating all nonexistent parent directories first. - * Unlike the {@link #createDirectory createDirectory} method, an exception - * is not thrown if the directory could not be created because it already - * exists. - * - *

The {@code attrs} parameter is optional {@link FileAttribute - * file-attributes} to set atomically when creating the nonexistent - * directories. Each file attribute is identified by its {@link - * FileAttribute#name name}. If more than one attribute of the same name is - * included in the array then all but the last occurrence is ignored. - * - *

If this method fails, then it may do so after creating some, but not - * all, of the parent directories. - * - * @param dir - * the directory to create - * - * @param attrs - * an optional list of file attributes to set atomically when - * creating the directory - * - * @return the directory - * - * @throws UnsupportedOperationException - * if the array contains an attribute that cannot be set atomically - * when creating the directory - * @throws FileAlreadyExistsException - * if {@code dir} exists but is not a directory (optional specific - * exception) - * @throws IOException - * if an I/O error occurs - * @throws SecurityException - * in the case of the default provider, and a security manager is - * installed, the {@link SecurityManager#checkWrite(String) checkWrite} - * method is invoked prior to attempting to create a directory and - * its {@link SecurityManager#checkRead(String) checkRead} is - * invoked for each parent directory that is checked. If {@code - * dir} is not an absolute path then its {@link Path#toAbsolutePath - * toAbsolutePath} may need to be invoked to get its absolute path. - * This may invoke the security manager's {@link - * SecurityManager#checkPropertyAccess(String) checkPropertyAccess} - * method to check access to the system property {@code user.dir} - */ - private static Path createDirectories(Path dir, final FileAttribute... attrs) - throws IOException - { - // attempt to create the directory - try { - createAndCheckIsDirectory(dir, attrs); - return dir; - } catch (final FileAlreadyExistsException x) { - // file exists and is not a directory - throw x; - } catch (final IOException x) { - // parent may not exist or other reason - } - SecurityException se = null; - try { - dir = dir.toAbsolutePath(); - } catch (final SecurityException x) { - // don't have permission to get absolute path - se = x; - } - // find a decendent that exists - Path parent = dir.getParent(); - while (parent != null) { - try { - parent.getFileSystem().provider().checkAccess(parent); - break; - } catch (final NoSuchFileException x) { - // does not exist - } - parent = parent.getParent(); - } - if (parent == null) { - // unable to find existing parent - if (se == null) { - throw new FileSystemException(dir.toString(), null, - "Unable to determine if root directory exists"); - } else { - throw se; - } - } - - // create directories - Path child = parent; - for (final Path name: parent.relativize(dir)) { - child = child.resolve(name); - createAndCheckIsDirectory(child, attrs); - } - return dir; - } - /** - * This is a copy of - * {@link Files#createAndCheckIsDirectory(Path, FileAttribute...)} that - * follows symlinks. - * - * Workaround for https://bugs.openjdk.java.net/browse/JDK-8130464 - * - * Used by createDirectories to attempt to create a directory. A no-op if - * the directory already exists. - */ - private static void createAndCheckIsDirectory( - final Path dir, - final FileAttribute... attrs) throws IOException { - - try { - Files.createDirectory(dir, attrs); - } catch (final FileAlreadyExistsException x) { - if (!Files.isDirectory(dir)) - throw x; - } - } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java new file mode 100644 index 00000000..a3d94d2b --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java @@ -0,0 +1,751 @@ +/** + * Copyright (c) 2017--2021, Stephan Saalfeld + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +package org.janelia.saalfeldlab.n5; + +import java.io.IOException; +import java.io.Reader; +import java.lang.reflect.Array; +import java.lang.reflect.Type; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonPrimitive; +import com.google.gson.reflect.TypeToken; + +/** + * {@link N5Reader} implementation through {@link KeyValueAccess} with JSON + * attributes parsed with {@link Gson}. + * + * @author Stephan Saalfeld + * @author Igor Pisarev + * @author Philipp Hanslovsky + */ +public class N5KeyValueReader implements N5Reader { + + /** + * Data object for caching meta data. Elements that are null are not yet + * cached. + */ + protected static class N5GroupInfo { + + public HashSet children = null; + public HashMap attributesCache = null; + public Boolean isDataset = null; + } + + protected static final N5GroupInfo emptyGroupInfo = new N5GroupInfo(); + + protected final KeyValueAccess keyValueAccess; + + protected final Gson gson; + + protected final HashMap metaCache = new HashMap<>(); + + protected final boolean cacheMeta; + + protected static final String jsonFile = "attributes.json"; + + protected final String basePath; + + /** + * Opens an {@link N5KeyValueReader} at a given base path with a custom + * {@link GsonBuilder} to support custom attributes. + * + * @param keyValueAccess + * @param basePath N5 base path + * @param gsonBuilder + * @param cacheMeta cache attributes and meta data + * Setting this to true avoids frequent reading and parsing of JSON + * encoded attributes and other meta data that requires accessing the + * store. This is most interesting for high latency backends. Changes + * of cached attributes and meta data by an independent writer will + * not be tracked. + * + * @throws IOException + * if the base path cannot be read or does not exist, + * if the N5 version of the container is not compatible with this + * implementation. + */ + public N5KeyValueReader( + final KeyValueAccess keyValueAccess, + final String basePath, + final GsonBuilder gsonBuilder, + final boolean cacheMeta) throws IOException { + + this.keyValueAccess = keyValueAccess; + this.basePath = keyValueAccess.normalize(basePath); + gsonBuilder.registerTypeAdapter(DataType.class, new DataType.JsonAdapter()); + gsonBuilder.registerTypeHierarchyAdapter(Compression.class, CompressionAdapter.getJsonAdapter()); + gsonBuilder.disableHtmlEscaping(); + this.gson = gsonBuilder.create(); + this.cacheMeta = cacheMeta; + if (exists("/")) { + final Version version = getVersion(); + if (!VERSION.isCompatible(version)) + throw new IOException("Incompatible version " + version + " (this is " + VERSION + ")."); + } + } + + public Gson getGson() { + + return gson; + } + + /** + * + * @return N5 base path + */ + public String getBasePath() { + + return this.basePath; + } + + + /** + * Parses an attribute from the given attributes map. + * + * @param map + * @param key + * @param clazz + * @return + * @throws IOException + */ + protected T parseAttribute( + final HashMap map, + final String key, + final Class clazz) throws IOException { + + final JsonElement attribute = map.get(key); + if (attribute != null) + return gson.fromJson(attribute, clazz); + else + return null; + } + + /** + * Parses an attribute from the given attributes map. + * + * @param map + * @param key + * @param type + * @return + * @throws IOException + */ + protected T parseAttribute( + final HashMap map, + final String key, + final Type type) throws IOException { + + final JsonElement attribute = map.get(key); + if (attribute != null) + return gson.fromJson(attribute, type); + else + return null; + } + + /** + * Reads the attributes map from a given {@link Reader}. + * + * @param reader + * @return + * @throws IOException + */ + protected HashMap readAttributes(final Reader reader) throws IOException { + + final Type mapType = new TypeToken>(){}.getType(); + final HashMap map = gson.fromJson(reader, mapType); + return map == null ? new HashMap<>() : map; + } + + /** + * Inserts new the JSON export of attributes into the given attributes map. + * + * @param map + * @param attributes + * @param gson + * @throws IOException + */ + protected void insertAttributes( + final HashMap map, + final Map attributes) throws IOException { + + for (final Entry entry : attributes.entrySet()) + map.put(entry.getKey(), gson.toJsonTree(entry.getValue())); + } + + /** + * Return a reasonable class for a {@link JsonPrimitive}. Possible return + * types are + *

    + *
  • boolean
    • + *
  • double
    • + *
  • long (if the number is an integer)
    • + *
  • String
    • + *
  • Object
    • + *
+ * + * @param jsonPrimitive + * @return + */ + protected static Class classForJsonPrimitive(final JsonPrimitive jsonPrimitive) { + + if (jsonPrimitive.isBoolean()) + return boolean.class; + else if (jsonPrimitive.isNumber()) { + final Number number = jsonPrimitive.getAsNumber(); + if (number.longValue() == number.doubleValue()) + return long.class; + else + return double.class; + } else if (jsonPrimitive.isString()) + return String.class; + else return Object.class; + } + + /** + * Best effort implementation of {@link N5Reader#listAttributes(String)} + * with limited type resolution fromJSON. Possible return types are + *
    + *
  • null
    • + *
  • boolean
    • + *
  • double
    • + *
  • long (if the number is an integer)
    • + *
  • String
    • + *
  • Object
    • + *
  • boolean[]
    • + *
  • double[]
    • + *
  • long[] (if all numbers in the array are integers)
    • + *
  • String[]
    • + *
  • Object[]
    • + *
+ */ + @Override + public Map> listAttributes(final String pathName) throws IOException { + + final HashMap jsonElementMap = getAttributes(pathName); + final HashMap> attributes = new HashMap<>(); + jsonElementMap.forEach( + (key, jsonElement) -> { + final Class clazz; + if (jsonElement.isJsonNull()) + clazz = null; + else if (jsonElement.isJsonPrimitive()) + clazz = classForJsonPrimitive((JsonPrimitive)jsonElement); + else if (jsonElement.isJsonArray()) { + final JsonArray jsonArray = (JsonArray)jsonElement; + Class arrayElementClass = Object.class; + if (jsonArray.size() > 0) { + final JsonElement firstElement = jsonArray.get(0); + if (firstElement.isJsonPrimitive()) { + arrayElementClass = classForJsonPrimitive(firstElement.getAsJsonPrimitive()); + for (int i = 1; i < jsonArray.size() && arrayElementClass != Object.class; ++i) { + final JsonElement element = jsonArray.get(i); + if (element.isJsonPrimitive()) { + final Class nextArrayElementClass = classForJsonPrimitive(element.getAsJsonPrimitive()); + if (nextArrayElementClass != arrayElementClass) + if (nextArrayElementClass == double.class && arrayElementClass == long.class) + arrayElementClass = double.class; + else { + arrayElementClass = Object.class; + break; + } + } else { + arrayElementClass = Object.class; + break; + } + } + } + clazz = Array.newInstance(arrayElementClass, 0).getClass(); + } else + clazz = Object[].class; + } + else + clazz = Object.class; + attributes.put(key, clazz); + }); + return attributes; + } + + @Override + public DatasetAttributes getDatasetAttributes(final String pathName) throws IOException { + + final long[] dimensions; + final DataType dataType; + int[] blockSize; + Compression compression; + final String compressionVersion0Name; + final String normalPathName = normalize(pathName); + if (cacheMeta) { + final N5GroupInfo info = getCachedN5GroupInfo(normalPathName); + if (info == emptyGroupInfo) + return null; + + final HashMap cachedMap; + if (info.isDataset == null) { + + synchronized (info) { + + cachedMap = getCachedAttributes(info, normalPathName); + if (cachedMap.isEmpty()) { + info.isDataset = false; + return null; + } + + dimensions = getAttribute(cachedMap, DatasetAttributes.dimensionsKey, long[].class); + if (dimensions == null) { + info.isDataset = false; + return null; + } + + dataType = getAttribute(cachedMap, DatasetAttributes.dataTypeKey, DataType.class); + if (dataType == null) { + info.isDataset = false; + return null; + } + + info.isDataset = true; + } + } else if (!info.isDataset) { + return null; + } else { + + cachedMap = getCachedAttributes(info, normalPathName); + dimensions = getAttribute(cachedMap, DatasetAttributes.dimensionsKey, long[].class); + dataType = getAttribute(cachedMap, DatasetAttributes.dataTypeKey, DataType.class); + } + + blockSize = getAttribute(cachedMap, DatasetAttributes.blockSizeKey, int[].class); + + compression = getAttribute(cachedMap, DatasetAttributes.compressionKey, Compression.class); + + /* version 0 */ + compressionVersion0Name = compression + == null + ? getAttribute(cachedMap, DatasetAttributes.compressionTypeKey, String.class) + : null; + + + } else { + final HashMap map = getAttributes(normalPathName); + + dimensions = parseAttribute(map, DatasetAttributes.dimensionsKey, long[].class); + if (dimensions == null) + return null; + + dataType = parseAttribute(map, DatasetAttributes.dataTypeKey, DataType.class); + if (dataType == null) + return null; + + blockSize = parseAttribute(map, DatasetAttributes.blockSizeKey, int[].class); + + compression = parseAttribute(map, DatasetAttributes.compressionKey, Compression.class); + + /* version 0 */ + compressionVersion0Name = compression == null + ? parseAttribute(map, DatasetAttributes.compressionTypeKey, String.class) + : null; + } + + if (blockSize == null) + blockSize = Arrays.stream(dimensions).mapToInt(a -> (int)a).toArray(); + + /* version 0 */ + if (compression == null) { + switch (compressionVersion0Name) { + case "raw": + compression = new RawCompression(); + break; + case "gzip": + compression = new GzipCompression(); + break; + case "bzip2": + compression = new Bzip2Compression(); + break; + case "lz4": + compression = new Lz4Compression(); + break; + case "xz": + compression = new XzCompression(); + break; + } + } + + return new DatasetAttributes(dimensions, blockSize, dataType, compression); + } + + protected HashMap readAttributes(final String absoluteNormalPath) throws IOException { + + if (!keyValueAccess.exists(absoluteNormalPath)) + return new HashMap<>(); + + try (final LockedChannel lockedChannel = keyValueAccess.lockForReading(absoluteNormalPath)) { + return readAttributes(lockedChannel.newReader()); + } + } + + /** + * Get and cache attributes for a group identified by an info object and a + * pathName. + * + * This helper method does not intelligently handle the case that the group + * does not exist (as indicated by info == emptyGroupInfo) which should be + * done in calling code. + * + * @param info + * @param pathName normalized group path without leading slash + * @return cached attributes + * empty map if the group exists but not attributes are set + * null if the group does not exist + * @throws IOException + */ + protected HashMap getCachedAttributes( + final N5GroupInfo info, + final String normalPath) throws IOException { + + HashMap cachedMap = info.attributesCache; + if (cachedMap == null) { + synchronized (info) { + cachedMap = info.attributesCache; + if (cachedMap == null) { + final String absoluteNormalPath = attributesPath(normalPath); + cachedMap = new HashMap<>(); + final HashMap map = readAttributes(absoluteNormalPath); + cachedMap.putAll(map); + info.attributesCache = cachedMap; + } + } + } + return cachedMap; + } + + @SuppressWarnings("unchecked") + protected T getAttribute( + final HashMap cachedMap, + final String key, + final Class clazz) { + + final Object cachedAttribute = cachedMap.get(key); + if (cachedAttribute == null) + return null; + else if (cachedAttribute instanceof JsonElement) { + final T attribute = gson.fromJson((JsonElement)cachedAttribute, clazz); + synchronized (cachedMap) { + cachedMap.put(key, attribute); + } + return attribute; + } else { + return (T)cachedAttribute; + } + } + + /** + * Helper method that returns or JSON decodes a cached attribute. + * + * @param + * @param cachedMap + * @param key + * @param type + * @return + */ + @SuppressWarnings("unchecked") + protected T getAttribute( + final HashMap cachedMap, + final String key, + final Type type) { + + final Object cachedAttribute = cachedMap.get(key); + if (cachedAttribute == null) + return null; + else if (cachedAttribute instanceof JsonElement) { + final T attribute = gson.fromJson((JsonElement)cachedAttribute, type); + synchronized (cachedMap) { + cachedMap.put(key, attribute); + } + return attribute; + } else { + return (T)cachedAttribute; + } + } + + @Override + public T getAttribute( + final String pathName, + final String key, + final Class clazz) throws IOException { + + final String normalPathName = normalize(pathName); + if (cacheMeta) { + final N5GroupInfo info = getCachedN5GroupInfo(normalPathName); + if (info == emptyGroupInfo) + return null; + final HashMap cachedMap = getCachedAttributes(info, normalPathName); + if (cachedMap.isEmpty()) + return null; + return getAttribute(cachedMap, key, clazz); + } else { + final HashMap map = getAttributes(normalPathName); + return parseAttribute(map, key, clazz); + } + } + + @Override + public T getAttribute( + final String pathName, + final String key, + final Type type) throws IOException { + + final String normalPathName = normalize(pathName); + if (cacheMeta) { + final N5GroupInfo info = getCachedN5GroupInfo(normalPathName); + if (info == emptyGroupInfo) + return null; + final HashMap cachedMap = getCachedAttributes(info, normalPathName); + if (cachedMap.isEmpty()) + return null; + return getAttribute(cachedMap, key, type); + } else { + final HashMap map = getAttributes(normalPathName); + return parseAttribute(map, key, type); + } + } + + protected boolean groupExists(final String absoluteNormalPath) { + + return keyValueAccess.exists(absoluteNormalPath) && keyValueAccess.isDirectory(absoluteNormalPath); + } + + /** + * Get an existing cached N5 group info object or create it. + * + * @param normalPathName normalized group path without leading slash + * @return + */ + protected N5GroupInfo getCachedN5GroupInfo(final String normalPathName) { + + N5GroupInfo info = metaCache.get(normalPathName); + if (info == null) { + + /* I do not have a better solution yet to allow parallel + * exists checks for independent paths than to accept the + * same exists check to potentially run multiple times. + */ + final boolean exists = groupExists(groupPath(normalPathName)); + + synchronized (metaCache) { + info = metaCache.get(normalPathName); + if (info == null) { + info = exists ? new N5GroupInfo() : emptyGroupInfo; + metaCache.put(normalPathName, info); + } + } + } + return info; + } + + @Override + public boolean exists(final String pathName) { + + final String normalPathName = normalize(pathName); + if (cacheMeta) + return getCachedN5GroupInfo(normalPathName) != emptyGroupInfo; + else + return groupExists(groupPath(normalPathName)); + } + + @Override + public boolean datasetExists(final String pathName) throws IOException { + + if (cacheMeta) { + final String normalPathName = normalize(pathName); + final N5GroupInfo info = getCachedN5GroupInfo(normalPathName); + if (info == emptyGroupInfo) + return false; + if (info.isDataset == null) { + synchronized (info) { + if (info.isDataset == null ) { + + } + else + return info.isDataset; + } + } else + return info.isDataset; + } + return exists(pathName) && getDatasetAttributes(pathName) != null; + } + + /** + * Reads or creates the attributes map of a group or dataset. + * + * @param pathName group path + * @return + * @throws IOException + */ + public HashMap getAttributes(final String pathName) throws IOException { + + final String path = attributesPath(normalize(pathName)); + if (exists(pathName) && !keyValueAccess.exists(path)) + return new HashMap<>(); + + try (final LockedChannel lockedChannel = keyValueAccess.lockForReading(path)) { + return readAttributes(lockedChannel.newReader()); + } + } + + @Override + public DataBlock readBlock( + final String pathName, + final DatasetAttributes datasetAttributes, + final long... gridPosition) throws IOException { + + final String path = getDataBlockPath(normalize(pathName), gridPosition); + if (!keyValueAccess.exists(path)) + return null; + + try (final LockedChannel lockedChannel = keyValueAccess.lockForReading(path)) { + return DefaultBlockReader.readBlock(lockedChannel.newInputStream(), datasetAttributes, gridPosition); + } + } + + /** + * + * @param normalPath normalized path name + * @return + * @throws IOException + */ + protected String[] normalList(final String normalPath) throws IOException { + + return keyValueAccess.listDirectories(groupPath(normalPath)); + } + + @Override + public String[] list(final String pathName) throws IOException { + + if (cacheMeta) { + final N5GroupInfo info = getCachedN5GroupInfo(normalize(pathName)); + if (info == emptyGroupInfo) + throw new IOException("Group '" + pathName +"' does not exist."); + else { + Set children = info.children; + final String[] list; + if (children == null) { + synchronized (info) { + children = info.children; + if (children == null) { + list = normalList(normalize(pathName)); + info.children = new HashSet<>(Arrays.asList(list)); + } else + list = children.toArray(new String[children.size()]); + } + } else + list = children.toArray(new String[children.size()]); + + return list; + } + } else { + return normalList(normalize(pathName)); + } + } + + /** + * Constructs the path for a data block in a dataset at a given grid position. + * + * The returned path is + * 
+	 * $basePath/datasetPathName/$gridPosition[0]/$gridPosition[1]/.../$gridPosition[n]
+	 *
+ * + * This is the file into which the data block will be stored. + * + * @param normalDatasetPathName normalized dataset path without leading slash + * @param gridPosition + * @return + */ + protected String getDataBlockPath( + final String normalDatasetPath, + final long... gridPosition) { + + final StringBuilder builder = new StringBuilder(groupPath(normalDatasetPath)); + + for (final long i : gridPosition) { + builder.append("/"); + builder.append(i); + } + + return builder.toString(); + } + + /** + * Constructs the path for the group or dataset. + * + * @param normalPath normalized group path without leading slash + * @return + */ + protected String groupPath(final String normalPath) { + + return basePath + "/" + normalPath; + } + + /** + * Constructs the path for the attributes file of a group or dataset. + * + * @param normalPath normalized group path without leading slash + * @return + */ + protected String attributesPath(final String normalPath) { + + return groupPath(normalPath) + "/" + jsonFile; + } + + /** + * Removes the leading slash from a given path and returns the normalized + * path. It ensures correctness on both Unix and Windows, otherwise + * {@code pathName} is treated as UNC path on Windows, and + * {@code Paths.get(pathName, ...)} fails with + * {@code InvalidPathException}. + * + * @param path + * @return + */ + protected String normalize(final String path) { + + return keyValueAccess.normalize(path.startsWith("/") || path.startsWith("\\") ? path.substring(1) : path); + } + + @Override + public String toString() { + + return String.format("%s[access=%s, basePath=%s]", getClass().getSimpleName(), keyValueAccess, basePath); + } +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriter.java b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriter.java new file mode 100644 index 00000000..9f526402 --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriter.java @@ -0,0 +1,344 @@ +/** + * Copyright (c) 2017--2021, Stephan Saalfeld + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +package org.janelia.saalfeldlab.n5; + +import java.io.IOException; +import java.io.Writer; +import java.lang.reflect.Type; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.stream.Stream; + +import org.janelia.saalfeldlab.n5.FileSystemKeyValueAccess.LockedFileChannel; + +import com.google.gson.GsonBuilder; +import com.google.gson.JsonElement; +import com.google.gson.reflect.TypeToken; + +/** + * Filesystem {@link N5Writer} implementation with version compatibility check. + * + * @author Stephan Saalfeld + */ +public class N5KeyValueWriter extends N5KeyValueReader implements N5Writer { + + /** + * Opens an {@link N5KeyValueWriter} at a given base path with a custom + * {@link GsonBuilder} to support custom attributes. + * + * If the base path does not exist, it will be created. + * + * If the base path exists and if the N5 version of the container is + * compatible with this implementation, the N5 version of this container + * will be set to the current N5 version of this implementation. + * + * @param fileSystem + * @param basePath n5 base path + * @param gsonBuilder + * @param cacheAttributes cache attributes + * Setting this to true avoids frequent reading and parsing of JSON + * encoded attributes, this is most interesting for high latency file + * systems. Changes of attributes by an independent writer will not be + * tracked. + * + * @throws IOException + * if the base path cannot be written to or cannot be created, + * if the N5 version of the container is not compatible with this + * implementation. + */ + public N5KeyValueWriter( + final KeyValueAccess keyValueAccess, + final String basePath, + final GsonBuilder gsonBuilder, + final boolean cacheAttributes) throws IOException { + + super(keyValueAccess, basePath, gsonBuilder, cacheAttributes); + createGroup("/"); + if (!VERSION.equals(getVersion())) + setAttribute("/", VERSION_KEY, VERSION.toString()); + } + + /** + * Writes the attributes map to a given {@link Writer}. + * + * @param writer + * @param map + * @throws IOException + */ + protected void writeAttributes( + final Writer writer, + final HashMap map) throws IOException { + + final Type mapType = new TypeToken>(){}.getType(); + gson.toJson(map, mapType, writer); + writer.flush(); + } + + /** + * Helper method to create and cache a group. + * + * @param normalPathName normalized group path without leading slash + * @return + * @throws IOException + */ + protected N5GroupInfo createCachedGroup(final String normalPathName) throws IOException { + + N5GroupInfo info = getCachedN5GroupInfo(normalPathName); + if (info == emptyGroupInfo) { + + /* The directories may be created multiple times concurrently, + * but a new cache entry is inserted only if none has been + * inserted in the meantime (because that may already include + * more cached data). + * + * This avoids synchronizing on the cache for independent + * group creation. + */ + keyValueAccess.createDirectories(groupPath(normalPathName)); + synchronized (metaCache) { + info = getCachedN5GroupInfo(normalPathName); + if (info == emptyGroupInfo) { + info = new N5GroupInfo(); + metaCache.put(normalPathName, info); + } + for (String childPathName = normalPathName; !childPathName.equals("");) { + final String parentPathName = keyValueAccess.parent(childPathName); + N5GroupInfo parentInfo = getCachedN5GroupInfo(parentPathName); + if (parentInfo == emptyGroupInfo) { + parentInfo = new N5GroupInfo(); + parentInfo.isDataset = false; + metaCache.put(parentPathName, parentInfo); + } + final HashSet children = parentInfo.children; + if (children != null) { + synchronized (children) { + children.add( + keyValueAccess.relativize(childPathName, parentPathName)); + } + } + childPathName = parentPathName; + } + } + } + return info; + } + + @Override + public void createGroup(final String pathName) throws IOException { + + final String normalPathName = normalize(pathName); + if (cacheMeta) { + final N5GroupInfo info = createCachedGroup(normalPathName); + synchronized (info) { + if (info.isDataset == null) + info.isDataset = false; + } + } else + keyValueAccess.createDirectories(groupPath(normalPathName)); + } + + @Override + public void createDataset( + final String pathName, + final DatasetAttributes datasetAttributes) throws IOException { + + final String normalPathName = normalize(pathName); + if (cacheMeta) { + final N5GroupInfo info = createCachedGroup(normalPathName); + synchronized (info) { + setDatasetAttributes(normalPathName, datasetAttributes); + info.isDataset = true; + } + } else { + createGroup(pathName); + setDatasetAttributes(normalPathName, datasetAttributes); + } + } + + /** + * Helper method that reads the existing map of attributes, JSON encodes, + * inserts and overrides the provided attributes, and writes them back into + * the attributes store. + * + * @param path + * @param attributes + * @throws IOException + */ + protected void writeAttributes( + final String path, + final Map attributes) throws IOException { + + final HashMap map = new HashMap<>(); + + try (final LockedChannel lock = keyValueAccess.lockForWriting(path)) { + map.putAll(readAttributes(lock.newReader())); + insertAttributes(map, attributes); + writeAttributes(lock.newWriter(), map); + } + } + + /** + * Check for attributes that are required for a group to be a dataset. + * + * @param cachedAttributes + * @return + */ + protected static boolean hasCachedDatasetAttributes(final Map cachedAttributes) { + + return cachedAttributes.keySet().contains(DatasetAttributes.dimensionsKey) && cachedAttributes.keySet().contains(DatasetAttributes.dataTypeKey); + } + + + /** + * Helper method to cache and write attributes. + * + * @param normalPathName normalized group path without leading slash + * @param attributes + * @param isDataset + * @return + * @throws IOException + */ + protected N5GroupInfo setCachedAttributes( + final String normalPathName, + final Map attributes) throws IOException { + + N5GroupInfo info = getCachedN5GroupInfo(normalPathName); + if (info == emptyGroupInfo) { + synchronized (metaCache) { + info = getCachedN5GroupInfo(normalPathName); + if (info == emptyGroupInfo) + throw new IOException("N5 group '" + normalPathName + "' does not exist. Cannot set attributes."); + } + } + final HashMap cachedMap = getCachedAttributes(info, normalPathName); + synchronized (info) { + cachedMap.putAll(attributes); + writeAttributes(attributesPath(normalPathName), attributes); + info.isDataset = hasCachedDatasetAttributes(cachedMap); + } + return info; + } + + @Override + public void setAttributes( + final String pathName, + final Map attributes) throws IOException { + + final String normalPathName = normalize(pathName); + if (cacheMeta) + setCachedAttributes(normalPathName, attributes); + else + writeAttributes(attributesPath(normalPathName), attributes); + } + + @Override + public void writeBlock( + final String pathName, + final DatasetAttributes datasetAttributes, + final DataBlock dataBlock) throws IOException { + + final String path = getDataBlockPath(normalize(pathName), dataBlock.getGridPosition()); + keyValueAccess.createDirectories(keyValueAccess.parent(path)); + try (final LockedChannel lock = keyValueAccess.lockForWriting(path)) { + + DefaultBlockWriter.writeBlock(lock.newOutputStream(), datasetAttributes, dataBlock); + } + } + + @Override + public boolean remove(final String pathName) throws IOException { + + final String normalPathName = normalize(pathName); + final String path = groupPath(normalPathName); + boolean exists = keyValueAccess.exists(path); + if (exists) { + + keyValueAccess.delete(normalPathName); + + + + final Path base = fileSystem.getPath(basePath); + try (final Stream pathStream = Files.walk(path)) { + pathStream.sorted(Comparator.reverseOrder()).forEach( + childPath -> { + if (Files.isRegularFile(childPath)) { + try (final LockedFileChannel channel = LockedFileChannel.openForWriting(childPath)) { + Files.delete(childPath); + } catch (final IOException e) { + e.printStackTrace(); + } + } else { + if (cacheMeta) { + synchronized (metaCache) { + metaCache.put(base.relativize(childPath).toString(), emptyGroupInfo); + tryDelete(childPath); + } + } else { + tryDelete(childPath); + } + } + }); + } + if (cacheMeta) { + if (!normalPathName.equals("")) { // not root + final Path parent = groupPath(normalPathName).getParent(); + final N5GroupInfo parentInfo = getCachedN5GroupInfo( + fileSystem.getPath(basePath).relativize(parent).toString()); // group must exist + final HashSet children = parentInfo.children; + if (children != null) { + synchronized (children) { + exists = Files.exists(path); + if (exists) + children.remove( + parent.relativize(fileSystem.getPath(normalPathName)).toString()); + } + } + } + } else + exists = Files.exists(path); + } + return !exists; + } + + @Override + public boolean deleteBlock( + final String pathName, + final long... gridPosition) throws IOException { + + final Path path = getDataBlockPath(normalize(pathName), gridPosition); + if (Files.exists(path)) + try (final LockedFileChannel channel = LockedFileChannel.openForWriting(path)) { + Files.deleteIfExists(path); + } + return !Files.exists(path); + } + + +} diff --git a/src/test/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccessTest.java b/src/test/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccessTest.java new file mode 100644 index 00000000..a95f15b7 --- /dev/null +++ b/src/test/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccessTest.java @@ -0,0 +1,69 @@ +/** + * + */ +package org.janelia.saalfeldlab.n5; + +import static org.junit.Assert.assertArrayEquals; + +import java.nio.file.FileSystems; +import java.util.Arrays; + +import org.junit.BeforeClass; +import org.junit.Test; + + +/** + * @author Stephan Saalfeld <saalfelds@janelia.hhmi.org> + * + */ +public class FileSystemKeyValueAccessTest { + + private static String[] testPaths = new String[] { + "/test/path/file", + "test/path/file", + "/test/path/file/", + "test/path/file/", + "/file", + "file", + "/file/", + "file/", + "/", + "" +// "", +// Paths.get("C:", "test", "path", "file").toString() + }; + + private static String[][] testPathComponents = new String[][] { + {"/", "test", "path", "file"}, + {"test", "path", "file"}, + {"/", "test", "path", "file"}, + {"test", "path", "file"}, + {"/", "file"}, + {"file"}, + {"/", "file"}, + {"file"}, + {"/"}, + {""} +// {""}, +// {"C:", "test", "path", "file"} + }; + + /** + * @throws java.lang.Exception + */ + @BeforeClass + public static void setUpBeforeClass() throws Exception {} + + @Test + public void testComponents() { + + final FileSystemKeyValueAccess access = new FileSystemKeyValueAccess(FileSystems.getDefault()); + + for (int i = 0; i < testPaths.length; ++i) { + + System.out.println(String.format("%d: %s -> %s", i, testPaths[i], Arrays.toString(access.components(testPaths[i])))); + + assertArrayEquals(testPathComponents[i], access.components(testPaths[i])); + } + } +} diff --git a/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java b/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java index c2e4f2bf..98aa4e3e 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java @@ -28,7 +28,19 @@ import static org.junit.Assert.fail; import java.io.IOException; +import java.nio.channels.Channels; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import org.janelia.saalfeldlab.n5.N5KeyValueReader.LockedFileChannel; import org.junit.Test; /** @@ -51,10 +63,162 @@ protected N5Writer createN5Writer() throws IOException { } @Test + public void testReadLock() throws IOException, InterruptedException { + + final Path path = Paths.get(testDirPath, "lock"); + try { + Files.delete(path); + } catch (final IOException e) {} + + LockedFileChannel lockedFileChannel = LockedFileChannel.openForWriting(path); + lockedFileChannel.close(); + lockedFileChannel = LockedFileChannel.openForReading(path); + System.out.println("locked"); + + final ExecutorService exec = Executors.newSingleThreadExecutor(); + final Future future = exec.submit(() -> { + LockedFileChannel.openForWriting(path).close(); + return null; + }); + + try { + System.out.println("Trying to acquire locked readable channel..."); + System.out.println(future.get(3, TimeUnit.SECONDS)); + fail("Lock broken!"); + } catch (final TimeoutException e) { + System.out.println("Lock held!"); + future.cancel(true); + } catch (final InterruptedException | ExecutionException e) { + future.cancel(true); + System.out.println("Test was interrupted!"); + } finally { + lockedFileChannel.close(); + Files.delete(path); + } + + exec.shutdownNow(); + } + + + @Test + public void testWriteLock() throws IOException { + + final Path path = Paths.get(testDirPath, "lock"); + try { + Files.delete(path); + } catch (final IOException e) {} + + final LockedFileChannel lockedFileChannel = LockedFileChannel.openForWriting(path); + System.out.println("locked"); + + final ExecutorService exec = Executors.newSingleThreadExecutor(); + final Future future = exec.submit(() -> { + LockedFileChannel.openForReading(path).close(); + return null; + }); + + try { + System.out.println("Trying to acquire locked writable channel..."); + System.out.println(future.get(3, TimeUnit.SECONDS)); + fail("Lock broken!"); + } catch (final TimeoutException e) { + System.out.println("Lock held!"); + future.cancel(true); + } catch (final InterruptedException | ExecutionException e) { + future.cancel(true); + System.out.println("Test was interrupted!"); + } finally { + lockedFileChannel.close(); + Files.delete(path); + } + + exec.shutdownNow(); + } + + @Test + public void testLockReleaseByReader() throws IOException { + + final Path path = Paths.get(testDirPath, "lock"); + try { + Files.delete(path); + } catch (final IOException e) {} + + final LockedFileChannel lockedFileChannel = LockedFileChannel.openForWriting(path); + System.out.println("locked"); + + Channels.newReader(lockedFileChannel.getFileChannel(), StandardCharsets.UTF_8.name()).close(); + System.out.println("reader released"); + + + final ExecutorService exec = Executors.newSingleThreadExecutor(); + final Future future = exec.submit(() -> { + LockedFileChannel.openForWriting(path).close(); + return null; + }); + + try { + System.out.println("Trying to acquire locked readable channel..."); + future.get(3, TimeUnit.SECONDS); + } catch (final TimeoutException e) { + fail("Lock not released!"); + future.cancel(true); + } catch (final InterruptedException | ExecutionException e) { + future.cancel(true); + System.out.println("Test was interrupted!"); + } finally { + lockedFileChannel.close(); + Files.delete(path); + } + + System.out.println("Lock was successfully released."); + + exec.shutdownNow(); + } + + @Test + public void testLockReleaseByInputStream() throws IOException { + + final Path path = Paths.get(testDirPath, "lock"); + try { + Files.delete(path); + } catch (final IOException e) {} + + final LockedFileChannel lockedFileChannel = LockedFileChannel.openForWriting(path); + System.out.println("locked"); + + Channels.newInputStream(lockedFileChannel.getFileChannel()).close(); + System.out.println("input stream released"); + + final ExecutorService exec = Executors.newSingleThreadExecutor(); + final Future future = exec.submit(() -> { + LockedFileChannel.openForWriting(path).close(); + return null; + }); + + try { + System.out.println("Trying to acquire locked readable channel..."); + future.get(3, TimeUnit.SECONDS); + } catch (final TimeoutException e) { + fail("Lock not released!"); + future.cancel(true); + } catch (final InterruptedException | ExecutionException e) { + future.cancel(true); + System.out.println("Test was interrupted!"); + } finally { + lockedFileChannel.close(); + Files.delete(path); + } + + System.out.println("Lock was successfully released."); + + exec.shutdownNow(); + } + +// @Test public void testCache() { final N5Writer n5Writer = n5; - try { + try { n5 = new N5FSWriter(testDirPath + "-cache", true); testAttributes(); From 1c4ed55d7ca6a425254bc8f742b37f6d5c3f4240 Mon Sep 17 00:00:00 2001 From: Stephan Saalfeld Date: Fri, 30 Jul 2021 14:08:31 -0400 Subject: [PATCH 023/243] WIP key value access interface (and file system implementation) --- .../n5/FileSystemKeyValueAccess.java | 47 +- .../saalfeldlab/n5/GzipCompression.java | 2 - .../saalfeldlab/n5/KeyValueAccess.java | 3 +- .../janelia/saalfeldlab/n5/N5FSReader.java | 757 +----------------- .../janelia/saalfeldlab/n5/N5FSWriter.java | 333 +------- .../saalfeldlab/n5/N5KeyValueReader.java | 10 +- .../saalfeldlab/n5/N5KeyValueWriter.java | 163 ++-- .../org/janelia/saalfeldlab/n5/N5FSTest.java | 43 +- 8 files changed, 149 insertions(+), 1209 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java index 4702c79a..40a89e60 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java @@ -45,6 +45,7 @@ import java.nio.file.StandardOpenOption; import java.nio.file.attribute.FileAttribute; import java.util.Comparator; +import java.util.Iterator; import java.util.stream.Stream; /** @@ -256,41 +257,43 @@ public void createDirectories(final String normalPath) throws IOException { public void delete(final String normalPath) throws IOException { final Path path = fileSystem.getPath(normalPath); - try (final Stream pathStream = Files.walk(path)) { - pathStream.sorted(Comparator.reverseOrder()).forEach( - childPath -> { - if (Files.isRegularFile(childPath)) { - try (final LockedChannel channel = lockForWriting(childPath)) { - Files.delete(childPath); - } catch (final IOException e) { - e.printStackTrace(); - } - } else { - tryDelete(childPath); + if (Files.isRegularFile(path)) { + try (final LockedChannel channel = lockForWriting(path)) { + Files.delete(path); + } + } else { + try (final Stream pathStream = Files.walk(path)) { + for (final Iterator i = pathStream.sorted(Comparator.reverseOrder()).iterator(); i.hasNext();) { + final Path childPath = i.next(); + if (Files.isRegularFile(childPath)) { + try (final LockedChannel channel = lockForWriting(childPath)) { + Files.delete(childPath); } - }); + } else { + tryDelete(childPath); + } + } + } } } - protected static void tryDelete(final Path childPath) { + protected static void tryDelete(final Path path) throws IOException { try { - Files.delete(childPath); + Files.delete(path); } catch (final DirectoryNotEmptyException e) { - // Even though childPath should be an empty directory, sometimes the deletion fails on network file - // when lock files are not cleared immediately after the leaves have been removed. + /* Even though path is expected to be an empty directory, sometimes + * deletion fails on network filesystems when lock files are not + * cleared immediately after the leaves have been removed. + */ try { - // wait and reattempt + /* wait and reattempt */ Thread.sleep(100); - Files.delete(childPath); + Files.delete(path); } catch (final InterruptedException ex) { e.printStackTrace(); Thread.currentThread().interrupt(); - } catch (final IOException ex) { - ex.printStackTrace(); } - } catch (final IOException e) { - e.printStackTrace(); } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GzipCompression.java b/src/main/java/org/janelia/saalfeldlab/n5/GzipCompression.java index f0574d15..79c76582 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GzipCompression.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GzipCompression.java @@ -29,8 +29,6 @@ import java.io.InputStream; import java.io.ObjectInputStream; import java.io.OutputStream; -import java.lang.reflect.Field; -import java.lang.reflect.Modifier; import java.util.zip.Deflater; import java.util.zip.DeflaterOutputStream; import java.util.zip.InflaterInputStream; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccess.java index 4dcb5964..872b0b88 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccess.java @@ -163,8 +163,7 @@ public interface KeyValueAccess { public void createDirectories(final String normalPath) throws IOException; /** - * Delete this path. This is only for empty leaves of the tree, not - * expected to work recursively. + * Delete a path. If the path is a directory, delete it recursively. * * @param normalPath is expected to be in normalized form, no further * efforts are made to normalize it. diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5FSReader.java b/src/main/java/org/janelia/saalfeldlab/n5/N5FSReader.java index 95f2987b..18314db7 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5FSReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5FSReader.java @@ -26,31 +26,10 @@ package org.janelia.saalfeldlab.n5; import java.io.IOException; -import java.io.Reader; -import java.io.Writer; -import java.lang.reflect.Array; -import java.lang.reflect.Type; -import java.nio.channels.Channels; -import java.nio.charset.StandardCharsets; import java.nio.file.FileSystems; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.Arrays; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Set; -import java.util.stream.Stream; - -import org.janelia.saalfeldlab.n5.FileSystemKeyValueAccess.LockedFileChannel; import com.google.gson.Gson; import com.google.gson.GsonBuilder; -import com.google.gson.JsonArray; -import com.google.gson.JsonElement; -import com.google.gson.JsonPrimitive; -import com.google.gson.reflect.TypeToken; /** * {@link N5Reader} implementation through {@link KeyValueAccess} with JSON @@ -60,84 +39,20 @@ * @author Igor Pisarev * @author Philipp Hanslovsky */ -public class N5FSReader implements N5Reader { - - /** - * Data object for caching meta data. Elements that are null are not yet - * cached. - */ - protected static class N5GroupInfo { - - public HashSet children = null; - public HashMap attributesCache = null; - public Boolean isDataset = null; - } - - protected static final N5GroupInfo emptyGroupInfo = new N5GroupInfo(); - - protected final KeyValueAccess keyValueAccess; - - protected final Gson gson; - - protected final HashMap metaCache = new HashMap<>(); - - protected final boolean cacheMeta; - - protected static final String jsonFile = "attributes.json"; - - protected final String basePath; +public class N5FSReader extends N5KeyValueReader { /** * Opens an {@link N5FSReader} at a given base path with a custom * {@link GsonBuilder} to support custom attributes. * - * @param keyValueAccess * @param basePath N5 base path * @param gsonBuilder * @param cacheMeta cache attributes and meta data * Setting this to true avoids frequent reading and parsing of JSON * encoded attributes and other meta data that requires accessing the * store. This is most interesting for high latency backends. Changes - * of cached attributes and meta data by an independent writer will - * not be tracked. - * - * @throws IOException - * if the base path cannot be read or does not exist, - * if the N5 version of the container is not compatible with this - * implementation. - */ - public N5FSReader( - final KeyValueAccess keyValueAccess, - final String basePath, - final GsonBuilder gsonBuilder, - final boolean cacheMeta) throws IOException { - - this.keyValueAccess = keyValueAccess; - this.basePath = basePath; - gsonBuilder.registerTypeAdapter(DataType.class, new DataType.JsonAdapter()); - gsonBuilder.registerTypeHierarchyAdapter(Compression.class, CompressionAdapter.getJsonAdapter()); - gsonBuilder.disableHtmlEscaping(); - this.gson = gsonBuilder.create(); - this.cacheMeta = cacheMeta; - if (exists("/")) { - final Version version = getVersion(); - if (!VERSION.isCompatible(version)) - throw new IOException("Incompatible version " + version + " (this is " + VERSION + ")."); - } - } - - /** - * Opens an {@link N5FSReader} at a given base path with a custom - * {@link GsonBuilder} to support custom attributes. - * - * @param basePath N5 base path - * @param gsonBuilder - * @param cacheMeta cache attributes and meta data - * Setting this to true avoids frequent reading and parsing of JSON - * encoded attributes and other meta data that requires accessing the - * store. This is most interesting for high latency backends. Changes - * of cached attributes and meta data by an independent writer will - * not be tracked. + * of cached attributes and meta data by an independent writer on the + * same container will not be tracked. * * @throws IOException * if the base path cannot be read or does not exist, @@ -146,7 +61,11 @@ public N5FSReader( */ public N5FSReader(final String basePath, final GsonBuilder gsonBuilder, final boolean cacheMeta) throws IOException { - this(FileSystems.getDefault(), basePath, gsonBuilder, cacheMeta); + super( + new FileSystemKeyValueAccess(FileSystems.getDefault()), + basePath, + gsonBuilder, + cacheMeta); } /** @@ -157,8 +76,8 @@ public N5FSReader(final String basePath, final GsonBuilder gsonBuilder, final bo * Setting this to true avoids frequent reading and parsing of JSON * encoded attributes and other meta data that requires accessing the * store. This is most interesting for high latency backends. Changes - * of cached attributes and meta data by an independent writer will - * not be tracked. + * of cached attributes and meta data by an independent writer on the + * same container will not be tracked. * * @throws IOException * if the base path cannot be read or does not exist, @@ -199,660 +118,4 @@ public N5FSReader(final String basePath) throws IOException { this(basePath, new GsonBuilder(), false); } - - public Gson getGson() { - - return gson; - } - - /** - * - * @return N5 base path - */ - public String getBasePath() { - - return this.basePath; - } - - - /** - * Parses an attribute from the given attributes map. - * - * @param map - * @param key - * @param clazz - * @return - * @throws IOException - */ - protected T parseAttribute( - final HashMap map, - final String key, - final Class clazz) throws IOException { - - final JsonElement attribute = map.get(key); - if (attribute != null) - return gson.fromJson(attribute, clazz); - else - return null; - } - - /** - * Parses an attribute from the given attributes map. - * - * @param map - * @param key - * @param type - * @return - * @throws IOException - */ - protected T parseAttribute( - final HashMap map, - final String key, - final Type type) throws IOException { - - final JsonElement attribute = map.get(key); - if (attribute != null) - return gson.fromJson(attribute, type); - else - return null; - } - - /** - * Reads the attributes map from a given {@link Reader}. - * - * @param reader - * @return - * @throws IOException - */ - protected HashMap readAttributes(final Reader reader) throws IOException { - - final Type mapType = new TypeToken>(){}.getType(); - final HashMap map = gson.fromJson(reader, mapType); - return map == null ? new HashMap<>() : map; - } - - /** - * Inserts new the JSON export of attributes into the given attributes map. - * - * @param map - * @param attributes - * @param gson - * @throws IOException - */ - protected void insertAttributes( - final HashMap map, - final Map attributes) throws IOException { - - for (final Entry entry : attributes.entrySet()) - map.put(entry.getKey(), gson.toJsonTree(entry.getValue())); - } - - /** - * Writes the attributes map to a given {@link Writer}. - * - * @param writer - * @param map - * @throws IOException - */ - protected void writeAttributes( - final Writer writer, - final HashMap map) throws IOException { - - final Type mapType = new TypeToken>(){}.getType(); - gson.toJson(map, mapType, writer); - writer.flush(); - } - - /** - * Return a reasonable class for a {@link JsonPrimitive}. Possible return - * types are - *
    - *
  • boolean
    • - *
  • double
    • - *
  • long (if the number is an integer)
    • - *
  • String
    • - *
  • Object
    • - *
- * - * @param jsonPrimitive - * @return - */ - protected static Class classForJsonPrimitive(final JsonPrimitive jsonPrimitive) { - - if (jsonPrimitive.isBoolean()) - return boolean.class; - else if (jsonPrimitive.isNumber()) { - final Number number = jsonPrimitive.getAsNumber(); - if (number.longValue() == number.doubleValue()) - return long.class; - else - return double.class; - } else if (jsonPrimitive.isString()) - return String.class; - else return Object.class; - } - - /** - * Best effort implementation of {@link N5Reader#listAttributes(String)} - * with limited type resolution fromJSON. Possible return types are - *
    - *
  • null
    • - *
  • boolean
    • - *
  • double
    • - *
  • long (if the number is an integer)
    • - *
  • String
    • - *
  • Object
    • - *
  • boolean[]
    • - *
  • double[]
    • - *
  • long[] (if all numbers in the array are integers)
    • - *
  • String[]
    • - *
  • Object[]
    • - *
- */ - @Override - public Map> listAttributes(final String pathName) throws IOException { - - final HashMap jsonElementMap = getAttributes(pathName); - final HashMap> attributes = new HashMap<>(); - jsonElementMap.forEach( - (key, jsonElement) -> { - final Class clazz; - if (jsonElement.isJsonNull()) - clazz = null; - else if (jsonElement.isJsonPrimitive()) - clazz = classForJsonPrimitive((JsonPrimitive)jsonElement); - else if (jsonElement.isJsonArray()) { - final JsonArray jsonArray = (JsonArray)jsonElement; - Class arrayElementClass = Object.class; - if (jsonArray.size() > 0) { - final JsonElement firstElement = jsonArray.get(0); - if (firstElement.isJsonPrimitive()) { - arrayElementClass = classForJsonPrimitive(firstElement.getAsJsonPrimitive()); - for (int i = 1; i < jsonArray.size() && arrayElementClass != Object.class; ++i) { - final JsonElement element = jsonArray.get(i); - if (element.isJsonPrimitive()) { - final Class nextArrayElementClass = classForJsonPrimitive(element.getAsJsonPrimitive()); - if (nextArrayElementClass != arrayElementClass) - if (nextArrayElementClass == double.class && arrayElementClass == long.class) - arrayElementClass = double.class; - else { - arrayElementClass = Object.class; - break; - } - } else { - arrayElementClass = Object.class; - break; - } - } - } - clazz = Array.newInstance(arrayElementClass, 0).getClass(); - } else - clazz = Object[].class; - } - else - clazz = Object.class; - attributes.put(key, clazz); - }); - return attributes; - } - - @Override - public DatasetAttributes getDatasetAttributes(final String pathName) throws IOException { - - final long[] dimensions; - final DataType dataType; - int[] blockSize; - Compression compression; - final String compressionVersion0Name; - final String normalPathName = normalize(pathName); - if (cacheMeta) { - final N5GroupInfo info = getCachedN5GroupInfo(normalPathName); - if (info == emptyGroupInfo) - return null; - - final HashMap cachedMap; - if (info.isDataset == null) { - - synchronized (info) { - - cachedMap = getCachedAttributes(info, normalPathName); - if (cachedMap.isEmpty()) { - info.isDataset = false; - return null; - } - - dimensions = getAttribute(cachedMap, DatasetAttributes.dimensionsKey, long[].class); - if (dimensions == null) { - info.isDataset = false; - return null; - } - - dataType = getAttribute(cachedMap, DatasetAttributes.dataTypeKey, DataType.class); - if (dataType == null) { - info.isDataset = false; - return null; - } - - info.isDataset = true; - } - } else if (!info.isDataset) { - return null; - } else { - - cachedMap = getCachedAttributes(info, normalPathName); - dimensions = getAttribute(cachedMap, DatasetAttributes.dimensionsKey, long[].class); - dataType = getAttribute(cachedMap, DatasetAttributes.dataTypeKey, DataType.class); - } - - blockSize = getAttribute(cachedMap, DatasetAttributes.blockSizeKey, int[].class); - - compression = getAttribute(cachedMap, DatasetAttributes.compressionKey, Compression.class); - - /* version 0 */ - compressionVersion0Name = compression - == null - ? getAttribute(cachedMap, DatasetAttributes.compressionTypeKey, String.class) - : null; - - - } else { - final HashMap map = getAttributes(normalPathName); - - dimensions = parseAttribute(map, DatasetAttributes.dimensionsKey, long[].class); - if (dimensions == null) - return null; - - dataType = parseAttribute(map, DatasetAttributes.dataTypeKey, DataType.class); - if (dataType == null) - return null; - - blockSize = parseAttribute(map, DatasetAttributes.blockSizeKey, int[].class); - - compression = parseAttribute(map, DatasetAttributes.compressionKey, Compression.class); - - /* version 0 */ - compressionVersion0Name = compression == null - ? parseAttribute(map, DatasetAttributes.compressionTypeKey, String.class) - : null; - } - - if (blockSize == null) - blockSize = Arrays.stream(dimensions).mapToInt(a -> (int)a).toArray(); - - /* version 0 */ - if (compression == null) { - switch (compressionVersion0Name) { - case "raw": - compression = new RawCompression(); - break; - case "gzip": - compression = new GzipCompression(); - break; - case "bzip2": - compression = new Bzip2Compression(); - break; - case "lz4": - compression = new Lz4Compression(); - break; - case "xz": - compression = new XzCompression(); - break; - } - } - - return new DatasetAttributes(dimensions, blockSize, dataType, compression); - } - - protected HashMap getAttributes(final Path path) throws IOException { - - if (!Files.exists(path)) - return new HashMap<>(); - - try (final LockedFileChannel lockedFileChannel = LockedFileChannel.openForReading(path)) { - return readAttributes(Channels.newReader(lockedFileChannel.getFileChannel(), StandardCharsets.UTF_8.name())); - } - } - - /** - * Get and cache attributes for a group identified by an info object and a - * pathName. - * - * This helper method does not intelligently handle the case that the group - * does not exist (as indicated by info == emptyGroupInfo) which should be - * done in calling code. - * - * @param info - * @param pathName normalized group path without leading slash - * @return cached attributes - * empty map if the group exists but not attributes are set - * null if the group does not exist - * @throws IOException - */ - protected HashMap getCachedAttributes( - final N5GroupInfo info, - final String pathName) throws IOException { - - HashMap cachedMap = info.attributesCache; - if (cachedMap == null) { - synchronized (info) { - cachedMap = info.attributesCache; - if (cachedMap == null) { - final Path path = getAttributesPath(pathName); - cachedMap = new HashMap<>(); - final HashMap map = getAttributes(path); - cachedMap.putAll(map); - info.attributesCache = cachedMap; - } - } - } - return cachedMap; - } - - @SuppressWarnings("unchecked") - protected T getAttribute( - final HashMap cachedMap, - final String key, - final Class clazz) { - - final Object cachedAttribute = cachedMap.get(key); - if (cachedAttribute == null) - return null; - else if (cachedAttribute instanceof JsonElement) { - final T attribute = gson.fromJson((JsonElement)cachedAttribute, clazz); - synchronized (cachedMap) { - cachedMap.put(key, attribute); - } - return attribute; - } else { - return (T)cachedAttribute; - } - } - - /** - * Helper method that returns or JSON decodes a cached attribute. - * - * @param - * @param cachedMap - * @param key - * @param type - * @return - */ - @SuppressWarnings("unchecked") - protected T getAttribute( - final HashMap cachedMap, - final String key, - final Type type) { - - final Object cachedAttribute = cachedMap.get(key); - if (cachedAttribute == null) - return null; - else if (cachedAttribute instanceof JsonElement) { - final T attribute = gson.fromJson((JsonElement)cachedAttribute, type); - synchronized (cachedMap) { - cachedMap.put(key, attribute); - } - return attribute; - } else { - return (T)cachedAttribute; - } - } - - @Override - public T getAttribute( - final String pathName, - final String key, - final Class clazz) throws IOException { - - final String normalPathName = normalize(pathName); - if (cacheMeta) { - final N5GroupInfo info = getCachedN5GroupInfo(normalPathName); - if (info == emptyGroupInfo) - return null; - final HashMap cachedMap = getCachedAttributes(info, normalPathName); - if (cachedMap.isEmpty()) - return null; - return getAttribute(cachedMap, key, clazz); - } else { - final HashMap map = getAttributes(normalPathName); - return parseAttribute(map, key, clazz); - } - } - - @Override - public T getAttribute( - final String pathName, - final String key, - final Type type) throws IOException { - - final String normalPathName = normalize(pathName); - if (cacheMeta) { - final N5GroupInfo info = getCachedN5GroupInfo(normalPathName); - if (info == emptyGroupInfo) - return null; - final HashMap cachedMap = getCachedAttributes(info, normalPathName); - if (cachedMap.isEmpty()) - return null; - return getAttribute(cachedMap, key, type); - } else { - final HashMap map = getAttributes(normalPathName); - return parseAttribute(map, key, type); - } - } - - protected boolean exists(final Path path) { - - return Files.exists(path) && Files.isDirectory(path); - } - - /** - * Get an existing cached N5 group info object or create it. - * - * @param normalPathName normalized group path without leading slash - * @return - */ - protected N5GroupInfo getCachedN5GroupInfo(final String normalPathName) { - - N5GroupInfo info = metaCache.get(normalPathName); - if (info == null) { - - /* I do not have a better solution yet to allow parallel - * exists checks for independent paths than to accept the - * same exists check to potentially run multiple times. - */ - final boolean exists = exists(getGroupPath(normalPathName)); - - synchronized (metaCache) { - info = metaCache.get(normalPathName); - if (info == null) { - info = exists ? new N5GroupInfo() : emptyGroupInfo; - metaCache.put(normalPathName, info); - } - } - } - return info; - } - - @Override - public boolean exists(final String pathName) { - - final String normalPathName = normalize(pathName); - if (cacheMeta) - return getCachedN5GroupInfo(normalPathName) != emptyGroupInfo; - else - return exists(getGroupPath(normalPathName)); - } - - @Override - public boolean datasetExists(final String pathName) throws IOException { - - if (cacheMeta) { - final String normalPathName = normalize(pathName); - final N5GroupInfo info = getCachedN5GroupInfo(normalPathName); - if (info == emptyGroupInfo) - return false; - if (info.isDataset == null) { - synchronized (info) { - if (info.isDataset == null ) { - - } - else - return info.isDataset; - } - } else - return info.isDataset; - } - return exists(pathName) && getDatasetAttributes(pathName) != null; - } - - /** - * Reads or creates the attributes map of a group or dataset. - * - * @param pathName group path - * @return - * @throws IOException - */ - public HashMap getAttributes(final String pathName) throws IOException { - - final Path path = getAttributesPath(normalize(pathName)); - if (exists(pathName) && !Files.exists(path)) - return new HashMap<>(); - - try (final LockedFileChannel lockedFileChannel = LockedFileChannel.openForReading(path)) { - return readAttributes(Channels.newReader(lockedFileChannel.getFileChannel(), StandardCharsets.UTF_8.name())); - } - } - - @Override - public DataBlock readBlock( - final String pathName, - final DatasetAttributes datasetAttributes, - final long... gridPosition) throws IOException { - - final Path path = getDataBlockPath(normalize(pathName), gridPosition); - if (!Files.exists(path)) - return null; - - try (final LockedFileChannel lockedChannel = LockedFileChannel.openForReading(path)) { - return DefaultBlockReader.readBlock(Channels.newInputStream(lockedChannel.getFileChannel()), datasetAttributes, gridPosition); - } - } - - /** - * - * @param normalPathName normalized path name - * @return - * @throws IOException - */ - protected String[] normalList(final String normalPathName) throws IOException { - - final Path path = getGroupPath(normalPathName); - try (final Stream pathStream = Files.list(path)) { - return pathStream - .filter(a -> Files.isDirectory(a)) - .map(a -> path.relativize(a).toString()) - .toArray(n -> new String[n]); - } - } - - @Override - public String[] list(final String pathName) throws IOException { - - if (cacheMeta) { - final N5GroupInfo info = getCachedN5GroupInfo(normalize(pathName)); - if (info == emptyGroupInfo) - throw new IOException("Group '" + pathName +"' does not exist."); - else { - Set children = info.children; - final String[] list; - if (children == null) { - synchronized (info) { - children = info.children; - if (children == null) { - list = normalList(normalize(pathName)); - info.children = new HashSet<>(Arrays.asList(list)); - } else - list = children.toArray(new String[children.size()]); - } - } else - list = children.toArray(new String[children.size()]); - - return list; - } - } else { - return normalList(normalize(pathName)); - } - } - - /** - * Constructs the path for a data block in a dataset at a given grid position. - * - * The returned path is - * 
-	 * $basePath/datasetPathName/$gridPosition[0]/$gridPosition[1]/.../$gridPosition[n]
-	 *
- * - * This is the file into which the data block will be stored. - * - * @param normalDatasetPathName normalized dataset path without leading slash - * @param gridPosition - * @return - */ - protected Path getDataBlockPath( - final String normalDatasetPathName, - final long... gridPosition) { - - final String[] pathComponents = new String[gridPosition.length + 1]; - pathComponents[0] = normalDatasetPathName; - for (int i = 1; i < pathComponents.length; ++i) - pathComponents[i] = Long.toString(gridPosition[i - 1]); - - return fileSystem.getPath(basePath, pathComponents); - } - - /** - * Constructs the path for the group or dataset. - * - * @param pathName normalized group path without leading slash - * @return - */ - protected Path getGroupPath(final String pathName) { - - return fileSystem.getPath(basePath, pathName); - } - - /** - * Constructs the path for the attributes file of a group or dataset. - * - * @param normalPathName normalized group path without leading slash - * @return - */ - protected Path getAttributesPath(final String normalPathName) { - - return fileSystem.getPath(basePath, normalPathName, jsonFile); - } - - /** - * Removes the leading slash from a given path and returns the corrected path. - * It ensures correctness on both Unix and Windows, otherwise {@code pathName} is treated - * as UNC path on Windows, and {@code Paths.get(pathName, ...)} fails with {@code InvalidPathException}. - * - * @param pathName - * @return - */ - protected String normalize(final String pathName) { - - return fileSystem.getPath(basePath) - .relativize( - fileSystem.getPath( - basePath, - pathName.startsWith("/") || pathName.startsWith("\\") ? pathName.substring(1) : pathName) - .normalize()) - .toString(); - } - - @Override - public String toString() { - - return String.format("%s[fileSystem=%s, basePath=%s]", getClass().getSimpleName(), fileSystem, basePath); - } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java b/src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java index d15a4b67..e63c6c8a 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java @@ -26,32 +26,16 @@ package org.janelia.saalfeldlab.n5; import java.io.IOException; -import java.io.Writer; -import java.lang.reflect.Type; -import java.nio.channels.Channels; -import java.nio.charset.StandardCharsets; -import java.nio.file.FileSystem; import java.nio.file.FileSystems; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.Comparator; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Map; -import java.util.stream.Stream; - -import org.janelia.saalfeldlab.n5.FileSystemKeyValueAccess.LockedFileChannel; import com.google.gson.GsonBuilder; -import com.google.gson.JsonElement; -import com.google.gson.reflect.TypeToken; /** * Filesystem {@link N5Writer} implementation with version compatibility check. * * @author Stephan Saalfeld */ -public class N5FSWriter extends N5KeyValueReader implements N5Writer { +public class N5FSWriter extends N5KeyValueWriter { /** * Opens an {@link N5FSWriter} at a given base path with a custom @@ -63,49 +47,14 @@ public class N5FSWriter extends N5KeyValueReader implements N5Writer { * compatible with this implementation, the N5 version of this container * will be set to the current N5 version of this implementation. * - * @param fileSystem * @param basePath n5 base path * @param gsonBuilder - * @param cacheAttributes cache attributes + * @param cacheMeta cache attributes and meta data * Setting this to true avoids frequent reading and parsing of JSON - * encoded attributes, this is most interesting for high latency file - * systems. Changes of attributes by an independent writer will not be - * tracked. - * - * @throws IOException - * if the base path cannot be written to or cannot be created, - * if the N5 version of the container is not compatible with this - * implementation. - */ - public N5FSWriter( - final FileSystem fileSystem, - final String basePath, - final GsonBuilder gsonBuilder, - final boolean cacheAttributes) throws IOException { - - super(fileSystem, basePath, gsonBuilder, cacheAttributes); - createGroup("/"); - if (!VERSION.equals(getVersion())) - setAttribute("/", VERSION_KEY, VERSION.toString()); - } - - /** - * Opens an {@link N5FSWriter} at a given base path with a custom - * {@link GsonBuilder} to support custom attributes. - * - * If the base path does not exist, it will be created. - * - * If the base path exists and if the N5 version of the container is - * compatible with this implementation, the N5 version of this container - * will be set to the current N5 version of this implementation. - * - * @param basePath n5 base path - * @param gsonBuilder - * @param cacheAttributes cache attributes - * Setting this to true avoids frequent reading and parsing of JSON - * encoded attributes, this is most interesting for high latency file - * systems. Changes of attributes by an independent writer will not be - * tracked. + * encoded attributes and other meta data that requires accessing the + * store. This is most interesting for high latency backends. Changes + * of cached attributes and meta data by an independent writer on the + * same container will not be tracked. * * @throws IOException * if the base path cannot be written to or cannot be created, @@ -114,7 +63,11 @@ public N5FSWriter( */ public N5FSWriter(final String basePath, final GsonBuilder gsonBuilder, final boolean cacheAttributes) throws IOException { - this(FileSystems.getDefault(), basePath, gsonBuilder, cacheAttributes); + super( + new FileSystemKeyValueAccess(FileSystems.getDefault()), + basePath, + gsonBuilder, + cacheAttributes); } /** @@ -128,11 +81,12 @@ public N5FSWriter(final String basePath, final GsonBuilder gsonBuilder, final bo * * @param basePath n5 base path * @param gsonBuilder - * @param cacheAttributes cache attributes + * @param cacheMeta cache attributes and meta data * Setting this to true avoids frequent reading and parsing of JSON - * encoded attributes, this is most interesting for high latency file - * systems. Changes of attributes by an independent writer will not be - * tracked. + * encoded attributes and other meta data that requires accessing the + * store. This is most interesting for high latency backends. Changes + * of cached attributes and meta data by an independent writer on the + * same container will not be tracked. * * @throws IOException * if the base path cannot be written to or cannot be created, @@ -188,259 +142,4 @@ public N5FSWriter(final String basePath) throws IOException { this(basePath, new GsonBuilder()); } - - /** - * Writes the attributes map to a given {@link Writer}. - * - * @param writer - * @param map - * @throws IOException - */ - protected void writeAttributes( - final Writer writer, - final HashMap map) throws IOException { - - final Type mapType = new TypeToken>(){}.getType(); - gson.toJson(map, mapType, writer); - writer.flush(); - } - - /** - * Helper method to create and cache a group. - * - * @param normalPathName normalized group path without leading slash - * @return - * @throws IOException - */ - protected N5GroupInfo createCachedGroup(final String normalPathName) throws IOException { - - N5GroupInfo info = getCachedN5GroupInfo(normalPathName); - if (info == emptyGroupInfo) { - - /* The directories may be created multiple times concurrently, - * but a new cache entry is inserted only if none has been - * inserted in the meantime (because that may already include - * more cached data). - * - * This avoids synchronizing on the cache for independent - * group creation. - */ - createDirectories(groupPath(normalPathName)); - synchronized (metaCache) { - info = getCachedN5GroupInfo(normalPathName); - if (info == emptyGroupInfo) { - info = new N5GroupInfo(); - metaCache.put(normalPathName, info); - } - for (String childPathName = normalPathName; !childPathName.equals("");) { - final Path parent = groupPath(childPathName).getParent(); - final String parentPathName = fileSystem.getPath(basePath).relativize(parent).toString(); - N5GroupInfo parentInfo = getCachedN5GroupInfo(parentPathName); - if (parentInfo == emptyGroupInfo) { - parentInfo = new N5GroupInfo(); - parentInfo.isDataset = false; - metaCache.put(parentPathName, parentInfo); - } - final HashSet children = parentInfo.children; - if (children != null) { - synchronized (children) { - children.add( - parent.relativize(groupPath(childPathName)).toString()); - } - } - childPathName = parentPathName; - } - } - } - return info; - } - - @Override - public void createGroup(final String pathName) throws IOException { - - final String normalPathName = normalize(pathName); - if (cacheMeta) { - final N5GroupInfo info = createCachedGroup(normalPathName); - synchronized (info) { - if (info.isDataset == null) - info.isDataset = false; - } - } else - createDirectories(groupPath(normalPathName)); - } - - @Override - public void createDataset( - final String pathName, - final DatasetAttributes datasetAttributes) throws IOException { - - final String normalPathName = normalize(pathName); - if (cacheMeta) { - final N5GroupInfo info = createCachedGroup(normalPathName); - synchronized (info) { - setDatasetAttributes(normalPathName, datasetAttributes); - info.isDataset = true; - } - } else { - createGroup(pathName); - setDatasetAttributes(normalPathName, datasetAttributes); - } - } - - /** - * Helper method that reads the existing map of attributes, JSON encodes, - * inserts and overrides the provided attributes, and writes them back into - * the attributes store. - * - * @param path - * @param attributes - * @throws IOException - */ - protected void writeAttributes( - final Path path, - final Map attributes) throws IOException { - - final HashMap map = new HashMap<>(); - - try (final LockedFileChannel lockedFileChannel = LockedFileChannel.openForWriting(path)) { - map.putAll(readAttributes(Channels.newReader(lockedFileChannel.getFileChannel(), StandardCharsets.UTF_8.name()))); - insertAttributes(map, attributes); - - lockedFileChannel.getFileChannel().truncate(0); - writeAttributes(Channels.newWriter(lockedFileChannel.getFileChannel(), StandardCharsets.UTF_8.name()), map); - } - } - - /** - * Check for attributes that are required for a group to be a dataset. - * - * @param cachedAttributes - * @return - */ - protected static boolean hasCachedDatasetAttributes(final Map cachedAttributes) { - - return cachedAttributes.keySet().contains(DatasetAttributes.dimensionsKey) && cachedAttributes.keySet().contains(DatasetAttributes.dataTypeKey); - } - - - /** - * Helper method to cache and write attributes. - * - * @param normalPathName normalized group path without leading slash - * @param attributes - * @param isDataset - * @return - * @throws IOException - */ - protected N5GroupInfo setCachedAttributes( - final String normalPathName, - final Map attributes) throws IOException { - - N5GroupInfo info = getCachedN5GroupInfo(normalPathName); - if (info == emptyGroupInfo) { - synchronized (metaCache) { - info = getCachedN5GroupInfo(normalPathName); - if (info == emptyGroupInfo) - throw new IOException("N5 group '" + normalPathName + "' does not exist. Cannot set attributes."); - } - } - final HashMap cachedMap = getCachedAttributes(info, normalPathName); - synchronized (info) { - cachedMap.putAll(attributes); - writeAttributes(attributesPath(normalPathName), attributes); - info.isDataset = hasCachedDatasetAttributes(cachedMap); - } - return info; - } - - @Override - public void setAttributes( - final String pathName, - final Map attributes) throws IOException { - - final String normalPathName = normalize(pathName); - if (cacheMeta) - setCachedAttributes(normalPathName, attributes); - else - writeAttributes(attributesPath(normalPathName), attributes); - } - - @Override - public void writeBlock( - final String pathName, - final DatasetAttributes datasetAttributes, - final DataBlock dataBlock) throws IOException { - - final Path path = getDataBlockPath(normalize(pathName), dataBlock.getGridPosition()); - createDirectories(path.getParent()); - try (final LockedFileChannel lockedChannel = LockedFileChannel.openForWriting(path)) { - lockedChannel.getFileChannel().truncate(0); - DefaultBlockWriter.writeBlock(Channels.newOutputStream(lockedChannel.getFileChannel()), datasetAttributes, dataBlock); - } - } - - @Override - public boolean remove(final String pathName) throws IOException { - - final String normalPathName = normalize(pathName); - final Path path = groupPath(normalPathName); - boolean exists = Files.exists(path); - if (exists) { - final Path base = fileSystem.getPath(basePath); - try (final Stream pathStream = Files.walk(path)) { - pathStream.sorted(Comparator.reverseOrder()).forEach( - childPath -> { - if (Files.isRegularFile(childPath)) { - try (final LockedFileChannel channel = LockedFileChannel.openForWriting(childPath)) { - Files.delete(childPath); - } catch (final IOException e) { - e.printStackTrace(); - } - } else { - if (cacheMeta) { - synchronized (metaCache) { - metaCache.put(base.relativize(childPath).toString(), emptyGroupInfo); - tryDelete(childPath); - } - } else { - tryDelete(childPath); - } - } - }); - } - if (cacheMeta) { - if (!normalPathName.equals("")) { // not root - final Path parent = groupPath(normalPathName).getParent(); - final N5GroupInfo parentInfo = getCachedN5GroupInfo( - fileSystem.getPath(basePath).relativize(parent).toString()); // group must exist - final HashSet children = parentInfo.children; - if (children != null) { - synchronized (children) { - exists = Files.exists(path); - if (exists) - children.remove( - parent.relativize(fileSystem.getPath(normalPathName)).toString()); - } - } - } - } else - exists = Files.exists(path); - } - return !exists; - } - - @Override - public boolean deleteBlock( - final String pathName, - final long... gridPosition) throws IOException { - - final Path path = getDataBlockPath(normalize(pathName), gridPosition); - if (Files.exists(path)) - try (final LockedFileChannel channel = LockedFileChannel.openForWriting(path)) { - Files.deleteIfExists(path); - } - return !Files.exists(path); - } - - } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java index a3d94d2b..3e608f02 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java @@ -693,10 +693,10 @@ public String[] list(final String pathName) throws IOException { * @return */ protected String getDataBlockPath( - final String normalDatasetPath, + final String normalPath, final long... gridPosition) { - final StringBuilder builder = new StringBuilder(groupPath(normalDatasetPath)); + final StringBuilder builder = new StringBuilder(groupPath(normalPath)); for (final long i : gridPosition) { builder.append("/"); @@ -707,7 +707,8 @@ protected String getDataBlockPath( } /** - * Constructs the path for the group or dataset. + * Constructs the absolute path (in terms of this store) for the group or + * dataset. * * @param normalPath normalized group path without leading slash * @return @@ -718,7 +719,8 @@ protected String groupPath(final String normalPath) { } /** - * Constructs the path for the attributes file of a group or dataset. + * Constructs the absolute path (in terms of this store) for the attributes + * file of a group or dataset. * * @param normalPath normalized group path without leading slash * @return diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriter.java b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriter.java index 9f526402..6c1721f6 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriter.java @@ -28,15 +28,9 @@ import java.io.IOException; import java.io.Writer; import java.lang.reflect.Type; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; import java.util.Map; -import java.util.stream.Stream; - -import org.janelia.saalfeldlab.n5.FileSystemKeyValueAccess.LockedFileChannel; import com.google.gson.GsonBuilder; import com.google.gson.JsonElement; @@ -104,13 +98,13 @@ protected void writeAttributes( /** * Helper method to create and cache a group. * - * @param normalPathName normalized group path without leading slash + * @param normalPath normalized group path without leading slash * @return * @throws IOException */ - protected N5GroupInfo createCachedGroup(final String normalPathName) throws IOException { + protected N5GroupInfo createCachedGroup(final String normalPath) throws IOException { - N5GroupInfo info = getCachedN5GroupInfo(normalPathName); + N5GroupInfo info = getCachedN5GroupInfo(normalPath); if (info == emptyGroupInfo) { /* The directories may be created multiple times concurrently, @@ -121,14 +115,14 @@ protected N5GroupInfo createCachedGroup(final String normalPathName) throws IOEx * This avoids synchronizing on the cache for independent * group creation. */ - keyValueAccess.createDirectories(groupPath(normalPathName)); + keyValueAccess.createDirectories(groupPath(normalPath)); synchronized (metaCache) { - info = getCachedN5GroupInfo(normalPathName); + info = getCachedN5GroupInfo(normalPath); if (info == emptyGroupInfo) { info = new N5GroupInfo(); - metaCache.put(normalPathName, info); + metaCache.put(normalPath, info); } - for (String childPathName = normalPathName; !childPathName.equals("");) { + for (String childPathName = normalPath; !childPathName.equals("");) { final String parentPathName = keyValueAccess.parent(childPathName); N5GroupInfo parentInfo = getCachedN5GroupInfo(parentPathName); if (parentInfo == emptyGroupInfo) { @@ -151,34 +145,34 @@ protected N5GroupInfo createCachedGroup(final String normalPathName) throws IOEx } @Override - public void createGroup(final String pathName) throws IOException { + public void createGroup(final String path) throws IOException { - final String normalPathName = normalize(pathName); + final String normalPath = normalize(path); if (cacheMeta) { - final N5GroupInfo info = createCachedGroup(normalPathName); + final N5GroupInfo info = createCachedGroup(normalPath); synchronized (info) { if (info.isDataset == null) info.isDataset = false; } } else - keyValueAccess.createDirectories(groupPath(normalPathName)); + keyValueAccess.createDirectories(groupPath(normalPath)); } @Override public void createDataset( - final String pathName, + final String path, final DatasetAttributes datasetAttributes) throws IOException { - final String normalPathName = normalize(pathName); + final String normalPath = normalize(path); if (cacheMeta) { - final N5GroupInfo info = createCachedGroup(normalPathName); + final N5GroupInfo info = createCachedGroup(normalPath); synchronized (info) { - setDatasetAttributes(normalPathName, datasetAttributes); + setDatasetAttributes(normalPath, datasetAttributes); info.isDataset = true; } } else { - createGroup(pathName); - setDatasetAttributes(normalPathName, datasetAttributes); + createGroup(path); + setDatasetAttributes(normalPath, datasetAttributes); } } @@ -219,28 +213,28 @@ protected static boolean hasCachedDatasetAttributes(final Map ca /** * Helper method to cache and write attributes. * - * @param normalPathName normalized group path without leading slash + * @param normalPath normalized group path without leading slash * @param attributes * @param isDataset * @return * @throws IOException */ protected N5GroupInfo setCachedAttributes( - final String normalPathName, + final String normalPath, final Map attributes) throws IOException { - N5GroupInfo info = getCachedN5GroupInfo(normalPathName); + N5GroupInfo info = getCachedN5GroupInfo(normalPath); if (info == emptyGroupInfo) { synchronized (metaCache) { - info = getCachedN5GroupInfo(normalPathName); + info = getCachedN5GroupInfo(normalPath); if (info == emptyGroupInfo) - throw new IOException("N5 group '" + normalPathName + "' does not exist. Cannot set attributes."); + throw new IOException("N5 group '" + normalPath + "' does not exist. Cannot set attributes."); } } - final HashMap cachedMap = getCachedAttributes(info, normalPathName); + final HashMap cachedMap = getCachedAttributes(info, normalPath); synchronized (info) { cachedMap.putAll(attributes); - writeAttributes(attributesPath(normalPathName), attributes); + writeAttributes(attributesPath(normalPath), attributes); info.isDataset = hasCachedDatasetAttributes(cachedMap); } return info; @@ -248,97 +242,80 @@ protected N5GroupInfo setCachedAttributes( @Override public void setAttributes( - final String pathName, + final String path, final Map attributes) throws IOException { - final String normalPathName = normalize(pathName); + final String normalPath = normalize(path); if (cacheMeta) - setCachedAttributes(normalPathName, attributes); + setCachedAttributes(normalPath, attributes); else - writeAttributes(attributesPath(normalPathName), attributes); + writeAttributes(attributesPath(normalPath), attributes); } @Override public void writeBlock( - final String pathName, + final String path, final DatasetAttributes datasetAttributes, final DataBlock dataBlock) throws IOException { - final String path = getDataBlockPath(normalize(pathName), dataBlock.getGridPosition()); - keyValueAccess.createDirectories(keyValueAccess.parent(path)); - try (final LockedChannel lock = keyValueAccess.lockForWriting(path)) { + final String blockPath = getDataBlockPath(normalize(path), dataBlock.getGridPosition()); + keyValueAccess.createDirectories(keyValueAccess.parent(blockPath)); + try (final LockedChannel lock = keyValueAccess.lockForWriting(blockPath)) { DefaultBlockWriter.writeBlock(lock.newOutputStream(), datasetAttributes, dataBlock); } } @Override - public boolean remove(final String pathName) throws IOException { - - final String normalPathName = normalize(pathName); - final String path = groupPath(normalPathName); - boolean exists = keyValueAccess.exists(path); - if (exists) { - - keyValueAccess.delete(normalPathName); - - - - final Path base = fileSystem.getPath(basePath); - try (final Stream pathStream = Files.walk(path)) { - pathStream.sorted(Comparator.reverseOrder()).forEach( - childPath -> { - if (Files.isRegularFile(childPath)) { - try (final LockedFileChannel channel = LockedFileChannel.openForWriting(childPath)) { - Files.delete(childPath); - } catch (final IOException e) { - e.printStackTrace(); - } - } else { - if (cacheMeta) { - synchronized (metaCache) { - metaCache.put(base.relativize(childPath).toString(), emptyGroupInfo); - tryDelete(childPath); - } - } else { - tryDelete(childPath); - } + public boolean remove(final String path) throws IOException { + + final String normalPath = normalize(path); + final String groupPath = groupPath(normalPath); + if (cacheMeta) { + synchronized (metaCache) { + if (keyValueAccess.exists(groupPath)) { + keyValueAccess.delete(groupPath); + + /* cache nonexistence for all prior children */ + for (final String key : metaCache.keySet()) { + if (key.startsWith(normalPath)) + metaCache.put(normalPath, emptyGroupInfo); + } + + /* remove child from parent */ + final String parentPath = keyValueAccess.parent(normalPath); + final N5GroupInfo parent = metaCache.get(parentPath); + if (parent != null) { + final HashSet children = parent.children; + if (children != null) { + synchronized (children) { + children.remove(keyValueAccess.relativize(normalPath, parentPath)); } - }); - } - if (cacheMeta) { - if (!normalPathName.equals("")) { // not root - final Path parent = groupPath(normalPathName).getParent(); - final N5GroupInfo parentInfo = getCachedN5GroupInfo( - fileSystem.getPath(basePath).relativize(parent).toString()); // group must exist - final HashSet children = parentInfo.children; - if (children != null) { - synchronized (children) { - exists = Files.exists(path); - if (exists) - children.remove( - parent.relativize(fileSystem.getPath(normalPathName)).toString()); } } } - } else - exists = Files.exists(path); + } + } else { + if (keyValueAccess.exists(groupPath)) + keyValueAccess.delete(groupPath); } - return !exists; + + /* an IOException should have occurred if anything had failed midway */ + return true; } @Override public boolean deleteBlock( - final String pathName, + final String path, final long... gridPosition) throws IOException { - final Path path = getDataBlockPath(normalize(pathName), gridPosition); - if (Files.exists(path)) - try (final LockedFileChannel channel = LockedFileChannel.openForWriting(path)) { - Files.deleteIfExists(path); + final String blockPath = getDataBlockPath(normalize(path), gridPosition); + if (keyValueAccess.exists(blockPath)) + try (final LockedChannel channel = keyValueAccess.lockForWriting(blockPath)) { + keyValueAccess.delete(blockPath); } - return !Files.exists(path); - } - + /* an IOException should have occurred if anything had failed midway */ + return true; + } } diff --git a/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java b/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java index 98aa4e3e..c53b1ec8 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java @@ -28,8 +28,7 @@ import static org.junit.Assert.fail; import java.io.IOException; -import java.nio.channels.Channels; -import java.nio.charset.StandardCharsets; +import java.nio.file.FileSystems; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; @@ -40,18 +39,18 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; -import org.janelia.saalfeldlab.n5.N5KeyValueReader.LockedFileChannel; import org.junit.Test; /** * Initiates testing of the filesystem-based N5 implementation. * - * @author Stephan Saalfeld <saalfelds@janelia.hhmi.org> - * @author Igor Pisarev <pisarevi@janelia.hhmi.org> + * @author Stephan Saalfeld + * @author Igor Pisarev */ public class N5FSTest extends AbstractN5Test { - static private String testDirPath = System.getProperty("user.home") + "/tmp/n5-test"; + private static FileSystemKeyValueAccess access = new FileSystemKeyValueAccess(FileSystems.getDefault()); + private static String testDirPath = System.getProperty("user.home") + "/tmp/n5-test"; /** * @throws IOException @@ -70,14 +69,14 @@ public void testReadLock() throws IOException, InterruptedException { Files.delete(path); } catch (final IOException e) {} - LockedFileChannel lockedFileChannel = LockedFileChannel.openForWriting(path); - lockedFileChannel.close(); - lockedFileChannel = LockedFileChannel.openForReading(path); + LockedChannel lock = access.lockForWriting(path); + lock.close(); + lock = access.lockForReading(path); System.out.println("locked"); final ExecutorService exec = Executors.newSingleThreadExecutor(); final Future future = exec.submit(() -> { - LockedFileChannel.openForWriting(path).close(); + access.lockForWriting(path).close(); return null; }); @@ -92,7 +91,7 @@ public void testReadLock() throws IOException, InterruptedException { future.cancel(true); System.out.println("Test was interrupted!"); } finally { - lockedFileChannel.close(); + lock.close(); Files.delete(path); } @@ -108,12 +107,12 @@ public void testWriteLock() throws IOException { Files.delete(path); } catch (final IOException e) {} - final LockedFileChannel lockedFileChannel = LockedFileChannel.openForWriting(path); + final LockedChannel lock = access.lockForWriting(path); System.out.println("locked"); final ExecutorService exec = Executors.newSingleThreadExecutor(); final Future future = exec.submit(() -> { - LockedFileChannel.openForReading(path).close(); + access.lockForReading(path).close(); return null; }); @@ -128,7 +127,7 @@ public void testWriteLock() throws IOException { future.cancel(true); System.out.println("Test was interrupted!"); } finally { - lockedFileChannel.close(); + lock.close(); Files.delete(path); } @@ -143,16 +142,16 @@ public void testLockReleaseByReader() throws IOException { Files.delete(path); } catch (final IOException e) {} - final LockedFileChannel lockedFileChannel = LockedFileChannel.openForWriting(path); + final LockedChannel lock = access.lockForWriting(path); System.out.println("locked"); - Channels.newReader(lockedFileChannel.getFileChannel(), StandardCharsets.UTF_8.name()).close(); + lock.newReader().close(); System.out.println("reader released"); final ExecutorService exec = Executors.newSingleThreadExecutor(); final Future future = exec.submit(() -> { - LockedFileChannel.openForWriting(path).close(); + access.lockForWriting(path).close(); return null; }); @@ -166,7 +165,7 @@ public void testLockReleaseByReader() throws IOException { future.cancel(true); System.out.println("Test was interrupted!"); } finally { - lockedFileChannel.close(); + lock.close(); Files.delete(path); } @@ -183,15 +182,15 @@ public void testLockReleaseByInputStream() throws IOException { Files.delete(path); } catch (final IOException e) {} - final LockedFileChannel lockedFileChannel = LockedFileChannel.openForWriting(path); + final LockedChannel lock = access.lockForWriting(path); System.out.println("locked"); - Channels.newInputStream(lockedFileChannel.getFileChannel()).close(); + lock.newInputStream().close(); System.out.println("input stream released"); final ExecutorService exec = Executors.newSingleThreadExecutor(); final Future future = exec.submit(() -> { - LockedFileChannel.openForWriting(path).close(); + access.lockForWriting(path).close(); return null; }); @@ -205,7 +204,7 @@ public void testLockReleaseByInputStream() throws IOException { future.cancel(true); System.out.println("Test was interrupted!"); } finally { - lockedFileChannel.close(); + lock.close(); Files.delete(path); } From b5a70af64efcaec4d326af84306d0a3c4027fc5b Mon Sep 17 00:00:00 2001 From: Stephan Saalfeld Date: Tue, 3 Aug 2021 17:11:14 -0400 Subject: [PATCH 024/243] This seems to work flawlessly --- .../n5/FileSystemKeyValueAccess.java | 10 +++--- .../saalfeldlab/n5/N5KeyValueWriter.java | 10 +++--- .../saalfeldlab/n5/AbstractN5Test.java | 5 +-- .../org/janelia/saalfeldlab/n5/N5FSTest.java | 31 +++++++------------ 4 files changed, 25 insertions(+), 31 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java index 40a89e60..adbe46ca 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java @@ -257,21 +257,21 @@ public void createDirectories(final String normalPath) throws IOException { public void delete(final String normalPath) throws IOException { final Path path = fileSystem.getPath(normalPath); - if (Files.isRegularFile(path)) { + + if (Files.isRegularFile(path)) try (final LockedChannel channel = lockForWriting(path)) { Files.delete(path); } - } else { + else { try (final Stream pathStream = Files.walk(path)) { for (final Iterator i = pathStream.sorted(Comparator.reverseOrder()).iterator(); i.hasNext();) { final Path childPath = i.next(); - if (Files.isRegularFile(childPath)) { + if (Files.isRegularFile(childPath)) try (final LockedChannel channel = lockForWriting(childPath)) { Files.delete(childPath); } - } else { + else tryDelete(childPath); - } } } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriter.java b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriter.java index 6c1721f6..debca5f4 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriter.java @@ -122,8 +122,10 @@ protected N5GroupInfo createCachedGroup(final String normalPath) throws IOExcept info = new N5GroupInfo(); metaCache.put(normalPath, info); } - for (String childPathName = normalPath; !childPathName.equals("");) { + for (String childPathName = normalPath; !(childPathName == null || childPathName.equals(""));) { final String parentPathName = keyValueAccess.parent(childPathName); + if (parentPathName == null) + break; N5GroupInfo parentInfo = getCachedN5GroupInfo(parentPathName); if (parentInfo == emptyGroupInfo) { parentInfo = new N5GroupInfo(); @@ -279,7 +281,7 @@ public boolean remove(final String path) throws IOException { /* cache nonexistence for all prior children */ for (final String key : metaCache.keySet()) { if (key.startsWith(normalPath)) - metaCache.put(normalPath, emptyGroupInfo); + metaCache.put(key, emptyGroupInfo); } /* remove child from parent */ @@ -311,9 +313,7 @@ public boolean deleteBlock( final String blockPath = getDataBlockPath(normalize(path), gridPosition); if (keyValueAccess.exists(blockPath)) - try (final LockedChannel channel = keyValueAccess.lockForWriting(blockPath)) { - keyValueAccess.delete(blockPath); - } + keyValueAccess.delete(blockPath); /* an IOException should have occurred if anything had failed midway */ return true; diff --git a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java index f75516b2..21cabde2 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java @@ -536,7 +536,7 @@ public void testDeepList() { final List datasetList = Arrays.asList(n5.deepList("/")); final N5Writer n5Writer = n5; - System.out.println(datasetList); + for (final String subGroup : subGroupNames) Assert.assertTrue("deepList contents", datasetList.contains(groupName.replaceFirst("/", "") + "/" + subGroup)); Assert.assertTrue("deepList contents", datasetList.contains(datasetName.replaceFirst("/", ""))); @@ -740,7 +740,7 @@ public void testVersion() throws NumberFormatException, IOException { final Version n5Version = n5.getVersion(); - System.out.println(n5Version); +// System.out.println(n5Version); Assert.assertTrue(n5Version.equals(N5Reader.VERSION)); @@ -784,6 +784,7 @@ public void testDelete() throws IOException { } protected boolean testDeleteIsBlockDeleted(final DataBlock dataBlock) { + return dataBlock == null; } } diff --git a/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java b/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java index c53b1ec8..ee56d01a 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java @@ -50,7 +50,7 @@ public class N5FSTest extends AbstractN5Test { private static FileSystemKeyValueAccess access = new FileSystemKeyValueAccess(FileSystems.getDefault()); - private static String testDirPath = System.getProperty("user.home") + "/tmp/n5-test"; + private static String testDirPath = System.getProperty("user.home") + "/tmp/n5-test.n5"; /** * @throws IOException @@ -61,7 +61,7 @@ protected N5Writer createN5Writer() throws IOException { return new N5FSWriter(testDirPath, false); } - @Test +// @Test public void testReadLock() throws IOException, InterruptedException { final Path path = Paths.get(testDirPath, "lock"); @@ -99,7 +99,7 @@ public void testReadLock() throws IOException, InterruptedException { } - @Test +// @Test public void testWriteLock() throws IOException { final Path path = Paths.get(testDirPath, "lock"); @@ -137,17 +137,16 @@ public void testWriteLock() throws IOException { @Test public void testLockReleaseByReader() throws IOException { + System.out.println("Testing lock release by Reader."); + final Path path = Paths.get(testDirPath, "lock"); try { Files.delete(path); } catch (final IOException e) {} final LockedChannel lock = access.lockForWriting(path); - System.out.println("locked"); lock.newReader().close(); - System.out.println("reader released"); - final ExecutorService exec = Executors.newSingleThreadExecutor(); final Future future = exec.submit(() -> { @@ -156,37 +155,34 @@ public void testLockReleaseByReader() throws IOException { }); try { - System.out.println("Trying to acquire locked readable channel..."); future.get(3, TimeUnit.SECONDS); } catch (final TimeoutException e) { - fail("Lock not released!"); + fail("... lock not released!"); future.cancel(true); } catch (final InterruptedException | ExecutionException e) { future.cancel(true); - System.out.println("Test was interrupted!"); + System.out.println("... test interrupted!"); } finally { lock.close(); Files.delete(path); } - System.out.println("Lock was successfully released."); - exec.shutdownNow(); } @Test public void testLockReleaseByInputStream() throws IOException { + System.out.println("Testing lock release by InputStream."); + final Path path = Paths.get(testDirPath, "lock"); try { Files.delete(path); } catch (final IOException e) {} final LockedChannel lock = access.lockForWriting(path); - System.out.println("locked"); lock.newInputStream().close(); - System.out.println("input stream released"); final ExecutorService exec = Executors.newSingleThreadExecutor(); final Future future = exec.submit(() -> { @@ -195,25 +191,22 @@ public void testLockReleaseByInputStream() throws IOException { }); try { - System.out.println("Trying to acquire locked readable channel..."); future.get(3, TimeUnit.SECONDS); } catch (final TimeoutException e) { - fail("Lock not released!"); + fail("... lock not released!"); future.cancel(true); } catch (final InterruptedException | ExecutionException e) { future.cancel(true); - System.out.println("Test was interrupted!"); + System.out.println("... test interrupted!"); } finally { lock.close(); Files.delete(path); } - System.out.println("Lock was successfully released."); - exec.shutdownNow(); } -// @Test + @Test public void testCache() { final N5Writer n5Writer = n5; From a0988c61f4be25c6951377a2c8961a5e6a1f5425 Mon Sep 17 00:00:00 2001 From: Stephan Saalfeld Date: Wed, 4 Aug 2021 11:26:29 -0400 Subject: [PATCH 025/243] feat: add and use KeyValuAccess path compose method abstracts further to platform dependent separation strings (e.g. "/" vs "\"). The resulting path string is now strictly a platform dependent path and should not be used in code where a platform independent string is expected. Thanks to the Windows FileSystem implementation accepting "/" separated paths, this should not cause pracitcal issues at this time but we may have to change the API to support other FileSystems (e.g. passing paths along as components instead of strings or settling on a nio Path like object). --- .../n5/FileSystemKeyValueAccess.java | 11 ++++++++++ .../saalfeldlab/n5/KeyValueAccess.java | 8 ++++++++ .../saalfeldlab/n5/N5KeyValueReader.java | 20 +++++++++---------- 3 files changed, 29 insertions(+), 10 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java index adbe46ca..7c215144 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java @@ -44,6 +44,7 @@ import java.nio.file.Path; import java.nio.file.StandardOpenOption; import java.nio.file.attribute.FileAttribute; +import java.util.Arrays; import java.util.Comparator; import java.util.Iterator; import java.util.stream.Stream; @@ -247,6 +248,16 @@ public String normalize(final String path) { return fileSystem.getPath(path).normalize().toString(); } + @Override + public String compose(final String... components) { + + if (components == null || components.length == 0) + return null; + if (components.length == 1) + return fileSystem.getPath(components[0]).toString(); + return fileSystem.getPath(components[0], Arrays.copyOfRange(components, 1, components.length)).toString(); + } + @Override public void createDirectories(final String normalPath) throws IOException { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccess.java index 872b0b88..52f23dba 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccess.java @@ -46,6 +46,14 @@ public interface KeyValueAccess { */ public String[] components(final String path); + /** + * Compose a path from components. + * + * @param components + * @return + */ + public String compose(final String... components); + /** * Get the parent of a path string. * diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java index 3e608f02..927f6a15 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java @@ -696,14 +696,14 @@ protected String getDataBlockPath( final String normalPath, final long... gridPosition) { - final StringBuilder builder = new StringBuilder(groupPath(normalPath)); - - for (final long i : gridPosition) { - builder.append("/"); - builder.append(i); - } - - return builder.toString(); + final String[] components = new String[gridPosition.length + 2]; + components[0] = basePath; + components[1] = normalPath; + int i = 1; + for (final long p : gridPosition) + components[++i] = Long.toString(p); + + return keyValueAccess.compose(components); } /** @@ -715,7 +715,7 @@ protected String getDataBlockPath( */ protected String groupPath(final String normalPath) { - return basePath + "/" + normalPath; + return keyValueAccess.compose(basePath, normalPath); } /** @@ -727,7 +727,7 @@ protected String groupPath(final String normalPath) { */ protected String attributesPath(final String normalPath) { - return groupPath(normalPath) + "/" + jsonFile; + return keyValueAccess.compose(basePath, normalPath, jsonFile); } /** From 9268b4c10a04b822601d079281bcee0c87b74244 Mon Sep 17 00:00:00 2001 From: Stephan Saalfeld Date: Sat, 4 Sep 2021 10:38:41 -0400 Subject: [PATCH 026/243] feat: LockedChannel.openForWriting creates the full path, this is better on key value stores that do not really have paths. TODO check for leftover redundant path creation, that wouldn't break things but makes an unnecessary exists test --- .../n5/FileSystemKeyValueAccess.java | 18 ++++++++++++++++-- .../janelia/saalfeldlab/n5/KeyValueAccess.java | 5 +++-- .../saalfeldlab/n5/N5KeyValueWriter.java | 1 - 3 files changed, 19 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java index 7c215144..9c2e9a24 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java @@ -75,8 +75,22 @@ protected LockedFileChannel(final String path, final boolean readOnly) throws IO protected LockedFileChannel(final Path path, final boolean readOnly) throws IOException { - final OpenOption[] options = readOnly ? new OpenOption[]{StandardOpenOption.READ} : new OpenOption[]{StandardOpenOption.READ, StandardOpenOption.WRITE, StandardOpenOption.CREATE}; - channel = FileChannel.open(path, options); + final OpenOption[] options; + if (readOnly) { + options = new OpenOption[]{StandardOpenOption.READ}; + channel = FileChannel.open(path, options); + } else { + options = new OpenOption[]{StandardOpenOption.READ, StandardOpenOption.WRITE, StandardOpenOption.CREATE}; + FileChannel tryChannel = null; + try { + tryChannel = FileChannel.open(path, options); + } catch (final NoSuchFileException e) { + createDirectories(path.getParent()); + tryChannel = FileChannel.open(path, options); + } finally { + channel = tryChannel; + } + } for (boolean waiting = true; waiting;) { waiting = false; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccess.java index 52f23dba..b8b2035e 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccess.java @@ -125,8 +125,9 @@ public interface KeyValueAccess { public LockedChannel lockForReading(final String normalPath) throws IOException; /** - * Create an exclusive lock on a path for writing. This isn't meant to be - * kept around. Create, use, [auto]close, e.g. + * Create an exclusive lock on a path for writing. If the file doesn't + * exist yet, it will be created, including all directories leading up to + * it. This lock isn't meant to be kept around. Create, use, [auto]close, e.g. * * try (final lock = store.lockForWriting()) { * ... diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriter.java b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriter.java index debca5f4..07287e33 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriter.java @@ -261,7 +261,6 @@ public void writeBlock( final DataBlock dataBlock) throws IOException { final String blockPath = getDataBlockPath(normalize(path), dataBlock.getGridPosition()); - keyValueAccess.createDirectories(keyValueAccess.parent(blockPath)); try (final LockedChannel lock = keyValueAccess.lockForWriting(blockPath)) { DefaultBlockWriter.writeBlock(lock.newOutputStream(), datasetAttributes, dataBlock); From 7e527eb4cb00f1f1c002185d9be2241a9dd6ffd9 Mon Sep 17 00:00:00 2001 From: Stephan Saalfeld Date: Tue, 14 Jun 2022 17:14:24 -0400 Subject: [PATCH 027/243] bump to pom-scijava-31.1.0 --- LICENSE.md | 2 +- pom.xml | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/LICENSE.md b/LICENSE.md index 1e1810e3..01cc4de4 100644 --- a/LICENSE.md +++ b/LICENSE.md @@ -1,6 +1,6 @@ ## BSD 2-Clause License -Copyright (c) 2017-2021, Stephan Saalfeld +Copyright (c) 2017-2022, Stephan Saalfeld All rights reserved. diff --git a/pom.xml b/pom.xml index 4ec43c52..a1329100 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ org.scijava pom-scijava - 30.0.0 + 31.1.0 @@ -105,6 +105,9 @@ 4.2 + 3.0.0-M3 + 0.8.6 + 2.6.1 From f849bdc90263910be4dbe48dd43d6741b9078f03 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Tue, 15 Nov 2022 09:51:55 -0500 Subject: [PATCH 028/243] feat: wip add failing test for getting attributes from urls --- .../saalfeldlab/n5/AbstractGsonReader.java | 20 +++ .../org/janelia/saalfeldlab/n5/N5Reader.java | 32 +++++ .../saalfeldlab/n5/url/UrlAttributeTest.java | 132 ++++++++++++++++++ .../urlAttributes.n5/a/aa/aaa/attributes.json | 3 + .../url/urlAttributes.n5/a/aa/attributes.json | 3 + .../url/urlAttributes.n5/a/attributes.json | 3 + .../url/urlAttributes.n5/attributes.json | 9 ++ 7 files changed, 202 insertions(+) create mode 100644 src/test/java/org/janelia/saalfeldlab/n5/url/UrlAttributeTest.java create mode 100644 src/test/resources/url/urlAttributes.n5/a/aa/aaa/attributes.json create mode 100644 src/test/resources/url/urlAttributes.n5/a/aa/attributes.json create mode 100644 src/test/resources/url/urlAttributes.n5/a/attributes.json create mode 100644 src/test/resources/url/urlAttributes.n5/attributes.json diff --git a/src/main/java/org/janelia/saalfeldlab/n5/AbstractGsonReader.java b/src/main/java/org/janelia/saalfeldlab/n5/AbstractGsonReader.java index cbda4b06..8ae6d210 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/AbstractGsonReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/AbstractGsonReader.java @@ -138,4 +138,24 @@ public T getAttribute( final HashMap map = getAttributes(pathName); return GsonAttributesParser.parseAttribute(map, key, type, getGson()); } + + @Override + public T getAttributeURL( + final String context, + final String url, + final Class clazz) throws IOException { + + // TODO implement me + return null; + } + + @Override + public T getAttributeURL( + final String context, + final String url, + final Type type) throws IOException { + + // TODO implement me + return null; + } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java index dc368446..c49699e2 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java @@ -249,6 +249,38 @@ public T getAttribute( final String key, final Type type) throws IOException; + /** + * Reads an attribute. + * + * @param context + * the context + * @param url + * @param clazz + * attribute class + * @return + * @throws IOException + */ + public T getAttributeURL( + final String context, + final String url, + final Class clazz) throws IOException; + + /** + * Reads an attribute. + * + * @param context + * the context + * @param url the url + * @param type + * attribute Type (use this for specifying generic types) + * @return + * @throws IOException + */ + public T getAttributeURL( + final String context, + final String url, + final Type type) throws IOException; + /** * Get mandatory dataset attributes. * diff --git a/src/test/java/org/janelia/saalfeldlab/n5/url/UrlAttributeTest.java b/src/test/java/org/janelia/saalfeldlab/n5/url/UrlAttributeTest.java new file mode 100644 index 00000000..45a41d02 --- /dev/null +++ b/src/test/java/org/janelia/saalfeldlab/n5/url/UrlAttributeTest.java @@ -0,0 +1,132 @@ +package org.janelia.saalfeldlab.n5.url; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.io.IOException; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import org.janelia.saalfeldlab.n5.N5FSReader; +import org.janelia.saalfeldlab.n5.N5Reader; +import org.junit.Before; +import org.junit.Test; + +public class UrlAttributeTest +{ + N5Reader n5; + String rootContext = ""; + int[] list; + HashMap obj; + Set rootKeys; + + @Before + public void before() + { + try + { + n5 = new N5FSReader( "src/test/resources/url/urlAttributes.n5" ); + rootContext = ""; + list = new int[] { 0, 1, 2, 3 }; + + obj = new HashMap<>(); + obj.put( "a", "aa" ); + obj.put( "b", "bb" ); + rootKeys = new HashSet<>(); + rootKeys.addAll( Stream.of("n5", "foo", "list", "object" ).collect( Collectors.toList() ) ); + } + catch ( IOException e ) + { + e.printStackTrace(); + } + } + + @SuppressWarnings( "unchecked" ) + @Test + public void testRootAttributes() + { + try + { + // get + HashMap everything = n5.getAttributeURL( "", "#/", HashMap.class ); + assertEquals( "2.5.1", (String) everything.get( "n5" )); + + assertEquals( "bar", n5.getAttributeURL( rootContext, "#foo", String.class ) ); + assertEquals( list, n5.getAttributeURL( "", "#list", int[].class ) ); + assertEquals( list, n5.getAttributeURL( "", "#/list", int[].class ) ); + + // list + assertEquals( list[0], (int)n5.getAttributeURL( rootContext, "#list[0]", Integer.class ) ); + assertEquals( list[1], (int)n5.getAttributeURL( rootContext, "#list[1]", Integer.class ) ); + assertEquals( list[2], (int)n5.getAttributeURL( rootContext, "#list[2]", Integer.class ) ); + + assertEquals( list[3], (int)n5.getAttributeURL( rootContext, "#list/[3]", Integer.class ) ); + assertEquals( list[3], (int)n5.getAttributeURL( rootContext, "#/list/[3]", Integer.class ) ); + + // object + assertTrue( mapsEqual( obj, n5.getAttributeURL( rootContext, "#object", Map.class ) ) ); + + } + catch ( IOException e ) + { + fail( e.getMessage() ); + } + + } + + public void testAbsolutePathAttributes() + { + final String a = "a"; + final String aa = "aa"; + final String aaa = "aaa"; + try + { + // name of a + assertEquals( "name of a from root", a, n5.getAttributeURL( "", "?a#name", String.class ) ); + assertEquals( "name of a from root", a, n5.getAttributeURL( "", "?/a#name", String.class ) ); + assertEquals( "name of a from a", a, n5.getAttributeURL( "a", "#name", String.class ) ); + assertEquals( "name of a from aa", a, n5.getAttributeURL( "aa", "?..#name", String.class ) ); + assertEquals( "name of a from aaa", a, n5.getAttributeURL( "aaa", "?../..#name", String.class ) ); + + // name of aa + assertEquals( "name of aa from root", aa, n5.getAttributeURL( "", "?a/aa#name", String.class ) ); + assertEquals( "name of aa from root", aa, n5.getAttributeURL( "", "?/a/aa#name", String.class ) ); + assertEquals( "name of aa from a", aa, n5.getAttributeURL( "a", "?aa#name", String.class ) ); + assertEquals( "name of aa from aa", aa, n5.getAttributeURL( "aa", "#name", String.class ) ); + assertEquals( "name of aa from aa", aa, n5.getAttributeURL( "aa", "?/#name", String.class ) ); + assertEquals( "name of aa from aaa", aa, n5.getAttributeURL( "aaa", "?..#name", String.class ) ); + + // name of aaa + assertEquals( "name of aaa from root", aaa, n5.getAttributeURL( "", "?a/aa/aaa#name", String.class ) ); + assertEquals( "name of aaa from root", aaa, n5.getAttributeURL( "", "?/a/aa/aaa#name", String.class ) ); + assertEquals( "name of aaa from a", aaa, n5.getAttributeURL( "a", "?aa/aaa#name", String.class ) ); + assertEquals( "name of aaa from aa", aaa, n5.getAttributeURL( "aa", "?aaa#name", String.class ) ); + assertEquals( "name of aaa from aaa", aaa, n5.getAttributeURL( "aaa", "#name", String.class ) ); + assertEquals( "name of aaa from aaa", aaa, n5.getAttributeURL( "aaa", "?/#name", String.class ) ); + } + catch ( IOException e ) + { + fail( e.getMessage() ); + } + } + + private boolean mapsEqual( Map a, Map b ) + { + if( ! a.keySet().equals( b.keySet() )) + return false; + + for( K k : a.keySet() ) + { + if( ! a.get( k ).equals( b.get( k ) )) + return false; + } + + return true; + } + +} diff --git a/src/test/resources/url/urlAttributes.n5/a/aa/aaa/attributes.json b/src/test/resources/url/urlAttributes.n5/a/aa/aaa/attributes.json new file mode 100644 index 00000000..0b7a477a --- /dev/null +++ b/src/test/resources/url/urlAttributes.n5/a/aa/aaa/attributes.json @@ -0,0 +1,3 @@ +{ + "name" : "aaa" +} diff --git a/src/test/resources/url/urlAttributes.n5/a/aa/attributes.json b/src/test/resources/url/urlAttributes.n5/a/aa/attributes.json new file mode 100644 index 00000000..47000178 --- /dev/null +++ b/src/test/resources/url/urlAttributes.n5/a/aa/attributes.json @@ -0,0 +1,3 @@ +{ + "name" : "aa" +} diff --git a/src/test/resources/url/urlAttributes.n5/a/attributes.json b/src/test/resources/url/urlAttributes.n5/a/attributes.json new file mode 100644 index 00000000..f1436a43 --- /dev/null +++ b/src/test/resources/url/urlAttributes.n5/a/attributes.json @@ -0,0 +1,3 @@ +{ + "name" : "a" +} diff --git a/src/test/resources/url/urlAttributes.n5/attributes.json b/src/test/resources/url/urlAttributes.n5/attributes.json new file mode 100644 index 00000000..b8af56bc --- /dev/null +++ b/src/test/resources/url/urlAttributes.n5/attributes.json @@ -0,0 +1,9 @@ +{ + "n5": "2.5.1", + "foo": "bar", + "list": [ 0, 1, 2, 3 ], + "object": { + "a": "aa", + "b": "bb" + } +} From 31f360cc1ad75f6b6201b118c4e42be2a525ea64 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Fri, 18 Nov 2022 10:04:23 -0500 Subject: [PATCH 029/243] feat: initial N5Url API --- .../saalfeldlab/n5/AbstractGsonReader.java | 11 +-- .../org/janelia/saalfeldlab/n5/N5Reader.java | 10 +-- .../org/janelia/saalfeldlab/n5/N5URL.java | 86 +++++++++++++++++++ .../saalfeldlab/n5/url/UrlAttributeTest.java | 54 ++++++------ 4 files changed, 121 insertions(+), 40 deletions(-) create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/N5URL.java diff --git a/src/main/java/org/janelia/saalfeldlab/n5/AbstractGsonReader.java b/src/main/java/org/janelia/saalfeldlab/n5/AbstractGsonReader.java index 8ae6d210..4b755607 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/AbstractGsonReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/AbstractGsonReader.java @@ -140,19 +140,16 @@ public T getAttribute( } @Override - public T getAttributeURL( - final String context, - final String url, + public T getAttribute( + final N5URL url, final Class clazz) throws IOException { - // TODO implement me return null; } @Override - public T getAttributeURL( - final String context, - final String url, + public T getAttribute( + final N5URL url, final Type type) throws IOException { // TODO implement me diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java index c49699e2..6d4a21b3 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java @@ -260,9 +260,8 @@ public T getAttribute( * @return * @throws IOException */ - public T getAttributeURL( - final String context, - final String url, + public T getAttribute( + final N5URL url, final Class clazz) throws IOException; /** @@ -276,9 +275,8 @@ public T getAttributeURL( * @return * @throws IOException */ - public T getAttributeURL( - final String context, - final String url, + public T getAttribute( + final N5URL url, final Type type) throws IOException; /** diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java b/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java new file mode 100644 index 00000000..505a39a4 --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java @@ -0,0 +1,86 @@ +package org.janelia.saalfeldlab.n5; + +import java.net.URI; +import java.net.URISyntaxException; + +public class N5URL { + + final URI uri; + private String scheme; + private String location; + private String dataset; + private String attribute; + + public N5URL(String uri) throws URISyntaxException { + this(encodeAsUri(uri)); + } + + public N5URL(URI uri) { + + this.uri = uri; + final int queryIdx = uri.getSchemeSpecificPart().indexOf('?'); + final String schemeSpecificPartWithoutQuery = uri.getSchemeSpecificPart().substring(0, queryIdx); + scheme = uri.getScheme() == null ? null : uri.getScheme(); + if (uri.getScheme() == null) { + location = schemeSpecificPartWithoutQuery.replaceFirst("//", ""); + } else { + location = uri.getScheme() + ":" + schemeSpecificPartWithoutQuery; + } + if (queryIdx > 0) { + dataset = uri.getSchemeSpecificPart().substring(queryIdx + 1); + } else { + dataset = null; + } + attribute = uri.getFragment(); + } + + @Override public String toString() { + + String scheme = ""; + if (uri.getScheme() != null) { + scheme = uri.getScheme() + ":"; + } + String fragment = ""; + if (uri.getFragment() != null) { + fragment = "#" + uri.getFragment(); + } + return scheme + uri.getSchemeSpecificPart() + fragment; + } + + public N5URL getRelative(URI relative) { + return null; + } + + public static URI encodeAsUri(String uri) throws URISyntaxException { + + /* find last # symbol to split fragment on. If we don't remove it first, then it will encode it, and not parse it separately + * after we remove the temporary _N5 scheme */ + + final int fragmentIdx = uri.lastIndexOf('#'); + final String uriWithoutFragment; + final String fragment; + if (fragmentIdx > 0) { + uriWithoutFragment = uri.substring(0, fragmentIdx); + fragment = uri.substring(fragmentIdx + 1); + } else { + uriWithoutFragment = uri; + fragment = null; + } + final URI _n5Uri = new URI("N5Internal", uriWithoutFragment, fragment); + + final URI n5Uri; + if (fragment == null) { + n5Uri = new URI(_n5Uri.getRawSchemeSpecificPart()); + } else { + n5Uri = new URI(_n5Uri.getRawSchemeSpecificPart() + "#" + _n5Uri.getRawFragment()); + } + return n5Uri; + } + + public static void main(String[] args) throws URISyntaxException { + +// System.out.println(new N5URL("/this/is/a/test?dataset#/a/b/attributes[0]")); + System.out.println(new N5URL("s3://janelia-co 1234 5t 56 sem-datasets/jrc_ma 445t 6crophage-2/jrc_macr 2d3 464 ophage-2.n5?d2 3d?aset#a d245 sdf")); + } + +} diff --git a/src/test/java/org/janelia/saalfeldlab/n5/url/UrlAttributeTest.java b/src/test/java/org/janelia/saalfeldlab/n5/url/UrlAttributeTest.java index 45a41d02..93da6e53 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/url/UrlAttributeTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/url/UrlAttributeTest.java @@ -53,23 +53,23 @@ public void testRootAttributes() try { // get - HashMap everything = n5.getAttributeURL( "", "#/", HashMap.class ); + HashMap everything = n5.getAttribute( "", HashMap.class ); assertEquals( "2.5.1", (String) everything.get( "n5" )); - assertEquals( "bar", n5.getAttributeURL( rootContext, "#foo", String.class ) ); - assertEquals( list, n5.getAttributeURL( "", "#list", int[].class ) ); - assertEquals( list, n5.getAttributeURL( "", "#/list", int[].class ) ); + assertEquals( "bar", n5.getAttribute( rootContext, String.class ) ); + assertEquals( list, n5.getAttribute( "", int[].class ) ); + assertEquals( list, n5.getAttribute( "", int[].class ) ); // list - assertEquals( list[0], (int)n5.getAttributeURL( rootContext, "#list[0]", Integer.class ) ); - assertEquals( list[1], (int)n5.getAttributeURL( rootContext, "#list[1]", Integer.class ) ); - assertEquals( list[2], (int)n5.getAttributeURL( rootContext, "#list[2]", Integer.class ) ); + assertEquals( list[0], (int)n5.getAttribute( rootContext, Integer.class ) ); + assertEquals( list[1], (int)n5.getAttribute( rootContext, Integer.class ) ); + assertEquals( list[2], (int)n5.getAttribute( rootContext, Integer.class ) ); - assertEquals( list[3], (int)n5.getAttributeURL( rootContext, "#list/[3]", Integer.class ) ); - assertEquals( list[3], (int)n5.getAttributeURL( rootContext, "#/list/[3]", Integer.class ) ); + assertEquals( list[3], (int)n5.getAttribute( rootContext, Integer.class ) ); + assertEquals( list[3], (int)n5.getAttribute( rootContext, Integer.class ) ); // object - assertTrue( mapsEqual( obj, n5.getAttributeURL( rootContext, "#object", Map.class ) ) ); + assertTrue( mapsEqual( obj, n5.getAttribute( rootContext, Map.class ) ) ); } catch ( IOException e ) @@ -87,27 +87,27 @@ public void testAbsolutePathAttributes() try { // name of a - assertEquals( "name of a from root", a, n5.getAttributeURL( "", "?a#name", String.class ) ); - assertEquals( "name of a from root", a, n5.getAttributeURL( "", "?/a#name", String.class ) ); - assertEquals( "name of a from a", a, n5.getAttributeURL( "a", "#name", String.class ) ); - assertEquals( "name of a from aa", a, n5.getAttributeURL( "aa", "?..#name", String.class ) ); - assertEquals( "name of a from aaa", a, n5.getAttributeURL( "aaa", "?../..#name", String.class ) ); + assertEquals( "name of a from root", a, n5.getAttribute( "", String.class ) ); + assertEquals( "name of a from root", a, n5.getAttribute( "", String.class ) ); + assertEquals( "name of a from a", a, n5.getAttribute( "a", String.class ) ); + assertEquals( "name of a from aa", a, n5.getAttribute( "aa", String.class ) ); + assertEquals( "name of a from aaa", a, n5.getAttribute( "aaa", String.class ) ); // name of aa - assertEquals( "name of aa from root", aa, n5.getAttributeURL( "", "?a/aa#name", String.class ) ); - assertEquals( "name of aa from root", aa, n5.getAttributeURL( "", "?/a/aa#name", String.class ) ); - assertEquals( "name of aa from a", aa, n5.getAttributeURL( "a", "?aa#name", String.class ) ); - assertEquals( "name of aa from aa", aa, n5.getAttributeURL( "aa", "#name", String.class ) ); - assertEquals( "name of aa from aa", aa, n5.getAttributeURL( "aa", "?/#name", String.class ) ); - assertEquals( "name of aa from aaa", aa, n5.getAttributeURL( "aaa", "?..#name", String.class ) ); + assertEquals( "name of aa from root", aa, n5.getAttribute( "", String.class ) ); + assertEquals( "name of aa from root", aa, n5.getAttribute( "", String.class ) ); + assertEquals( "name of aa from a", aa, n5.getAttribute( "a", String.class ) ); + assertEquals( "name of aa from aa", aa, n5.getAttribute( "aa", String.class ) ); + assertEquals( "name of aa from aa", aa, n5.getAttribute( "aa", String.class ) ); + assertEquals( "name of aa from aaa", aa, n5.getAttribute( "aaa", String.class ) ); // name of aaa - assertEquals( "name of aaa from root", aaa, n5.getAttributeURL( "", "?a/aa/aaa#name", String.class ) ); - assertEquals( "name of aaa from root", aaa, n5.getAttributeURL( "", "?/a/aa/aaa#name", String.class ) ); - assertEquals( "name of aaa from a", aaa, n5.getAttributeURL( "a", "?aa/aaa#name", String.class ) ); - assertEquals( "name of aaa from aa", aaa, n5.getAttributeURL( "aa", "?aaa#name", String.class ) ); - assertEquals( "name of aaa from aaa", aaa, n5.getAttributeURL( "aaa", "#name", String.class ) ); - assertEquals( "name of aaa from aaa", aaa, n5.getAttributeURL( "aaa", "?/#name", String.class ) ); + assertEquals( "name of aaa from root", aaa, n5.getAttribute( "", String.class ) ); + assertEquals( "name of aaa from root", aaa, n5.getAttribute( "", String.class ) ); + assertEquals( "name of aaa from a", aaa, n5.getAttribute( "a", String.class ) ); + assertEquals( "name of aaa from aa", aaa, n5.getAttribute( "aa", String.class ) ); + assertEquals( "name of aaa from aaa", aaa, n5.getAttribute( "aaa", String.class ) ); + assertEquals( "name of aaa from aaa", aaa, n5.getAttribute( "aaa", String.class ) ); } catch ( IOException e ) { From fa78a76b9e308a0a3557d4927ff66d5f260363ca Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Fri, 18 Nov 2022 15:28:13 -0500 Subject: [PATCH 030/243] test: update test for getting attributes to use N5URL --- .../saalfeldlab/n5/url/UrlAttributeTest.java | 105 ++++++++++++------ 1 file changed, 73 insertions(+), 32 deletions(-) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/url/UrlAttributeTest.java b/src/test/java/org/janelia/saalfeldlab/n5/url/UrlAttributeTest.java index 93da6e53..ec7ad2c7 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/url/UrlAttributeTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/url/UrlAttributeTest.java @@ -1,10 +1,13 @@ package org.janelia.saalfeldlab.n5.url; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertThrows; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import java.io.IOException; +import java.net.URISyntaxException; import java.util.HashMap; import java.util.HashSet; import java.util.Map; @@ -14,6 +17,7 @@ import org.janelia.saalfeldlab.n5.N5FSReader; import org.janelia.saalfeldlab.n5.N5Reader; +import org.janelia.saalfeldlab.n5.N5URL; import org.junit.Before; import org.junit.Test; @@ -53,66 +57,103 @@ public void testRootAttributes() try { // get - HashMap everything = n5.getAttribute( "", HashMap.class ); - assertEquals( "2.5.1", (String) everything.get( "n5" )); + HashMap everything = n5.getAttribute( new N5URL(""), HashMap.class ); + assertEquals( "empty url", "2.5.1", (String) everything.get( "n5" )); - assertEquals( "bar", n5.getAttribute( rootContext, String.class ) ); - assertEquals( list, n5.getAttribute( "", int[].class ) ); - assertEquals( list, n5.getAttribute( "", int[].class ) ); + HashMap everything2 = n5.getAttribute( new N5URL("#/"), HashMap.class ); + assertEquals( "root attribute", "2.5.1", (String) everything2.get( "n5" )); + + assertEquals( "url to attribute", "bar", n5.getAttribute( new N5URL("#foo"), String.class ) ); + assertEquals( "url to attribute absolute", "bar", n5.getAttribute( new N5URL("#/foo"), String.class ) ); + + assertEquals( "#foo", "bar", n5.getAttribute( new N5URL("#foo"), String.class ) ); + assertEquals( "#/foo", "bar", n5.getAttribute( new N5URL("#/foo"), String.class ) ); + assertEquals( "?#foo", "bar", n5.getAttribute( new N5URL("?#foo"), String.class ) ); + assertEquals( "?#/foo", "bar", n5.getAttribute( new N5URL("?#/foo"), String.class ) ); + assertEquals( "?/#/foo", "bar", n5.getAttribute( new N5URL("?/#/foo"), String.class ) ); + assertEquals( "?/.#/foo", "bar", n5.getAttribute( new N5URL("?/.#/foo"), String.class ) ); + assertEquals( "?./#/foo", "bar", n5.getAttribute( new N5URL("?./#/foo"), String.class ) ); + assertEquals( "?.#foo", "bar", n5.getAttribute( new N5URL("?.#foo"), String.class ) ); + assertEquals( "?/a/..#foo", "bar", n5.getAttribute( new N5URL("?/a/..#foo"), String.class ) ); + assertEquals( "?/a/../.#foo", "bar", n5.getAttribute( new N5URL("?/a/../.#foo"), String.class ) ); + + assertEquals( "url list", list, n5.getAttribute( new N5URL("#list"), int[].class ) ); // list - assertEquals( list[0], (int)n5.getAttribute( rootContext, Integer.class ) ); - assertEquals( list[1], (int)n5.getAttribute( rootContext, Integer.class ) ); - assertEquals( list[2], (int)n5.getAttribute( rootContext, Integer.class ) ); + assertEquals( "url list[0]", list[0], (int)n5.getAttribute( new N5URL("#list[0]"), Integer.class ) ); + assertEquals( "url list[1]", list[1], (int)n5.getAttribute( new N5URL("#list[1]"), Integer.class ) ); + assertEquals( "url list[2]", list[2], (int)n5.getAttribute( new N5URL("#list[2]"), Integer.class ) ); + + assertEquals( "url list[3]", list[3], (int)n5.getAttribute( new N5URL("#list[3]"), Integer.class ) ); + assertEquals( "url list/[3]", list[3], (int)n5.getAttribute( new N5URL("#list/[3]"), Integer.class ) ); + assertEquals( "url list//[3]", list[3], (int)n5.getAttribute( new N5URL("#list//[3]"), Integer.class ) ); + assertEquals( "url //list//[3]", list[3], (int)n5.getAttribute( new N5URL("#//list//[3]"), Integer.class ) ); + assertEquals( "url //list//[3]//", list[3], (int)n5.getAttribute( new N5URL("#//list////[3]//"), Integer.class ) ); - assertEquals( list[3], (int)n5.getAttribute( rootContext, Integer.class ) ); - assertEquals( list[3], (int)n5.getAttribute( rootContext, Integer.class ) ); - // object - assertTrue( mapsEqual( obj, n5.getAttribute( rootContext, Map.class ) ) ); - + assertTrue( "url object", mapsEqual( obj, n5.getAttribute( new N5URL("#object"), Map.class ))); + assertEquals( "url object/a", "aa", n5.getAttribute( new N5URL("#object/a"), Map.class )); + assertEquals( "url object/b", "bb", n5.getAttribute( new N5URL("#object/b"), Map.class )); + + // failures + // or assertThrows? + assertNull( "url to attribute", n5.getAttribute( new N5URL("file://garbage.n5?trash#foo"), String.class ) ); + assertNull( "url to attribute", n5.getAttribute( new N5URL("file://garbage.n5?/#foo"), String.class ) ); } catch ( IOException e ) { fail( e.getMessage() ); } + catch ( URISyntaxException e ) + { + fail( e.getMessage() ); + } } - public void testAbsolutePathAttributes() + public void testPathAttributes() { final String a = "a"; final String aa = "aa"; final String aaa = "aaa"; + try { + final N5URL aUrl = new N5URL( "?/a" ); + final N5URL aaUrl = new N5URL( "?/a/aa" ); + final N5URL aaaUrl = new N5URL( "?/a/aa/aaa" ); + // name of a - assertEquals( "name of a from root", a, n5.getAttribute( "", String.class ) ); - assertEquals( "name of a from root", a, n5.getAttribute( "", String.class ) ); - assertEquals( "name of a from a", a, n5.getAttribute( "a", String.class ) ); - assertEquals( "name of a from aa", a, n5.getAttribute( "aa", String.class ) ); - assertEquals( "name of a from aaa", a, n5.getAttribute( "aaa", String.class ) ); - + assertEquals( "name of a from root", a, n5.getAttribute( new N5URL("?/a#name"), String.class ) ); + assertEquals( "name of a from root", a, n5.getAttribute( new N5URL("?a#name"), String.class ) ); + assertEquals( "name of a from a", a, n5.getAttribute( aUrl.getRelative( new N5URL("?/a#name")), String.class )); + assertEquals( "name of a from aa", a, n5.getAttribute( aaUrl.getRelative( new N5URL("?..#name")), String.class )); + assertEquals( "name of a from aaa", a, n5.getAttribute( aaaUrl.getRelative( new N5URL("?../..#name")), String.class )); + // name of aa - assertEquals( "name of aa from root", aa, n5.getAttribute( "", String.class ) ); - assertEquals( "name of aa from root", aa, n5.getAttribute( "", String.class ) ); - assertEquals( "name of aa from a", aa, n5.getAttribute( "a", String.class ) ); - assertEquals( "name of aa from aa", aa, n5.getAttribute( "aa", String.class ) ); - assertEquals( "name of aa from aa", aa, n5.getAttribute( "aa", String.class ) ); - assertEquals( "name of aa from aaa", aa, n5.getAttribute( "aaa", String.class ) ); + assertEquals( "name of aa from root", aa, n5.getAttribute( new N5URL("?/a/aa#name"), String.class ) ); + assertEquals( "name of aa from root", aa, n5.getAttribute( new N5URL("?a/aa#name"), String.class ) ); + assertEquals( "name of aa from a", aa, n5.getAttribute( aUrl.getRelative( new N5URL("?aa#name")), String.class )); + assertEquals( "name of aa from aa", aa, n5.getAttribute( aaUrl.getRelative( new N5URL("?/#name")), String.class )); + assertEquals( "name of aa from aa", aa, n5.getAttribute( aaUrl.getRelative( new N5URL("#name")), String.class )); + assertEquals( "name of aa from aaa", aa, n5.getAttribute( aaaUrl.getRelative( new N5URL("?..#name")), String.class )); // name of aaa - assertEquals( "name of aaa from root", aaa, n5.getAttribute( "", String.class ) ); - assertEquals( "name of aaa from root", aaa, n5.getAttribute( "", String.class ) ); - assertEquals( "name of aaa from a", aaa, n5.getAttribute( "a", String.class ) ); - assertEquals( "name of aaa from aa", aaa, n5.getAttribute( "aa", String.class ) ); - assertEquals( "name of aaa from aaa", aaa, n5.getAttribute( "aaa", String.class ) ); - assertEquals( "name of aaa from aaa", aaa, n5.getAttribute( "aaa", String.class ) ); + assertEquals( "name of aaa from root", aaa, n5.getAttribute( new N5URL("?/a/aa/aaa#name"), String.class ) ); + assertEquals( "name of aaa from root", aaa, n5.getAttribute( new N5URL("?a/aa/aaa#name"), String.class ) ); + assertEquals( "name of aaa from a", aaa, n5.getAttribute( aUrl.getRelative( new N5URL("?aa/aaa#name")), String.class )); + assertEquals( "name of aaa from aa", aaa, n5.getAttribute( aaUrl.getRelative( new N5URL("?aaa#name")), String.class )); + assertEquals( "name of aaa from aaa", aaa, n5.getAttribute( aaaUrl.getRelative( new N5URL("#name")), String.class )); + assertEquals( "name of aaa from aaa", aaa, n5.getAttribute( aaaUrl.getRelative( new N5URL("?/#name")), String.class )); } catch ( IOException e ) { fail( e.getMessage() ); } + catch ( URISyntaxException e ) + { + fail( e.getMessage() ); + } } private boolean mapsEqual( Map a, Map b ) From 52e4add31faa4e6671a3260180506fb536b67b75 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Fri, 18 Nov 2022 15:33:31 -0500 Subject: [PATCH 031/243] feat: initial N5URL relativize support --- .../org/janelia/saalfeldlab/n5/N5URL.java | 144 +++++++++++++++--- .../org/janelia/saalfeldlab/n5/N5URLTest.java | 25 +++ 2 files changed, 144 insertions(+), 25 deletions(-) create mode 100644 src/test/java/org/janelia/saalfeldlab/n5/N5URLTest.java diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java b/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java index 505a39a4..92ccdd0c 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java @@ -1,65 +1,160 @@ package org.janelia.saalfeldlab.n5; +import org.apache.commons.compress.compressors.FileNameUtil; +import org.apache.commons.compress.utils.FileNameUtils; + +import java.io.File; import java.net.URI; import java.net.URISyntaxException; public class N5URL { final URI uri; - private String scheme; - private String location; - private String dataset; - private String attribute; + private final String scheme; + private final String location; + private final String dataset; + private final String attribute; public N5URL(String uri) throws URISyntaxException { + this(encodeAsUri(uri)); } public N5URL(URI uri) { this.uri = uri; - final int queryIdx = uri.getSchemeSpecificPart().indexOf('?'); - final String schemeSpecificPartWithoutQuery = uri.getSchemeSpecificPart().substring(0, queryIdx); scheme = uri.getScheme() == null ? null : uri.getScheme(); + final String schemeSpecificPartWithoutQuery = getSchemeSpecificPartWithoutQuery(); if (uri.getScheme() == null) { location = schemeSpecificPartWithoutQuery.replaceFirst("//", ""); } else { location = uri.getScheme() + ":" + schemeSpecificPartWithoutQuery; } - if (queryIdx > 0) { - dataset = uri.getSchemeSpecificPart().substring(queryIdx + 1); - } else { - dataset = null; - } + dataset = uri.getQuery(); attribute = uri.getFragment(); } + public String getLocation() { + + return location; + } + + public String getDataset() { + + return dataset; + } + + public String getAttribute() { + + return attribute; + } + + private String getSchemePart() { + + return scheme == null ? "" : scheme + "://"; + } + + private String getLocationPart() { + + return location; + } + + private String getDatasetPart() { + + return dataset == null ? "" : "?" + dataset; + } + + private String getAttributePart() { + + return attribute == null ? "" : "#" + attribute; + } + @Override public String toString() { - String scheme = ""; - if (uri.getScheme() != null) { - scheme = uri.getScheme() + ":"; + return getLocationPart() + getDatasetPart() + getAttributePart(); + } + + private String getSchemeSpecificPartWithoutQuery() { + + return uri.getSchemeSpecificPart().replace("?" + uri.getQuery(), ""); + } + + public N5URL getRelative(N5URL relative) throws URISyntaxException { + + final URI thisUri = uri; + final URI relativeUri = relative.uri; + + final StringBuilder newUri = new StringBuilder(); + + if (relativeUri.getScheme() != null) { + return relative; + } + final String thisScheme = thisUri.getScheme(); + if (thisScheme != null) { + newUri.append(thisScheme).append(":"); + } + + if (relativeUri.getAuthority() != null) { + newUri + .append(relativeUri.getAuthority()) + .append(relativeUri.getPath()) + .append(relative.getDatasetPart()) + .append(relative.getAttributePart()); + return new N5URL(newUri.toString()); } - String fragment = ""; - if (uri.getFragment() != null) { - fragment = "#" + uri.getFragment(); + final String thisAuthority = thisUri.getAuthority(); + if (thisAuthority != null) { + newUri.append("//").append(thisAuthority); } - return scheme + uri.getSchemeSpecificPart() + fragment; + + if (relativeUri.getPath() != null) { + if (!relativeUri.getPath().startsWith("/")) { + newUri.append(thisUri.getPath()).append('/'); + } + newUri + .append(relativeUri.getPath()) + .append(relative.getDatasetPart()) + .append(relative.getAttributePart()); + return new N5URL(newUri.toString()); + } + newUri.append(thisUri.getPath()); + + if (relativeUri.getQuery() != null) { + newUri + .append(relative.getDatasetPart()) + .append(relative.getAttributePart()); + return new N5URL(newUri.toString()); + } + newUri.append(this.getDatasetPart()); + + if (relativeUri.getFragment() != null) { + newUri.append(relative.getAttributePart()); + return new N5URL(newUri.toString()); + } + newUri.append(this.getAttributePart()); + + return new N5URL(newUri.toString()); } - public N5URL getRelative(URI relative) { - return null; + public N5URL getRelative(URI relative) throws URISyntaxException { + + return getRelative(new N5URL(relative)); + } + + public N5URL getRelative(String relative) throws URISyntaxException { + + return getRelative(new N5URL(relative)); } public static URI encodeAsUri(String uri) throws URISyntaxException { /* find last # symbol to split fragment on. If we don't remove it first, then it will encode it, and not parse it separately - * after we remove the temporary _N5 scheme */ + * after we remove the temporary _N5 scheme */ - final int fragmentIdx = uri.lastIndexOf('#'); + final int fragmentIdx = uri.lastIndexOf('#'); final String uriWithoutFragment; final String fragment; - if (fragmentIdx > 0) { + if (fragmentIdx >= 0) { uriWithoutFragment = uri.substring(0, fragmentIdx); fragment = uri.substring(fragmentIdx + 1); } else { @@ -79,8 +174,7 @@ public static URI encodeAsUri(String uri) throws URISyntaxException { public static void main(String[] args) throws URISyntaxException { -// System.out.println(new N5URL("/this/is/a/test?dataset#/a/b/attributes[0]")); - System.out.println(new N5URL("s3://janelia-co 1234 5t 56 sem-datasets/jrc_ma 445t 6crophage-2/jrc_macr 2d3 464 ophage-2.n5?d2 3d?aset#a d245 sdf")); + new N5URL("/a/b/c").getRelative("d?a#test"); } } diff --git a/src/test/java/org/janelia/saalfeldlab/n5/N5URLTest.java b/src/test/java/org/janelia/saalfeldlab/n5/N5URLTest.java new file mode 100644 index 00000000..67abac2e --- /dev/null +++ b/src/test/java/org/janelia/saalfeldlab/n5/N5URLTest.java @@ -0,0 +1,25 @@ +package org.janelia.saalfeldlab.n5; + +import junit.framework.TestCase; + +import java.net.URISyntaxException; + +public class N5URLTest extends TestCase { + + public void testGetLocation() { + + } + + public void testGetDataset() { + + } + + public void testGetAttribute() { + + } + + public void testGetRelative() throws URISyntaxException { + assertEquals("/a/b/c/d?e#f", new N5URL("/a/b/c").getRelative("d?e#f").toString()); + + } +} \ No newline at end of file From 6c4e70d53f917afc7de5b1b4922560671da0d668 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Mon, 21 Nov 2022 16:34:39 -0500 Subject: [PATCH 032/243] feat: getAttribute from url --- .../saalfeldlab/n5/AbstractGsonReader.java | 177 ++++++++++++++-- .../org/janelia/saalfeldlab/n5/N5URL.java | 197 ++++++++++++++++-- 2 files changed, 340 insertions(+), 34 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/AbstractGsonReader.java b/src/main/java/org/janelia/saalfeldlab/n5/AbstractGsonReader.java index 4b755607..7369f574 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/AbstractGsonReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/AbstractGsonReader.java @@ -1,16 +1,16 @@ /** * Copyright (c) 2017, Stephan Saalfeld * All rights reserved. - * + *

* Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: - * + *

* 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. + * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + *

* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE @@ -25,14 +25,28 @@ */ package org.janelia.saalfeldlab.n5; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonSyntaxException; +import com.google.gson.TypeAdapter; +import com.google.gson.reflect.TypeToken; +import com.google.gson.stream.JsonReader; +import com.google.gson.stream.JsonWriter; + import java.io.IOException; +import java.io.PipedReader; +import java.io.PipedWriter; import java.lang.reflect.Type; +import java.math.BigDecimal; +import java.math.BigInteger; import java.util.Arrays; import java.util.HashMap; - -import com.google.gson.Gson; -import com.google.gson.GsonBuilder; -import com.google.gson.JsonElement; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; /** * Abstract base class implementing {@link N5Reader} with JSON attributes @@ -44,6 +58,8 @@ */ public abstract class AbstractGsonReader implements GsonAttributesParser, N5Reader { + private static final Pattern ARRAY_INDEX = Pattern.compile("\\[([0-9]+)]"); + protected final Gson gson; /** @@ -144,7 +160,145 @@ public T getAttribute( final N5URL url, final Class clazz) throws IOException { - return null; + Type mapType = new TypeToken>() { + + }.getType(); + TypeAdapter jsonElementTypeAdapter = gson.getAdapter(JsonElement.class); + HashMap map = getAttributes(url.resolveDataset()); + + final String attributePath = url.resolveAttribute(); + JsonElement json = null; + for (final String pathPart : attributePath.split("/")) { + if (pathPart.isEmpty()) + continue; + if (map.containsKey(pathPart)) { + json = map.get(pathPart); + } else if (json instanceof JsonObject && json.getAsJsonObject().get(pathPart) != null) { + json = json.getAsJsonObject().get(pathPart); + } else { + final Matcher matcher = ARRAY_INDEX.matcher(pathPart); + if (json != null && json.isJsonArray() && matcher.matches()) { + final int index = Integer.parseInt(matcher.group().replace("[", "").replace("]", "")); + json = json.getAsJsonArray().get(index); + } + } + } + if (json == null && clazz.isAssignableFrom(HashMap.class)) { + /* reconstruct the tree to parse as Object */ + final PipedWriter writer = new PipedWriter(); + //TODO: writer blocks if there is not enough space for new content. What do we want to do here? + final PipedReader in = new PipedReader(writer, 1000); + + final JsonWriter out = new JsonWriter(writer); + out.beginObject(); + for (Map.Entry entry : map.entrySet()) { + String k = entry.getKey(); + JsonElement val = entry.getValue(); + out.name(k); + jsonElementTypeAdapter.write(out, val); + } + out.endObject(); + + Map retMap = gson.fromJson(new JsonReader(in), mapType); + + return (T)retMap; + } + if (json instanceof JsonArray) { + final JsonArray array = json.getAsJsonArray(); + if (clazz == boolean[].class) { + final boolean[] retArray = new boolean[array.size()]; + for (int i = 0; i < array.size(); i++) { + final Boolean value = gson.fromJson(array.get(i), boolean.class); + retArray[i] = value; + } + return (T)retArray; + } else if (clazz == Number[].class) { + final Number[] retArray = new Number[array.size()]; + for (int i = 0; i < array.size(); i++) { + final Number value = gson.fromJson(array.get(i), Number.class); + retArray[i] = value; + } + return (T)retArray; + } else if (clazz == String[].class) { + final String[] retArray = new String[array.size()]; + for (int i = 0; i < array.size(); i++) { + final String value = gson.fromJson(array.get(i), String.class); + retArray[i] = value; + } + return (T)retArray; + } else if (clazz == double[].class) { + final double[] retArray = new double[array.size()]; + for (int i = 0; i < array.size(); i++) { + final double value = gson.fromJson(array.get(i), double.class); + retArray[i] = value; + } + return (T)retArray; + } else if (clazz == BigDecimal[].class) { + final BigDecimal[] retArray = new BigDecimal[array.size()]; + for (int i = 0; i < array.size(); i++) { + final BigDecimal value = gson.fromJson(array.get(i), BigDecimal.class); + retArray[i] = value; + } + return (T)retArray; + } else if (clazz == BigInteger[].class) { + final BigInteger[] retArray = new BigInteger[array.size()]; + for (int i = 0; i < array.size(); i++) { + final BigInteger value = gson.fromJson(array.get(i), BigInteger.class); + retArray[i] = value; + } + return (T)retArray; + } else if (clazz == float[].class) { + final float[] retArray = new float[array.size()]; + for (int i = 0; i < array.size(); i++) { + final float value = gson.fromJson(array.get(i), float.class); + retArray[i] = value; + } + return (T)retArray; + } else if (clazz == long[].class) { + final long[] retArray = new long[array.size()]; + for (int i = 0; i < array.size(); i++) { + final long value = gson.fromJson(array.get(i), long.class); + retArray[i] = value; + } + return (T)retArray; + } else if (clazz == short[].class) { + final short[] retArray = new short[array.size()]; + for (int i = 0; i < array.size(); i++) { + final short value = gson.fromJson(array.get(i), short.class); + retArray[i] = value; + } + return (T)retArray; + } else if (clazz == int[].class) { + final int[] retArray = new int[array.size()]; + for (int i = 0; i < array.size(); i++) { + final int value = gson.fromJson(array.get(i), int.class); + retArray[i] = value; + } + return (T)retArray; + } else if (clazz == byte[].class) { + final byte[] retArray = new byte[array.size()]; + for (int i = 0; i < array.size(); i++) { + final byte value = gson.fromJson(array.get(i), byte.class); + retArray[i] = value; + } + return (T)retArray; + } else if (clazz == char[].class) { + final char[] retArray = new char[array.size()]; + for (int i = 0; i < array.size(); i++) { + final char value = gson.fromJson(array.get(i), char.class); + retArray[i] = value; + } + return (T)retArray; + } + } + + try { + return gson.fromJson(json, clazz); + } catch ( + JsonSyntaxException e) { + return null; + } + } @Override @@ -152,7 +306,6 @@ public T getAttribute( final N5URL url, final Type type) throws IOException { - // TODO implement me return null; } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java b/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java index 92ccdd0c..b847ebce 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java @@ -1,11 +1,10 @@ package org.janelia.saalfeldlab.n5; -import org.apache.commons.compress.compressors.FileNameUtil; -import org.apache.commons.compress.utils.FileNameUtils; - -import java.io.File; import java.net.URI; import java.net.URISyntaxException; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; public class N5URL { @@ -44,11 +43,21 @@ public String getDataset() { return dataset; } + public String resolveDataset() { + + return resolveDatasetPath( getDataset() == null ? "" : getDataset() ); + } + public String getAttribute() { return attribute; } + public String resolveAttribute() { + + return resolveAttributePath( getAttribute() == null ? "" : getAttribute() ); + } + private String getSchemePart() { return scheme == null ? "" : scheme + "://"; @@ -107,28 +116,42 @@ public N5URL getRelative(N5URL relative) throws URISyntaxException { newUri.append("//").append(thisAuthority); } - if (relativeUri.getPath() != null) { - if (!relativeUri.getPath().startsWith("/")) { + if (!relativeUri.getPath().isEmpty()) { + final String path = relativeUri.getPath(); + final char char0 = path.charAt(0); + final boolean isAbsolute = char0 == '/' || (path.length() >= 2 && path.charAt(1) == ':' && char0 >= 'A' && char0 <= 'Z'); + if (!isAbsolute) { newUri.append(thisUri.getPath()).append('/'); } newUri - .append(relativeUri.getPath()) + .append(path) .append(relative.getDatasetPart()) .append(relative.getAttributePart()); return new N5URL(newUri.toString()); } newUri.append(thisUri.getPath()); - if (relativeUri.getQuery() != null) { - newUri - .append(relative.getDatasetPart()) - .append(relative.getAttributePart()); + final String query = relativeUri.getQuery(); + if (query != null) { + if (query.charAt(0) != '/' && thisUri.getQuery() != null) { + newUri.append(this.getDatasetPart()).append('/'); + newUri.append(relativeUri.getQuery()); + } else { + newUri.append(relative.getDatasetPart()); + } + newUri.append(relative.getAttributePart()); return new N5URL(newUri.toString()); } newUri.append(this.getDatasetPart()); - if (relativeUri.getFragment() != null) { - newUri.append(relative.getAttributePart()); + final String fragment = relativeUri.getFragment(); + if (fragment != null) { + if (fragment.charAt(0) != '/' && thisUri.getFragment() != null) { + newUri.append(this.getAttributePart()).append('/'); + } else { + newUri.append(relative.getAttributePart()); + } + return new N5URL(newUri.toString()); } newUri.append(this.getAttributePart()); @@ -146,11 +169,137 @@ public N5URL getRelative(String relative) throws URISyntaxException { return getRelative(new N5URL(relative)); } + public static String resolveDatasetPath(String path) { + final char[] pathChars = path.toCharArray(); + + final List tokens = new ArrayList<>(); + StringBuilder curToken = new StringBuilder(); + boolean escape = false; + for (final char character : pathChars) { + /* Skip if we last saw escape*/ + if (escape) { + escape = false; + curToken.append(character); + continue; + } + /* Check if we are escape character */ + if (character == '\\') { + escape = true; + } else if (character == '/') { + if (tokens.isEmpty() && curToken.length() == 0) { + /* If we are root, and the first token, then add the '/' */ + curToken.append(character); + } + + /* The current token is complete, add it to the list, if it isn't empty */ + final String newToken = curToken.toString(); + if (!newToken.isEmpty()) { + /* If our token is '..' then remove the last token instead of adding a new one */ + if (newToken.equals("..")) { + tokens.remove(tokens.size() - 1); + } else { + tokens.add(newToken); + } + } + /* reset for the next token */ + curToken.setLength(0); + } else { + curToken.append(character); + } + } + final String lastToken = curToken.toString(); + if ( !lastToken.isEmpty() ) { + if (lastToken.equals( ".." )) { + tokens.remove( tokens.size() - 1 ); + } else { + tokens.add( lastToken ); + } + } + if (tokens.isEmpty()) return ""; + String root = ""; + if (tokens.get(0).equals("/")) { + tokens.remove(0); + root = "/"; + } + return root + tokens.stream() + .filter(it -> !it.equals(".")) + .filter(it ->!it.isEmpty()) + .reduce((l,r) -> l + "/" + r).orElse( "" ); + } + + public static String resolveAttributePath(String path) { + final char[] pathChars = path.toCharArray(); + + final List tokens = new ArrayList<>(); + StringBuilder curToken = new StringBuilder(); + boolean escape = false; + for (final char character : pathChars) { + /* Skip if we last saw escape*/ + if (escape) { + escape = false; + curToken.append(character); + continue; + } + /* Check if we are escape character */ + if (character == '\\') { + escape = true; + } else if (character == '/' || character == '[' || character == ']') { + if (character == '/' && tokens.isEmpty() && curToken.length() == 0) { + /* If we are root, and the first token, then add the '/' */ + curToken.append(character); + } else if (character == ']') { + /* If ']' add before terminating the token */ + curToken.append(character); + } + + /* The current token is complete, add it to the list, if it isn't empty */ + final String newToken = curToken.toString(); + if (!newToken.isEmpty()) { + /* If our token is '..' then remove the last token instead of adding a new one */ + if (newToken.equals("..")) { + tokens.remove(tokens.size() - 1); + } else { + tokens.add(newToken); + } + } + /* reset for the next token */ + curToken.setLength(0); + + /* if '[' add to the start of the next token */ + if (character == '[') { + curToken.append('['); + } + } else { + curToken.append(character); + } + } + final String lastToken = curToken.toString(); + if ( !lastToken.isEmpty() ) { + if (lastToken.equals( ".." )) { + tokens.remove( tokens.size() - 1 ); + } else { + tokens.add( lastToken ); + } + } + if (tokens.isEmpty()) return ""; + String root = ""; + if (tokens.get(0).equals("/")) { + tokens.remove(0); + root = "/"; + } + return root + tokens.stream() + .filter(it -> !it.equals(".")) + .filter(it ->!it.isEmpty()) + .reduce((l,r) -> l + "/" + r).orElse( "" ); + } + public static URI encodeAsUri(String uri) throws URISyntaxException { + if (uri.trim().length() == 0) { + return new URI(uri); + } /* find last # symbol to split fragment on. If we don't remove it first, then it will encode it, and not parse it separately * after we remove the temporary _N5 scheme */ - final int fragmentIdx = uri.lastIndexOf('#'); final String uriWithoutFragment; final String fragment; @@ -161,20 +310,24 @@ public static URI encodeAsUri(String uri) throws URISyntaxException { uriWithoutFragment = uri; fragment = null; } - final URI _n5Uri = new URI("N5Internal", uriWithoutFragment, fragment); + /* Edge case to handle when uriWithoutFragment is empty */ + final URI _n5Uri; + if (uriWithoutFragment.length() == 0 && fragment != null && fragment.length() > 0 ) { + _n5Uri = new URI("N5Internal", "//STAND_IN", fragment); + } else { + _n5Uri = new URI("N5Internal", uriWithoutFragment, fragment); + } final URI n5Uri; if (fragment == null) { n5Uri = new URI(_n5Uri.getRawSchemeSpecificPart()); } else { - n5Uri = new URI(_n5Uri.getRawSchemeSpecificPart() + "#" + _n5Uri.getRawFragment()); + if (Objects.equals(_n5Uri.getPath(), "") && Objects.equals(_n5Uri.getAuthority(), "STAND_IN")) { + n5Uri = new URI("#" + _n5Uri.getRawFragment()); + } else { + n5Uri = new URI(_n5Uri.getRawSchemeSpecificPart() + "#" + _n5Uri.getRawFragment()); + } } return n5Uri; } - - public static void main(String[] args) throws URISyntaxException { - - new N5URL("/a/b/c").getRelative("d?a#test"); - } - } From 02cc78368e7e1c7084666781640741851a73f399 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Mon, 21 Nov 2022 16:35:07 -0500 Subject: [PATCH 033/243] test: N5URL tests --- .../org/janelia/saalfeldlab/n5/N5URLTest.java | 66 +++++-- .../saalfeldlab/n5/url/UrlAttributeTest.java | 172 ++++++++---------- 2 files changed, 129 insertions(+), 109 deletions(-) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/N5URLTest.java b/src/test/java/org/janelia/saalfeldlab/n5/N5URLTest.java index 67abac2e..520e7512 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/N5URLTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/N5URLTest.java @@ -1,25 +1,63 @@ package org.janelia.saalfeldlab.n5; -import junit.framework.TestCase; +import org.junit.Test; import java.net.URISyntaxException; -public class N5URLTest extends TestCase { - - public void testGetLocation() { - - } - - public void testGetDataset() { - - } - - public void testGetAttribute() { - +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; + +public class N5URLTest { + @Test + public void testAttributePath() { + assertEquals("/a/b/c/d/e", N5URL.resolveAttributePath("/a/b/c/d/e")); + assertEquals("/a/b/c/d/e", N5URL.resolveAttributePath("/a/b/c/d/e/")); + assertEquals("/", N5URL.resolveAttributePath("/")); + assertEquals("", N5URL.resolveAttributePath("")); + assertEquals("/", N5URL.resolveAttributePath("/a/..")); + assertEquals("/", N5URL.resolveAttributePath("/a/../b/../c/d/../..")); + assertEquals("/", N5URL.resolveAttributePath("/a/../b/../c/d/../..")); + assertEquals("/", N5URL.resolveAttributePath("/a/../b/../c/d/../..")); + assertEquals("/", N5URL.resolveAttributePath("/./././././")); + assertEquals("", N5URL.resolveAttributePath("./././././")); + + assertEquals("/a/[0]/b/[0]", N5URL.resolveAttributePath("/a/[0]/b[0]/")); + assertEquals("[0]", N5URL.resolveAttributePath("[0]")); + assertEquals("/[0]", N5URL.resolveAttributePath("/[0]/")); + assertEquals("/a/[0]", N5URL.resolveAttributePath("/a[0]/")); + assertEquals("/[0]/b", N5URL.resolveAttributePath("/[0]b/")); + + assertEquals("let's/try/a/real/case/with spaces", N5URL.resolveAttributePath("let's/try/a/real/case/with spaces/")); + assertEquals("let's/try/a/real/case/with spaces", N5URL.resolveAttributePath("let's/try/a/real/////case////with spaces/")); + assertThrows( IndexOutOfBoundsException.class, () -> N5URL.resolveAttributePath("../first/relative/../not/allowed")); } + @Test public void testGetRelative() throws URISyntaxException { - assertEquals("/a/b/c/d?e#f", new N5URL("/a/b/c").getRelative("d?e#f").toString()); + assertEquals( + "/a/b/c/d?e#f", + new N5URL("/a/b/c").getRelative("d?e#f").toString()); + assertEquals( + "/d?e#f", + new N5URL("/a/b/c").getRelative("/d?e#f").toString()); + assertEquals( + "file:/a/b/c", + new N5URL("s3://janelia-cosem-datasets/jrc_hela-3/jrc_hela-3.n5").getRelative("file:/a/b/c").toString()); + assertEquals( + "s3://janelia-cosem-datasets/a/b/c", + new N5URL("s3://janelia-cosem-datasets/jrc_hela-3/jrc_hela-3.n5").getRelative("/a/b/c").toString()); + assertEquals( + "s3://janelia-cosem-datasets/a/b/c?d/e#f/g", + new N5URL("s3://janelia-cosem-datasets/jrc_hela-3/jrc_hela-3.n5").getRelative("/a/b/c?d/e#f/g").toString()); + assertEquals( + "s3://janelia-cosem-datasets/jrc_hela-3/jrc_hela-3.n5/a/b/c?d/e#f/g", + new N5URL("s3://janelia-cosem-datasets/jrc_hela-3/jrc_hela-3.n5").getRelative("a/b/c?d/e#f/g").toString()); + assertEquals( + "s3://janelia-cosem-datasets/jrc_hela-3/jrc_hela-3.n5?d/e#f/g", + new N5URL("s3://janelia-cosem-datasets/jrc_hela-3/jrc_hela-3.n5").getRelative("?d/e#f/g").toString()); + assertEquals( + "s3://janelia-cosem-datasets/jrc_hela-3/jrc_hela-3.n5#f/g", + new N5URL("s3://janelia-cosem-datasets/jrc_hela-3/jrc_hela-3.n5").getRelative("#f/g").toString()); } } \ No newline at end of file diff --git a/src/test/java/org/janelia/saalfeldlab/n5/url/UrlAttributeTest.java b/src/test/java/org/janelia/saalfeldlab/n5/url/UrlAttributeTest.java index ec7ad2c7..26199d6c 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/url/UrlAttributeTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/url/UrlAttributeTest.java @@ -1,13 +1,12 @@ package org.janelia.saalfeldlab.n5.url; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; import static org.junit.Assert.assertThrows; import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; import java.io.IOException; import java.net.URISyntaxException; +import java.nio.file.NoSuchFileException; import java.util.HashMap; import java.util.HashSet; import java.util.Map; @@ -18,6 +17,7 @@ import org.janelia.saalfeldlab.n5.N5FSReader; import org.janelia.saalfeldlab.n5.N5Reader; import org.janelia.saalfeldlab.n5.N5URL; +import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -28,7 +28,7 @@ public class UrlAttributeTest int[] list; HashMap obj; Set rootKeys; - + @Before public void before() { @@ -40,7 +40,7 @@ public void before() obj = new HashMap<>(); obj.put( "a", "aa" ); - obj.put( "b", "bb" ); + obj.put( "b", "bb" ); rootKeys = new HashSet<>(); rootKeys.addAll( Stream.of("n5", "foo", "list", "object" ).collect( Collectors.toList() ) ); } @@ -49,111 +49,93 @@ public void before() e.printStackTrace(); } } - - @SuppressWarnings( "unchecked" ) + + @SuppressWarnings("unchecked") @Test - public void testRootAttributes() - { - try - { - // get - HashMap everything = n5.getAttribute( new N5URL(""), HashMap.class ); - assertEquals( "empty url", "2.5.1", (String) everything.get( "n5" )); - - HashMap everything2 = n5.getAttribute( new N5URL("#/"), HashMap.class ); - assertEquals( "root attribute", "2.5.1", (String) everything2.get( "n5" )); - - assertEquals( "url to attribute", "bar", n5.getAttribute( new N5URL("#foo"), String.class ) ); - assertEquals( "url to attribute absolute", "bar", n5.getAttribute( new N5URL("#/foo"), String.class ) ); - - assertEquals( "#foo", "bar", n5.getAttribute( new N5URL("#foo"), String.class ) ); - assertEquals( "#/foo", "bar", n5.getAttribute( new N5URL("#/foo"), String.class ) ); - assertEquals( "?#foo", "bar", n5.getAttribute( new N5URL("?#foo"), String.class ) ); - assertEquals( "?#/foo", "bar", n5.getAttribute( new N5URL("?#/foo"), String.class ) ); - assertEquals( "?/#/foo", "bar", n5.getAttribute( new N5URL("?/#/foo"), String.class ) ); - assertEquals( "?/.#/foo", "bar", n5.getAttribute( new N5URL("?/.#/foo"), String.class ) ); - assertEquals( "?./#/foo", "bar", n5.getAttribute( new N5URL("?./#/foo"), String.class ) ); - assertEquals( "?.#foo", "bar", n5.getAttribute( new N5URL("?.#foo"), String.class ) ); - assertEquals( "?/a/..#foo", "bar", n5.getAttribute( new N5URL("?/a/..#foo"), String.class ) ); - assertEquals( "?/a/../.#foo", "bar", n5.getAttribute( new N5URL("?/a/../.#foo"), String.class ) ); - - assertEquals( "url list", list, n5.getAttribute( new N5URL("#list"), int[].class ) ); - - // list - assertEquals( "url list[0]", list[0], (int)n5.getAttribute( new N5URL("#list[0]"), Integer.class ) ); - assertEquals( "url list[1]", list[1], (int)n5.getAttribute( new N5URL("#list[1]"), Integer.class ) ); - assertEquals( "url list[2]", list[2], (int)n5.getAttribute( new N5URL("#list[2]"), Integer.class ) ); - - assertEquals( "url list[3]", list[3], (int)n5.getAttribute( new N5URL("#list[3]"), Integer.class ) ); - assertEquals( "url list/[3]", list[3], (int)n5.getAttribute( new N5URL("#list/[3]"), Integer.class ) ); - assertEquals( "url list//[3]", list[3], (int)n5.getAttribute( new N5URL("#list//[3]"), Integer.class ) ); - assertEquals( "url //list//[3]", list[3], (int)n5.getAttribute( new N5URL("#//list//[3]"), Integer.class ) ); - assertEquals( "url //list//[3]//", list[3], (int)n5.getAttribute( new N5URL("#//list////[3]//"), Integer.class ) ); - - // object - assertTrue( "url object", mapsEqual( obj, n5.getAttribute( new N5URL("#object"), Map.class ))); - assertEquals( "url object/a", "aa", n5.getAttribute( new N5URL("#object/a"), Map.class )); - assertEquals( "url object/b", "bb", n5.getAttribute( new N5URL("#object/b"), Map.class )); - - // failures - // or assertThrows? - assertNull( "url to attribute", n5.getAttribute( new N5URL("file://garbage.n5?trash#foo"), String.class ) ); - assertNull( "url to attribute", n5.getAttribute( new N5URL("file://garbage.n5?/#foo"), String.class ) ); - } - catch ( IOException e ) - { - fail( e.getMessage() ); - } - catch ( URISyntaxException e ) - { - fail( e.getMessage() ); - } + public void testRootAttributes() throws URISyntaxException, IOException { + // get + Map everything = n5.getAttribute(new N5URL(""), Map.class); + assertEquals("empty url", "2.5.1", (String)everything.get("n5")); + + Map everything2 = n5.getAttribute(new N5URL("#/"), Map.class); + assertEquals("root attribute", "2.5.1", (String)everything2.get("n5")); + + assertEquals("url to attribute", "bar", n5.getAttribute(new N5URL("#foo"), String.class)); + assertEquals("url to attribute absolute", "bar", n5.getAttribute(new N5URL("#/foo"), String.class)); + + assertEquals("#foo", "bar", n5.getAttribute(new N5URL("#foo"), String.class)); + assertEquals("#/foo", "bar", n5.getAttribute(new N5URL("#/foo"), String.class)); + assertEquals("?#foo", "bar", n5.getAttribute(new N5URL("?#foo"), String.class)); + assertEquals("?#/foo", "bar", n5.getAttribute(new N5URL("?#/foo"), String.class)); + assertEquals("?/#/foo", "bar", n5.getAttribute(new N5URL("?/#/foo"), String.class)); + assertEquals("?/.#/foo", "bar", n5.getAttribute(new N5URL("?/.#/foo"), String.class)); + assertEquals("?./#/foo", "bar", n5.getAttribute(new N5URL("?./#/foo"), String.class)); + assertEquals("?.#foo", "bar", n5.getAttribute(new N5URL("?.#foo"), String.class)); + assertEquals("?/a/..#foo", "bar", n5.getAttribute(new N5URL("?/a/..#foo"), String.class)); + assertEquals("?/a/../.#foo", "bar", n5.getAttribute(new N5URL("?/a/../.#foo"), String.class)); + + Assert.assertArrayEquals( "url list", list, n5.getAttribute(new N5URL("#list"), int[].class)); + + // list + assertEquals("url list[0]", list[0], (int)n5.getAttribute(new N5URL("#list[0]"), Integer.class)); + assertEquals("url list[1]", list[1], (int)n5.getAttribute(new N5URL("#list[1]"), Integer.class)); + assertEquals("url list[2]", list[2], (int)n5.getAttribute(new N5URL("#list[2]"), Integer.class)); + + assertEquals("url list[3]", list[3], (int)n5.getAttribute(new N5URL("#list[3]"), Integer.class)); + assertEquals("url list/[3]", list[3], (int)n5.getAttribute(new N5URL("#list/[3]"), Integer.class)); + assertEquals("url list//[3]", list[3], (int)n5.getAttribute(new N5URL("#list//[3]"), Integer.class)); + assertEquals("url //list//[3]", list[3], (int)n5.getAttribute(new N5URL("#//list//[3]"), Integer.class)); + assertEquals("url //list//[3]//", list[3], (int)n5.getAttribute(new N5URL("#//list////[3]//"), Integer.class)); + + // object + assertTrue("url object", mapsEqual(obj, n5.getAttribute(new N5URL("#object"), Map.class))); + assertEquals("url object/a", "aa", n5.getAttribute(new N5URL("#object/a"), String.class)); + assertEquals("url object/b", "bb", n5.getAttribute(new N5URL("#object/b"), String.class)); + + // failures + // or assertThrows? + // Currently, they pass, but only because we define the test by the current results. We should discuss what we want though. + assertThrows("url to attribute", NoSuchFileException.class, () -> n5.getAttribute(new N5URL("file://garbage.n5?trash#foo"), String.class)); + assertEquals("url to attribute", "bar", n5.getAttribute(new N5URL("file://garbage.n5?/#foo"), String.class)); } - public void testPathAttributes() - { + @Test + public void testPathAttributes() throws URISyntaxException, IOException { + final String a = "a"; final String aa = "aa"; final String aaa = "aaa"; - try - { - final N5URL aUrl = new N5URL( "?/a" ); - final N5URL aaUrl = new N5URL( "?/a/aa" ); - final N5URL aaaUrl = new N5URL( "?/a/aa/aaa" ); + final N5URL aUrl = new N5URL("?/a"); + final N5URL aaUrl = new N5URL("?/a/aa"); + final N5URL aaaUrl = new N5URL("?/a/aa/aaa"); // name of a assertEquals( "name of a from root", a, n5.getAttribute( new N5URL("?/a#name"), String.class ) ); assertEquals( "name of a from root", a, n5.getAttribute( new N5URL("?a#name"), String.class ) ); assertEquals( "name of a from a", a, n5.getAttribute( aUrl.getRelative( new N5URL("?/a#name")), String.class )); - assertEquals( "name of a from aa", a, n5.getAttribute( aaUrl.getRelative( new N5URL("?..#name")), String.class )); + assertEquals( "name of a from aa", a, n5.getAttribute(aaUrl.getRelative("?..#name"), String.class )); assertEquals( "name of a from aaa", a, n5.getAttribute( aaaUrl.getRelative( new N5URL("?../..#name")), String.class )); - // name of aa - assertEquals( "name of aa from root", aa, n5.getAttribute( new N5URL("?/a/aa#name"), String.class ) ); - assertEquals( "name of aa from root", aa, n5.getAttribute( new N5URL("?a/aa#name"), String.class ) ); - assertEquals( "name of aa from a", aa, n5.getAttribute( aUrl.getRelative( new N5URL("?aa#name")), String.class )); - assertEquals( "name of aa from aa", aa, n5.getAttribute( aaUrl.getRelative( new N5URL("?/#name")), String.class )); - assertEquals( "name of aa from aa", aa, n5.getAttribute( aaUrl.getRelative( new N5URL("#name")), String.class )); - assertEquals( "name of aa from aaa", aa, n5.getAttribute( aaaUrl.getRelative( new N5URL("?..#name")), String.class )); - - // name of aaa - assertEquals( "name of aaa from root", aaa, n5.getAttribute( new N5URL("?/a/aa/aaa#name"), String.class ) ); - assertEquals( "name of aaa from root", aaa, n5.getAttribute( new N5URL("?a/aa/aaa#name"), String.class ) ); - assertEquals( "name of aaa from a", aaa, n5.getAttribute( aUrl.getRelative( new N5URL("?aa/aaa#name")), String.class )); - assertEquals( "name of aaa from aa", aaa, n5.getAttribute( aaUrl.getRelative( new N5URL("?aaa#name")), String.class )); - assertEquals( "name of aaa from aaa", aaa, n5.getAttribute( aaaUrl.getRelative( new N5URL("#name")), String.class )); - assertEquals( "name of aaa from aaa", aaa, n5.getAttribute( aaaUrl.getRelative( new N5URL("?/#name")), String.class )); - } - catch ( IOException e ) - { - fail( e.getMessage() ); - } - catch ( URISyntaxException e ) - { - fail( e.getMessage() ); - } + // name of aa + assertEquals("name of aa from root", aa, n5.getAttribute(new N5URL("?/a/aa#name"), String.class)); + assertEquals("name of aa from root", aa, n5.getAttribute(new N5URL("?a/aa#name"), String.class)); + assertEquals("name of aa from a", aa, n5.getAttribute(aUrl.getRelative("?aa#name"), String.class)); + + assertEquals("name of aa from aa", aa, n5.getAttribute(aaUrl.getRelative("?./#name"), String.class)); + + assertEquals("name of aa from aa", aa, n5.getAttribute(aaUrl.getRelative("#name"), String.class)); + assertEquals("name of aa from aaa", aa, n5.getAttribute(aaaUrl.getRelative("?..#name"), String.class)); + + // name of aaa + assertEquals("name of aaa from root", aaa, n5.getAttribute(new N5URL("?/a/aa/aaa#name"), String.class)); + assertEquals("name of aaa from root", aaa, n5.getAttribute(new N5URL("?a/aa/aaa#name"), String.class)); + assertEquals("name of aaa from a", aaa, n5.getAttribute(aUrl.getRelative("?aa/aaa#name"), String.class)); + assertEquals("name of aaa from aa", aaa, n5.getAttribute(aaUrl.getRelative("?aaa#name"), String.class)); + assertEquals("name of aaa from aaa", aaa, n5.getAttribute(aaaUrl.getRelative("#name"), String.class)); + + assertEquals("name of aaa from aaa", aaa, n5.getAttribute(aaaUrl.getRelative("?./#name"), String.class)); } private boolean mapsEqual( Map a, Map b ) @@ -162,7 +144,7 @@ private boolean mapsEqual( Map a, Map b ) return false; for( K k : a.keySet() ) - { + { if( ! a.get( k ).equals( b.get( k ) )) return false; } From ead786db1bc3d01eaaae961185207c39730a5d9b Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Tue, 22 Nov 2022 09:25:40 -0500 Subject: [PATCH 034/243] test: test object parsing from urls --- .../saalfeldlab/n5/url/UrlAttributeTest.java | 52 +++++++++++++++++++ .../url/urlAttributes.n5/objs/attributes.json | 12 +++++ 2 files changed, 64 insertions(+) create mode 100644 src/test/resources/url/urlAttributes.n5/objs/attributes.json diff --git a/src/test/java/org/janelia/saalfeldlab/n5/url/UrlAttributeTest.java b/src/test/java/org/janelia/saalfeldlab/n5/url/UrlAttributeTest.java index 26199d6c..2ae60c80 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/url/UrlAttributeTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/url/UrlAttributeTest.java @@ -1,5 +1,6 @@ package org.janelia.saalfeldlab.n5.url; +import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertThrows; import static org.junit.Assert.assertTrue; @@ -29,6 +30,9 @@ public class UrlAttributeTest HashMap obj; Set rootKeys; + TestInts testObjInts; + TestDoubles testObjDoubles; + @Before public void before() { @@ -48,6 +52,9 @@ public void before() { e.printStackTrace(); } + + testObjInts = new TestInts( "ints", "intsName", new int[] { 5, 4, 3 } ); + testObjDoubles = new TestDoubles( "doubles", "doublesName", new double[] { 5.5, 4.4, 3.3 } ); } @SuppressWarnings("unchecked") @@ -138,6 +145,20 @@ public void testPathAttributes() throws URISyntaxException, IOException { assertEquals("name of aaa from aaa", aaa, n5.getAttribute(aaaUrl.getRelative("?./#name"), String.class)); } + @Test + public void testPathObject() throws IOException, URISyntaxException + { + final TestInts ints = n5.getAttribute( new N5URL( "?objs#intsKey" ), TestInts.class ); + assertEquals( testObjInts.name, ints.name ); + assertEquals( testObjInts.type, ints.type ); + assertArrayEquals( testObjInts.t(), ints.t() ); + + final TestDoubles doubles = n5.getAttribute( new N5URL( "?objs#doublesKey" ), TestDoubles.class ); + assertEquals( testObjDoubles.name, doubles.name ); + assertEquals( testObjDoubles.type, doubles.type ); + assertArrayEquals( testObjDoubles.t(), doubles.t(), 1e-9 ); + } + private boolean mapsEqual( Map a, Map b ) { if( ! a.keySet().equals( b.keySet() )) @@ -152,4 +173,35 @@ private boolean mapsEqual( Map a, Map b ) return true; } + private static class TestObject< T > + { + String type; + String name; + T t; + + public TestObject( String type, String name, T t ) + { + this.name = name; + this.type = type; + this.t = t; + } + public T t() { return t; } + } + + private static class TestDoubles extends TestObject< double[] > + { + public TestDoubles( String type, String name, double[] t ) + { + super( type, name, t ); + } + } + + private static class TestInts extends TestObject< int[] > + { + public TestInts( String type, String name, int[] t ) + { + super( type, name, t ); + } + } + } diff --git a/src/test/resources/url/urlAttributes.n5/objs/attributes.json b/src/test/resources/url/urlAttributes.n5/objs/attributes.json new file mode 100644 index 00000000..0a9addd3 --- /dev/null +++ b/src/test/resources/url/urlAttributes.n5/objs/attributes.json @@ -0,0 +1,12 @@ +{ + "intsKey": { + "type": "ints", + "name": "intsName", + "t": [ 5, 4, 3 ] + }, + "doublesKey": { + "type": "doubles", + "name": "doublesName", + "t": [ 5.5, 4.4, 3.3 ] + } +} From 9b6e90dbd4e529823052dfc80ab680de47c7d305 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Tue, 22 Nov 2022 11:36:06 -0500 Subject: [PATCH 035/243] feat: support generic arrays when getting attribute from N5URL --- .../saalfeldlab/n5/AbstractGsonReader.java | 178 +++++++++--------- .../saalfeldlab/n5/url/UrlAttributeTest.java | 53 ++++++ .../url/urlAttributes.n5/objs/attributes.json | 27 ++- 3 files changed, 165 insertions(+), 93 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/AbstractGsonReader.java b/src/main/java/org/janelia/saalfeldlab/n5/AbstractGsonReader.java index 7369f574..5a5b869d 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/AbstractGsonReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/AbstractGsonReader.java @@ -39,6 +39,7 @@ import java.io.IOException; import java.io.PipedReader; import java.io.PipedWriter; +import java.lang.reflect.Array; import java.lang.reflect.Type; import java.math.BigDecimal; import java.math.BigInteger; @@ -158,8 +159,9 @@ public T getAttribute( @Override public T getAttribute( final N5URL url, - final Class clazz) throws IOException { + final Type type) throws IOException { + final Class clazz = (type instanceof Class) ? ((Class)type) : null; Type mapType = new TypeToken>() { }.getType(); @@ -183,7 +185,8 @@ public T getAttribute( } } } - if (json == null && clazz.isAssignableFrom(HashMap.class)) { + if (json == null && clazz != null && clazz.isAssignableFrom(HashMap.class)) { + //NOTE: Would not need to do this if we could get the root `JsonElement` attribute, instead of just the `Map T getAttribute( out.endObject(); Map retMap = gson.fromJson(new JsonReader(in), mapType); - return (T)retMap; } if (json instanceof JsonArray) { final JsonArray array = json.getAsJsonArray(); - if (clazz == boolean[].class) { - final boolean[] retArray = new boolean[array.size()]; - for (int i = 0; i < array.size(); i++) { - final Boolean value = gson.fromJson(array.get(i), boolean.class); - retArray[i] = value; - } - return (T)retArray; - } else if (clazz == Number[].class) { - final Number[] retArray = new Number[array.size()]; - for (int i = 0; i < array.size(); i++) { - final Number value = gson.fromJson(array.get(i), Number.class); - retArray[i] = value; - } - return (T)retArray; - } else if (clazz == String[].class) { - final String[] retArray = new String[array.size()]; - for (int i = 0; i < array.size(); i++) { - final String value = gson.fromJson(array.get(i), String.class); - retArray[i] = value; - } - return (T)retArray; - } else if (clazz == double[].class) { - final double[] retArray = new double[array.size()]; - for (int i = 0; i < array.size(); i++) { - final double value = gson.fromJson(array.get(i), double.class); - retArray[i] = value; - } - return (T)retArray; - } else if (clazz == BigDecimal[].class) { - final BigDecimal[] retArray = new BigDecimal[array.size()]; - for (int i = 0; i < array.size(); i++) { - final BigDecimal value = gson.fromJson(array.get(i), BigDecimal.class); - retArray[i] = value; - } - return (T)retArray; - } else if (clazz == BigInteger[].class) { - final BigInteger[] retArray = new BigInteger[array.size()]; - for (int i = 0; i < array.size(); i++) { - final BigInteger value = gson.fromJson(array.get(i), BigInteger.class); - retArray[i] = value; - } - return (T)retArray; - } else if (clazz == float[].class) { - final float[] retArray = new float[array.size()]; - for (int i = 0; i < array.size(); i++) { - final float value = gson.fromJson(array.get(i), float.class); - retArray[i] = value; - } - return (T)retArray; - } else if (clazz == long[].class) { - final long[] retArray = new long[array.size()]; - for (int i = 0; i < array.size(); i++) { - final long value = gson.fromJson(array.get(i), long.class); - retArray[i] = value; - } - return (T)retArray; - } else if (clazz == short[].class) { - final short[] retArray = new short[array.size()]; - for (int i = 0; i < array.size(); i++) { - final short value = gson.fromJson(array.get(i), short.class); - retArray[i] = value; - } - return (T)retArray; - } else if (clazz == int[].class) { - final int[] retArray = new int[array.size()]; - for (int i = 0; i < array.size(); i++) { - final int value = gson.fromJson(array.get(i), int.class); - retArray[i] = value; - } - return (T)retArray; - } else if (clazz == byte[].class) { - final byte[] retArray = new byte[array.size()]; - for (int i = 0; i < array.size(); i++) { - final byte value = gson.fromJson(array.get(i), byte.class); - retArray[i] = value; - } - return (T)retArray; - } else if (clazz == char[].class) { - final char[] retArray = new char[array.size()]; - for (int i = 0; i < array.size(); i++) { - final char value = gson.fromJson(array.get(i), char.class); - retArray[i] = value; - } - return (T)retArray; - } + T retArray = getJsonAsArray(array, type); + if (retArray != null) + return retArray; } - try { - return gson.fromJson(json, clazz); + return gson.fromJson(json, type); } catch ( JsonSyntaxException e) { return null; } - } @Override public T getAttribute( final N5URL url, - final Type type) throws IOException { + final Class clazz) throws IOException { + + return getAttribute(url, (Type)clazz); + } + + private T getJsonAsArray(JsonArray array, Class clazz) { + return getJsonAsArray(array, (Type)clazz); + } + private T getJsonAsArray(JsonArray array, Type type) { + + + final Class clazz = (type instanceof Class) ? ((Class)type) : null; + + if (type == boolean[].class) { + final boolean[] retArray = new boolean[array.size()]; + for (int i = 0; i < array.size(); i++) { + final Boolean value = gson.fromJson(array.get(i), boolean.class); + retArray[i] = value; + } + return (T)retArray; + } else if (type == double[].class) { + final double[] retArray = new double[array.size()]; + for (int i = 0; i < array.size(); i++) { + final double value = gson.fromJson(array.get(i), double.class); + retArray[i] = value; + } + return (T)retArray; + } else if (type == float[].class) { + final float[] retArray = new float[array.size()]; + for (int i = 0; i < array.size(); i++) { + final float value = gson.fromJson(array.get(i), float.class); + retArray[i] = value; + } + return (T)retArray; + } else if (type == long[].class) { + final long[] retArray = new long[array.size()]; + for (int i = 0; i < array.size(); i++) { + final long value = gson.fromJson(array.get(i), long.class); + retArray[i] = value; + } + return (T)retArray; + } else if (type == short[].class) { + final short[] retArray = new short[array.size()]; + for (int i = 0; i < array.size(); i++) { + final short value = gson.fromJson(array.get(i), short.class); + retArray[i] = value; + } + return (T)retArray; + } else if (type == int[].class) { + final int[] retArray = new int[array.size()]; + for (int i = 0; i < array.size(); i++) { + final int value = gson.fromJson(array.get(i), int.class); + retArray[i] = value; + } + return (T)retArray; + } else if (type == byte[].class) { + final byte[] retArray = new byte[array.size()]; + for (int i = 0; i < array.size(); i++) { + final byte value = gson.fromJson(array.get(i), byte.class); + retArray[i] = value; + } + return (T)retArray; + } else if (type == char[].class) { + final char[] retArray = new char[array.size()]; + for (int i = 0; i < array.size(); i++) { + final char value = gson.fromJson(array.get(i), char.class); + retArray[i] = value; + } + return (T)retArray; + } else if(clazz != null && clazz.isArray()) { + final Class componentCls = clazz.getComponentType(); + final Object[] clsArray = (Object[] )Array.newInstance(componentCls, array.size()); + for (int i = 0; i < array.size(); i++) { + clsArray[i] = gson.fromJson(array.get(i), componentCls); + } + return (T)clsArray; + } return null; } } diff --git a/src/test/java/org/janelia/saalfeldlab/n5/url/UrlAttributeTest.java b/src/test/java/org/janelia/saalfeldlab/n5/url/UrlAttributeTest.java index 2ae60c80..a0748194 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/url/UrlAttributeTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/url/UrlAttributeTest.java @@ -8,9 +8,11 @@ import java.io.IOException; import java.net.URISyntaxException; import java.nio.file.NoSuchFileException; +import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -157,6 +159,24 @@ public void testPathObject() throws IOException, URISyntaxException assertEquals( testObjDoubles.name, doubles.name ); assertEquals( testObjDoubles.type, doubles.type ); assertArrayEquals( testObjDoubles.t(), doubles.t(), 1e-9 ); + + final TestDoubles[] doubleArray = n5.getAttribute( new N5URL( "?objs#array" ), TestDoubles[].class ); + final TestDoubles doubles1 = new TestDoubles("doubles", "doubles1", new double[]{ 5.7, 4.5, 3.4 }); + final TestDoubles doubles2 = new TestDoubles("doubles", "doubles2", new double[]{ 5.8, 4.6, 3.5 }); + final TestDoubles doubles3 = new TestDoubles("doubles", "doubles3", new double[]{ 5.9, 4.7, 3.6 }); + final TestDoubles doubles4 = new TestDoubles("doubles", "doubles4", new double[]{ 5.10, 4.8, 3.7 }); + final TestDoubles[] expectedDoubles = new TestDoubles[] {doubles1, doubles2, doubles3, doubles4}; + assertArrayEquals(expectedDoubles, doubleArray); + + final String[] stringArray = n5.getAttribute( new N5URL( "?objs#String" ), String[].class ); + final String[] expectedString = new String[] {"This", "is", "a", "test"}; + + final Integer[] integerArray = n5.getAttribute( new N5URL( "?objs#Integer" ), Integer[].class ); + final Integer[] expectedInteger = new Integer[] {1,2,3,4}; + + final int[] intArray = n5.getAttribute( new N5URL( "?objs#int" ), int[].class ); + final int[] expectedInt = new int[] {1,2,3,4}; + assertArrayEquals(expectedInt, intArray); } private boolean mapsEqual( Map a, Map b ) @@ -186,6 +206,17 @@ public TestObject( String type, String name, T t ) this.t = t; } public T t() { return t; } + + @Override public boolean equals(Object obj) { + + if (obj.getClass() == this.getClass()) { + final TestObject otherTestObject = (TestObject)obj; + return Objects.equals(name, otherTestObject.name) + && Objects.equals(type, otherTestObject.type) + && Objects.equals(t, otherTestObject.t); + } + return false; + } } private static class TestDoubles extends TestObject< double[] > @@ -194,6 +225,17 @@ public TestDoubles( String type, String name, double[] t ) { super( type, name, t ); } + + @Override public boolean equals(Object obj) { + + if (obj.getClass() == this.getClass()) { + final TestDoubles otherTestObject = (TestDoubles)obj; + return Objects.equals(name, otherTestObject.name) + && Objects.equals(type, otherTestObject.type) + && Arrays.equals(t, otherTestObject.t); + } + return false; + } } private static class TestInts extends TestObject< int[] > @@ -202,6 +244,17 @@ public TestInts( String type, String name, int[] t ) { super( type, name, t ); } + @Override public boolean equals(Object obj) { + + if (obj.getClass() == this.getClass()) { + final TestInts otherTestObject = (TestInts)obj; + return Objects.equals(name, otherTestObject.name) + && Objects.equals(type, otherTestObject.type) + && Arrays.equals(t, otherTestObject.t); + } + return false; + } + } } diff --git a/src/test/resources/url/urlAttributes.n5/objs/attributes.json b/src/test/resources/url/urlAttributes.n5/objs/attributes.json index 0a9addd3..deab148c 100644 --- a/src/test/resources/url/urlAttributes.n5/objs/attributes.json +++ b/src/test/resources/url/urlAttributes.n5/objs/attributes.json @@ -8,5 +8,30 @@ "type": "doubles", "name": "doublesName", "t": [ 5.5, 4.4, 3.3 ] - } + }, + "int": [1, 2,3,4], + "Integer": [1, 2,3,4], + "String": ["This", "is", "a", "test"], + "array": [ + { + "type": "doubles", + "name": "doubles1", + "t": [ 5.7, 4.5, 3.4 ] + }, + { + "type": "doubles", + "name": "doubles2", + "t": [ 5.8, 4.6, 3.5 ] + }, + { + "type": "doubles", + "name": "doubles3", + "t": [ 5.9, 4.7, 3.6 ] + }, + { + "type": "doubles", + "name": "doubles4", + "t": [ 5.10, 4.8, 3.7 ] + } + ] } From bc059795110aecf8ea32502c2da28a8b69c626a2 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Mon, 28 Nov 2022 17:00:27 -0500 Subject: [PATCH 036/243] feat: remove N5Url getAttributes methods in favor of handling internally in existing `getAttributes` --- .../org/janelia/saalfeldlab/n5/N5Reader.java | 30 ------------------- 1 file changed, 30 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java index 6d4a21b3..dc368446 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java @@ -249,36 +249,6 @@ public T getAttribute( final String key, final Type type) throws IOException; - /** - * Reads an attribute. - * - * @param context - * the context - * @param url - * @param clazz - * attribute class - * @return - * @throws IOException - */ - public T getAttribute( - final N5URL url, - final Class clazz) throws IOException; - - /** - * Reads an attribute. - * - * @param context - * the context - * @param url the url - * @param type - * attribute Type (use this for specifying generic types) - * @return - * @throws IOException - */ - public T getAttribute( - final N5URL url, - final Type type) throws IOException; - /** * Get mandatory dataset attributes. * From f12345fb50cce67ff73207a22e6b2aabea1da2ae Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Mon, 28 Nov 2022 17:01:47 -0500 Subject: [PATCH 037/243] feat: initial handling of structured fragments in `getAttributes`; WIP --- .../saalfeldlab/n5/AbstractGsonReader.java | 35 +++++++++++++------ 1 file changed, 25 insertions(+), 10 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/AbstractGsonReader.java b/src/main/java/org/janelia/saalfeldlab/n5/AbstractGsonReader.java index 5a5b869d..08264294 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/AbstractGsonReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/AbstractGsonReader.java @@ -41,8 +41,7 @@ import java.io.PipedWriter; import java.lang.reflect.Array; import java.lang.reflect.Type; -import java.math.BigDecimal; -import java.math.BigInteger; +import java.net.URISyntaxException; import java.util.Arrays; import java.util.HashMap; import java.util.Map; @@ -142,8 +141,18 @@ public T getAttribute( final String key, final Class clazz) throws IOException { - final HashMap map = getAttributes(pathName); - return GsonAttributesParser.parseAttribute(map, key, clazz, getGson()); + // final HashMap map = getAttributes(pathName); + try + { + + final String dataset = pathName != null ? "?" + pathName : "?"; + final String fragment = key != null ? "#" + key : "#"; + return getAttribute( new N5URL( dataset + fragment ), clazz ); + } + catch ( URISyntaxException e ) + { + throw new IOException( e ); + } } @Override @@ -152,12 +161,19 @@ public T getAttribute( final String key, final Type type) throws IOException { - final HashMap map = getAttributes(pathName); - return GsonAttributesParser.parseAttribute(map, key, type, getGson()); + try + { + final String dataset = pathName != null ? "?" + pathName : "?"; + final String fragment = key != null ? "#" + key : "#"; + return getAttribute( new N5URL( dataset + fragment ), type ); + } + catch ( URISyntaxException e ) + { + throw new IOException( e ); + } } - @Override - public T getAttribute( + private T getAttribute( final N5URL url, final Type type) throws IOException { @@ -219,8 +235,7 @@ public T getAttribute( } } - @Override - public T getAttribute( + private T getAttribute( final N5URL url, final Class clazz) throws IOException { From cd9eddfa4d231c8d00dd8d74704f0d945e3ed4b6 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Mon, 28 Nov 2022 17:02:30 -0500 Subject: [PATCH 038/243] refactor(test): migrate away from remove `getAttributes(N5Url, Class)` method --- .../saalfeldlab/n5/url/UrlAttributeTest.java | 251 ++++++++++-------- 1 file changed, 143 insertions(+), 108 deletions(-) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/url/UrlAttributeTest.java b/src/test/java/org/janelia/saalfeldlab/n5/url/UrlAttributeTest.java index a0748194..29e3605e 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/url/UrlAttributeTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/url/UrlAttributeTest.java @@ -1,13 +1,7 @@ package org.janelia.saalfeldlab.n5.url; -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertThrows; -import static org.junit.Assert.assertTrue; - import java.io.IOException; import java.net.URISyntaxException; -import java.nio.file.NoSuchFileException; import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; @@ -16,7 +10,6 @@ import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; - import org.janelia.saalfeldlab.n5.N5FSReader; import org.janelia.saalfeldlab.n5.N5Reader; import org.janelia.saalfeldlab.n5.N5URL; @@ -24,15 +17,24 @@ import org.junit.Before; import org.junit.Test; +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + public class UrlAttributeTest { N5Reader n5; + String rootContext = ""; + int[] list; - HashMap obj; - Set rootKeys; + + HashMap< String, String > obj; + + Set< String > rootKeys; TestInts testObjInts; + TestDoubles testObjDoubles; @Before @@ -48,7 +50,7 @@ public void before() obj.put( "a", "aa" ); obj.put( "b", "bb" ); rootKeys = new HashSet<>(); - rootKeys.addAll( Stream.of("n5", "foo", "list", "object" ).collect( Collectors.toList() ) ); + rootKeys.addAll( Stream.of( "n5", "foo", "list", "object" ).collect( Collectors.toList() ) ); } catch ( IOException e ) { @@ -59,134 +61,151 @@ public void before() testObjDoubles = new TestDoubles( "doubles", "doublesName", new double[] { 5.5, 4.4, 3.3 } ); } - @SuppressWarnings("unchecked") + @SuppressWarnings( "unchecked" ) @Test - public void testRootAttributes() throws URISyntaxException, IOException { + public void testRootAttributes() throws URISyntaxException, IOException + { // get - Map everything = n5.getAttribute(new N5URL(""), Map.class); - assertEquals("empty url", "2.5.1", (String)everything.get("n5")); + Map< String, Object > everything = getAttribute( n5, new N5URL( "" ), Map.class ); + assertEquals( "empty url", "2.5.1", ( String ) everything.get( "n5" ) ); - Map everything2 = n5.getAttribute(new N5URL("#/"), Map.class); - assertEquals("root attribute", "2.5.1", (String)everything2.get("n5")); + Map< String, Object > everything2 = getAttribute( n5, new N5URL( "#/" ), Map.class ); + assertEquals( "root attribute", "2.5.1", ( String ) everything2.get( "n5" ) ); - assertEquals("url to attribute", "bar", n5.getAttribute(new N5URL("#foo"), String.class)); - assertEquals("url to attribute absolute", "bar", n5.getAttribute(new N5URL("#/foo"), String.class)); + assertEquals( "url to attribute", "bar", getAttribute( n5, new N5URL( "#foo" ), String.class ) ); + assertEquals( "url to attribute absolute", "bar", getAttribute( n5, new N5URL( "#/foo" ), String.class ) ); - assertEquals("#foo", "bar", n5.getAttribute(new N5URL("#foo"), String.class)); - assertEquals("#/foo", "bar", n5.getAttribute(new N5URL("#/foo"), String.class)); - assertEquals("?#foo", "bar", n5.getAttribute(new N5URL("?#foo"), String.class)); - assertEquals("?#/foo", "bar", n5.getAttribute(new N5URL("?#/foo"), String.class)); - assertEquals("?/#/foo", "bar", n5.getAttribute(new N5URL("?/#/foo"), String.class)); - assertEquals("?/.#/foo", "bar", n5.getAttribute(new N5URL("?/.#/foo"), String.class)); - assertEquals("?./#/foo", "bar", n5.getAttribute(new N5URL("?./#/foo"), String.class)); - assertEquals("?.#foo", "bar", n5.getAttribute(new N5URL("?.#foo"), String.class)); - assertEquals("?/a/..#foo", "bar", n5.getAttribute(new N5URL("?/a/..#foo"), String.class)); - assertEquals("?/a/../.#foo", "bar", n5.getAttribute(new N5URL("?/a/../.#foo"), String.class)); + assertEquals( "#foo", "bar", getAttribute( n5, new N5URL( "#foo" ), String.class ) ); + assertEquals( "#/foo", "bar", getAttribute( n5, new N5URL( "#/foo" ), String.class ) ); + assertEquals( "?#foo", "bar", getAttribute( n5, new N5URL( "?#foo" ), String.class ) ); + assertEquals( "?#/foo", "bar", getAttribute( n5, new N5URL( "?#/foo" ), String.class ) ); + assertEquals( "?/#/foo", "bar", getAttribute( n5, new N5URL( "?/#/foo" ), String.class ) ); + assertEquals( "?/.#/foo", "bar", getAttribute( n5, new N5URL( "?/.#/foo" ), String.class ) ); + assertEquals( "?./#/foo", "bar", getAttribute( n5, new N5URL( "?./#/foo" ), String.class ) ); + assertEquals( "?.#foo", "bar", getAttribute( n5, new N5URL( "?.#foo" ), String.class ) ); + assertEquals( "?/a/..#foo", "bar", getAttribute( n5, new N5URL( "?/a/..#foo" ), String.class ) ); + assertEquals( "?/a/../.#foo", "bar", getAttribute( n5, new N5URL( "?/a/../.#foo" ), String.class ) ); - Assert.assertArrayEquals( "url list", list, n5.getAttribute(new N5URL("#list"), int[].class)); + Assert.assertArrayEquals( "url list", list, getAttribute( n5, new N5URL( "#list" ), int[].class ) ); // list - assertEquals("url list[0]", list[0], (int)n5.getAttribute(new N5URL("#list[0]"), Integer.class)); - assertEquals("url list[1]", list[1], (int)n5.getAttribute(new N5URL("#list[1]"), Integer.class)); - assertEquals("url list[2]", list[2], (int)n5.getAttribute(new N5URL("#list[2]"), Integer.class)); + assertEquals( "url list[0]", list[ 0 ], ( int ) getAttribute( n5, new N5URL( "#list[0]" ), Integer.class ) ); + assertEquals( "url list[1]", list[ 1 ], ( int ) getAttribute( n5, new N5URL( "#list[1]" ), Integer.class ) ); + assertEquals( "url list[2]", list[ 2 ], ( int ) getAttribute( n5, new N5URL( "#list[2]" ), Integer.class ) ); - assertEquals("url list[3]", list[3], (int)n5.getAttribute(new N5URL("#list[3]"), Integer.class)); - assertEquals("url list/[3]", list[3], (int)n5.getAttribute(new N5URL("#list/[3]"), Integer.class)); - assertEquals("url list//[3]", list[3], (int)n5.getAttribute(new N5URL("#list//[3]"), Integer.class)); - assertEquals("url //list//[3]", list[3], (int)n5.getAttribute(new N5URL("#//list//[3]"), Integer.class)); - assertEquals("url //list//[3]//", list[3], (int)n5.getAttribute(new N5URL("#//list////[3]//"), Integer.class)); + assertEquals( "url list[3]", list[ 3 ], ( int ) getAttribute( n5, new N5URL( "#list[3]" ), Integer.class ) ); + assertEquals( "url list/[3]", list[ 3 ], ( int ) getAttribute( n5, new N5URL( "#list/[3]" ), Integer.class ) ); + assertEquals( "url list//[3]", list[ 3 ], ( int ) getAttribute( n5, new N5URL( "#list//[3]" ), Integer.class ) ); + assertEquals( "url //list//[3]", list[ 3 ], ( int ) getAttribute( n5, new N5URL( "#//list//[3]" ), Integer.class ) ); + assertEquals( "url //list//[3]//", list[ 3 ], ( int ) getAttribute( n5, new N5URL( "#//list////[3]//" ), Integer.class ) ); // object - assertTrue("url object", mapsEqual(obj, n5.getAttribute(new N5URL("#object"), Map.class))); - assertEquals("url object/a", "aa", n5.getAttribute(new N5URL("#object/a"), String.class)); - assertEquals("url object/b", "bb", n5.getAttribute(new N5URL("#object/b"), String.class)); - - // failures - // or assertThrows? - // Currently, they pass, but only because we define the test by the current results. We should discuss what we want though. - assertThrows("url to attribute", NoSuchFileException.class, () -> n5.getAttribute(new N5URL("file://garbage.n5?trash#foo"), String.class)); - assertEquals("url to attribute", "bar", n5.getAttribute(new N5URL("file://garbage.n5?/#foo"), String.class)); - + assertTrue( "url object", mapsEqual( obj, getAttribute( n5, new N5URL( "#object" ), Map.class ) ) ); + assertEquals( "url object/a", "aa", getAttribute( n5, new N5URL( "#object/a" ), String.class ) ); + assertEquals( "url object/b", "bb", getAttribute( n5, new N5URL( "#object/b" ), String.class ) ); } @Test - public void testPathAttributes() throws URISyntaxException, IOException { + public void testPathAttributes() throws URISyntaxException, IOException + { final String a = "a"; final String aa = "aa"; final String aaa = "aaa"; - final N5URL aUrl = new N5URL("?/a"); - final N5URL aaUrl = new N5URL("?/a/aa"); - final N5URL aaaUrl = new N5URL("?/a/aa/aaa"); + final N5URL aUrl = new N5URL( "?/a" ); + final N5URL aaUrl = new N5URL( "?/a/aa" ); + final N5URL aaaUrl = new N5URL( "?/a/aa/aaa" ); - // name of a - assertEquals( "name of a from root", a, n5.getAttribute( new N5URL("?/a#name"), String.class ) ); - assertEquals( "name of a from root", a, n5.getAttribute( new N5URL("?a#name"), String.class ) ); - assertEquals( "name of a from a", a, n5.getAttribute( aUrl.getRelative( new N5URL("?/a#name")), String.class )); - assertEquals( "name of a from aa", a, n5.getAttribute(aaUrl.getRelative("?..#name"), String.class )); - assertEquals( "name of a from aaa", a, n5.getAttribute( aaaUrl.getRelative( new N5URL("?../..#name")), String.class )); + // name of a + assertEquals( "name of a from root", a, getAttribute( n5, "?/a#name", String.class ) ); + assertEquals( "name of a from root", a, getAttribute( n5, new N5URL( "?a#name" ), String.class ) ); + assertEquals( "name of a from a", a, getAttribute( n5, aUrl.getRelative( new N5URL( "?/a#name" ) ), String.class ) ); + assertEquals( "name of a from aa", a, getAttribute( n5, aaUrl.getRelative( "?..#name" ), String.class ) ); + assertEquals( "name of a from aaa", a, getAttribute( n5, aaaUrl.getRelative( new N5URL( "?../..#name" ) ), String.class ) ); // name of aa - assertEquals("name of aa from root", aa, n5.getAttribute(new N5URL("?/a/aa#name"), String.class)); - assertEquals("name of aa from root", aa, n5.getAttribute(new N5URL("?a/aa#name"), String.class)); - assertEquals("name of aa from a", aa, n5.getAttribute(aUrl.getRelative("?aa#name"), String.class)); + assertEquals( "name of aa from root", aa, getAttribute( n5, new N5URL( "?/a/aa#name" ), String.class ) ); + assertEquals( "name of aa from root", aa, getAttribute( n5, new N5URL( "?a/aa#name" ), String.class ) ); + assertEquals( "name of aa from a", aa, getAttribute( n5, aUrl.getRelative( "?aa#name" ), String.class ) ); - assertEquals("name of aa from aa", aa, n5.getAttribute(aaUrl.getRelative("?./#name"), String.class)); + assertEquals( "name of aa from aa", aa, getAttribute( n5, aaUrl.getRelative( "?./#name" ), String.class ) ); - assertEquals("name of aa from aa", aa, n5.getAttribute(aaUrl.getRelative("#name"), String.class)); - assertEquals("name of aa from aaa", aa, n5.getAttribute(aaaUrl.getRelative("?..#name"), String.class)); + assertEquals( "name of aa from aa", aa, getAttribute( n5, aaUrl.getRelative( "#name" ), String.class ) ); + assertEquals( "name of aa from aaa", aa, getAttribute( n5, aaaUrl.getRelative( "?..#name" ), String.class ) ); // name of aaa - assertEquals("name of aaa from root", aaa, n5.getAttribute(new N5URL("?/a/aa/aaa#name"), String.class)); - assertEquals("name of aaa from root", aaa, n5.getAttribute(new N5URL("?a/aa/aaa#name"), String.class)); - assertEquals("name of aaa from a", aaa, n5.getAttribute(aUrl.getRelative("?aa/aaa#name"), String.class)); - assertEquals("name of aaa from aa", aaa, n5.getAttribute(aaUrl.getRelative("?aaa#name"), String.class)); - assertEquals("name of aaa from aaa", aaa, n5.getAttribute(aaaUrl.getRelative("#name"), String.class)); + assertEquals( "name of aaa from root", aaa, getAttribute( n5, new N5URL( "?/a/aa/aaa#name" ), String.class ) ); + assertEquals( "name of aaa from root", aaa, getAttribute( n5, new N5URL( "?a/aa/aaa#name" ), String.class ) ); + assertEquals( "name of aaa from a", aaa, getAttribute( n5, aUrl.getRelative( "?aa/aaa#name" ), String.class ) ); + assertEquals( "name of aaa from aa", aaa, getAttribute( n5, aaUrl.getRelative( "?aaa#name" ), String.class ) ); + assertEquals( "name of aaa from aaa", aaa, getAttribute( n5, aaaUrl.getRelative( "#name" ), String.class ) ); - assertEquals("name of aaa from aaa", aaa, n5.getAttribute(aaaUrl.getRelative("?./#name"), String.class)); + assertEquals( "name of aaa from aaa", aaa, getAttribute( n5, aaaUrl.getRelative( "?./#name" ), String.class ) ); + } + + private < T > T getAttribute( final N5Reader n5, final String url1, Class< T > clazz ) throws URISyntaxException, IOException + { + return getAttribute( n5, url1, null, clazz ); + } + + private < T > T getAttribute( final N5Reader n5, final String url1, final String url2, Class< T > clazz ) throws URISyntaxException, IOException + { + final N5URL n5URL = url2 == null ? new N5URL( url1 ) : new N5URL( url1 ).getRelative( url2 ); + return getAttribute( n5, n5URL, clazz ); + } + + private < T > T getAttribute( final N5Reader n5, final N5URL url1, Class< T > clazz ) throws URISyntaxException, IOException + { + return getAttribute( n5, url1, null, clazz ); + } + + private < T > T getAttribute( final N5Reader n5, final N5URL url1, final String url2, Class< T > clazz ) throws URISyntaxException, IOException + { + final N5URL n5URL = url2 == null ? url1 : url1.getRelative( url2 ); + return n5.getAttribute( n5URL.getDataset(), n5URL.getAttribute(), clazz ); } @Test public void testPathObject() throws IOException, URISyntaxException { - final TestInts ints = n5.getAttribute( new N5URL( "?objs#intsKey" ), TestInts.class ); + final TestInts ints = getAttribute( n5, new N5URL( "?objs#intsKey" ), TestInts.class ); assertEquals( testObjInts.name, ints.name ); assertEquals( testObjInts.type, ints.type ); assertArrayEquals( testObjInts.t(), ints.t() ); - final TestDoubles doubles = n5.getAttribute( new N5URL( "?objs#doublesKey" ), TestDoubles.class ); + final TestDoubles doubles = getAttribute( n5, new N5URL( "?objs#doublesKey" ), TestDoubles.class ); assertEquals( testObjDoubles.name, doubles.name ); assertEquals( testObjDoubles.type, doubles.type ); assertArrayEquals( testObjDoubles.t(), doubles.t(), 1e-9 ); - final TestDoubles[] doubleArray = n5.getAttribute( new N5URL( "?objs#array" ), TestDoubles[].class ); - final TestDoubles doubles1 = new TestDoubles("doubles", "doubles1", new double[]{ 5.7, 4.5, 3.4 }); - final TestDoubles doubles2 = new TestDoubles("doubles", "doubles2", new double[]{ 5.8, 4.6, 3.5 }); - final TestDoubles doubles3 = new TestDoubles("doubles", "doubles3", new double[]{ 5.9, 4.7, 3.6 }); - final TestDoubles doubles4 = new TestDoubles("doubles", "doubles4", new double[]{ 5.10, 4.8, 3.7 }); - final TestDoubles[] expectedDoubles = new TestDoubles[] {doubles1, doubles2, doubles3, doubles4}; - assertArrayEquals(expectedDoubles, doubleArray); + final TestDoubles[] doubleArray = getAttribute( n5, new N5URL( "?objs#array" ), TestDoubles[].class ); + final TestDoubles doubles1 = new TestDoubles( "doubles", "doubles1", new double[] { 5.7, 4.5, 3.4 } ); + final TestDoubles doubles2 = new TestDoubles( "doubles", "doubles2", new double[] { 5.8, 4.6, 3.5 } ); + final TestDoubles doubles3 = new TestDoubles( "doubles", "doubles3", new double[] { 5.9, 4.7, 3.6 } ); + final TestDoubles doubles4 = new TestDoubles( "doubles", "doubles4", new double[] { 5.10, 4.8, 3.7 } ); + final TestDoubles[] expectedDoubles = new TestDoubles[] { doubles1, doubles2, doubles3, doubles4 }; + assertArrayEquals( expectedDoubles, doubleArray ); - final String[] stringArray = n5.getAttribute( new N5URL( "?objs#String" ), String[].class ); - final String[] expectedString = new String[] {"This", "is", "a", "test"}; + final String[] stringArray = getAttribute( n5, new N5URL( "?objs#String" ), String[].class ); + final String[] expectedString = new String[] { "This", "is", "a", "test" }; - final Integer[] integerArray = n5.getAttribute( new N5URL( "?objs#Integer" ), Integer[].class ); - final Integer[] expectedInteger = new Integer[] {1,2,3,4}; + final Integer[] integerArray = getAttribute( n5, new N5URL( "?objs#Integer" ), Integer[].class ); + final Integer[] expectedInteger = new Integer[] { 1, 2, 3, 4 }; - final int[] intArray = n5.getAttribute( new N5URL( "?objs#int" ), int[].class ); - final int[] expectedInt = new int[] {1,2,3,4}; - assertArrayEquals(expectedInt, intArray); + final int[] intArray = getAttribute( n5, new N5URL( "?objs#int" ), int[].class ); + final int[] expectedInt = new int[] { 1, 2, 3, 4 }; + assertArrayEquals( expectedInt, intArray ); } - private boolean mapsEqual( Map a, Map b ) + private < K, V > boolean mapsEqual( Map< K, V > a, Map< K, V > b ) { - if( ! a.keySet().equals( b.keySet() )) + if ( !a.keySet().equals( b.keySet() ) ) return false; - for( K k : a.keySet() ) + for ( K k : a.keySet() ) { - if( ! a.get( k ).equals( b.get( k ) )) + if ( !a.get( k ).equals( b.get( k ) ) ) return false; } @@ -196,7 +215,9 @@ private boolean mapsEqual( Map a, Map b ) private static class TestObject< T > { String type; + String name; + T t; public TestObject( String type, String name, T t ) @@ -205,15 +226,22 @@ public TestObject( String type, String name, T t ) this.type = type; this.t = t; } - public T t() { return t; } - @Override public boolean equals(Object obj) { + public T t() + { + return t; + } - if (obj.getClass() == this.getClass()) { - final TestObject otherTestObject = (TestObject)obj; - return Objects.equals(name, otherTestObject.name) - && Objects.equals(type, otherTestObject.type) - && Objects.equals(t, otherTestObject.t); + @Override + public boolean equals( Object obj ) + { + + if ( obj.getClass() == this.getClass() ) + { + final TestObject< ? > otherTestObject = ( TestObject< ? > ) obj; + return Objects.equals( name, otherTestObject.name ) + && Objects.equals( type, otherTestObject.type ) + && Objects.equals( t, otherTestObject.t ); } return false; } @@ -226,13 +254,16 @@ public TestDoubles( String type, String name, double[] t ) super( type, name, t ); } - @Override public boolean equals(Object obj) { + @Override + public boolean equals( Object obj ) + { - if (obj.getClass() == this.getClass()) { - final TestDoubles otherTestObject = (TestDoubles)obj; - return Objects.equals(name, otherTestObject.name) - && Objects.equals(type, otherTestObject.type) - && Arrays.equals(t, otherTestObject.t); + if ( obj.getClass() == this.getClass() ) + { + final TestDoubles otherTestObject = ( TestDoubles ) obj; + return Objects.equals( name, otherTestObject.name ) + && Objects.equals( type, otherTestObject.type ) + && Arrays.equals( t, otherTestObject.t ); } return false; } @@ -244,13 +275,17 @@ public TestInts( String type, String name, int[] t ) { super( type, name, t ); } - @Override public boolean equals(Object obj) { - if (obj.getClass() == this.getClass()) { - final TestInts otherTestObject = (TestInts)obj; - return Objects.equals(name, otherTestObject.name) - && Objects.equals(type, otherTestObject.type) - && Arrays.equals(t, otherTestObject.t); + @Override + public boolean equals( Object obj ) + { + + if ( obj.getClass() == this.getClass() ) + { + final TestInts otherTestObject = ( TestInts ) obj; + return Objects.equals( name, otherTestObject.name ) + && Objects.equals( type, otherTestObject.type ) + && Arrays.equals( t, otherTestObject.t ); } return false; } From 62fb3f5dff20c712f48e990c8e3f882f58ce9afb Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Thu, 1 Dec 2022 11:12:45 -0500 Subject: [PATCH 039/243] feat: add readAttributesJson --- .../saalfeldlab/n5/GsonAttributesParser.java | 23 +++++++++++++++++++ .../janelia/saalfeldlab/n5/N5FSReader.java | 13 +++++++++++ 2 files changed, 36 insertions(+) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonAttributesParser.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonAttributesParser.java index 16b93df4..3f9b7550 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonAttributesParser.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonAttributesParser.java @@ -25,6 +25,7 @@ */ package org.janelia.saalfeldlab.n5; +import com.google.gson.JsonObject; import java.io.IOException; import java.io.Reader; import java.io.Writer; @@ -58,6 +59,15 @@ public interface GsonAttributesParser extends N5Reader { */ public HashMap getAttributes(final String pathName) throws IOException; + /** + * Reads the attributes a group or dataset. + * + * @param pathName group path + * @return the root {@link JsonObject} of the attributes + * @throws IOException + */ + public JsonObject getAttributesJson(final String pathName) throws IOException; + /** * Parses an attribute from the given attributes map. * @@ -116,6 +126,19 @@ public static HashMap readAttributes(final Reader reader, f return map == null ? new HashMap<>() : map; } + /** + * Reads the attributes json from a given {@link Reader}. + * + * @param reader + * @return the root {@link JsonObject} of the attributes + * @throws IOException + */ + public static JsonObject readAttributesJson(final Reader reader, final Gson gson) throws IOException { + + final JsonObject json = gson.fromJson(reader, JsonObject.class); + return json; + } + /** * Inserts new the JSON export of attributes into the given attributes map. * diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5FSReader.java b/src/main/java/org/janelia/saalfeldlab/n5/N5FSReader.java index f48f168e..0cc2e5a5 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5FSReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5FSReader.java @@ -25,6 +25,7 @@ */ package org.janelia.saalfeldlab.n5; +import com.google.gson.JsonObject; import java.io.Closeable; import java.io.IOException; import java.nio.channels.Channels; @@ -163,6 +164,18 @@ public HashMap getAttributes(final String pathName) throws } } + @Override + public JsonObject getAttributesJson(final String pathName) throws IOException { + + final Path path = Paths.get(basePath, getAttributesPath(pathName).toString()); + if (exists(pathName) && !Files.exists(path)) + return null; + + try (final LockedFileChannel lockedFileChannel = LockedFileChannel.openForReading(path)) { + return GsonAttributesParser.readAttributesJson(Channels.newReader(lockedFileChannel.getFileChannel(), StandardCharsets.UTF_8.name()), getGson()); + } + } + @Override public DataBlock readBlock( final String pathName, From b3b0b0bcbb3cdac43f4a7dcfd72294c49bdde19d Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Thu, 1 Dec 2022 11:16:44 -0500 Subject: [PATCH 040/243] feat: static N5URL constructor from container/group/path --- .../saalfeldlab/n5/AbstractGsonReader.java | 32 ++++++------------- .../org/janelia/saalfeldlab/n5/N5URL.java | 7 ++++ 2 files changed, 16 insertions(+), 23 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/AbstractGsonReader.java b/src/main/java/org/janelia/saalfeldlab/n5/AbstractGsonReader.java index 08264294..501812cd 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/AbstractGsonReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/AbstractGsonReader.java @@ -31,10 +31,7 @@ import com.google.gson.JsonElement; import com.google.gson.JsonObject; import com.google.gson.JsonSyntaxException; -import com.google.gson.TypeAdapter; import com.google.gson.reflect.TypeToken; -import com.google.gson.stream.JsonReader; -import com.google.gson.stream.JsonWriter; import java.io.IOException; import java.io.PipedReader; @@ -141,17 +138,10 @@ public T getAttribute( final String key, final Class clazz) throws IOException { - // final HashMap map = getAttributes(pathName); - try - { - - final String dataset = pathName != null ? "?" + pathName : "?"; - final String fragment = key != null ? "#" + key : "#"; - return getAttribute( new N5URL( dataset + fragment ), clazz ); - } - catch ( URISyntaxException e ) - { - throw new IOException( e ); + try { + return getAttribute(N5URL.from(null, pathName, key), clazz); + } catch (URISyntaxException e) { + throw new IOException(e); } } @@ -161,15 +151,10 @@ public T getAttribute( final String key, final Type type) throws IOException { - try - { - final String dataset = pathName != null ? "?" + pathName : "?"; - final String fragment = key != null ? "#" + key : "#"; - return getAttribute( new N5URL( dataset + fragment ), type ); - } - catch ( URISyntaxException e ) - { - throw new IOException( e ); + try { + return getAttribute(N5URL.from(null, pathName, key), type); + } catch (URISyntaxException e) { + throw new IOException(e); } } @@ -313,6 +298,7 @@ private T getJsonAsArray(JsonArray array, Type type) { for (int i = 0; i < array.size(); i++) { clsArray[i] = gson.fromJson(array.get(i), componentCls); } + //noinspection unchecked return (T)clsArray; } return null; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java b/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java index b847ebce..6b555123 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java @@ -330,4 +330,11 @@ public static URI encodeAsUri(String uri) throws URISyntaxException { } return n5Uri; } + + public static N5URL from(String container, String group, String attribute) throws URISyntaxException { + final String containerPart = container != null ? container : ""; + final String groupPart = group != null ? "?" + group : "?"; + final String attributePart = attribute != null ? "#" + attribute : "#"; + return new N5URL(containerPart + groupPart + attributePart); + } } From f5571cbc8016dadac5c95458ede3026835e43020 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Thu, 1 Dec 2022 11:16:53 -0500 Subject: [PATCH 041/243] refactor: use getAttributesJson --- .../saalfeldlab/n5/AbstractGsonReader.java | 42 ++++--------------- 1 file changed, 9 insertions(+), 33 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/AbstractGsonReader.java b/src/main/java/org/janelia/saalfeldlab/n5/AbstractGsonReader.java index 501812cd..837dd262 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/AbstractGsonReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/AbstractGsonReader.java @@ -34,8 +34,6 @@ import com.google.gson.reflect.TypeToken; import java.io.IOException; -import java.io.PipedReader; -import java.io.PipedWriter; import java.lang.reflect.Array; import java.lang.reflect.Type; import java.net.URISyntaxException; @@ -163,20 +161,12 @@ private T getAttribute( final Type type) throws IOException { final Class clazz = (type instanceof Class) ? ((Class)type) : null; - Type mapType = new TypeToken>() { - - }.getType(); - TypeAdapter jsonElementTypeAdapter = gson.getAdapter(JsonElement.class); - HashMap map = getAttributes(url.resolveDataset()); - final String attributePath = url.resolveAttribute(); - JsonElement json = null; + JsonElement json = getAttributesJson(url.resolveDataset()); for (final String pathPart : attributePath.split("/")) { if (pathPart.isEmpty()) continue; - if (map.containsKey(pathPart)) { - json = map.get(pathPart); - } else if (json instanceof JsonObject && json.getAsJsonObject().get(pathPart) != null) { + if (json instanceof JsonObject && json.getAsJsonObject().get(pathPart) != null) { json = json.getAsJsonObject().get(pathPart); } else { final Matcher matcher = ARRAY_INDEX.matcher(pathPart); @@ -186,24 +176,10 @@ private T getAttribute( } } } - if (json == null && clazz != null && clazz.isAssignableFrom(HashMap.class)) { - //NOTE: Would not need to do this if we could get the root `JsonElement` attribute, instead of just the `Map entry : map.entrySet()) { - String k = entry.getKey(); - JsonElement val = entry.getValue(); - out.name(k); - jsonElementTypeAdapter.write(out, val); - } - out.endObject(); - - Map retMap = gson.fromJson(new JsonReader(in), mapType); + if (clazz != null && clazz.isAssignableFrom(HashMap.class)) { + Type mapType = new TypeToken>() {}.getType(); + Map retMap = gson.fromJson(json, mapType); + //noinspection unchecked return (T)retMap; } if (json instanceof JsonArray) { @@ -228,12 +204,12 @@ private T getAttribute( } private T getJsonAsArray(JsonArray array, Class clazz) { + return getJsonAsArray(array, (Type)clazz); } private T getJsonAsArray(JsonArray array, Type type) { - final Class clazz = (type instanceof Class) ? ((Class)type) : null; if (type == boolean[].class) { @@ -292,9 +268,9 @@ private T getJsonAsArray(JsonArray array, Type type) { retArray[i] = value; } return (T)retArray; - } else if(clazz != null && clazz.isArray()) { + } else if (clazz != null && clazz.isArray()) { final Class componentCls = clazz.getComponentType(); - final Object[] clsArray = (Object[] )Array.newInstance(componentCls, array.size()); + final Object[] clsArray = (Object[])Array.newInstance(componentCls, array.size()); for (int i = 0; i < array.size(); i++) { clsArray[i] = gson.fromJson(array.get(i), componentCls); } From a3368867e6642ae65c59ae302cab08d6197d2568 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Thu, 1 Dec 2022 11:17:01 -0500 Subject: [PATCH 042/243] refactor: rename fields/methods --- .../org/janelia/saalfeldlab/n5/N5URL.java | 38 +++++++++---------- .../org/janelia/saalfeldlab/n5/N5URLTest.java | 16 ++++---- .../saalfeldlab/n5/url/UrlAttributeTest.java | 28 +++++++------- 3 files changed, 41 insertions(+), 41 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java b/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java index 6b555123..26685470 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java @@ -10,8 +10,8 @@ public class N5URL { final URI uri; private final String scheme; - private final String location; - private final String dataset; + private final String container; + private final String group; private final String attribute; public N5URL(String uri) throws URISyntaxException { @@ -25,37 +25,37 @@ public N5URL(URI uri) { scheme = uri.getScheme() == null ? null : uri.getScheme(); final String schemeSpecificPartWithoutQuery = getSchemeSpecificPartWithoutQuery(); if (uri.getScheme() == null) { - location = schemeSpecificPartWithoutQuery.replaceFirst("//", ""); + container = schemeSpecificPartWithoutQuery.replaceFirst("//", ""); } else { - location = uri.getScheme() + ":" + schemeSpecificPartWithoutQuery; + container = uri.getScheme() + ":" + schemeSpecificPartWithoutQuery; } - dataset = uri.getQuery(); + group = uri.getQuery(); attribute = uri.getFragment(); } - public String getLocation() { + public String getContainerPath() { - return location; + return container; } - public String getDataset() { + public String getGroupPath() { - return dataset; + return group; } public String resolveDataset() { - return resolveDatasetPath( getDataset() == null ? "" : getDataset() ); + return resolveDatasetPath( getGroupPath() == null ? "" : getGroupPath() ); } - public String getAttribute() { + public String getAttributePath() { return attribute; } public String resolveAttribute() { - return resolveAttributePath( getAttribute() == null ? "" : getAttribute() ); + return resolveAttributePath( getAttributePath() == null ? "" : getAttributePath() ); } private String getSchemePart() { @@ -65,12 +65,12 @@ private String getSchemePart() { private String getLocationPart() { - return location; + return container; } private String getDatasetPart() { - return dataset == null ? "" : "?" + dataset; + return group == null ? "" : "?" + group; } private String getAttributePart() { @@ -88,7 +88,7 @@ private String getSchemeSpecificPartWithoutQuery() { return uri.getSchemeSpecificPart().replace("?" + uri.getQuery(), ""); } - public N5URL getRelative(N5URL relative) throws URISyntaxException { + public N5URL resolveRelative(N5URL relative) throws URISyntaxException { final URI thisUri = uri; final URI relativeUri = relative.uri; @@ -159,14 +159,14 @@ public N5URL getRelative(N5URL relative) throws URISyntaxException { return new N5URL(newUri.toString()); } - public N5URL getRelative(URI relative) throws URISyntaxException { + public N5URL resolveRelative(URI relative) throws URISyntaxException { - return getRelative(new N5URL(relative)); + return resolveRelative(new N5URL(relative)); } - public N5URL getRelative(String relative) throws URISyntaxException { + public N5URL resolveRelative(String relative) throws URISyntaxException { - return getRelative(new N5URL(relative)); + return resolveRelative(new N5URL(relative)); } public static String resolveDatasetPath(String path) { diff --git a/src/test/java/org/janelia/saalfeldlab/n5/N5URLTest.java b/src/test/java/org/janelia/saalfeldlab/n5/N5URLTest.java index 520e7512..97a0caaa 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/N5URLTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/N5URLTest.java @@ -37,27 +37,27 @@ public void testGetRelative() throws URISyntaxException { assertEquals( "/a/b/c/d?e#f", - new N5URL("/a/b/c").getRelative("d?e#f").toString()); + new N5URL("/a/b/c").resolveRelative("d?e#f").toString()); assertEquals( "/d?e#f", - new N5URL("/a/b/c").getRelative("/d?e#f").toString()); + new N5URL("/a/b/c").resolveRelative("/d?e#f").toString()); assertEquals( "file:/a/b/c", - new N5URL("s3://janelia-cosem-datasets/jrc_hela-3/jrc_hela-3.n5").getRelative("file:/a/b/c").toString()); + new N5URL("s3://janelia-cosem-datasets/jrc_hela-3/jrc_hela-3.n5").resolveRelative("file:/a/b/c").toString()); assertEquals( "s3://janelia-cosem-datasets/a/b/c", - new N5URL("s3://janelia-cosem-datasets/jrc_hela-3/jrc_hela-3.n5").getRelative("/a/b/c").toString()); + new N5URL("s3://janelia-cosem-datasets/jrc_hela-3/jrc_hela-3.n5").resolveRelative("/a/b/c").toString()); assertEquals( "s3://janelia-cosem-datasets/a/b/c?d/e#f/g", - new N5URL("s3://janelia-cosem-datasets/jrc_hela-3/jrc_hela-3.n5").getRelative("/a/b/c?d/e#f/g").toString()); + new N5URL("s3://janelia-cosem-datasets/jrc_hela-3/jrc_hela-3.n5").resolveRelative("/a/b/c?d/e#f/g").toString()); assertEquals( "s3://janelia-cosem-datasets/jrc_hela-3/jrc_hela-3.n5/a/b/c?d/e#f/g", - new N5URL("s3://janelia-cosem-datasets/jrc_hela-3/jrc_hela-3.n5").getRelative("a/b/c?d/e#f/g").toString()); + new N5URL("s3://janelia-cosem-datasets/jrc_hela-3/jrc_hela-3.n5").resolveRelative("a/b/c?d/e#f/g").toString()); assertEquals( "s3://janelia-cosem-datasets/jrc_hela-3/jrc_hela-3.n5?d/e#f/g", - new N5URL("s3://janelia-cosem-datasets/jrc_hela-3/jrc_hela-3.n5").getRelative("?d/e#f/g").toString()); + new N5URL("s3://janelia-cosem-datasets/jrc_hela-3/jrc_hela-3.n5").resolveRelative("?d/e#f/g").toString()); assertEquals( "s3://janelia-cosem-datasets/jrc_hela-3/jrc_hela-3.n5#f/g", - new N5URL("s3://janelia-cosem-datasets/jrc_hela-3/jrc_hela-3.n5").getRelative("#f/g").toString()); + new N5URL("s3://janelia-cosem-datasets/jrc_hela-3/jrc_hela-3.n5").resolveRelative("#f/g").toString()); } } \ No newline at end of file diff --git a/src/test/java/org/janelia/saalfeldlab/n5/url/UrlAttributeTest.java b/src/test/java/org/janelia/saalfeldlab/n5/url/UrlAttributeTest.java index 29e3605e..82dd9202 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/url/UrlAttributeTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/url/UrlAttributeTest.java @@ -120,28 +120,28 @@ public void testPathAttributes() throws URISyntaxException, IOException // name of a assertEquals( "name of a from root", a, getAttribute( n5, "?/a#name", String.class ) ); assertEquals( "name of a from root", a, getAttribute( n5, new N5URL( "?a#name" ), String.class ) ); - assertEquals( "name of a from a", a, getAttribute( n5, aUrl.getRelative( new N5URL( "?/a#name" ) ), String.class ) ); - assertEquals( "name of a from aa", a, getAttribute( n5, aaUrl.getRelative( "?..#name" ), String.class ) ); - assertEquals( "name of a from aaa", a, getAttribute( n5, aaaUrl.getRelative( new N5URL( "?../..#name" ) ), String.class ) ); + assertEquals( "name of a from a", a, getAttribute( n5, aUrl.resolveRelative( new N5URL( "?/a#name" ) ), String.class ) ); + assertEquals( "name of a from aa", a, getAttribute( n5, aaUrl.resolveRelative( "?..#name" ), String.class ) ); + assertEquals( "name of a from aaa", a, getAttribute( n5, aaaUrl.resolveRelative( new N5URL( "?../..#name" ) ), String.class ) ); // name of aa assertEquals( "name of aa from root", aa, getAttribute( n5, new N5URL( "?/a/aa#name" ), String.class ) ); assertEquals( "name of aa from root", aa, getAttribute( n5, new N5URL( "?a/aa#name" ), String.class ) ); - assertEquals( "name of aa from a", aa, getAttribute( n5, aUrl.getRelative( "?aa#name" ), String.class ) ); + assertEquals( "name of aa from a", aa, getAttribute( n5, aUrl.resolveRelative( "?aa#name" ), String.class ) ); - assertEquals( "name of aa from aa", aa, getAttribute( n5, aaUrl.getRelative( "?./#name" ), String.class ) ); + assertEquals( "name of aa from aa", aa, getAttribute( n5, aaUrl.resolveRelative( "?./#name" ), String.class ) ); - assertEquals( "name of aa from aa", aa, getAttribute( n5, aaUrl.getRelative( "#name" ), String.class ) ); - assertEquals( "name of aa from aaa", aa, getAttribute( n5, aaaUrl.getRelative( "?..#name" ), String.class ) ); + assertEquals( "name of aa from aa", aa, getAttribute( n5, aaUrl.resolveRelative( "#name" ), String.class ) ); + assertEquals( "name of aa from aaa", aa, getAttribute( n5, aaaUrl.resolveRelative( "?..#name" ), String.class ) ); // name of aaa assertEquals( "name of aaa from root", aaa, getAttribute( n5, new N5URL( "?/a/aa/aaa#name" ), String.class ) ); assertEquals( "name of aaa from root", aaa, getAttribute( n5, new N5URL( "?a/aa/aaa#name" ), String.class ) ); - assertEquals( "name of aaa from a", aaa, getAttribute( n5, aUrl.getRelative( "?aa/aaa#name" ), String.class ) ); - assertEquals( "name of aaa from aa", aaa, getAttribute( n5, aaUrl.getRelative( "?aaa#name" ), String.class ) ); - assertEquals( "name of aaa from aaa", aaa, getAttribute( n5, aaaUrl.getRelative( "#name" ), String.class ) ); + assertEquals( "name of aaa from a", aaa, getAttribute( n5, aUrl.resolveRelative( "?aa/aaa#name" ), String.class ) ); + assertEquals( "name of aaa from aa", aaa, getAttribute( n5, aaUrl.resolveRelative( "?aaa#name" ), String.class ) ); + assertEquals( "name of aaa from aaa", aaa, getAttribute( n5, aaaUrl.resolveRelative( "#name" ), String.class ) ); - assertEquals( "name of aaa from aaa", aaa, getAttribute( n5, aaaUrl.getRelative( "?./#name" ), String.class ) ); + assertEquals( "name of aaa from aaa", aaa, getAttribute( n5, aaaUrl.resolveRelative( "?./#name" ), String.class ) ); } private < T > T getAttribute( final N5Reader n5, final String url1, Class< T > clazz ) throws URISyntaxException, IOException @@ -151,7 +151,7 @@ private < T > T getAttribute( final N5Reader n5, final String url1, Class< T > c private < T > T getAttribute( final N5Reader n5, final String url1, final String url2, Class< T > clazz ) throws URISyntaxException, IOException { - final N5URL n5URL = url2 == null ? new N5URL( url1 ) : new N5URL( url1 ).getRelative( url2 ); + final N5URL n5URL = url2 == null ? new N5URL( url1 ) : new N5URL( url1 ).resolveRelative( url2 ); return getAttribute( n5, n5URL, clazz ); } @@ -162,8 +162,8 @@ private < T > T getAttribute( final N5Reader n5, final N5URL url1, Class< T > cl private < T > T getAttribute( final N5Reader n5, final N5URL url1, final String url2, Class< T > clazz ) throws URISyntaxException, IOException { - final N5URL n5URL = url2 == null ? url1 : url1.getRelative( url2 ); - return n5.getAttribute( n5URL.getDataset(), n5URL.getAttribute(), clazz ); + final N5URL n5URL = url2 == null ? url1 : url1.resolveRelative( url2 ); + return n5.getAttribute( n5URL.getGroupPath(), n5URL.getAttributePath(), clazz ); } @Test From 520dbcb5b39e8db599421edc928f99cdcaab4fa9 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Thu, 1 Dec 2022 11:24:25 -0500 Subject: [PATCH 043/243] refactor: more N5URL renaming --- .../saalfeldlab/n5/AbstractGsonReader.java | 2 +- .../org/janelia/saalfeldlab/n5/N5URL.java | 22 +++++++++---------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/AbstractGsonReader.java b/src/main/java/org/janelia/saalfeldlab/n5/AbstractGsonReader.java index 837dd262..8d0021bf 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/AbstractGsonReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/AbstractGsonReader.java @@ -162,7 +162,7 @@ private T getAttribute( final Class clazz = (type instanceof Class) ? ((Class)type) : null; final String attributePath = url.resolveAttribute(); - JsonElement json = getAttributesJson(url.resolveDataset()); + JsonElement json = getAttributesJson(url.resolveGroup()); for (final String pathPart : attributePath.split("/")) { if (pathPart.isEmpty()) continue; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java b/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java index 26685470..c6cd9169 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java @@ -43,9 +43,9 @@ public String getGroupPath() { return group; } - public String resolveDataset() { + public String resolveGroup() { - return resolveDatasetPath( getGroupPath() == null ? "" : getGroupPath() ); + return resolveGroupPath( getGroupPath() == null ? "" : getGroupPath() ); } public String getAttributePath() { @@ -63,12 +63,12 @@ private String getSchemePart() { return scheme == null ? "" : scheme + "://"; } - private String getLocationPart() { + private String getContainerPart() { return container; } - private String getDatasetPart() { + private String getGroupPart() { return group == null ? "" : "?" + group; } @@ -80,7 +80,7 @@ private String getAttributePart() { @Override public String toString() { - return getLocationPart() + getDatasetPart() + getAttributePart(); + return getContainerPart() + getGroupPart() + getAttributePart(); } private String getSchemeSpecificPartWithoutQuery() { @@ -107,7 +107,7 @@ public N5URL resolveRelative(N5URL relative) throws URISyntaxException { newUri .append(relativeUri.getAuthority()) .append(relativeUri.getPath()) - .append(relative.getDatasetPart()) + .append(relative.getGroupPart()) .append(relative.getAttributePart()); return new N5URL(newUri.toString()); } @@ -125,7 +125,7 @@ public N5URL resolveRelative(N5URL relative) throws URISyntaxException { } newUri .append(path) - .append(relative.getDatasetPart()) + .append(relative.getGroupPart()) .append(relative.getAttributePart()); return new N5URL(newUri.toString()); } @@ -134,15 +134,15 @@ public N5URL resolveRelative(N5URL relative) throws URISyntaxException { final String query = relativeUri.getQuery(); if (query != null) { if (query.charAt(0) != '/' && thisUri.getQuery() != null) { - newUri.append(this.getDatasetPart()).append('/'); + newUri.append(this.getGroupPart()).append('/'); newUri.append(relativeUri.getQuery()); } else { - newUri.append(relative.getDatasetPart()); + newUri.append(relative.getGroupPart()); } newUri.append(relative.getAttributePart()); return new N5URL(newUri.toString()); } - newUri.append(this.getDatasetPart()); + newUri.append(this.getGroupPart()); final String fragment = relativeUri.getFragment(); if (fragment != null) { @@ -169,7 +169,7 @@ public N5URL resolveRelative(String relative) throws URISyntaxException { return resolveRelative(new N5URL(relative)); } - public static String resolveDatasetPath(String path) { + public static String resolveGroupPath(String path) { final char[] pathChars = path.toCharArray(); final List tokens = new ArrayList<>(); From 88d58f35b572c260378a7921a15da200ee84b831 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Thu, 1 Dec 2022 11:29:41 -0500 Subject: [PATCH 044/243] refactor: align method naming with Path conventions --- .../saalfeldlab/n5/AbstractGsonReader.java | 4 +- .../org/janelia/saalfeldlab/n5/N5URL.java | 22 +++---- .../org/janelia/saalfeldlab/n5/N5URLTest.java | 58 ++++++++++--------- .../saalfeldlab/n5/url/UrlAttributeTest.java | 26 ++++----- 4 files changed, 56 insertions(+), 54 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/AbstractGsonReader.java b/src/main/java/org/janelia/saalfeldlab/n5/AbstractGsonReader.java index 8d0021bf..8b765af8 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/AbstractGsonReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/AbstractGsonReader.java @@ -161,8 +161,8 @@ private T getAttribute( final Type type) throws IOException { final Class clazz = (type instanceof Class) ? ((Class)type) : null; - final String attributePath = url.resolveAttribute(); - JsonElement json = getAttributesJson(url.resolveGroup()); + final String attributePath = url.normalizeAttributePath(); + JsonElement json = getAttributesJson(url.normalizeGroupPath()); for (final String pathPart : attributePath.split("/")) { if (pathPart.isEmpty()) continue; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java b/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java index c6cd9169..29fa831f 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java @@ -43,9 +43,9 @@ public String getGroupPath() { return group; } - public String resolveGroup() { + public String normalizeGroupPath() { - return resolveGroupPath( getGroupPath() == null ? "" : getGroupPath() ); + return normalizeGroupPath( getGroupPath() == null ? "" : getGroupPath() ); } public String getAttributePath() { @@ -53,9 +53,9 @@ public String getAttributePath() { return attribute; } - public String resolveAttribute() { + public String normalizeAttributePath() { - return resolveAttributePath( getAttributePath() == null ? "" : getAttributePath() ); + return normalizeAttributePath( getAttributePath() == null ? "" : getAttributePath() ); } private String getSchemePart() { @@ -88,7 +88,7 @@ private String getSchemeSpecificPartWithoutQuery() { return uri.getSchemeSpecificPart().replace("?" + uri.getQuery(), ""); } - public N5URL resolveRelative(N5URL relative) throws URISyntaxException { + public N5URL resolve(N5URL relative) throws URISyntaxException { final URI thisUri = uri; final URI relativeUri = relative.uri; @@ -159,17 +159,17 @@ public N5URL resolveRelative(N5URL relative) throws URISyntaxException { return new N5URL(newUri.toString()); } - public N5URL resolveRelative(URI relative) throws URISyntaxException { + public N5URL resolve(URI relative) throws URISyntaxException { - return resolveRelative(new N5URL(relative)); + return resolve(new N5URL(relative)); } - public N5URL resolveRelative(String relative) throws URISyntaxException { + public N5URL resolve(String relative) throws URISyntaxException { - return resolveRelative(new N5URL(relative)); + return resolve(new N5URL(relative)); } - public static String resolveGroupPath(String path) { + public static String normalizeGroupPath(String path) { final char[] pathChars = path.toCharArray(); final List tokens = new ArrayList<>(); @@ -227,7 +227,7 @@ public static String resolveGroupPath(String path) { .reduce((l,r) -> l + "/" + r).orElse( "" ); } - public static String resolveAttributePath(String path) { + public static String normalizeAttributePath(String path) { final char[] pathChars = path.toCharArray(); final List tokens = new ArrayList<>(); diff --git a/src/test/java/org/janelia/saalfeldlab/n5/N5URLTest.java b/src/test/java/org/janelia/saalfeldlab/n5/N5URLTest.java index 97a0caaa..1252f43d 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/N5URLTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/N5URLTest.java @@ -10,54 +10,56 @@ public class N5URLTest { @Test public void testAttributePath() { - assertEquals("/a/b/c/d/e", N5URL.resolveAttributePath("/a/b/c/d/e")); - assertEquals("/a/b/c/d/e", N5URL.resolveAttributePath("/a/b/c/d/e/")); - assertEquals("/", N5URL.resolveAttributePath("/")); - assertEquals("", N5URL.resolveAttributePath("")); - assertEquals("/", N5URL.resolveAttributePath("/a/..")); - assertEquals("/", N5URL.resolveAttributePath("/a/../b/../c/d/../..")); - assertEquals("/", N5URL.resolveAttributePath("/a/../b/../c/d/../..")); - assertEquals("/", N5URL.resolveAttributePath("/a/../b/../c/d/../..")); - assertEquals("/", N5URL.resolveAttributePath("/./././././")); - assertEquals("", N5URL.resolveAttributePath("./././././")); - - assertEquals("/a/[0]/b/[0]", N5URL.resolveAttributePath("/a/[0]/b[0]/")); - assertEquals("[0]", N5URL.resolveAttributePath("[0]")); - assertEquals("/[0]", N5URL.resolveAttributePath("/[0]/")); - assertEquals("/a/[0]", N5URL.resolveAttributePath("/a[0]/")); - assertEquals("/[0]/b", N5URL.resolveAttributePath("/[0]b/")); - - assertEquals("let's/try/a/real/case/with spaces", N5URL.resolveAttributePath("let's/try/a/real/case/with spaces/")); - assertEquals("let's/try/a/real/case/with spaces", N5URL.resolveAttributePath("let's/try/a/real/////case////with spaces/")); - assertThrows( IndexOutOfBoundsException.class, () -> N5URL.resolveAttributePath("../first/relative/../not/allowed")); + assertEquals("/a/b/c/d/e", N5URL.normalizeAttributePath("/a/b/c/d/e")); + assertEquals("/a/b/c/d/e", N5URL.normalizeAttributePath("/a/b/c/d/e/")); + assertEquals("/", N5URL.normalizeAttributePath("/")); + assertEquals("", N5URL.normalizeAttributePath("")); + assertEquals("/", N5URL.normalizeAttributePath("/a/..")); + assertEquals("/", N5URL.normalizeAttributePath("/a/../b/../c/d/../..")); + assertEquals("/", N5URL.normalizeAttributePath("/a/../b/../c/d/../..")); + assertEquals("/", N5URL.normalizeAttributePath("/a/../b/../c/d/../..")); + assertEquals("/", N5URL.normalizeAttributePath("/./././././")); + assertEquals("", N5URL.normalizeAttributePath("./././././")); + + assertEquals("/a/[0]/b/[0]", N5URL.normalizeAttributePath("/a/[0]/b[0]/")); + assertEquals("[0]", N5URL.normalizeAttributePath("[0]")); + assertEquals("/[0]", N5URL.normalizeAttributePath("/[0]/")); + assertEquals("/a/[0]", N5URL.normalizeAttributePath("/a[0]/")); + assertEquals("/[0]/b", N5URL.normalizeAttributePath("/[0]b/")); + + assertEquals("let's/try/a/real/case/with spaces", N5URL.normalizeAttributePath("let's/try/a/real/case/with spaces/")); + assertEquals("let's/try/a/real/case/with spaces", N5URL.normalizeAttributePath("let's/try/a/real/////case////with spaces/")); + assertThrows( IndexOutOfBoundsException.class, () -> N5URL.normalizeAttributePath("../first/relative/../not/allowed")); } @Test public void testGetRelative() throws URISyntaxException { + //TODO: nio path interface? + // Path assertEquals( "/a/b/c/d?e#f", - new N5URL("/a/b/c").resolveRelative("d?e#f").toString()); + new N5URL("/a/b/c").resolve("d?e#f").toString()); assertEquals( "/d?e#f", - new N5URL("/a/b/c").resolveRelative("/d?e#f").toString()); + new N5URL("/a/b/c").resolve("/d?e#f").toString()); assertEquals( "file:/a/b/c", - new N5URL("s3://janelia-cosem-datasets/jrc_hela-3/jrc_hela-3.n5").resolveRelative("file:/a/b/c").toString()); + new N5URL("s3://janelia-cosem-datasets/jrc_hela-3/jrc_hela-3.n5").resolve("file:/a/b/c").toString()); assertEquals( "s3://janelia-cosem-datasets/a/b/c", - new N5URL("s3://janelia-cosem-datasets/jrc_hela-3/jrc_hela-3.n5").resolveRelative("/a/b/c").toString()); + new N5URL("s3://janelia-cosem-datasets/jrc_hela-3/jrc_hela-3.n5").resolve("/a/b/c").toString()); assertEquals( "s3://janelia-cosem-datasets/a/b/c?d/e#f/g", - new N5URL("s3://janelia-cosem-datasets/jrc_hela-3/jrc_hela-3.n5").resolveRelative("/a/b/c?d/e#f/g").toString()); + new N5URL("s3://janelia-cosem-datasets/jrc_hela-3/jrc_hela-3.n5").resolve("/a/b/c?d/e#f/g").toString()); assertEquals( "s3://janelia-cosem-datasets/jrc_hela-3/jrc_hela-3.n5/a/b/c?d/e#f/g", - new N5URL("s3://janelia-cosem-datasets/jrc_hela-3/jrc_hela-3.n5").resolveRelative("a/b/c?d/e#f/g").toString()); + new N5URL("s3://janelia-cosem-datasets/jrc_hela-3/jrc_hela-3.n5").resolve("a/b/c?d/e#f/g").toString()); assertEquals( "s3://janelia-cosem-datasets/jrc_hela-3/jrc_hela-3.n5?d/e#f/g", - new N5URL("s3://janelia-cosem-datasets/jrc_hela-3/jrc_hela-3.n5").resolveRelative("?d/e#f/g").toString()); + new N5URL("s3://janelia-cosem-datasets/jrc_hela-3/jrc_hela-3.n5").resolve("?d/e#f/g").toString()); assertEquals( "s3://janelia-cosem-datasets/jrc_hela-3/jrc_hela-3.n5#f/g", - new N5URL("s3://janelia-cosem-datasets/jrc_hela-3/jrc_hela-3.n5").resolveRelative("#f/g").toString()); + new N5URL("s3://janelia-cosem-datasets/jrc_hela-3/jrc_hela-3.n5").resolve("#f/g").toString()); } } \ No newline at end of file diff --git a/src/test/java/org/janelia/saalfeldlab/n5/url/UrlAttributeTest.java b/src/test/java/org/janelia/saalfeldlab/n5/url/UrlAttributeTest.java index 82dd9202..16694fde 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/url/UrlAttributeTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/url/UrlAttributeTest.java @@ -120,28 +120,28 @@ public void testPathAttributes() throws URISyntaxException, IOException // name of a assertEquals( "name of a from root", a, getAttribute( n5, "?/a#name", String.class ) ); assertEquals( "name of a from root", a, getAttribute( n5, new N5URL( "?a#name" ), String.class ) ); - assertEquals( "name of a from a", a, getAttribute( n5, aUrl.resolveRelative( new N5URL( "?/a#name" ) ), String.class ) ); - assertEquals( "name of a from aa", a, getAttribute( n5, aaUrl.resolveRelative( "?..#name" ), String.class ) ); - assertEquals( "name of a from aaa", a, getAttribute( n5, aaaUrl.resolveRelative( new N5URL( "?../..#name" ) ), String.class ) ); + assertEquals( "name of a from a", a, getAttribute( n5, aUrl.resolve( new N5URL( "?/a#name" ) ), String.class ) ); + assertEquals( "name of a from aa", a, getAttribute( n5, aaUrl.resolve( "?..#name" ), String.class ) ); + assertEquals( "name of a from aaa", a, getAttribute( n5, aaaUrl.resolve( new N5URL( "?../..#name" ) ), String.class ) ); // name of aa assertEquals( "name of aa from root", aa, getAttribute( n5, new N5URL( "?/a/aa#name" ), String.class ) ); assertEquals( "name of aa from root", aa, getAttribute( n5, new N5URL( "?a/aa#name" ), String.class ) ); - assertEquals( "name of aa from a", aa, getAttribute( n5, aUrl.resolveRelative( "?aa#name" ), String.class ) ); + assertEquals( "name of aa from a", aa, getAttribute( n5, aUrl.resolve( "?aa#name" ), String.class ) ); - assertEquals( "name of aa from aa", aa, getAttribute( n5, aaUrl.resolveRelative( "?./#name" ), String.class ) ); + assertEquals( "name of aa from aa", aa, getAttribute( n5, aaUrl.resolve( "?./#name" ), String.class ) ); - assertEquals( "name of aa from aa", aa, getAttribute( n5, aaUrl.resolveRelative( "#name" ), String.class ) ); - assertEquals( "name of aa from aaa", aa, getAttribute( n5, aaaUrl.resolveRelative( "?..#name" ), String.class ) ); + assertEquals( "name of aa from aa", aa, getAttribute( n5, aaUrl.resolve( "#name" ), String.class ) ); + assertEquals( "name of aa from aaa", aa, getAttribute( n5, aaaUrl.resolve( "?..#name" ), String.class ) ); // name of aaa assertEquals( "name of aaa from root", aaa, getAttribute( n5, new N5URL( "?/a/aa/aaa#name" ), String.class ) ); assertEquals( "name of aaa from root", aaa, getAttribute( n5, new N5URL( "?a/aa/aaa#name" ), String.class ) ); - assertEquals( "name of aaa from a", aaa, getAttribute( n5, aUrl.resolveRelative( "?aa/aaa#name" ), String.class ) ); - assertEquals( "name of aaa from aa", aaa, getAttribute( n5, aaUrl.resolveRelative( "?aaa#name" ), String.class ) ); - assertEquals( "name of aaa from aaa", aaa, getAttribute( n5, aaaUrl.resolveRelative( "#name" ), String.class ) ); + assertEquals( "name of aaa from a", aaa, getAttribute( n5, aUrl.resolve( "?aa/aaa#name" ), String.class ) ); + assertEquals( "name of aaa from aa", aaa, getAttribute( n5, aaUrl.resolve( "?aaa#name" ), String.class ) ); + assertEquals( "name of aaa from aaa", aaa, getAttribute( n5, aaaUrl.resolve( "#name" ), String.class ) ); - assertEquals( "name of aaa from aaa", aaa, getAttribute( n5, aaaUrl.resolveRelative( "?./#name" ), String.class ) ); + assertEquals( "name of aaa from aaa", aaa, getAttribute( n5, aaaUrl.resolve( "?./#name" ), String.class ) ); } private < T > T getAttribute( final N5Reader n5, final String url1, Class< T > clazz ) throws URISyntaxException, IOException @@ -151,7 +151,7 @@ private < T > T getAttribute( final N5Reader n5, final String url1, Class< T > c private < T > T getAttribute( final N5Reader n5, final String url1, final String url2, Class< T > clazz ) throws URISyntaxException, IOException { - final N5URL n5URL = url2 == null ? new N5URL( url1 ) : new N5URL( url1 ).resolveRelative( url2 ); + final N5URL n5URL = url2 == null ? new N5URL( url1 ) : new N5URL( url1 ).resolve( url2 ); return getAttribute( n5, n5URL, clazz ); } @@ -162,7 +162,7 @@ private < T > T getAttribute( final N5Reader n5, final N5URL url1, Class< T > cl private < T > T getAttribute( final N5Reader n5, final N5URL url1, final String url2, Class< T > clazz ) throws URISyntaxException, IOException { - final N5URL n5URL = url2 == null ? url1 : url1.resolveRelative( url2 ); + final N5URL n5URL = url2 == null ? url1 : url1.resolve( url2 ); return n5.getAttribute( n5URL.getGroupPath(), n5URL.getAttributePath(), clazz ); } From bb9bd61a03010c314fe86d8fc36b7e7abe36d5f9 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Thu, 1 Dec 2022 14:41:03 -0500 Subject: [PATCH 045/243] feat: add method to check if N5URL is absolute --- src/main/java/org/janelia/saalfeldlab/n5/N5URL.java | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java b/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java index 29fa831f..fd9c5ad7 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java @@ -88,6 +88,16 @@ private String getSchemeSpecificPartWithoutQuery() { return uri.getSchemeSpecificPart().replace("?" + uri.getQuery(), ""); } + public boolean isAbsolute() { + + final String path = uri.getPath(); + if (!path.isEmpty()) { + final char char0 = path.charAt(0); + final boolean isAbsolute = char0 == '/' || (path.length() >= 2 && path.charAt(1) == ':' && char0 >= 'A' && char0 <= 'Z'); + } + return false; + } + public N5URL resolve(N5URL relative) throws URISyntaxException { final URI thisUri = uri; From 5f0df9ca653516b937f7899f59f98e469321b9a5 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Thu, 1 Dec 2022 14:41:06 -0500 Subject: [PATCH 046/243] feat: add method to normalize container path --- src/main/java/org/janelia/saalfeldlab/n5/N5URL.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java b/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java index fd9c5ad7..dfbcd8bb 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java @@ -43,6 +43,11 @@ public String getGroupPath() { return group; } + public String normalizeContainerPath() { + + return normalizePath(getContainerPath()); + } + public String normalizeGroupPath() { return normalizeGroupPath( getGroupPath() == null ? "" : getGroupPath() ); From 36cc96fc6ba152f0968e227db691d27a5e41dc17 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Thu, 1 Dec 2022 14:41:09 -0500 Subject: [PATCH 047/243] refactor: rename and use new methods --- .../org/janelia/saalfeldlab/n5/N5URL.java | 52 ++++++++++--------- 1 file changed, 28 insertions(+), 24 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java b/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java index dfbcd8bb..00764576 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java @@ -50,7 +50,7 @@ public String normalizeContainerPath() { public String normalizeGroupPath() { - return normalizeGroupPath( getGroupPath() == null ? "" : getGroupPath() ); + return normalizePath(getGroupPath()); } public String getAttributePath() { @@ -60,7 +60,7 @@ public String getAttributePath() { public String normalizeAttributePath() { - return normalizeAttributePath( getAttributePath() == null ? "" : getAttributePath() ); + return normalizeAttributePath(getAttributePath()); } private String getSchemePart() { @@ -131,11 +131,9 @@ public N5URL resolve(N5URL relative) throws URISyntaxException { newUri.append("//").append(thisAuthority); } - if (!relativeUri.getPath().isEmpty()) { - final String path = relativeUri.getPath(); - final char char0 = path.charAt(0); - final boolean isAbsolute = char0 == '/' || (path.length() >= 2 && path.charAt(1) == ':' && char0 >= 'A' && char0 <= 'Z'); - if (!isAbsolute) { + final String path = relativeUri.getPath(); + if (!path.isEmpty()) { + if (!relative.isAbsolute()) { newUri.append(thisUri.getPath()).append('/'); } newUri @@ -184,7 +182,9 @@ public N5URL resolve(String relative) throws URISyntaxException { return resolve(new N5URL(relative)); } - public static String normalizeGroupPath(String path) { + public static String normalizePath(String path) { + + path = path == null ? "" : path; final char[] pathChars = path.toCharArray(); final List tokens = new ArrayList<>(); @@ -223,14 +223,15 @@ public static String normalizeGroupPath(String path) { } } final String lastToken = curToken.toString(); - if ( !lastToken.isEmpty() ) { - if (lastToken.equals( ".." )) { - tokens.remove( tokens.size() - 1 ); + if (!lastToken.isEmpty()) { + if (lastToken.equals("..")) { + tokens.remove(tokens.size() - 1); } else { - tokens.add( lastToken ); + tokens.add(lastToken); } } - if (tokens.isEmpty()) return ""; + if (tokens.isEmpty()) + return ""; String root = ""; if (tokens.get(0).equals("/")) { tokens.remove(0); @@ -238,11 +239,12 @@ public static String normalizeGroupPath(String path) { } return root + tokens.stream() .filter(it -> !it.equals(".")) - .filter(it ->!it.isEmpty()) - .reduce((l,r) -> l + "/" + r).orElse( "" ); + .filter(it -> !it.isEmpty()) + .reduce((l, r) -> l + "/" + r).orElse(""); } public static String normalizeAttributePath(String path) { + final char[] pathChars = path.toCharArray(); final List tokens = new ArrayList<>(); @@ -262,7 +264,7 @@ public static String normalizeAttributePath(String path) { if (character == '/' && tokens.isEmpty() && curToken.length() == 0) { /* If we are root, and the first token, then add the '/' */ curToken.append(character); - } else if (character == ']') { + } else if (character == ']') { /* If ']' add before terminating the token */ curToken.append(character); } @@ -289,14 +291,15 @@ public static String normalizeAttributePath(String path) { } } final String lastToken = curToken.toString(); - if ( !lastToken.isEmpty() ) { - if (lastToken.equals( ".." )) { - tokens.remove( tokens.size() - 1 ); + if (!lastToken.isEmpty()) { + if (lastToken.equals("..")) { + tokens.remove(tokens.size() - 1); } else { - tokens.add( lastToken ); + tokens.add(lastToken); } } - if (tokens.isEmpty()) return ""; + if (tokens.isEmpty()) + return ""; String root = ""; if (tokens.get(0).equals("/")) { tokens.remove(0); @@ -304,8 +307,8 @@ public static String normalizeAttributePath(String path) { } return root + tokens.stream() .filter(it -> !it.equals(".")) - .filter(it ->!it.isEmpty()) - .reduce((l,r) -> l + "/" + r).orElse( "" ); + .filter(it -> !it.isEmpty()) + .reduce((l, r) -> l + "/" + r).orElse(""); } public static URI encodeAsUri(String uri) throws URISyntaxException { @@ -327,7 +330,7 @@ public static URI encodeAsUri(String uri) throws URISyntaxException { } /* Edge case to handle when uriWithoutFragment is empty */ final URI _n5Uri; - if (uriWithoutFragment.length() == 0 && fragment != null && fragment.length() > 0 ) { + if (uriWithoutFragment.length() == 0 && fragment != null && fragment.length() > 0) { _n5Uri = new URI("N5Internal", "//STAND_IN", fragment); } else { _n5Uri = new URI("N5Internal", uriWithoutFragment, fragment); @@ -347,6 +350,7 @@ public static URI encodeAsUri(String uri) throws URISyntaxException { } public static N5URL from(String container, String group, String attribute) throws URISyntaxException { + final String containerPart = container != null ? container : ""; final String groupPart = group != null ? "?" + group : "?"; final String attributePart = attribute != null ? "#" + attribute : "#"; From 0fde153f9020dfdf2256160897e3e8a8e4c949e4 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Thu, 1 Dec 2022 15:19:46 -0500 Subject: [PATCH 048/243] fix: properly return when the N5Url is absolute --- src/main/java/org/janelia/saalfeldlab/n5/N5URL.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java b/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java index 00764576..4470c40d 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java @@ -98,7 +98,7 @@ public boolean isAbsolute() { final String path = uri.getPath(); if (!path.isEmpty()) { final char char0 = path.charAt(0); - final boolean isAbsolute = char0 == '/' || (path.length() >= 2 && path.charAt(1) == ':' && char0 >= 'A' && char0 <= 'Z'); + return char0 == '/' || (path.length() >= 2 && path.charAt(1) == ':' && char0 >= 'A' && char0 <= 'Z'); } return false; } From ffd5260814586683ce6dbfc035a13de45e48a59f Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Fri, 2 Dec 2022 14:48:02 -0500 Subject: [PATCH 049/243] refactor: Move ARRAY_INDEX --- .../java/org/janelia/saalfeldlab/n5/AbstractGsonReader.java | 5 +---- src/main/java/org/janelia/saalfeldlab/n5/N5URL.java | 1 + 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/AbstractGsonReader.java b/src/main/java/org/janelia/saalfeldlab/n5/AbstractGsonReader.java index 8b765af8..81cf1a6a 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/AbstractGsonReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/AbstractGsonReader.java @@ -41,7 +41,6 @@ import java.util.HashMap; import java.util.Map; import java.util.regex.Matcher; -import java.util.regex.Pattern; /** * Abstract base class implementing {@link N5Reader} with JSON attributes @@ -53,8 +52,6 @@ */ public abstract class AbstractGsonReader implements GsonAttributesParser, N5Reader { - private static final Pattern ARRAY_INDEX = Pattern.compile("\\[([0-9]+)]"); - protected final Gson gson; /** @@ -169,7 +166,7 @@ private T getAttribute( if (json instanceof JsonObject && json.getAsJsonObject().get(pathPart) != null) { json = json.getAsJsonObject().get(pathPart); } else { - final Matcher matcher = ARRAY_INDEX.matcher(pathPart); + final Matcher matcher = N5URL.ARRAY_INDEX.matcher(pathPart); if (json != null && json.isJsonArray() && matcher.matches()) { final int index = Integer.parseInt(matcher.group().replace("[", "").replace("]", "")); json = json.getAsJsonArray().get(index); diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java b/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java index 4470c40d..efcc1f2d 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java @@ -8,6 +8,7 @@ public class N5URL { + static final Pattern ARRAY_INDEX = Pattern.compile("\\[([0-9]+)]"); final URI uri; private final String scheme; private final String container; From d8f208ac43150713b605e11dad80f99e3a9f4878 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Fri, 2 Dec 2022 14:48:20 -0500 Subject: [PATCH 050/243] feat: allow getAttributes root to be JsonElement --- .../org/janelia/saalfeldlab/n5/GsonAttributesParser.java | 8 ++++---- src/main/java/org/janelia/saalfeldlab/n5/N5FSReader.java | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonAttributesParser.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonAttributesParser.java index 3f9b7550..d833bf1d 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonAttributesParser.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonAttributesParser.java @@ -63,10 +63,10 @@ public interface GsonAttributesParser extends N5Reader { * Reads the attributes a group or dataset. * * @param pathName group path - * @return the root {@link JsonObject} of the attributes + * @return the root {@link JsonElement} of the attributes * @throws IOException */ - public JsonObject getAttributesJson(final String pathName) throws IOException; + public JsonElement getAttributesJson(final String pathName) throws IOException; /** * Parses an attribute from the given attributes map. @@ -133,9 +133,9 @@ public static HashMap readAttributes(final Reader reader, f * @return the root {@link JsonObject} of the attributes * @throws IOException */ - public static JsonObject readAttributesJson(final Reader reader, final Gson gson) throws IOException { + public static JsonElement readAttributesJson(final Reader reader, final Gson gson) throws IOException { - final JsonObject json = gson.fromJson(reader, JsonObject.class); + final JsonElement json = gson.fromJson(reader, JsonElement.class); return json; } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5FSReader.java b/src/main/java/org/janelia/saalfeldlab/n5/N5FSReader.java index 0cc2e5a5..327904f9 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5FSReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5FSReader.java @@ -165,7 +165,7 @@ public HashMap getAttributes(final String pathName) throws } @Override - public JsonObject getAttributesJson(final String pathName) throws IOException { + public JsonElement getAttributesJson(final String pathName) throws IOException { final Path path = Paths.get(basePath, getAttributesPath(pathName).toString()); if (exists(pathName) && !Files.exists(path)) From e0f3c468c976140ed25a54e0d251019a9ffe812f Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Fri, 2 Dec 2022 14:50:18 -0500 Subject: [PATCH 051/243] feat: support set attribute by attribute path --- .../janelia/saalfeldlab/n5/N5FSWriter.java | 51 +++- .../org/janelia/saalfeldlab/n5/N5URL.java | 255 ++++++++++++++++++ 2 files changed, 304 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java b/src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java index c961bcb7..47484a1c 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java @@ -25,7 +25,12 @@ */ package org.janelia.saalfeldlab.n5; +import com.google.gson.Gson; +import com.google.gson.JsonElement; + import java.io.IOException; +import java.io.Writer; +import java.net.URISyntaxException; import java.nio.channels.Channels; import java.nio.charset.StandardCharsets; import java.nio.file.DirectoryNotEmptyException; @@ -38,6 +43,7 @@ import java.nio.file.attribute.FileAttribute; import java.util.Comparator; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.stream.Stream; @@ -86,7 +92,6 @@ public N5FSWriter(final String basePath, final GsonBuilder gsonBuilder) throws I * will be set to the current N5 version of this implementation. * * @param basePath n5 base path - * @param gsonBuilder * @throws IOException * if the base path cannot be written to or cannot be created, * if the N5 version of the container is not compatible with this @@ -104,6 +109,48 @@ public void createGroup(final String pathName) throws IOException { createDirectories(path); } + @Override public void setAttribute(String pathName, String key, T attribute) throws IOException { + + final N5URL attributeUrl; + try { + attributeUrl = N5URL.from(null, pathName, key); + } catch (URISyntaxException e) { + throw new IOException(e); + } + + final String groupPath = attributeUrl.normalizeGroupPath(); + final String attributePath = attributeUrl.getAttributePath(); + + final Path path = Paths.get(basePath, getAttributesPath(pathName).toString()); + createDirectories(path.getParent()); + + JsonElement attributesRoot = getAttributesJson(pathName); + try (final LockedFileChannel lockedFileChannel = LockedFileChannel.openForWriting(path)) { + JsonElement json = attributesRoot; + final Gson gson = getGson(); + final List attributePathTokens = attributeUrl.getAttributePathTokens(gson, attribute); + for (N5URL.N5UrlAttributePathToken token : attributePathTokens) { + if (token.canNavigate(json) && !(token.child instanceof N5URL.N5UrlAttributePathLeaf)) { + json = token.navigateJsonElement(json); + if (token.child instanceof N5URL.N5UrlAttributePathLeaf) { + token.writeLeaf(json); + } + } else { + json = token.createJsonElement(json); + } + if (attributesRoot == null) { + assert token.getParent() != null; + attributesRoot = token.getParent(); + } + } + + lockedFileChannel.getFileChannel().truncate(0); + final Writer writer = Channels.newWriter(lockedFileChannel.getFileChannel(), StandardCharsets.UTF_8.name()); + gson.toJson(attributesRoot, writer); + writer.flush(); + } + } + @Override public void setAttributes( final String pathName, @@ -199,7 +246,7 @@ public boolean deleteBlock( * Workaround for https://bugs.openjdk.java.net/browse/JDK-8130464 * * Creates a directory by creating all nonexistent parent directories first. - * Unlike the {@link #createDirectory createDirectory} method, an exception + * Unlike the {@link Files#createDirectory(Path, FileAttribute[])}} method, an exception * is not thrown if the directory could not be created because it already * exists. * diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java b/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java index efcc1f2d..2b265611 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java @@ -1,10 +1,18 @@ package org.janelia.saalfeldlab.n5; +import com.google.gson.Gson; +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonNull; +import com.google.gson.JsonObject; + import java.net.URI; import java.net.URISyntaxException; import java.util.ArrayList; import java.util.List; import java.util.Objects; +import java.util.regex.Matcher; +import java.util.regex.Pattern; public class N5URL { @@ -64,6 +72,54 @@ public String normalizeAttributePath() { return normalizeAttributePath(getAttributePath()); } + public ArrayList getAttributePathTokens(Gson gson) { + return getAttributePathTokens(gson, null); + } + public ArrayList getAttributePathTokens(Gson gson, T value) { + + final String[] attributePaths = normalizeAttributePath().replaceAll("^/", "").split("/"); + + if (attributePaths.length == 0) { + return new ArrayList<>(); + } else if ( attributePaths.length == 1 && attributePaths[0].isEmpty()) { + final N5UrlAttributePathLeaf leaf = new N5UrlAttributePathLeaf(gson, value); + final ArrayList tokens = new ArrayList<>(); + tokens.add(leaf); + return tokens; + } + final ArrayList n5UrlAttributePathTokens = new ArrayList<>(); + for (int i = 0; i < attributePaths.length; i++) { + final String pathToken = attributePaths[i]; + final Matcher matcher = ARRAY_INDEX.matcher(pathToken); + final N5UrlAttributePathToken newToken; + if (matcher.matches()) { + final int index = Integer.parseInt(matcher.group().replace("[", "").replace("]", "")); + newToken = new N5UrlAttributePathArray(gson, index); + n5UrlAttributePathTokens.add(newToken); + } else { + newToken = new N5UrlAttributePathObject(gson, pathToken); + n5UrlAttributePathTokens.add(newToken); + } + + if (i > 0) { + final N5UrlAttributePathToken parent = n5UrlAttributePathTokens.get(i - 1); + parent.setChild(newToken); + } + } + + if (value != null) { + final N5UrlAttributePathLeaf leaf = new N5UrlAttributePathLeaf(gson, value); + /* get last token, if present */ + if (n5UrlAttributePathTokens.size() >= 1) { + final N5UrlAttributePathToken lastToken = n5UrlAttributePathTokens.get(n5UrlAttributePathTokens.size() - 1); + lastToken.setChild(leaf); + } + n5UrlAttributePathTokens.add(leaf); + } + + return n5UrlAttributePathTokens; + } + private String getSchemePart() { return scheme == null ? "" : scheme + "://"; @@ -350,6 +406,205 @@ public static URI encodeAsUri(String uri) throws URISyntaxException { return n5Uri; } + public static N5UrlAttributePathToken getAttributePathToken(Gson gson, String attributePathToken) { + + final Matcher matcher = ARRAY_INDEX.matcher(attributePathToken); + if (matcher.matches()) { + final int index = Integer.parseInt(matcher.group().replace("[", "").replace("]", "")); + + return new N5UrlAttributePathArray(gson, index); + } else { + return new N5UrlAttributePathObject(gson, attributePathToken); + } + } + + public static abstract class N5UrlAttributePathToken { + + protected final Gson gson; + + protected JsonElement parent; + protected N5UrlAttributePathToken child; + + public N5UrlAttributePathToken(Gson gson) { + + this.gson = gson; + } + + public void setChild(N5UrlAttributePathToken child) { + + this.child = child; + } + + public abstract boolean canNavigate(JsonElement json); + + public abstract JsonElement navigateJsonElement(JsonElement json); + + public abstract JsonElement createJsonElement(JsonElement json); + + public abstract JsonElement writeLeaf(JsonElement json); + + public JsonElement getParent() { + + return parent; + } + } + + public static class N5UrlAttributePathObject extends N5UrlAttributePathToken { + + public final String key; + + public N5UrlAttributePathObject(Gson gson, String key) { + + super(gson); + this.key = key; + } + + @Override public boolean canNavigate(JsonElement json) { + if (json == null) { + return false; + } + return json.isJsonObject() && json.getAsJsonObject().has(key); + } + + @Override public JsonElement navigateJsonElement(JsonElement json) { + + return json.getAsJsonObject().get(key); + } + + @Override public JsonElement createJsonElement(JsonElement json) { + + final JsonObject object; + if (json == null) { + object = new JsonObject(); + } else { + object = json.getAsJsonObject(); + } + parent = object; + + final JsonElement newJsonElement; + if (child instanceof N5UrlAttributePathArray) { + newJsonElement = new JsonArray(); + object.add(key, newJsonElement); + } else if (child instanceof N5UrlAttributePathObject) { + newJsonElement = new JsonObject(); + object.add(key, newJsonElement); + } else { + newJsonElement = new JsonObject(); + writeLeaf(object); + } + return newJsonElement; + } + + @Override public JsonElement writeLeaf(JsonElement json) { + + final N5UrlAttributePathLeaf leaf = ((N5UrlAttributePathLeaf)child); + final JsonElement leafJson = leaf.createJsonElement(null); + json.getAsJsonObject().add(key, leafJson); + return leafJson; + } + } + + public static class N5UrlAttributePathArray extends N5UrlAttributePathToken { + + private final int index; + + public N5UrlAttributePathArray(Gson gson, int index) { + + super(gson); + this.index = index; + } + + @Override public boolean canNavigate(JsonElement json) { + if (json == null) { + return false; + } + + return json.isJsonArray() && json.getAsJsonArray().size() > index; + } + + @Override public JsonElement navigateJsonElement(JsonElement json) { + + return json.getAsJsonArray().get(index); + } + + @Override public JsonElement createJsonElement(JsonElement json) { + + final JsonArray array; + if (json == null) { + array = new JsonArray(); + } else { + array = json.getAsJsonArray(); + } + parent = array; + + for (int i = array.size(); i <= index; i++) { + array.add(JsonNull.INSTANCE); + } + final JsonElement newJsonElement; + if (child instanceof N5UrlAttributePathArray) { + newJsonElement = new JsonArray(); + array.set(index, newJsonElement); + } else if (child instanceof N5UrlAttributePathObject) { + newJsonElement = new JsonObject(); + array.set(index, newJsonElement); + } else { + newJsonElement = child.createJsonElement(null); + array.set(index, newJsonElement); + } + return newJsonElement; + } + + @Override public JsonElement writeLeaf(JsonElement json) { + + final N5UrlAttributePathLeaf leaf = ((N5UrlAttributePathLeaf)child); + final JsonElement leafJson = leaf.createJsonElement(null); + final JsonArray array = json.getAsJsonArray(); + + for (int i = array.size(); i <= index; i++) { + array.add(JsonNull.INSTANCE); + } + + array.set(index, leafJson); + return leafJson; + } + } + + public static class N5UrlAttributePathLeaf extends N5UrlAttributePathToken { + + private final T value; + + public N5UrlAttributePathLeaf(Gson gson, T value) { + + super(gson); + this.value = value; + } + + @Override public boolean canNavigate(JsonElement json) { + + return false; + } + + @Override public JsonElement navigateJsonElement(JsonElement json) { + + throw new UnsupportedOperationException("Cannot Navigate from a leaf "); + } + + @Override public JsonElement createJsonElement(JsonElement json) { + /* Leaf is weird. It is never added manually, also via the previous path token. + * HOWEVER, when there is no path token, and the leaf is the root, than it is called manually, + * and we wish to be our own parent (i.e. the root of the json tree). */ + parent = gson.toJsonTree(value); + return parent; + } + + @Override public JsonElement writeLeaf(JsonElement json) { + /* Practically, this method doesn't mean much when you are a leaf. */ + return createJsonElement(null); + } + + + } + public static N5URL from(String container, String group, String attribute) throws URISyntaxException { final String containerPart = container != null ? container : ""; From 84ae29f519395217f9c512a7459865662ad0752d Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Fri, 2 Dec 2022 14:50:29 -0500 Subject: [PATCH 052/243] fix: don't store temp files in user.home (#88) --- .../java/org/janelia/saalfeldlab/n5/N5Benchmark.java | 11 ++++++++++- .../java/org/janelia/saalfeldlab/n5/N5FSTest.java | 10 +++++++++- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/N5Benchmark.java b/src/test/java/org/janelia/saalfeldlab/n5/N5Benchmark.java index 512d82b4..9d12d9b3 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/N5Benchmark.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/N5Benchmark.java @@ -20,6 +20,7 @@ import java.io.File; import java.io.IOException; +import java.nio.file.Files; import java.util.ArrayList; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; @@ -53,7 +54,15 @@ */ public class N5Benchmark { - private static String testDirPath = System.getProperty("user.home") + "/tmp/n5-benchmark"; + private static String testDirPath; + + static { + try { + testDirPath = Files.createTempDirectory("n5-benchmark-").toFile().getCanonicalPath(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } private static String datasetName = "/dataset"; diff --git a/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java b/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java index 8fa240c3..7809def6 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java @@ -26,7 +26,15 @@ */ public class N5FSTest extends AbstractN5Test { - static private String testDirPath = System.getProperty("user.home") + "/tmp/n5-test"; + static private String testDirPath; + + static { + try { + testDirPath = Files.createTempDirectory("n5-test-").toFile().getCanonicalPath(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } /** * @throws IOException From 4c3ab8cb7699bd1846af008cf904c6f6d02f86f6 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Fri, 2 Dec 2022 14:50:36 -0500 Subject: [PATCH 053/243] test: set attribute by path --- .../org/janelia/saalfeldlab/n5/N5FSTest.java | 122 ++++++++++++++++++ .../saalfeldlab/n5/url/UrlAttributeTest.java | 2 +- 2 files changed, 123 insertions(+), 1 deletion(-) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java b/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java index 7809def6..888b104c 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java @@ -16,7 +16,16 @@ */ package org.janelia.saalfeldlab.n5; +import com.google.gson.reflect.TypeToken; +import org.janelia.saalfeldlab.n5.url.UrlAttributeTest; +import org.junit.Assert; +import org.junit.Test; + import java.io.IOException; +import java.net.URISyntaxException; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.function.Consumer; /** * Initiates testing of the filesystem-based N5 implementation. @@ -44,4 +53,117 @@ protected N5Writer createN5Writer() throws IOException { return new N5FSWriter(testDirPath); } + + + + private class TestData { + public String key; + public T value; + public Class cls; + + public TestData(String key, T value) { + + this.key = key; + this.value = value; + this.cls = (Class) value.getClass(); + } + } + + @Test + public void testAttributePaths() throws IOException { + String testGroup = "test"; + n5.createGroup(testGroup); + + final ArrayList> existingTests = new ArrayList<>(); + + Consumer> addAndTest = testData -> { + /* test a new value on existing path */ + try { + n5.setAttribute(testGroup, testData.key, testData.value); + Assert.assertEquals(testData.value, n5.getAttribute(testGroup, testData.key, testData.cls)); + Assert.assertEquals(testData.value, n5.getAttribute(testGroup, testData.key, TypeToken.get(testData.cls).getType())); + + /* previous values should still be there, but we remove first if the test we just added overwrites. */ + existingTests.removeIf(test -> { + try { + final String normalizedTestKey = N5URL.from(null, "", test.key).normalizeAttributePath().replaceAll("^/", ""); + final String normalizedTestDataKey = N5URL.from(null, "", testData.key).normalizeAttributePath().replaceAll("^/", ""); + return normalizedTestKey.equals(normalizedTestDataKey); + } catch (URISyntaxException e) { + throw new RuntimeException(e); + } + }); + for (TestData test : existingTests) { + Assert.assertEquals(test.value, n5.getAttribute(testGroup, test.key, test.cls)); + Assert.assertEquals(test.value, n5.getAttribute(testGroup, test.key, TypeToken.get(test.cls).getType())); + } + existingTests.add(testData); + } catch (IOException e) { + throw new RuntimeException(e); + } + }; + + /* Test a new value by path */ + addAndTest.accept(new TestData<>("/a/b/c/key1", "value1")); + /* test a new value on existing path */ + addAndTest.accept(new TestData<>("/a/b/key2", "value2")); + /* test replacing an existing value */ + addAndTest.accept(new TestData<>("/a/b/c/key1", "new_value1")); + + /* Test a new value with arrays */ + addAndTest.accept(new TestData<>("/array[0]/b/c/key1", "array_value1")); + /* test replacing an existing value */ + addAndTest.accept(new TestData<>("/array[0]/b/c/key1", "new_array_value1")); + /* test a new value on existing path with arrays */ + addAndTest.accept(new TestData<>("/array[0]/d[3]/key2", "array_value2")); + /* test a new value on existing path with nested arrays */ + addAndTest.accept(new TestData<>("/array[1][2]/[3]key2", "array2_value2")); + /* test with syntax variants */ + addAndTest.accept(new TestData<>("/array[1][2]/[3]key2", "array3_value3")); + addAndTest.accept(new TestData<>("array[1]/[2][3]/key2", "array3_value4")); + addAndTest.accept(new TestData<>("/array/[1]/[2]/[3]/key2", "array3_value5")); + + /* Non String tests */ + + addAndTest.accept(new TestData<>("/an/integer/test", 1)); + addAndTest.accept(new TestData<>("/a/double/test", 1.0)); + addAndTest.accept(new TestData<>("/a/float/test", 1.0F)); + addAndTest.accept(new TestData<>("/a/boolean/test", true)); + + + + final UrlAttributeTest.TestDoubles doubles1 = new UrlAttributeTest.TestDoubles( "doubles", "doubles1", new double[] { 5.7, 4.5, 3.4 } ); + final UrlAttributeTest.TestDoubles doubles2 = new UrlAttributeTest.TestDoubles( "doubles", "doubles2", new double[] { 5.8, 4.6, 3.5 } ); + final UrlAttributeTest.TestDoubles doubles3 = new UrlAttributeTest.TestDoubles( "doubles", "doubles3", new double[] { 5.9, 4.7, 3.6 } ); + final UrlAttributeTest.TestDoubles doubles4 = new UrlAttributeTest.TestDoubles( "doubles", "doubles4", new double[] { 5.10, 4.8, 3.7 } ); + addAndTest.accept(new TestData<>("/doubles[1]", doubles1)); + addAndTest.accept(new TestData<>("/doubles[2]", doubles2)); + addAndTest.accept(new TestData<>("/doubles[3]", doubles3)); + addAndTest.accept(new TestData<>("/doubles[4]", doubles4)); + + /* Test overwrite custom */ + addAndTest.accept(new TestData<>("/doubles[1]", doubles4)); + + n5.remove(testGroup); + } + + @Test + public void testRootLeaves() throws IOException { + //TODO: Currently, this fails if you try to overwrite an existing root; should we support that? + // In general, the current situation is such that you can only replace leaf nodes; + + final ArrayList> tests = new ArrayList<>(); + tests.add(new TestData<>("", "empty_root")); + tests.add(new TestData<>("/", "replace_empty_root")); + tests.add(new TestData<>("[0]", "array_root")); + + for (TestData testData : tests) { + final String canonicalPath = Files.createTempDirectory("root-leaf-test-").toFile().getCanonicalPath(); + try (final N5FSWriter writer = new N5FSWriter(canonicalPath)) { + writer.setAttribute(groupName, testData.key, testData.value); + Assert.assertEquals(testData.value, writer.getAttribute(groupName, testData.key, testData.cls)); + Assert.assertEquals(testData.value, writer.getAttribute(groupName, testData.key, TypeToken.get(testData.cls).getType())); + } + } + } } diff --git a/src/test/java/org/janelia/saalfeldlab/n5/url/UrlAttributeTest.java b/src/test/java/org/janelia/saalfeldlab/n5/url/UrlAttributeTest.java index 16694fde..c2f2708e 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/url/UrlAttributeTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/url/UrlAttributeTest.java @@ -247,7 +247,7 @@ public boolean equals( Object obj ) } } - private static class TestDoubles extends TestObject< double[] > + public static class TestDoubles extends TestObject< double[] > { public TestDoubles( String type, String name, double[] t ) { From f269285ee1b8e2f1dd1c8aeb79c4e15887c5cf95 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Mon, 5 Dec 2022 13:56:02 -0500 Subject: [PATCH 054/243] feat: support replacing incompatible structures based on the attribute path --- .../org/janelia/saalfeldlab/n5/N5URL.java | 118 ++++++++++++------ 1 file changed, 83 insertions(+), 35 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java b/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java index 2b265611..44fb0402 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java @@ -6,6 +6,7 @@ import com.google.gson.JsonNull; import com.google.gson.JsonObject; +import com.google.gson.JsonPrimitive; import java.net.URI; import java.net.URISyntaxException; import java.util.ArrayList; @@ -77,12 +78,17 @@ public ArrayList getAttributePathTokens(Gson gson) { } public ArrayList getAttributePathTokens(Gson gson, T value) { - final String[] attributePaths = normalizeAttributePath().replaceAll("^/", "").split("/"); + return N5URL.getAttributePathTokens(gson, normalizeAttributePath(), value); + } + + public static ArrayList getAttributePathTokens(Gson gson, String attributePath, T value) { + + final String[] attributePaths = attributePath.replaceAll("^/", "").split("/"); if (attributePaths.length == 0) { return new ArrayList<>(); } else if ( attributePaths.length == 1 && attributePaths[0].isEmpty()) { - final N5UrlAttributePathLeaf leaf = new N5UrlAttributePathLeaf(gson, value); + final N5UrlAttributePathLeaf leaf = new N5UrlAttributePathLeaf<>(gson, value); final ArrayList tokens = new ArrayList<>(); tokens.add(leaf); return tokens; @@ -104,19 +110,20 @@ public ArrayList getAttributePathTokens(Gson gson, if (i > 0) { final N5UrlAttributePathToken parent = n5UrlAttributePathTokens.get(i - 1); parent.setChild(newToken); + newToken.setParent(parent); } } if (value != null) { - final N5UrlAttributePathLeaf leaf = new N5UrlAttributePathLeaf(gson, value); + final N5UrlAttributePathLeaf leaf = new N5UrlAttributePathLeaf<>(gson, value); /* get last token, if present */ if (n5UrlAttributePathTokens.size() >= 1) { final N5UrlAttributePathToken lastToken = n5UrlAttributePathTokens.get(n5UrlAttributePathTokens.size() - 1); lastToken.setChild(leaf); + leaf.setParent(lastToken); } n5UrlAttributePathTokens.add(leaf); } - return n5UrlAttributePathTokens; } @@ -405,24 +412,13 @@ public static URI encodeAsUri(String uri) throws URISyntaxException { } return n5Uri; } - - public static N5UrlAttributePathToken getAttributePathToken(Gson gson, String attributePathToken) { - - final Matcher matcher = ARRAY_INDEX.matcher(attributePathToken); - if (matcher.matches()) { - final int index = Integer.parseInt(matcher.group().replace("[", "").replace("]", "")); - - return new N5UrlAttributePathArray(gson, index); - } else { - return new N5UrlAttributePathObject(gson, attributePathToken); - } - } - public static abstract class N5UrlAttributePathToken { protected final Gson gson; - protected JsonElement parent; + protected JsonElement root; + + protected N5UrlAttributePathToken parent; protected N5UrlAttributePathToken child; public N5UrlAttributePathToken(Gson gson) { @@ -435,18 +431,33 @@ public void setChild(N5UrlAttributePathToken child) { this.child = child; } + public void setParent(N5UrlAttributePathToken parent) { + + this.parent = parent; + } + + public JsonElement getRoot() { + + return this.root; + } + public abstract boolean canNavigate(JsonElement json); public abstract JsonElement navigateJsonElement(JsonElement json); + public JsonElement replaceJsonElement(JsonElement jsonParent) { + + parent.removeJsonElement(jsonParent); + return parent.createJsonElement(jsonParent); + } + public abstract JsonElement createJsonElement(JsonElement json); - public abstract JsonElement writeLeaf(JsonElement json); + public abstract void writeLeaf(JsonElement json); - public JsonElement getParent() { + protected abstract void removeJsonElement(JsonElement jsonParent); - return parent; - } + public abstract boolean jsonIncompatible(JsonElement json); } public static class N5UrlAttributePathObject extends N5UrlAttributePathToken { @@ -459,6 +470,11 @@ public N5UrlAttributePathObject(Gson gson, String key) { this.key = key; } + @Override public boolean jsonIncompatible(JsonElement json) { + + return json != null && !json.isJsonObject(); + } + @Override public boolean canNavigate(JsonElement json) { if (json == null) { return false; @@ -471,15 +487,18 @@ public N5UrlAttributePathObject(Gson gson, String key) { return json.getAsJsonObject().get(key); } - @Override public JsonElement createJsonElement(JsonElement json) { + @Override protected void removeJsonElement(JsonElement jsonParent) { + jsonParent.getAsJsonObject().remove(key); + } + @Override public JsonElement createJsonElement(JsonElement json) { final JsonObject object; if (json == null) { object = new JsonObject(); + root = object; } else { object = json.getAsJsonObject(); } - parent = object; final JsonElement newJsonElement; if (child instanceof N5UrlAttributePathArray) { @@ -495,17 +514,25 @@ public N5UrlAttributePathObject(Gson gson, String key) { return newJsonElement; } - @Override public JsonElement writeLeaf(JsonElement json) { + @Override public void writeLeaf(JsonElement json) { final N5UrlAttributePathLeaf leaf = ((N5UrlAttributePathLeaf)child); final JsonElement leafJson = leaf.createJsonElement(null); json.getAsJsonObject().add(key, leafJson); - return leafJson; } } public static class N5UrlAttributePathArray extends N5UrlAttributePathToken { + @Override public boolean jsonIncompatible(JsonElement json) { + + return json!= null && !json.isJsonArray(); + } + + @Override protected void removeJsonElement(JsonElement jsonParent) { + jsonParent.getAsJsonArray().set(index, JsonNull.INSTANCE); + } + private final int index; public N5UrlAttributePathArray(Gson gson, int index) { @@ -532,13 +559,22 @@ public N5UrlAttributePathArray(Gson gson, int index) { final JsonArray array; if (json == null) { array = new JsonArray(); + root = array; } else { array = json.getAsJsonArray(); } - parent = array; + + JsonElement fillArrayValue = JsonNull.INSTANCE; + + if (child instanceof N5UrlAttributePathLeaf) { + final N5UrlAttributePathLeaf< ? > leaf = ( N5UrlAttributePathLeaf< ? > ) child; + if (leaf.value instanceof Number) { + fillArrayValue = new JsonPrimitive( 0 ); + } + } for (int i = array.size(); i <= index; i++) { - array.add(JsonNull.INSTANCE); + array.add(fillArrayValue); } final JsonElement newJsonElement; if (child instanceof N5UrlAttributePathArray) { @@ -554,7 +590,7 @@ public N5UrlAttributePathArray(Gson gson, int index) { return newJsonElement; } - @Override public JsonElement writeLeaf(JsonElement json) { + @Override public void writeLeaf(JsonElement json) { final N5UrlAttributePathLeaf leaf = ((N5UrlAttributePathLeaf)child); final JsonElement leafJson = leaf.createJsonElement(null); @@ -565,7 +601,6 @@ public N5UrlAttributePathArray(Gson gson, int index) { } array.set(index, leafJson); - return leafJson; } } @@ -584,22 +619,35 @@ public N5UrlAttributePathLeaf(Gson gson, T value) { return false; } + @Override public boolean jsonIncompatible(JsonElement json) { + + return false; + } + @Override public JsonElement navigateJsonElement(JsonElement json) { throw new UnsupportedOperationException("Cannot Navigate from a leaf "); } + @Override protected void removeJsonElement(JsonElement jsonParent) { + throw new UnsupportedOperationException("Leaf Removal Unnecessary, leaves should alway be overwritable"); + + } + @Override public JsonElement createJsonElement(JsonElement json) { /* Leaf is weird. It is never added manually, also via the previous path token. * HOWEVER, when there is no path token, and the leaf is the root, than it is called manually, * and we wish to be our own parent (i.e. the root of the json tree). */ - parent = gson.toJsonTree(value); - return parent; + final JsonElement leaf = gson.toJsonTree(value); + if (json == null) { + root = leaf; + } + return leaf; } - @Override public JsonElement writeLeaf(JsonElement json) { - /* Practically, this method doesn't mean much when you are a leaf. */ - return createJsonElement(null); + @Override public void writeLeaf(JsonElement json) { + /* Practically, this method doesn't mean much when you are a leaf. + * It's primarily meant to have the parent write their children, if their child is a leaf. */ } From a4ef8663920978fe9a2797223124e0ec82c84ae5 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Mon, 5 Dec 2022 13:56:40 -0500 Subject: [PATCH 055/243] refactor: move setAttribute logic to GsonAttributesParser --- .../saalfeldlab/n5/AbstractGsonReader.java | 110 ++------------- .../saalfeldlab/n5/GsonAttributesParser.java | 127 ++++++++++++++++++ .../janelia/saalfeldlab/n5/N5FSWriter.java | 34 +---- 3 files changed, 138 insertions(+), 133 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/AbstractGsonReader.java b/src/main/java/org/janelia/saalfeldlab/n5/AbstractGsonReader.java index 81cf1a6a..9ee2a0d5 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/AbstractGsonReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/AbstractGsonReader.java @@ -34,7 +34,6 @@ import com.google.gson.reflect.TypeToken; import java.io.IOException; -import java.lang.reflect.Array; import java.lang.reflect.Type; import java.net.URISyntaxException; import java.util.Arrays; @@ -133,11 +132,7 @@ public T getAttribute( final String key, final Class clazz) throws IOException { - try { - return getAttribute(N5URL.from(null, pathName, key), clazz); - } catch (URISyntaxException e) { - throw new IOException(e); - } + return getAttribute(pathName, key, TypeToken.get(clazz).getType()); } @Override @@ -146,21 +141,18 @@ public T getAttribute( final String key, final Type type) throws IOException { + final String normalizedGroupPath; + final String normalizedAttributePath; try { - return getAttribute(N5URL.from(null, pathName, key), type); + final N5URL n5url = N5URL.from(null, pathName, key); + normalizedGroupPath = n5url.normalizeGroupPath(); + normalizedAttributePath = n5url.normalizeAttributePath(); } catch (URISyntaxException e) { throw new IOException(e); } - } - - private T getAttribute( - final N5URL url, - final Type type) throws IOException { - final Class clazz = (type instanceof Class) ? ((Class)type) : null; - final String attributePath = url.normalizeAttributePath(); - JsonElement json = getAttributesJson(url.normalizeGroupPath()); - for (final String pathPart : attributePath.split("/")) { + JsonElement json = getAttributesJson(normalizedGroupPath); + for (final String pathPart : normalizedAttributePath.split("/")) { if (pathPart.isEmpty()) continue; if (json instanceof JsonObject && json.getAsJsonObject().get(pathPart) != null) { @@ -181,7 +173,7 @@ private T getAttribute( } if (json instanceof JsonArray) { final JsonArray array = json.getAsJsonArray(); - T retArray = getJsonAsArray(array, type); + T retArray = GsonAttributesParser.getJsonAsArray(gson, array, type); if (retArray != null) return retArray; } @@ -192,88 +184,4 @@ private T getAttribute( return null; } } - - private T getAttribute( - final N5URL url, - final Class clazz) throws IOException { - - return getAttribute(url, (Type)clazz); - } - - private T getJsonAsArray(JsonArray array, Class clazz) { - - return getJsonAsArray(array, (Type)clazz); - } - - private T getJsonAsArray(JsonArray array, Type type) { - - final Class clazz = (type instanceof Class) ? ((Class)type) : null; - - if (type == boolean[].class) { - final boolean[] retArray = new boolean[array.size()]; - for (int i = 0; i < array.size(); i++) { - final Boolean value = gson.fromJson(array.get(i), boolean.class); - retArray[i] = value; - } - return (T)retArray; - } else if (type == double[].class) { - final double[] retArray = new double[array.size()]; - for (int i = 0; i < array.size(); i++) { - final double value = gson.fromJson(array.get(i), double.class); - retArray[i] = value; - } - return (T)retArray; - } else if (type == float[].class) { - final float[] retArray = new float[array.size()]; - for (int i = 0; i < array.size(); i++) { - final float value = gson.fromJson(array.get(i), float.class); - retArray[i] = value; - } - return (T)retArray; - } else if (type == long[].class) { - final long[] retArray = new long[array.size()]; - for (int i = 0; i < array.size(); i++) { - final long value = gson.fromJson(array.get(i), long.class); - retArray[i] = value; - } - return (T)retArray; - } else if (type == short[].class) { - final short[] retArray = new short[array.size()]; - for (int i = 0; i < array.size(); i++) { - final short value = gson.fromJson(array.get(i), short.class); - retArray[i] = value; - } - return (T)retArray; - } else if (type == int[].class) { - final int[] retArray = new int[array.size()]; - for (int i = 0; i < array.size(); i++) { - final int value = gson.fromJson(array.get(i), int.class); - retArray[i] = value; - } - return (T)retArray; - } else if (type == byte[].class) { - final byte[] retArray = new byte[array.size()]; - for (int i = 0; i < array.size(); i++) { - final byte value = gson.fromJson(array.get(i), byte.class); - retArray[i] = value; - } - return (T)retArray; - } else if (type == char[].class) { - final char[] retArray = new char[array.size()]; - for (int i = 0; i < array.size(); i++) { - final char value = gson.fromJson(array.get(i), char.class); - retArray[i] = value; - } - return (T)retArray; - } else if (clazz != null && clazz.isArray()) { - final Class componentCls = clazz.getComponentType(); - final Object[] clsArray = (Object[])Array.newInstance(componentCls, array.size()); - for (int i = 0; i < array.size(); i++) { - clsArray[i] = gson.fromJson(array.get(i), componentCls); - } - //noinspection unchecked - return (T)clsArray; - } - return null; - } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonAttributesParser.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonAttributesParser.java index d833bf1d..6390314d 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonAttributesParser.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonAttributesParser.java @@ -32,6 +32,7 @@ import java.lang.reflect.Array; import java.lang.reflect.Type; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Map.Entry; @@ -173,6 +174,132 @@ public static void writeAttributes( writer.flush(); } + public static void writeAttribute( + final Writer writer, + JsonElement root, + final String attributePath, + final T attribute, + final Gson gson) throws IOException { + + + JsonElement parent = null; + JsonElement json = root; + final List attributePathTokens = N5URL.getAttributePathTokens(gson, attributePath, attribute); + for (N5URL.N5UrlAttributePathToken token : attributePathTokens) { + /* The json is incompatible if it needs to be replaced; json == null is NOT incompatible. + * The two cases are if either the JsonElement is different than expected, or we are a leaf + * with no parent, in which case we always clear the existing root/json if any exists. */ + if (token.jsonIncompatible(json) || (token instanceof N5URL.N5UrlAttributePathLeaf && parent == null) ) { + if (parent == null) { + /* We are replacing the root, so just set the root to null, and continue */ + root = null; + json = null; + } else { + json = token.replaceJsonElement(parent); + } + } + + /* If we can dive into this jsonElement for the current token, do so, + * Otherwise, create the element */ + if (token.canNavigate(json) && !(token.child instanceof N5URL.N5UrlAttributePathLeaf)) { + parent = json; + json = token.navigateJsonElement(json); + if (token.child instanceof N5URL.N5UrlAttributePathLeaf) { + token.writeLeaf(json); + } + } else { + parent = json; + json = token.createJsonElement(json); + /* If the root is null, we need to set it based on the newly created element */ + if (root == null) { + if (token instanceof N5URL.N5UrlAttributePathLeaf ) { + /* if it's a leaf, we are the root*/ + root = json; + } else { + /* Otherwise, creating the element will have specified the root, and we can grab it now. */ + assert token.getRoot() != null; + root = token.getRoot(); + parent = root; + } + } + } + } + gson.toJson(root, writer); + writer.flush(); + } + + static T getJsonAsArray(Gson gson, JsonArray array, Type type) { + + final Class clazz = (type instanceof Class) ? ((Class)type) : null; + + if (type == boolean[].class) { + final boolean[] retArray = new boolean[array.size()]; + for (int i = 0; i < array.size(); i++) { + final Boolean value = gson.fromJson(array.get(i), boolean.class); + retArray[i] = value; + } + return (T)retArray; + } else if (type == double[].class) { + final double[] retArray = new double[array.size()]; + for (int i = 0; i < array.size(); i++) { + final double value = gson.fromJson(array.get(i), double.class); + retArray[i] = value; + } + return (T)retArray; + } else if (type == float[].class) { + final float[] retArray = new float[array.size()]; + for (int i = 0; i < array.size(); i++) { + final float value = gson.fromJson(array.get(i), float.class); + retArray[i] = value; + } + return (T)retArray; + } else if (type == long[].class) { + final long[] retArray = new long[array.size()]; + for (int i = 0; i < array.size(); i++) { + final long value = gson.fromJson(array.get(i), long.class); + retArray[i] = value; + } + return (T)retArray; + } else if (type == short[].class) { + final short[] retArray = new short[array.size()]; + for (int i = 0; i < array.size(); i++) { + final short value = gson.fromJson(array.get(i), short.class); + retArray[i] = value; + } + return (T)retArray; + } else if (type == int[].class) { + final int[] retArray = new int[array.size()]; + for (int i = 0; i < array.size(); i++) { + final int value = gson.fromJson(array.get(i), int.class); + retArray[i] = value; + } + return (T)retArray; + } else if (type == byte[].class) { + final byte[] retArray = new byte[array.size()]; + for (int i = 0; i < array.size(); i++) { + final byte value = gson.fromJson(array.get(i), byte.class); + retArray[i] = value; + } + return (T)retArray; + } else if (type == char[].class) { + final char[] retArray = new char[array.size()]; + for (int i = 0; i < array.size(); i++) { + final char value = gson.fromJson(array.get(i), char.class); + retArray[i] = value; + } + return (T)retArray; + } else if (clazz != null && clazz.isArray()) { + final Class componentCls = clazz.getComponentType(); + final Object[] clsArray = (Object[])Array.newInstance(componentCls, array.size()); + for (int i = 0; i < array.size(); i++) { + clsArray[i] = gson.fromJson(array.get(i), componentCls); + } + //noinspection unchecked + return (T)clsArray; + } + return null; + } + /** * Return a reasonable class for a {@link JsonPrimitive}. Possible return * types are diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java b/src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java index 47484a1c..04e02377 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java @@ -111,43 +111,13 @@ public void createGroup(final String pathName) throws IOException { @Override public void setAttribute(String pathName, String key, T attribute) throws IOException { - final N5URL attributeUrl; - try { - attributeUrl = N5URL.from(null, pathName, key); - } catch (URISyntaxException e) { - throw new IOException(e); - } - - final String groupPath = attributeUrl.normalizeGroupPath(); - final String attributePath = attributeUrl.getAttributePath(); - final Path path = Paths.get(basePath, getAttributesPath(pathName).toString()); createDirectories(path.getParent()); - JsonElement attributesRoot = getAttributesJson(pathName); try (final LockedFileChannel lockedFileChannel = LockedFileChannel.openForWriting(path)) { - JsonElement json = attributesRoot; - final Gson gson = getGson(); - final List attributePathTokens = attributeUrl.getAttributePathTokens(gson, attribute); - for (N5URL.N5UrlAttributePathToken token : attributePathTokens) { - if (token.canNavigate(json) && !(token.child instanceof N5URL.N5UrlAttributePathLeaf)) { - json = token.navigateJsonElement(json); - if (token.child instanceof N5URL.N5UrlAttributePathLeaf) { - token.writeLeaf(json); - } - } else { - json = token.createJsonElement(json); - } - if (attributesRoot == null) { - assert token.getParent() != null; - attributesRoot = token.getParent(); - } - } - + final String attributePath = N5URL.normalizeAttributePath(key); lockedFileChannel.getFileChannel().truncate(0); - final Writer writer = Channels.newWriter(lockedFileChannel.getFileChannel(), StandardCharsets.UTF_8.name()); - gson.toJson(attributesRoot, writer); - writer.flush(); + GsonAttributesParser.writeAttribute(Channels.newWriter(lockedFileChannel.getFileChannel(), StandardCharsets.UTF_8.name()), attributesRoot, attributePath, attribute, getGson()); } } From 8258d3ca3aee8bd99091361d0d8614f62c2d0a03 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Mon, 5 Dec 2022 13:57:07 -0500 Subject: [PATCH 056/243] refactor(test): move setAttribute test to AbstractN5Test --- .../org/janelia/saalfeldlab/n5/N5FSTest.java | 119 ++---------------- 1 file changed, 11 insertions(+), 108 deletions(-) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java b/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java index 888b104c..60ffb3b0 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java @@ -16,16 +16,12 @@ */ package org.janelia.saalfeldlab.n5; -import com.google.gson.reflect.TypeToken; import org.janelia.saalfeldlab.n5.url.UrlAttributeTest; -import org.junit.Assert; import org.junit.Test; import java.io.IOException; -import java.net.URISyntaxException; import java.nio.file.Files; import java.util.ArrayList; -import java.util.function.Consumer; /** * Initiates testing of the filesystem-based N5 implementation. @@ -54,116 +50,23 @@ protected N5Writer createN5Writer() throws IOException { return new N5FSWriter(testDirPath); } - - - private class TestData { - public String key; - public T value; - public Class cls; - - public TestData(String key, T value) { - - this.key = key; - this.value = value; - this.cls = (Class) value.getClass(); - } - } - @Test - public void testAttributePaths() throws IOException { - String testGroup = "test"; - n5.createGroup(testGroup); + public void customObjectTest() { + final String testGroup = "test"; final ArrayList> existingTests = new ArrayList<>(); - Consumer> addAndTest = testData -> { - /* test a new value on existing path */ - try { - n5.setAttribute(testGroup, testData.key, testData.value); - Assert.assertEquals(testData.value, n5.getAttribute(testGroup, testData.key, testData.cls)); - Assert.assertEquals(testData.value, n5.getAttribute(testGroup, testData.key, TypeToken.get(testData.cls).getType())); - - /* previous values should still be there, but we remove first if the test we just added overwrites. */ - existingTests.removeIf(test -> { - try { - final String normalizedTestKey = N5URL.from(null, "", test.key).normalizeAttributePath().replaceAll("^/", ""); - final String normalizedTestDataKey = N5URL.from(null, "", testData.key).normalizeAttributePath().replaceAll("^/", ""); - return normalizedTestKey.equals(normalizedTestDataKey); - } catch (URISyntaxException e) { - throw new RuntimeException(e); - } - }); - for (TestData test : existingTests) { - Assert.assertEquals(test.value, n5.getAttribute(testGroup, test.key, test.cls)); - Assert.assertEquals(test.value, n5.getAttribute(testGroup, test.key, TypeToken.get(test.cls).getType())); - } - existingTests.add(testData); - } catch (IOException e) { - throw new RuntimeException(e); - } - }; - - /* Test a new value by path */ - addAndTest.accept(new TestData<>("/a/b/c/key1", "value1")); - /* test a new value on existing path */ - addAndTest.accept(new TestData<>("/a/b/key2", "value2")); - /* test replacing an existing value */ - addAndTest.accept(new TestData<>("/a/b/c/key1", "new_value1")); - - /* Test a new value with arrays */ - addAndTest.accept(new TestData<>("/array[0]/b/c/key1", "array_value1")); - /* test replacing an existing value */ - addAndTest.accept(new TestData<>("/array[0]/b/c/key1", "new_array_value1")); - /* test a new value on existing path with arrays */ - addAndTest.accept(new TestData<>("/array[0]/d[3]/key2", "array_value2")); - /* test a new value on existing path with nested arrays */ - addAndTest.accept(new TestData<>("/array[1][2]/[3]key2", "array2_value2")); - /* test with syntax variants */ - addAndTest.accept(new TestData<>("/array[1][2]/[3]key2", "array3_value3")); - addAndTest.accept(new TestData<>("array[1]/[2][3]/key2", "array3_value4")); - addAndTest.accept(new TestData<>("/array/[1]/[2]/[3]/key2", "array3_value5")); - - /* Non String tests */ - - addAndTest.accept(new TestData<>("/an/integer/test", 1)); - addAndTest.accept(new TestData<>("/a/double/test", 1.0)); - addAndTest.accept(new TestData<>("/a/float/test", 1.0F)); - addAndTest.accept(new TestData<>("/a/boolean/test", true)); - - - - final UrlAttributeTest.TestDoubles doubles1 = new UrlAttributeTest.TestDoubles( "doubles", "doubles1", new double[] { 5.7, 4.5, 3.4 } ); - final UrlAttributeTest.TestDoubles doubles2 = new UrlAttributeTest.TestDoubles( "doubles", "doubles2", new double[] { 5.8, 4.6, 3.5 } ); - final UrlAttributeTest.TestDoubles doubles3 = new UrlAttributeTest.TestDoubles( "doubles", "doubles3", new double[] { 5.9, 4.7, 3.6 } ); - final UrlAttributeTest.TestDoubles doubles4 = new UrlAttributeTest.TestDoubles( "doubles", "doubles4", new double[] { 5.10, 4.8, 3.7 } ); - addAndTest.accept(new TestData<>("/doubles[1]", doubles1)); - addAndTest.accept(new TestData<>("/doubles[2]", doubles2)); - addAndTest.accept(new TestData<>("/doubles[3]", doubles3)); - addAndTest.accept(new TestData<>("/doubles[4]", doubles4)); + final UrlAttributeTest.TestDoubles doubles1 = new UrlAttributeTest.TestDoubles("doubles", "doubles1", new double[]{5.7, 4.5, 3.4}); + final UrlAttributeTest.TestDoubles doubles2 = new UrlAttributeTest.TestDoubles("doubles", "doubles2", new double[]{5.8, 4.6, 3.5}); + final UrlAttributeTest.TestDoubles doubles3 = new UrlAttributeTest.TestDoubles("doubles", "doubles3", new double[]{5.9, 4.7, 3.6}); + final UrlAttributeTest.TestDoubles doubles4 = new UrlAttributeTest.TestDoubles("doubles", "doubles4", new double[]{5.10, 4.8, 3.7}); + addAndTest(existingTests, new TestData<>(testGroup, "/doubles[1]", doubles1)); + addAndTest(existingTests, new TestData<>(testGroup, "/doubles[2]", doubles2)); + addAndTest(existingTests, new TestData<>(testGroup, "/doubles[3]", doubles3)); + addAndTest(existingTests, new TestData<>(testGroup, "/doubles[4]", doubles4)); /* Test overwrite custom */ - addAndTest.accept(new TestData<>("/doubles[1]", doubles4)); - - n5.remove(testGroup); + addAndTest(existingTests, new TestData<>(testGroup, "/doubles[1]", doubles4)); } - @Test - public void testRootLeaves() throws IOException { - //TODO: Currently, this fails if you try to overwrite an existing root; should we support that? - // In general, the current situation is such that you can only replace leaf nodes; - - final ArrayList> tests = new ArrayList<>(); - tests.add(new TestData<>("", "empty_root")); - tests.add(new TestData<>("/", "replace_empty_root")); - tests.add(new TestData<>("[0]", "array_root")); - - for (TestData testData : tests) { - final String canonicalPath = Files.createTempDirectory("root-leaf-test-").toFile().getCanonicalPath(); - try (final N5FSWriter writer = new N5FSWriter(canonicalPath)) { - writer.setAttribute(groupName, testData.key, testData.value); - Assert.assertEquals(testData.value, writer.getAttribute(groupName, testData.key, testData.cls)); - Assert.assertEquals(testData.value, writer.getAttribute(groupName, testData.key, TypeToken.get(testData.cls).getType())); - } - } - } } From fcc6c4766665c4f704b5b464f238353369d48fa1 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Mon, 5 Dec 2022 13:57:42 -0500 Subject: [PATCH 057/243] feat(test): add setAttribute tests for replacing intermediate members from an attribute path --- .../janelia/saalfeldlab/n5/N5FSWriter.java | 5 - .../saalfeldlab/n5/AbstractN5Test.java | 156 ++++++++++++++++++ 2 files changed, 156 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java b/src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java index 04e02377..46159342 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java @@ -25,12 +25,9 @@ */ package org.janelia.saalfeldlab.n5; -import com.google.gson.Gson; import com.google.gson.JsonElement; import java.io.IOException; -import java.io.Writer; -import java.net.URISyntaxException; import java.nio.channels.Channels; import java.nio.charset.StandardCharsets; import java.nio.file.DirectoryNotEmptyException; @@ -43,12 +40,10 @@ import java.nio.file.attribute.FileAttribute; import java.util.Comparator; import java.util.HashMap; -import java.util.List; import java.util.Map; import java.util.stream.Stream; import com.google.gson.GsonBuilder; -import com.google.gson.JsonElement; /** * Filesystem {@link N5Writer} implementation with version compatibility check. diff --git a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java index 572f1f7e..62f4233a 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java @@ -18,7 +18,11 @@ import static org.junit.Assert.fail; +import com.google.gson.JsonNull; +import java.io.File; import java.io.IOException; +import java.net.URISyntaxException; +import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; @@ -763,4 +767,156 @@ public void testDelete() throws IOException { protected boolean testDeleteIsBlockDeleted(final DataBlock dataBlock) { return dataBlock == null; } + + public class TestData { + + public String groupPath; + public String attributePath; + public T attributeValue; + public Class attributeClass; + + public TestData(String groupPath, String key, T attributeValue) { + + this.groupPath = groupPath; + this.attributePath = key; + this.attributeValue = attributeValue; + this.attributeClass = (Class)attributeValue.getClass(); + } + } + + protected void addAndTest(ArrayList> existingTests, TestData testData) { + /* test a new value on existing path */ + try { + n5.setAttribute(testData.groupPath, testData.attributePath, testData.attributeValue); + Assert.assertEquals(testData.attributeValue, n5.getAttribute(testData.groupPath, testData.attributePath, testData.attributeClass)); + Assert.assertEquals(testData.attributeValue, + n5.getAttribute(testData.groupPath, testData.attributePath, TypeToken.get(testData.attributeClass).getType())); + + /* previous values should still be there, but we remove first if the test we just added overwrites. */ + existingTests.removeIf(test -> { + try { + final String normalizedTestKey = N5URL.from(null, "", test.attributePath).normalizeAttributePath().replaceAll("^/", ""); + final String normalizedTestDataKey = N5URL.from(null, "", testData.attributePath).normalizeAttributePath().replaceAll("^/", ""); + return normalizedTestKey.equals(normalizedTestDataKey); + } catch (URISyntaxException e) { + throw new RuntimeException(e); + } + }); + for (TestData test : existingTests) { + Assert.assertEquals(test.attributeValue, n5.getAttribute(test.groupPath, test.attributePath, test.attributeClass)); + Assert.assertEquals(test.attributeValue, n5.getAttribute(test.groupPath, test.attributePath, TypeToken.get(test.attributeClass).getType())); + } + existingTests.add(testData); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + @Test + public void testAttributePaths() throws IOException { + + String testGroup = "test"; + n5.createGroup(testGroup); + + final ArrayList> existingTests = new ArrayList<>(); + + /* Test a new value by path */ + addAndTest(existingTests, new TestData<>(testGroup, "/a/b/c/key1", "value1")); + /* test a new value on existing path */ + addAndTest(existingTests, new TestData<>(testGroup, "/a/b/key2", "value2")); + /* test replacing an existing value */ + addAndTest(existingTests, new TestData<>(testGroup, "/a/b/c/key1", "new_value1")); + + /* Test a new value with arrays */ + addAndTest(existingTests, new TestData<>(testGroup, "/array[0]/b/c/key1", "array_value1")); + /* test replacing an existing value */ + addAndTest(existingTests, new TestData<>(testGroup, "/array[0]/b/c/key1", "new_array_value1")); + /* test a new value on existing path with arrays */ + addAndTest(existingTests, new TestData<>(testGroup, "/array[0]/d[3]/key2", "array_value2")); + /* test a new value on existing path with nested arrays */ + addAndTest(existingTests, new TestData<>(testGroup, "/array[1][2]/[3]key2", "array2_value2")); + /* test with syntax variants */ + addAndTest(existingTests, new TestData<>(testGroup, "/array[1][2]/[3]key2", "array3_value3")); + addAndTest(existingTests, new TestData<>(testGroup, "array[1]/[2][3]/key2", "array3_value4")); + addAndTest(existingTests, new TestData<>(testGroup, "/array/[1]/[2]/[3]/key2", "array3_value5")); + + /* Non String tests */ + + addAndTest(existingTests, new TestData<>(testGroup, "/an/integer/test", 1)); + addAndTest(existingTests, new TestData<>(testGroup, "/a/double/test", 1.0)); + addAndTest(existingTests, new TestData<>(testGroup, "/a/float/test", 1.0F)); + final TestData booleanTest = new TestData<>(testGroup, "/a/boolean/test", true); + addAndTest(existingTests, booleanTest); + + /* overwrite structure*/ + existingTests.remove(booleanTest); + addAndTest(existingTests, new TestData<>(testGroup, "/a/boolean[2]/test", true)); + + /* Fill an array with number */ + addAndTest(existingTests, new TestData<>(testGroup, "/filled/double_array[5]", 5.0)); + addAndTest(existingTests, new TestData<>(testGroup, "/filled/double_array[1]", 1.0)); + addAndTest(existingTests, new TestData<>(testGroup, "/filled/double_array[2]", 2.0)); + + addAndTest(existingTests, new TestData<>(testGroup, "/filled/double_array[4]", 4.0)); + addAndTest(existingTests, new TestData<>(testGroup, "/filled/double_array[0]", 0.0)); + + /* We intentionally skipped index 3, it should be `0` */ + Assert.assertEquals((Integer)0, n5.getAttribute(testGroup, "/filled/double_array[3]", Integer.class)); + + /* Fill an array with Object */ + addAndTest(existingTests, new TestData<>(testGroup, "/filled/string_array[5]", "f")); + addAndTest(existingTests, new TestData<>(testGroup, "/filled/string_array[1]", "b")); + addAndTest(existingTests, new TestData<>(testGroup, "/filled/string_array[2]", "c")); + + addAndTest(existingTests, new TestData<>(testGroup, "/filled/string_array[4]", "e")); + addAndTest(existingTests, new TestData<>(testGroup, "/filled/string_array[0]", "a")); + + /* We intentionally skipped index 3, it should be null */ + Assert.assertNull( n5.getAttribute( testGroup, "/filled/double_array[3]", JsonNull.class ) ); + + n5.remove(testGroup); + } + + @Test + public void testRootLeaves() throws IOException { + //TODO: Currently, this fails if you try to overwrite an existing root; should we support that? + // In general, the current situation is such that you can only replace leaf nodes; + + + /* Test with new root's each time */ + final ArrayList> tests = new ArrayList<>(); + tests.add(new TestData<>(groupName, "", "empty_root")); + tests.add(new TestData<>(groupName, "/", "replace_empty_root")); + tests.add(new TestData<>(groupName, "[0]", "array_root")); + + for (TestData testData : tests) { + final File tmpFile = Files.createTempDirectory( "root-leaf-test-" ).toFile(); + tmpFile.deleteOnExit(); + final String canonicalPath = tmpFile.getCanonicalPath(); + try (final N5FSWriter writer = new N5FSWriter(canonicalPath)) { + writer.setAttribute(testData.groupPath, testData.attributePath, testData.attributeValue); + Assert.assertEquals(testData.attributeValue, writer.getAttribute(testData.groupPath, testData.attributePath, testData.attributeClass)); + Assert.assertEquals(testData.attributeValue, + writer.getAttribute(testData.groupPath, testData.attributePath, TypeToken.get(testData.attributeClass).getType())); + } + } + + /* Test with replacing the root */ + tests.clear(); + tests.add(new TestData<>(groupName, "", "empty_root")); + tests.add(new TestData<>(groupName, "/", "replace_empty_root")); + tests.add(new TestData<>(groupName, "[0]", "array_root")); + + final File tmpFile = Files.createTempDirectory( "reuse-root-leaf-test-" ).toFile(); + tmpFile.deleteOnExit(); + final String canonicalPath = tmpFile.getCanonicalPath(); + try (final N5FSWriter writer = new N5FSWriter(canonicalPath)) { + for (TestData testData : tests) { + writer.setAttribute(testData.groupPath, testData.attributePath, testData.attributeValue); + Assert.assertEquals(testData.attributeValue, writer.getAttribute(testData.groupPath, testData.attributePath, testData.attributeClass)); + Assert.assertEquals(testData.attributeValue, + writer.getAttribute(testData.groupPath, testData.attributePath, TypeToken.get(testData.attributeClass).getType())); + } + } + } } From db0c4d68a98c7edc12abd01e206d910fd80ccfc5 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Wed, 7 Dec 2022 16:12:54 -0500 Subject: [PATCH 058/243] feat: expose key --- src/main/java/org/janelia/saalfeldlab/n5/N5URL.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java b/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java index 44fb0402..a7720bc6 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java @@ -470,6 +470,11 @@ public N5UrlAttributePathObject(Gson gson, String key) { this.key = key; } + public String getKey() + { + return key; + } + @Override public boolean jsonIncompatible(JsonElement json) { return json != null && !json.isJsonObject(); From fe9a659658388b5c565d3fbfe84f40dd4f7e4367 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Wed, 7 Dec 2022 16:14:38 -0500 Subject: [PATCH 059/243] refactor: separate write/insertAttribute, move readAttribute --- .../saalfeldlab/n5/AbstractGsonReader.java | 43 +------------ .../saalfeldlab/n5/GsonAttributesParser.java | 61 +++++++++++++++++-- 2 files changed, 59 insertions(+), 45 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/AbstractGsonReader.java b/src/main/java/org/janelia/saalfeldlab/n5/AbstractGsonReader.java index 9ee2a0d5..c8472b12 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/AbstractGsonReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/AbstractGsonReader.java @@ -27,10 +27,7 @@ import com.google.gson.Gson; import com.google.gson.GsonBuilder; -import com.google.gson.JsonArray; import com.google.gson.JsonElement; -import com.google.gson.JsonObject; -import com.google.gson.JsonSyntaxException; import com.google.gson.reflect.TypeToken; import java.io.IOException; @@ -38,8 +35,6 @@ import java.net.URISyntaxException; import java.util.Arrays; import java.util.HashMap; -import java.util.Map; -import java.util.regex.Matcher; /** * Abstract base class implementing {@link N5Reader} with JSON attributes @@ -140,48 +135,16 @@ public T getAttribute( final String pathName, final String key, final Type type) throws IOException { - final String normalizedGroupPath; final String normalizedAttributePath; try { - final N5URL n5url = N5URL.from(null, pathName, key); + final N5URL n5url = N5URL.from(null, pathName, key ); normalizedGroupPath = n5url.normalizeGroupPath(); normalizedAttributePath = n5url.normalizeAttributePath(); } catch (URISyntaxException e) { throw new IOException(e); } - final Class clazz = (type instanceof Class) ? ((Class)type) : null; - JsonElement json = getAttributesJson(normalizedGroupPath); - for (final String pathPart : normalizedAttributePath.split("/")) { - if (pathPart.isEmpty()) - continue; - if (json instanceof JsonObject && json.getAsJsonObject().get(pathPart) != null) { - json = json.getAsJsonObject().get(pathPart); - } else { - final Matcher matcher = N5URL.ARRAY_INDEX.matcher(pathPart); - if (json != null && json.isJsonArray() && matcher.matches()) { - final int index = Integer.parseInt(matcher.group().replace("[", "").replace("]", "")); - json = json.getAsJsonArray().get(index); - } - } - } - if (clazz != null && clazz.isAssignableFrom(HashMap.class)) { - Type mapType = new TypeToken>() {}.getType(); - Map retMap = gson.fromJson(json, mapType); - //noinspection unchecked - return (T)retMap; - } - if (json instanceof JsonArray) { - final JsonArray array = json.getAsJsonArray(); - T retArray = GsonAttributesParser.getJsonAsArray(gson, array, type); - if (retArray != null) - return retArray; - } - try { - return gson.fromJson(json, type); - } catch ( - JsonSyntaxException e) { - return null; - } + return GsonAttributesParser.readAttribute( getAttributesJson( normalizedGroupPath ), normalizedAttributePath, type, gson); } + } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonAttributesParser.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonAttributesParser.java index 6390314d..4d11063a 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonAttributesParser.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonAttributesParser.java @@ -26,6 +26,7 @@ package org.janelia.saalfeldlab.n5; import com.google.gson.JsonObject; +import com.google.gson.JsonSyntaxException; import java.io.IOException; import java.io.Reader; import java.io.Writer; @@ -41,6 +42,7 @@ import com.google.gson.JsonElement; import com.google.gson.JsonPrimitive; import com.google.gson.reflect.TypeToken; +import java.util.regex.Matcher; /** * {@link N5Reader} for JSON attributes parsed by {@link Gson}. @@ -140,6 +142,47 @@ public static JsonElement readAttributesJson(final Reader reader, final Gson gso return json; } + static < T > T readAttribute( final JsonElement root, final String normalizedAttributePath, final Class cls, final Gson gson ) { + + return readAttribute(root, normalizedAttributePath, TypeToken.get( cls ).getType(), gson ); + } + static < T > T readAttribute( final JsonElement root, final String normalizedAttributePath, final Type type, final Gson gson ) { + final Class clazz = ( type instanceof Class) ? ((Class) type ) : null; + JsonElement json = root; + for (final String pathPart : normalizedAttributePath.split("/")) { + if (pathPart.isEmpty()) + continue; + if (json instanceof JsonObject && json.getAsJsonObject().get(pathPart) != null) { + json = json.getAsJsonObject().get(pathPart); + } else { + final Matcher matcher = N5URL.ARRAY_INDEX.matcher(pathPart); + if (json != null && json.isJsonArray() && matcher.matches()) { + final int index = Integer.parseInt(matcher.group().replace("[", "").replace("]", "")); + json = json.getAsJsonArray().get(index); + } + } + } + if (clazz != null && clazz.isAssignableFrom(HashMap.class)) { + Type mapType = new TypeToken>() {}.getType(); + Map retMap = gson.fromJson(json, mapType); + //noinspection unchecked + return ( T ) retMap; + } + if (json instanceof JsonArray) { + final JsonArray array = json.getAsJsonArray(); + T retArray = GsonAttributesParser.getJsonAsArray(gson, array, type ); + if (retArray != null) + return retArray; + } + try { + return gson.fromJson( json, type ); + } catch ( JsonSyntaxException e) { + if (type == String.class) + return (T) gson.toJson(json); + return null; + } + } + /** * Inserts new the JSON export of attributes into the given attributes map. * @@ -181,14 +224,20 @@ public static void writeAttribute( final T attribute, final Gson gson) throws IOException { + root = insertAttribute(root, attributePath, attribute, gson); + gson.toJson(root, writer); + writer.flush(); + } + + public static JsonElement insertAttribute(JsonElement root, String attributePath, T attribute, Gson gson) { JsonElement parent = null; JsonElement json = root; final List attributePathTokens = N5URL.getAttributePathTokens(gson, attributePath, attribute); for (N5URL.N5UrlAttributePathToken token : attributePathTokens) { /* The json is incompatible if it needs to be replaced; json == null is NOT incompatible. - * The two cases are if either the JsonElement is different than expected, or we are a leaf - * with no parent, in which case we always clear the existing root/json if any exists. */ + * The two cases are if either the JsonElement is different than expected, or we are a leaf + * with no parent, in which case we always clear the existing root/json if any exists. */ if (token.jsonIncompatible(json) || (token instanceof N5URL.N5UrlAttributePathLeaf && parent == null) ) { if (parent == null) { /* We are replacing the root, so just set the root to null, and continue */ @@ -200,7 +249,7 @@ public static void writeAttribute( } /* If we can dive into this jsonElement for the current token, do so, - * Otherwise, create the element */ + * Otherwise, create the element */ if (token.canNavigate(json) && !(token.child instanceof N5URL.N5UrlAttributePathLeaf)) { parent = json; json = token.navigateJsonElement(json); @@ -224,10 +273,12 @@ public static void writeAttribute( } } } - gson.toJson(root, writer); - writer.flush(); + return root; } + static T getJsonAsArray(Gson gson, JsonArray array, Class cls) { + return getJsonAsArray( gson, array, TypeToken.get( cls ).getType() ); + } static T getJsonAsArray(Gson gson, JsonArray array, Type type) { final Class clazz = (type instanceof Class) ? ((Class)type) : null; From f6bc4605b6c3ff6a2aeba230afadc61912fe62cd Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Wed, 21 Dec 2022 10:48:46 -0500 Subject: [PATCH 060/243] refactor: clarify that the input should be normalized --- .../org/janelia/saalfeldlab/n5/GsonAttributesParser.java | 8 ++++---- src/main/java/org/janelia/saalfeldlab/n5/N5URL.java | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonAttributesParser.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonAttributesParser.java index 4d11063a..5a06377e 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonAttributesParser.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonAttributesParser.java @@ -220,20 +220,20 @@ public static void writeAttributes( public static void writeAttribute( final Writer writer, JsonElement root, - final String attributePath, + final String normalizedAttributePath, final T attribute, final Gson gson) throws IOException { - root = insertAttribute(root, attributePath, attribute, gson); + root = insertAttribute(root, normalizedAttributePath, attribute, gson); gson.toJson(root, writer); writer.flush(); } - public static JsonElement insertAttribute(JsonElement root, String attributePath, T attribute, Gson gson) { + public static JsonElement insertAttribute(JsonElement root, String normalizedAttributePath, T attribute, Gson gson) { JsonElement parent = null; JsonElement json = root; - final List attributePathTokens = N5URL.getAttributePathTokens(gson, attributePath, attribute); + final List attributePathTokens = N5URL.getAttributePathTokens(gson, normalizedAttributePath, attribute); for (N5URL.N5UrlAttributePathToken token : attributePathTokens) { /* The json is incompatible if it needs to be replaced; json == null is NOT incompatible. * The two cases are if either the JsonElement is different than expected, or we are a leaf diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java b/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java index a7720bc6..0f72d09f 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java @@ -81,9 +81,9 @@ public ArrayList getAttributePathTokens(Gson gson, return N5URL.getAttributePathTokens(gson, normalizeAttributePath(), value); } - public static ArrayList getAttributePathTokens(Gson gson, String attributePath, T value) { + public static ArrayList getAttributePathTokens(Gson gson, String normalizedAttributePath, T value) { - final String[] attributePaths = attributePath.replaceAll("^/", "").split("/"); + final String[] attributePaths = normalizedAttributePath.replaceAll("^/", "").split("/"); if (attributePaths.length == 0) { return new ArrayList<>(); From ab433d8f1e1f68d8ac5d28d21fe9cdd1ace7e702 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Wed, 21 Dec 2022 10:48:51 -0500 Subject: [PATCH 061/243] fix: handle JsonPrimitive values as leaf --- src/main/java/org/janelia/saalfeldlab/n5/N5URL.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java b/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java index 0f72d09f..d9cdc94f 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java @@ -573,7 +573,7 @@ public N5UrlAttributePathArray(Gson gson, int index) { if (child instanceof N5UrlAttributePathLeaf) { final N5UrlAttributePathLeaf< ? > leaf = ( N5UrlAttributePathLeaf< ? > ) child; - if (leaf.value instanceof Number) { + if (leaf.value instanceof Number || (leaf.value instanceof JsonPrimitive && ((JsonPrimitive)leaf.value).isNumber())) { fillArrayValue = new JsonPrimitive( 0 ); } } From fed78a9b67e948306354ba142bc7f6e9785c8c64 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Wed, 21 Dec 2022 10:48:57 -0500 Subject: [PATCH 062/243] fix: support when attributes.json doesn't contain JsonObject --- .../org/janelia/saalfeldlab/n5/GsonAttributesParser.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonAttributesParser.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonAttributesParser.java index 5a06377e..9c7c0656 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonAttributesParser.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonAttributesParser.java @@ -124,8 +124,11 @@ public static T parseAttribute( */ public static HashMap readAttributes(final Reader reader, final Gson gson) throws IOException { + /* Handle that case where the attributes.json file is valid json, but not a JsonObject, but returning an empty map. */ + final JsonElement attributes = readAttributesJson(reader, gson); + if (attributes == null || !attributes.isJsonObject()) return new HashMap<>(); final Type mapType = new TypeToken>(){}.getType(); - final HashMap map = gson.fromJson(reader, mapType); + final HashMap map = gson.fromJson(attributes, mapType); return map == null ? new HashMap<>() : map; } @@ -159,6 +162,8 @@ static < T > T readAttribute( final JsonElement root, final String normalizedAtt if (json != null && json.isJsonArray() && matcher.matches()) { final int index = Integer.parseInt(matcher.group().replace("[", "").replace("]", "")); json = json.getAsJsonArray().get(index); + } else { + return null; } } } From fbc1b0285d543b4e5d118c93f16dfc861a2b1759 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Wed, 21 Dec 2022 10:49:03 -0500 Subject: [PATCH 063/243] feat(test): add method to createN5Write from specified location --- .../janelia/saalfeldlab/n5/AbstractN5Test.java | 4 ++++ .../org/janelia/saalfeldlab/n5/N5FSTest.java | 18 +++++++++--------- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java index 62f4233a..3633e807 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java @@ -19,6 +19,7 @@ import static org.junit.Assert.fail; import com.google.gson.JsonNull; +import com.google.gson.JsonObject; import java.io.File; import java.io.IOException; import java.net.URISyntaxException; @@ -43,6 +44,9 @@ import com.google.gson.reflect.TypeToken; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; + /** * Abstract base class for testing N5 functionality. * Subclasses are expected to provide a specific N5 implementation to be tested by defining the {@link #createN5Writer()} method. diff --git a/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java b/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java index 60ffb3b0..c0cd8a81 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java @@ -31,23 +31,23 @@ */ public class N5FSTest extends AbstractN5Test { - static private String testDirPath; - - static { + /** + * @throws IOException + */ + @Override + protected N5Writer createN5Writer() throws IOException { + String testDirPath; try { testDirPath = Files.createTempDirectory("n5-test-").toFile().getCanonicalPath(); } catch (IOException e) { throw new RuntimeException(e); } + return createN5Writer(testDirPath); } - /** - * @throws IOException - */ @Override - protected N5Writer createN5Writer() throws IOException { - - return new N5FSWriter(testDirPath); + protected N5Writer createN5Writer(String location) throws IOException { + return new N5FSWriter(location); } @Test From 0759c3746c0907cedfae1736270d2e2631b173be Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Wed, 21 Dec 2022 10:54:52 -0500 Subject: [PATCH 064/243] feat(test): add method for createN5Writer with location --- src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java index 3633e807..7ca456c8 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java @@ -74,6 +74,8 @@ public abstract class AbstractN5Test { protected abstract N5Writer createN5Writer() throws IOException; + protected abstract N5Writer createN5Writer(String location) throws IOException; + protected Compression[] getCompressions() { return new Compression[] { From 201d595dc56652bd2e098297686554729bdb84ee Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Wed, 21 Dec 2022 10:55:17 -0500 Subject: [PATCH 065/243] feat: support writing/reading opaque keys via `getAttributes`/`setAttributes` --- .../janelia/saalfeldlab/n5/AbstractGsonReader.java | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/AbstractGsonReader.java b/src/main/java/org/janelia/saalfeldlab/n5/AbstractGsonReader.java index c8472b12..ea1f8da0 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/AbstractGsonReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/AbstractGsonReader.java @@ -28,6 +28,7 @@ import com.google.gson.Gson; import com.google.gson.GsonBuilder; import com.google.gson.JsonElement; +import com.google.gson.JsonSyntaxException; import com.google.gson.reflect.TypeToken; import java.io.IOException; @@ -135,6 +136,18 @@ public T getAttribute( final String pathName, final String key, final Type type) throws IOException { + /* Short Circuit if the key exists exactly as is */ + final HashMap attributes = getAttributes(pathName); + if (attributes.containsKey(key)) { + try { + return gson.fromJson(attributes.get(key), type); + } catch (JsonSyntaxException e) { + /* This can happen for example when you have an attribute object with the exact key, but ALSO + * have a structured json path that you want to query. In this case, it will fail here if the types are different + * but we may want it to continue to try below. */ + } + } + final String normalizedGroupPath; final String normalizedAttributePath; try { From ed1dc191ad4f8e34f7b06cdf09a5db5d25362514 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Wed, 21 Dec 2022 10:55:23 -0500 Subject: [PATCH 066/243] test: test adding attributes via opaque keys --- .../saalfeldlab/n5/AbstractN5Test.java | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java index 7ca456c8..3d230be9 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java @@ -880,6 +880,28 @@ public void testAttributePaths() throws IOException { /* We intentionally skipped index 3, it should be null */ Assert.assertNull( n5.getAttribute( testGroup, "/filled/double_array[3]", JsonNull.class ) ); + /* Lastly, we wish to ensure that adding via a map does NOT interpret the json path structure, but rather it adds the keys opaquely*/ + final HashMap testAttributes = new HashMap<>(); + testAttributes.put("/z/y/x", 10); + testAttributes.put("q/r/t", 11); + testAttributes.put("/l/m[10]/n", 12); + testAttributes.put("/", 13); + /* intentionally the same as above, but this time it should be added as an opaque key*/ + testAttributes.put("/a/b/key2", "value2"); + n5.setAttributes(testGroup, testAttributes); + + assertEquals((Integer)10, n5.getAttribute(testGroup, "/z/y/x", Integer.class)); + assertEquals((Integer)11, n5.getAttribute(testGroup, "q/r/t", Integer.class)); + assertEquals((Integer)12, n5.getAttribute(testGroup, "/l/m[10]/n", Integer.class)); + assertEquals((Integer)13, n5.getAttribute(testGroup, "/", Integer.class)); + /* We are passing a different type for the same key ("/"). + * This means it will try ot grab the exact match first, but then fail, and continuat on + * to try and grab the value as a json structure. I should grab the root, and match the empty string case */ + assertEquals(n5.getAttribute(testGroup, "", JsonObject.class), n5.getAttribute(testGroup, "/", JsonObject.class)); + + /* Lastly, ensure grabing nonsence results in an exception */ + assertNull(n5.getAttribute(testGroup, "/this/key/does/not/exist", Object.class)); + n5.remove(testGroup); } From e21fbad38d18f2506a39aead5b755df9244ab91a Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Wed, 21 Dec 2022 10:55:28 -0500 Subject: [PATCH 067/243] test: test replacement of existing root/structure of attributes --- .../saalfeldlab/n5/AbstractN5Test.java | 35 ++++++++++++++++--- 1 file changed, 30 insertions(+), 5 deletions(-) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java index 3d230be9..2e1479cc 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java @@ -918,10 +918,10 @@ public void testRootLeaves() throws IOException { tests.add(new TestData<>(groupName, "[0]", "array_root")); for (TestData testData : tests) { - final File tmpFile = Files.createTempDirectory( "root-leaf-test-" ).toFile(); + final File tmpFile = Files.createTempDirectory("root-leaf-test-").toFile(); tmpFile.deleteOnExit(); final String canonicalPath = tmpFile.getCanonicalPath(); - try (final N5FSWriter writer = new N5FSWriter(canonicalPath)) { + try (final N5Writer writer = createN5Writer(canonicalPath)) { writer.setAttribute(testData.groupPath, testData.attributePath, testData.attributeValue); Assert.assertEquals(testData.attributeValue, writer.getAttribute(testData.groupPath, testData.attributePath, testData.attributeClass)); Assert.assertEquals(testData.attributeValue, @@ -929,16 +929,16 @@ public void testRootLeaves() throws IOException { } } - /* Test with replacing the root */ + /* Test with replacing an existing root-leaf */ tests.clear(); tests.add(new TestData<>(groupName, "", "empty_root")); tests.add(new TestData<>(groupName, "/", "replace_empty_root")); tests.add(new TestData<>(groupName, "[0]", "array_root")); - final File tmpFile = Files.createTempDirectory( "reuse-root-leaf-test-" ).toFile(); + final File tmpFile = Files.createTempDirectory("reuse-root-leaf-test-").toFile(); tmpFile.deleteOnExit(); final String canonicalPath = tmpFile.getCanonicalPath(); - try (final N5FSWriter writer = new N5FSWriter(canonicalPath)) { + try (final N5Writer writer = createN5Writer(canonicalPath)) { for (TestData testData : tests) { writer.setAttribute(testData.groupPath, testData.attributePath, testData.attributeValue); Assert.assertEquals(testData.attributeValue, writer.getAttribute(testData.groupPath, testData.attributePath, testData.attributeClass)); @@ -946,5 +946,30 @@ public void testRootLeaves() throws IOException { writer.getAttribute(testData.groupPath, testData.attributePath, TypeToken.get(testData.attributeClass).getType())); } } + + /* Test with replacing an existing root non-leaf*/ + tests.clear(); + final TestData rootAsObject = new TestData<>(groupName, "/some/non/leaf[3]/structure", 100); + final TestData rootAsPrimitive = new TestData<>(groupName, "", 200); + final TestData rootAsArray = new TestData<>(groupName, "/", 300); + tests.add(rootAsPrimitive); + tests.add(rootAsArray); + final File tmpFile2 = Files.createTempDirectory("reuse-root-leaf-test-").toFile(); + tmpFile2.deleteOnExit(); + final String canonicalPath2 = tmpFile2.getCanonicalPath(); + try (final N5Writer writer = createN5Writer(canonicalPath2)) { + for (TestData test : tests) { + /* Set the root as Object*/ + writer.setAttribute(rootAsObject.groupPath, rootAsObject.attributePath, rootAsObject.attributeValue); + Assert.assertEquals(rootAsObject.attributeValue, writer.getAttribute(rootAsObject.groupPath, rootAsObject.attributePath, rootAsObject.attributeClass)); + + /* Override the root with something else */ + writer.setAttribute(test.groupPath, test.attributePath, test.attributeValue); + /* Verify original root is gone */ + Assert.assertNull(writer.getAttribute(rootAsObject.groupPath, rootAsObject.attributePath, rootAsObject.attributeClass)); + /* verify new root exists */ + Assert.assertEquals(test.attributeValue, writer.getAttribute(test.groupPath, test.attributePath, test.attributeClass)); + } + } } } From 0f6a29afdd37838db676d91b76ee468921790b2c Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Wed, 21 Dec 2022 10:55:43 -0500 Subject: [PATCH 068/243] test: add normalize identity check --- src/test/java/org/janelia/saalfeldlab/n5/N5URLTest.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/N5URLTest.java b/src/test/java/org/janelia/saalfeldlab/n5/N5URLTest.java index 1252f43d..c3bbb16f 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/N5URLTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/N5URLTest.java @@ -30,6 +30,9 @@ public void testAttributePath() { assertEquals("let's/try/a/real/case/with spaces", N5URL.normalizeAttributePath("let's/try/a/real/case/with spaces/")); assertEquals("let's/try/a/real/case/with spaces", N5URL.normalizeAttributePath("let's/try/a/real/////case////with spaces/")); assertThrows( IndexOutOfBoundsException.class, () -> N5URL.normalizeAttributePath("../first/relative/../not/allowed")); + + String normalizedPath = N5URL.normalizeAttributePath("let's/try/a/real/////case////with spaces/"); + assertEquals("Normalizing a normal path should be the identity", normalizedPath, N5URL.normalizeAttributePath(normalizedPath)); } @Test From 1f15bbb2ae45f230ab098abcef133e7d97e2fc5e Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Wed, 21 Dec 2022 16:32:26 -0500 Subject: [PATCH 069/243] fix: return root for group and attribute if they weren't specified --- src/main/java/org/janelia/saalfeldlab/n5/N5URL.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java b/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java index d9cdc94f..fef318d2 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java @@ -50,7 +50,7 @@ public String getContainerPath() { public String getGroupPath() { - return group; + return group != null ? group : "/"; } public String normalizeContainerPath() { @@ -65,7 +65,7 @@ public String normalizeGroupPath() { public String getAttributePath() { - return attribute; + return attribute != null ? attribute : "/"; } public String normalizeAttributePath() { From 254b46da3a0eae9e1e276981eb39688c512affd3 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Wed, 21 Dec 2022 16:33:14 -0500 Subject: [PATCH 070/243] fix: always absolute if scheme is specified --- src/main/java/org/janelia/saalfeldlab/n5/N5URL.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java b/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java index fef318d2..9b3b2b60 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java @@ -159,6 +159,7 @@ private String getSchemeSpecificPartWithoutQuery() { public boolean isAbsolute() { + if (scheme != null) return true; final String path = uri.getPath(); if (!path.isEmpty()) { final char char0 = path.charAt(0); From b5751240ef507817bf3bdca6db0a7138a88e1ec8 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Wed, 21 Dec 2022 16:33:54 -0500 Subject: [PATCH 071/243] refactor: rename parameters --- .../org/janelia/saalfeldlab/n5/N5URL.java | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java b/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java index 9b3b2b60..c259bbab 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java @@ -171,12 +171,12 @@ public boolean isAbsolute() { public N5URL resolve(N5URL relative) throws URISyntaxException { final URI thisUri = uri; - final URI relativeUri = relative.uri; + final URI relativeUri = relativeN5Url.uri; final StringBuilder newUri = new StringBuilder(); if (relativeUri.getScheme() != null) { - return relative; + return relativeN5Url; } final String thisScheme = thisUri.getScheme(); if (thisScheme != null) { @@ -187,8 +187,8 @@ public N5URL resolve(N5URL relative) throws URISyntaxException { newUri .append(relativeUri.getAuthority()) .append(relativeUri.getPath()) - .append(relative.getGroupPart()) - .append(relative.getAttributePart()); + .append(relativeN5Url.getGroupPart()) + .append(relativeN5Url.getAttributePart()); return new N5URL(newUri.toString()); } final String thisAuthority = thisUri.getAuthority(); @@ -198,13 +198,13 @@ public N5URL resolve(N5URL relative) throws URISyntaxException { final String path = relativeUri.getPath(); if (!path.isEmpty()) { - if (!relative.isAbsolute()) { + if (!relativeN5Url.isAbsolute()) { newUri.append(thisUri.getPath()).append('/'); } newUri .append(path) - .append(relative.getGroupPart()) - .append(relative.getAttributePart()); + .append(relativeN5Url.getGroupPart()) + .append(relativeN5Url.getAttributePart()); return new N5URL(newUri.toString()); } newUri.append(thisUri.getPath()); @@ -215,9 +215,9 @@ public N5URL resolve(N5URL relative) throws URISyntaxException { newUri.append(this.getGroupPart()).append('/'); newUri.append(relativeUri.getQuery()); } else { - newUri.append(relative.getGroupPart()); + newUri.append(relativeN5Url.getGroupPart()); } - newUri.append(relative.getAttributePart()); + newUri.append(relativeN5Url.getAttributePart()); return new N5URL(newUri.toString()); } newUri.append(this.getGroupPart()); @@ -227,7 +227,7 @@ public N5URL resolve(N5URL relative) throws URISyntaxException { if (fragment.charAt(0) != '/' && thisUri.getFragment() != null) { newUri.append(this.getAttributePart()).append('/'); } else { - newUri.append(relative.getAttributePart()); + newUri.append(relativeN5Url.getAttributePart()); } return new N5URL(newUri.toString()); @@ -376,7 +376,7 @@ public static String normalizeAttributePath(String path) { .reduce((l, r) -> l + "/" + r).orElse(""); } - public static URI encodeAsUri(String uri) throws URISyntaxException { + private static URI encodeAsUri(String uri) throws URISyntaxException { if (uri.trim().length() == 0) { return new URI(uri); From cf5c71b102b8c1b96bc186c2847a34b6b48b6e6c Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Wed, 21 Dec 2022 16:34:17 -0500 Subject: [PATCH 072/243] doc: add N5URL documentation --- .../org/janelia/saalfeldlab/n5/N5URL.java | 141 ++++++++++++++++-- 1 file changed, 131 insertions(+), 10 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java b/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java index c259bbab..270891ce 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java @@ -43,44 +43,86 @@ public N5URL(URI uri) { attribute = uri.getFragment(); } + /** + * @return the container path + */ public String getContainerPath() { return container; } + /** + * @return the group path, or root ("/") if none was provided + */ public String getGroupPath() { return group != null ? group : "/"; } + /** + * @return the normalized container path + */ public String normalizeContainerPath() { return normalizePath(getContainerPath()); } + /** + * @return the normalized group path + */ public String normalizeGroupPath() { return normalizePath(getGroupPath()); } + /** + * @return the attribute path, or root ("/") if none was provided + */ public String getAttributePath() { return attribute != null ? attribute : "/"; } + /** + * @return the normalized attribute path + */ public String normalizeAttributePath() { return normalizeAttributePath(getAttributePath()); } + /** + * + * Parse this {@link N5URL} as a list of {@link N5UrlAttributePathToken}. + * + * @see N5URL#getAttributePathTokens(Gson, String, Object) + */ public ArrayList getAttributePathTokens(Gson gson) { return getAttributePathTokens(gson, null); } + + /** + * + * Parse this {@link N5URL} as a list of {@link N5UrlAttributePathToken}. + * + * @see N5URL#getAttributePathTokens(Gson, String, Object) + */ public ArrayList getAttributePathTokens(Gson gson, T value) { return N5URL.getAttributePathTokens(gson, normalizeAttributePath(), value); } + /** + * Parses the {@link String normalizedAttributePath} to a list of {@link N5UrlAttributePathToken}. + * This is useful for traversing or constructing a json representation of the provided {@link String normalizedAttributePath}. + * Note that {@link String normalizedAttributePath} should be normalized prior to generating this list + * + * @param gson used to parse {@link T value} + * @param normalizedAttributePath to parse into {@link N5UrlAttributePathToken N5UrlAttributePathTokens} + * @param value to be stored in the last {@link N5UrlAttributePathToken} + * @param type of the value representing the last {@link N5UrlAttributePathToken} + * @return a list of {@link N5UrlAttributePathToken N5AttributePathTokens} + */ public static ArrayList getAttributePathTokens(Gson gson, String normalizedAttributePath, T value) { final String[] attributePaths = normalizedAttributePath.replaceAll("^/", "").split("/"); @@ -157,6 +199,12 @@ private String getSchemeSpecificPartWithoutQuery() { return uri.getSchemeSpecificPart().replace("?" + uri.getQuery(), ""); } + /** + * N5URL is always considered absolute if a scheme is provided. + * If no scheme is provided, the N5URL is absolute if it starts with either "/" or "[A-Z]:" + * + * @return if the path for this N5URL is absolute + */ public boolean isAbsolute() { if (scheme != null) return true; @@ -168,7 +216,15 @@ public boolean isAbsolute() { return false; } - public N5URL resolve(N5URL relative) throws URISyntaxException { + /** + * Generate a new N5URL which is the result of resolving {@link N5URL relativeN5Url} to this {@link N5URL}. + * If relativeN5Url is not relative to this N5URL, then the resulting N5URL is equivalent to relativeN5Url. + * + * @param relativeN5Url N5URL to resolve against ourselves + * @return the result of the resolution. + * @throws URISyntaxException + */ + public N5URL resolve(N5URL relativeN5Url) throws URISyntaxException { final URI thisUri = uri; final URI relativeUri = relativeN5Url.uri; @@ -237,16 +293,38 @@ public N5URL resolve(N5URL relative) throws URISyntaxException { return new N5URL(newUri.toString()); } - public N5URL resolve(URI relative) throws URISyntaxException { - - return resolve(new N5URL(relative)); + /** + * Generate a new N5URL which is the result of resolving {@link URI relativeUri} to this {@link N5URL}. + * If relativeUri is not relative to this N5URL, then the resulting N5URL is equivalent to relativeUri. + * + * @param relativeUri URI to resolve against ourselves + * @return the result of the resolution. + * @throws URISyntaxException + */ + public N5URL resolve(URI relativeUri) throws URISyntaxException { + + return resolve(new N5URL(relativeUri)); } - public N5URL resolve(String relative) throws URISyntaxException { - - return resolve(new N5URL(relative)); + /** + * Generate a new N5URL which is the result of resolving {@link String relativeString} to this {@link N5URL} + * If relativeString is not relative to this N5URL, then the resulting N5URL is equivalent to relativeString. + * + * @param relativeString String to resolve against ourselves + * @return the result of the resolution. + * @throws URISyntaxException + */ + public N5URL resolve(String relativeString) throws URISyntaxException { + + return resolve(new N5URL(relativeString)); } + /** + * Normalize a path, resulting in removal of redundant "/", "./", and resolution of relative "../". + * + * @param path to normalize + * @return the normalized path + */ public static String normalizePath(String path) { path = path == null ? "" : path; @@ -308,9 +386,43 @@ public static String normalizePath(String path) { .reduce((l, r) -> l + "/" + r).orElse(""); } - public static String normalizeAttributePath(String path) { - - final char[] pathChars = path.toCharArray(); + /** + * Normalize the {@link String attributePath}. + *

+ * Attribute paths have a few of special characters: + *

    + *
  • "." which represents the current element
    • + *
  • ".." which represent the previous elemnt
    • + *
  • "/" which is used to separate elements in the json tree
    • + *
  • [N] where N is an integer, refer to an index in the previous element in the tree; the previous element must be an array.
    • + * Note: [N] also separates the previous and following elements, regardless of whether it is preceded by "/" or not. + *
+ *

+ * When normalizing: + *

    + *
  • "/" are added before and after any indexing brackets [N]
    • + *
  • any redundant "/" are removed
    • + *
  • any relative ".." and "." are resolved
    • + *
+ * + * Examples of valid attribute paths, and their normalizations + *
    + *
  • /a/b/c -> /a/b/c
    • + *
  • /a/b/c/ -> /a/b/c
    • + *
  • ///a///b///c -> /a/b/c
    • + *
  • /a/././b/c -> /a/b/c
    • + *
  • /a/b[1]c -> /a/b/[1]/c
    • + *
  • /a/b/[1]c -> /a/b/[1]/c
    • + *
  • /a/b[1]/c -> /a/b/[1]/c
    • + *
  • /a/b[1]/c/.. -> /a/b/[1]
    • + *
+ * + * @param attributePath to normalize + * @return the normalized attribute path + */ + public static String normalizeAttributePath(String attributePath) { + + final char[] pathChars = attributePath.toCharArray(); final List tokens = new ArrayList<>(); StringBuilder curToken = new StringBuilder(); @@ -659,6 +771,15 @@ public N5UrlAttributePathLeaf(Gson gson, T value) { } + /** + * Generate an {@link N5URL} from a container, group, and attribute + * + * @param container of the N5Url + * @param group of the N5Url + * @param attribute of the N5Url + * @return the {@link N5URL} + * @throws URISyntaxException + */ public static N5URL from(String container, String group, String attribute) throws URISyntaxException { final String containerPart = container != null ? container : ""; From c69d850d091a246c713d7bf5c3365f345060b8f5 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Wed, 21 Dec 2022 16:34:30 -0500 Subject: [PATCH 073/243] test: add isAbsolute tests --- .../org/janelia/saalfeldlab/n5/N5URLTest.java | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/N5URLTest.java b/src/test/java/org/janelia/saalfeldlab/n5/N5URLTest.java index c3bbb16f..02a6612c 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/N5URLTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/N5URLTest.java @@ -5,7 +5,9 @@ import java.net.URISyntaxException; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; public class N5URLTest { @Test @@ -35,6 +37,21 @@ public void testAttributePath() { assertEquals("Normalizing a normal path should be the identity", normalizedPath, N5URL.normalizeAttributePath(normalizedPath)); } + @Test + public void testIsAbsolute() throws URISyntaxException { + /* Always true if scheme provided */ + assertTrue(new N5URL("file:///a/b/c").isAbsolute()); + assertTrue(new N5URL("file://C:\\\\a\\\\b\\\\c").isAbsolute()); + + /* Unix Paths*/ + assertTrue(new N5URL("/a/b/c").isAbsolute()); + assertFalse(new N5URL("a/b/c").isAbsolute()); + + /* Windows Paths*/ + assertTrue(new N5URL("C:\\\\a\\\\b\\\\c").isAbsolute()); + assertFalse(new N5URL("a\\\\b\\\\c").isAbsolute()); + } + @Test public void testGetRelative() throws URISyntaxException { From a81cfc4271ae1b9a9e6b562cfb09ea0107ec68ae Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Thu, 22 Dec 2022 14:41:18 -0500 Subject: [PATCH 074/243] fix: N5FSReader should fail if base path does not exist (#89) --- src/main/java/org/janelia/saalfeldlab/n5/N5FSReader.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5FSReader.java b/src/main/java/org/janelia/saalfeldlab/n5/N5FSReader.java index 327904f9..8ad58bf2 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5FSReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5FSReader.java @@ -28,6 +28,8 @@ import com.google.gson.JsonObject; import java.io.Closeable; import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; import java.nio.channels.Channels; import java.nio.channels.FileChannel; import java.nio.channels.OverlappingFileLockException; @@ -120,6 +122,8 @@ public N5FSReader(final String basePath, final GsonBuilder gsonBuilder) throws I final Version version = getVersion(); if (!VERSION.isCompatible(version)) throw new IOException("Incompatible version " + version + " (this is " + VERSION + ")."); + } else { + throw new IOException("Base Path for N5Reader does not exist: " + basePath); } } From fa0f1ccf307e31b5f11537edad406fd6b9da3b96 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Thu, 22 Dec 2022 14:43:03 -0500 Subject: [PATCH 075/243] feat: expose encodeAsUri --- src/main/java/org/janelia/saalfeldlab/n5/N5URL.java | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java b/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java index 270891ce..42ae5ad5 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java @@ -488,7 +488,16 @@ public static String normalizeAttributePath(String attributePath) { .reduce((l, r) -> l + "/" + r).orElse(""); } - private static URI encodeAsUri(String uri) throws URISyntaxException { + /** + * Encode the inpurt {@link String uri} so that illegal characters are properly escaped prior to generating the resulting {@link URI}. + * + * @param uri to encode + * + * @return the {@link URI} created from encoding the {@link String uri} + * + * @throws URISyntaxException if {@link String uri} is not valid + */ + public static URI encodeAsUri(String uri) throws URISyntaxException { if (uri.trim().length() == 0) { return new URI(uri); From c1abdfc0e4ecde84ee4590d95792a5b5278e164a Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Tue, 3 Jan 2023 13:48:47 -0500 Subject: [PATCH 076/243] feat: inline createDirectories now that non-existent path throws IOException during reader construction (#89) --- src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java b/src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java index 7fbf8c2e..38908a44 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java @@ -73,8 +73,7 @@ public class N5FSWriter extends N5FSReader implements N5Writer { */ public N5FSWriter(final String basePath, final GsonBuilder gsonBuilder) throws IOException { - super(basePath, gsonBuilder); - createDirectories(Paths.get(basePath)); + super(createDirectories(Paths.get(basePath)).toString(), gsonBuilder); if (!VERSION.equals(getVersion())) setAttribute("/", VERSION_KEY, VERSION.toString()); } From 023dfed35171f25bb1c5a7096b41fc69090f22f3 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Tue, 3 Jan 2023 17:09:13 -0500 Subject: [PATCH 077/243] fix(test): use new writer --- .../saalfeldlab/n5/AbstractN5Test.java | 68 +++++++++---------- 1 file changed, 31 insertions(+), 37 deletions(-) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java index caf31cf6..4005f9c9 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java @@ -505,35 +505,32 @@ public void testList() { } @Test - public void testDeepList() { - try { + public void testDeepList() throws IOException, ExecutionException, InterruptedException { - // clear container to start - for (final String g : n5.list("/")) - n5.remove(g); + try (N5Writer writer = createN5Writer()) { - n5.createGroup(groupName); + writer.createGroup(groupName); for (final String subGroup : subGroupNames) - n5.createGroup(groupName + "/" + subGroup); + writer.createGroup(groupName + "/" + subGroup); - final List groupsList = Arrays.asList(n5.deepList("/")); + final List groupsList = Arrays.asList(writer.deepList("/")); for (final String subGroup : subGroupNames) Assert.assertTrue("deepList contents", groupsList.contains(groupName.replaceFirst("/", "") + "/" + subGroup)); for (final String subGroup : subGroupNames) - Assert.assertTrue("deepList contents", Arrays.asList(n5.deepList("")).contains(groupName.replaceFirst("/", "") + "/" + subGroup)); + Assert.assertTrue("deepList contents", Arrays.asList(writer.deepList("")).contains(groupName.replaceFirst("/", "") + "/" + subGroup)); final DatasetAttributes datasetAttributes = new DatasetAttributes(dimensions, blockSize, DataType.UINT64, new RawCompression()); - final LongArrayDataBlock dataBlock = new LongArrayDataBlock( blockSize, new long[]{0,0,0}, new long[blockNumElements] ); - n5.createDataset(datasetName, datasetAttributes ); - n5.writeBlock(datasetName, datasetAttributes, dataBlock); + final LongArrayDataBlock dataBlock = new LongArrayDataBlock(blockSize, new long[]{0, 0, 0}, new long[blockNumElements]); + writer.createDataset(datasetName, datasetAttributes); + writer.writeBlock(datasetName, datasetAttributes, dataBlock); - final List datasetList = Arrays.asList(n5.deepList("/")); + final List datasetList = Arrays.asList(writer.deepList("/")); for (final String subGroup : subGroupNames) Assert.assertTrue("deepList contents", datasetList.contains(groupName.replaceFirst("/", "") + "/" + subGroup)); Assert.assertFalse("deepList stops at datasets", datasetList.contains(datasetName + "/0")); - final List datasetList2 = Arrays.asList(n5.deepList("")); + final List datasetList2 = Arrays.asList(writer.deepList("")); for (final String subGroup : subGroupNames) Assert.assertTrue("deepList contents", datasetList2.contains(groupName.replaceFirst("/", "") + "/" + subGroup)); Assert.assertTrue("deepList contents", datasetList2.contains(datasetName.replaceFirst("/", ""))); @@ -541,25 +538,25 @@ public void testDeepList() { final String prefix = "/test"; final String datasetSuffix = "group/dataset"; - final List datasetList3 = Arrays.asList(n5.deepList(prefix)); + final List datasetList3 = Arrays.asList(writer.deepList(prefix)); for (final String subGroup : subGroupNames) Assert.assertTrue("deepList contents", datasetList3.contains("group/" + subGroup)); Assert.assertTrue("deepList contents", datasetList3.contains(datasetName.replaceFirst(prefix + "/", ""))); // parallel deepList tests - final List datasetListP = Arrays.asList(n5.deepList("/", Executors.newFixedThreadPool(2))); + final List datasetListP = Arrays.asList(writer.deepList("/", Executors.newFixedThreadPool(2))); for (final String subGroup : subGroupNames) Assert.assertTrue("deepList contents", datasetListP.contains(groupName.replaceFirst("/", "") + "/" + subGroup)); Assert.assertTrue("deepList contents", datasetListP.contains(datasetName.replaceFirst("/", ""))); Assert.assertFalse("deepList stops at datasets", datasetListP.contains(datasetName + "/0")); - final List datasetListP2 = Arrays.asList(n5.deepList("", Executors.newFixedThreadPool(2))); + final List datasetListP2 = Arrays.asList(writer.deepList("", Executors.newFixedThreadPool(2))); for (final String subGroup : subGroupNames) Assert.assertTrue("deepList contents", datasetListP2.contains(groupName.replaceFirst("/", "") + "/" + subGroup)); Assert.assertTrue("deepList contents", datasetListP2.contains(datasetName.replaceFirst("/", ""))); Assert.assertFalse("deepList stops at datasets", datasetListP2.contains(datasetName + "/0")); - final List datasetListP3 = Arrays.asList(n5.deepList(prefix, Executors.newFixedThreadPool(2))); + final List datasetListP3 = Arrays.asList(writer.deepList(prefix, Executors.newFixedThreadPool(2))); for (final String subGroup : subGroupNames) Assert.assertTrue("deepList contents", datasetListP3.contains("group/" + subGroup)); Assert.assertTrue("deepList contents", datasetListP3.contains(datasetName.replaceFirst(prefix + "/", ""))); @@ -573,83 +570,80 @@ public void testDeepList() { return d.matches(".*/[bc]$"); }; - final List datasetListFilter1 = Arrays.asList(n5.deepList(prefix, isCalledDataset)); + final List datasetListFilter1 = Arrays.asList(writer.deepList(prefix, isCalledDataset)); Assert.assertTrue( "deepList filter \"dataset\"", datasetListFilter1.stream().map(x -> prefix + x).allMatch(isCalledDataset)); - final List datasetListFilter2 = Arrays.asList(n5.deepList(prefix, isBorC)); + final List datasetListFilter2 = Arrays.asList(writer.deepList(prefix, isBorC)); Assert.assertTrue( "deepList filter \"b or c\"", datasetListFilter2.stream().map(x -> prefix + x).allMatch(isBorC)); final List datasetListFilterP1 = - Arrays.asList(n5.deepList(prefix, isCalledDataset, Executors.newFixedThreadPool(2))); + Arrays.asList(writer.deepList(prefix, isCalledDataset, Executors.newFixedThreadPool(2))); Assert.assertTrue( "deepList filter \"dataset\"", datasetListFilterP1.stream().map(x -> prefix + x).allMatch(isCalledDataset)); final List datasetListFilterP2 = - Arrays.asList(n5.deepList(prefix, isBorC, Executors.newFixedThreadPool(2))); + Arrays.asList(writer.deepList(prefix, isBorC, Executors.newFixedThreadPool(2))); Assert.assertTrue( "deepList filter \"b or c\"", datasetListFilterP2.stream().map(x -> prefix + x).allMatch(isBorC)); // test dataset filtering - final List datasetListFilterD = Arrays.asList(n5.deepListDatasets(prefix)); + final List datasetListFilterD = Arrays.asList(writer.deepListDatasets(prefix)); Assert.assertTrue( "deepListDataset", datasetListFilterD.size() == 1 && (prefix + "/" + datasetListFilterD.get(0)).equals(datasetName)); Assert.assertArrayEquals( datasetListFilterD.toArray(), - n5.deepList( + writer.deepList( prefix, a -> { - try { return n5.datasetExists(a); } + try { return writer.datasetExists(a); } catch (final IOException e) { return false; } })); - final List datasetListFilterDandBC = Arrays.asList(n5.deepListDatasets(prefix, isBorC)); + final List datasetListFilterDandBC = Arrays.asList(writer.deepListDatasets(prefix, isBorC)); Assert.assertTrue("deepListDatasetFilter", datasetListFilterDandBC.size() == 0); Assert.assertArrayEquals( datasetListFilterDandBC.toArray(), - n5.deepList( + writer.deepList( prefix, a -> { - try { return n5.datasetExists(a) && isBorC.test(a); } + try { return writer.datasetExists(a) && isBorC.test(a); } catch (final IOException e) { return false; } })); final List datasetListFilterDP = - Arrays.asList(n5.deepListDatasets(prefix, Executors.newFixedThreadPool(2))); + Arrays.asList(writer.deepListDatasets(prefix, Executors.newFixedThreadPool(2))); Assert.assertTrue( "deepListDataset Parallel", datasetListFilterDP.size() == 1 && (prefix + "/" + datasetListFilterDP.get(0)).equals(datasetName)); Assert.assertArrayEquals( datasetListFilterDP.toArray(), - n5.deepList( + writer.deepList( prefix, a -> { - try { return n5.datasetExists(a); } + try { return writer.datasetExists(a); } catch (final IOException e) { return false; } }, Executors.newFixedThreadPool(2))); final List datasetListFilterDandBCP = - Arrays.asList(n5.deepListDatasets(prefix, isBorC, Executors.newFixedThreadPool(2))); + Arrays.asList(writer.deepListDatasets(prefix, isBorC, Executors.newFixedThreadPool(2))); Assert.assertTrue("deepListDatasetFilter Parallel", datasetListFilterDandBCP.size() == 0); Assert.assertArrayEquals( datasetListFilterDandBCP.toArray(), - n5.deepList( + writer.deepList( prefix, a -> { - try { return n5.datasetExists(a) && isBorC.test(a); } + try { return writer.datasetExists(a) && isBorC.test(a); } catch (final IOException e) { return false; } }, Executors.newFixedThreadPool(2))); - - } catch (final IOException | InterruptedException | ExecutionException e) { - fail(e.getMessage()); } } From 2d023a523aec8f8a85c2ea16010668a7c9300516 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Tue, 3 Jan 2023 17:09:21 -0500 Subject: [PATCH 078/243] fix(test): test for RawCompression inheritance (used by zarr) --- .../saalfeldlab/n5/AbstractN5Test.java | 28 ++++++++----------- 1 file changed, 11 insertions(+), 17 deletions(-) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java index 4005f9c9..f2db4928 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java @@ -144,27 +144,21 @@ public void testCreateGroup() { } @Test - public void testCreateDataset() { + public void testCreateDataset() throws IOException { - try { - n5.createDataset(datasetName, dimensions, blockSize, DataType.UINT64, new RawCompression()); - } catch (final IOException e) { - fail(e.getMessage()); - } + final DatasetAttributes info; + try (N5Writer writer = createN5Writer()) { + writer.createDataset(datasetName, dimensions, blockSize, DataType.UINT64, new RawCompression()); - if (!n5.exists(datasetName)) - fail("Dataset does not exist"); + if (!writer.exists(datasetName)) + fail("Dataset does not exist"); - try { - final DatasetAttributes info = n5.getDatasetAttributes(datasetName); - Assert.assertArrayEquals(dimensions, info.getDimensions()); - Assert.assertArrayEquals(blockSize, info.getBlockSize()); - Assert.assertEquals(DataType.UINT64, info.getDataType()); - Assert.assertEquals(RawCompression.class, info.getCompression().getClass()); - } catch (final IOException e) { - fail("Dataset info cannot be opened"); - e.printStackTrace(); + info = writer.getDatasetAttributes(datasetName); } + Assert.assertArrayEquals(dimensions, info.getDimensions()); + Assert.assertArrayEquals(blockSize, info.getBlockSize()); + Assert.assertEquals(DataType.UINT64, info.getDataType()); + Assert.assertTrue(info.getCompression() instanceof RawCompression); } @Test From 17c90b8167c8f2f85f65d4ed47188d8af651e15a Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Tue, 7 Feb 2023 14:42:56 -0500 Subject: [PATCH 079/243] fix(test): specify separate n5containers for Version and deeplist tests previously, reusing containers resulted in subsequent test failing --- .../janelia/saalfeldlab/n5/AbstractN5Test.java | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java index f2db4928..109230b9 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java @@ -501,7 +501,10 @@ public void testList() { @Test public void testDeepList() throws IOException, ExecutionException, InterruptedException { - try (N5Writer writer = createN5Writer()) { + final File tmpFile = Files.createTempDirectory("deeplist-test-").toFile(); + tmpFile.deleteOnExit(); + final String canonicalPath = tmpFile.getCanonicalPath(); + try (final N5Writer writer = createN5Writer(canonicalPath)) { writer.createGroup(groupName); for (final String subGroup : subGroupNames) @@ -722,9 +725,14 @@ public void testVersion() throws NumberFormatException, IOException { Assert.assertTrue(n5Version.equals(N5Reader.VERSION)); - n5.setAttribute("/", N5Reader.VERSION_KEY, new Version(N5Reader.VERSION.getMajor() + 1, N5Reader.VERSION.getMinor(), N5Reader.VERSION.getPatch()).toString()); - Assert.assertFalse(N5Reader.VERSION.isCompatible(n5.getVersion())); + final File tmpFile = Files.createTempDirectory("aws-version-test-").toFile(); + tmpFile.deleteOnExit(); + final String canonicalPath = tmpFile.getCanonicalPath(); + try (N5Writer writer = createN5Writer(canonicalPath)) { + writer.setAttribute("/", N5Reader.VERSION_KEY, new Version(N5Reader.VERSION.getMajor() + 1, N5Reader.VERSION.getMinor(), N5Reader.VERSION.getPatch()).toString()); + Assert.assertFalse(N5Reader.VERSION.isCompatible(writer.getVersion())); + } } @Test @@ -896,9 +904,6 @@ public void testAttributePaths() throws IOException { @Test public void testRootLeaves() throws IOException { - //TODO: Currently, this fails if you try to overwrite an existing root; should we support that? - // In general, the current situation is such that you can only replace leaf nodes; - /* Test with new root's each time */ final ArrayList> tests = new ArrayList<>(); From 5198fedaaab4600485b5ef7c7c185718c22b378e Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Wed, 8 Feb 2023 15:43:15 -0500 Subject: [PATCH 080/243] feat(test): add whitespace encoding and nested array tests --- .../saalfeldlab/n5/url/UrlAttributeTest.java | 19 +++++++++++++++---- .../url/urlAttributes.n5/attributes.json | 7 +++++++ 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/url/UrlAttributeTest.java b/src/test/java/org/janelia/saalfeldlab/n5/url/UrlAttributeTest.java index c2f2708e..8c458803 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/url/UrlAttributeTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/url/UrlAttributeTest.java @@ -10,7 +10,8 @@ import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; -import org.janelia.saalfeldlab.n5.N5FSReader; + +import org.janelia.saalfeldlab.n5.N5FSWriter; import org.janelia.saalfeldlab.n5.N5Reader; import org.janelia.saalfeldlab.n5.N5URL; import org.junit.Assert; @@ -42,7 +43,7 @@ public void before() { try { - n5 = new N5FSReader( "src/test/resources/url/urlAttributes.n5" ); + n5 = new N5FSWriter( "src/test/resources/url/urlAttributes.n5" ); rootContext = ""; list = new int[] { 0, 1, 2, 3 }; @@ -67,10 +68,11 @@ public void testRootAttributes() throws URISyntaxException, IOException { // get Map< String, Object > everything = getAttribute( n5, new N5URL( "" ), Map.class ); - assertEquals( "empty url", "2.5.1", ( String ) everything.get( "n5" ) ); + final String version = N5Reader.VERSION.toString(); + assertEquals( "empty url", version, ( String ) everything.get( "n5" ) ); Map< String, Object > everything2 = getAttribute( n5, new N5URL( "#/" ), Map.class ); - assertEquals( "root attribute", "2.5.1", ( String ) everything2.get( "n5" ) ); + assertEquals( "root attribute", version, ( String ) everything2.get( "n5" ) ); assertEquals( "url to attribute", "bar", getAttribute( n5, new N5URL( "#foo" ), String.class ) ); assertEquals( "url to attribute absolute", "bar", getAttribute( n5, new N5URL( "#/foo" ), String.class ) ); @@ -86,6 +88,9 @@ public void testRootAttributes() throws URISyntaxException, IOException assertEquals( "?/a/..#foo", "bar", getAttribute( n5, new N5URL( "?/a/..#foo" ), String.class ) ); assertEquals( "?/a/../.#foo", "bar", getAttribute( n5, new N5URL( "?/a/../.#foo" ), String.class ) ); + /* whitespace-encoding-necesary, fragment-only test*/ + assertEquals( "#f o o", "b a r", getAttribute( n5, new N5URL( "#f o o" ), String.class ) ); + Assert.assertArrayEquals( "url list", list, getAttribute( n5, new N5URL( "#list" ), int[].class ) ); // list @@ -142,6 +147,12 @@ public void testPathAttributes() throws URISyntaxException, IOException assertEquals( "name of aaa from aaa", aaa, getAttribute( n5, aaaUrl.resolve( "#name" ), String.class ) ); assertEquals( "name of aaa from aaa", aaa, getAttribute( n5, aaaUrl.resolve( "?./#name" ), String.class ) ); + + assertEquals( "nested list 1", (Integer)1, getAttribute( n5, new N5URL( "#nestedList[0][0][0]" ), Integer.class ) ); + assertEquals( "nested list 1", (Integer)1, getAttribute( n5, new N5URL( "#/nestedList/[0][0][0]" ), Integer.class ) ); + assertEquals( "nested list 1", (Integer)1, getAttribute( n5, new N5URL( "#nestedList//[0]/[0]///[0]" ), Integer.class ) ); + assertEquals( "nested list 1", (Integer)1, getAttribute( n5, new N5URL( "#/nestedList[0]//[0][0]" ), Integer.class ) ); + } private < T > T getAttribute( final N5Reader n5, final String url1, Class< T > clazz ) throws URISyntaxException, IOException diff --git a/src/test/resources/url/urlAttributes.n5/attributes.json b/src/test/resources/url/urlAttributes.n5/attributes.json index b8af56bc..74b95426 100644 --- a/src/test/resources/url/urlAttributes.n5/attributes.json +++ b/src/test/resources/url/urlAttributes.n5/attributes.json @@ -1,7 +1,14 @@ { "n5": "2.5.1", "foo": "bar", + "f o o": "b a r", "list": [ 0, 1, 2, 3 ], + "nestedList": [ + [ [ 1, 2, 3, 4 ] ], + [ [ 10, 20, 30, 40 ] ], + [ [ 100, 200, 300, 400 ] ], + [ [ 1000, 2000, 3000, 4000 ] ] + ], "object": { "a": "aa", "b": "bb" From f4c3d71b1f6f2d535803fb3dfeeb18c460aa5e71 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Wed, 8 Feb 2023 15:57:06 -0500 Subject: [PATCH 081/243] feat!,refactor: update how path tokens behave --- .../saalfeldlab/n5/GsonAttributesParser.java | 61 ++- .../n5/LinkedAttributePathToken.java | 257 +++++++++++++ .../org/janelia/saalfeldlab/n5/N5URL.java | 347 ++---------------- 3 files changed, 315 insertions(+), 350 deletions(-) create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/LinkedAttributePathToken.java diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonAttributesParser.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonAttributesParser.java index 9c7c0656..eab8ef58 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonAttributesParser.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonAttributesParser.java @@ -33,7 +33,6 @@ import java.lang.reflect.Array; import java.lang.reflect.Type; import java.util.HashMap; -import java.util.List; import java.util.Map; import java.util.Map.Entry; @@ -161,7 +160,11 @@ static < T > T readAttribute( final JsonElement root, final String normalizedAtt final Matcher matcher = N5URL.ARRAY_INDEX.matcher(pathPart); if (json != null && json.isJsonArray() && matcher.matches()) { final int index = Integer.parseInt(matcher.group().replace("[", "").replace("]", "")); - json = json.getAsJsonArray().get(index); + final JsonArray jsonArray = json.getAsJsonArray(); + if (index >= jsonArray.size()) { + return null; + } + json = jsonArray.get(index); } else { return null; } @@ -236,47 +239,25 @@ public static void writeAttribute( public static JsonElement insertAttribute(JsonElement root, String normalizedAttributePath, T attribute, Gson gson) { - JsonElement parent = null; + LinkedAttributePathToken pathToken = N5URL.getAttributePathTokens(normalizedAttributePath); + /* No path to traverse or build; just write the value */ + if (pathToken == null) + return gson.toJsonTree(attribute); + JsonElement json = root; - final List attributePathTokens = N5URL.getAttributePathTokens(gson, normalizedAttributePath, attribute); - for (N5URL.N5UrlAttributePathToken token : attributePathTokens) { - /* The json is incompatible if it needs to be replaced; json == null is NOT incompatible. - * The two cases are if either the JsonElement is different than expected, or we are a leaf - * with no parent, in which case we always clear the existing root/json if any exists. */ - if (token.jsonIncompatible(json) || (token instanceof N5URL.N5UrlAttributePathLeaf && parent == null) ) { - if (parent == null) { - /* We are replacing the root, so just set the root to null, and continue */ - root = null; - json = null; - } else { - json = token.replaceJsonElement(parent); - } - } + while (pathToken != null) { - /* If we can dive into this jsonElement for the current token, do so, - * Otherwise, create the element */ - if (token.canNavigate(json) && !(token.child instanceof N5URL.N5UrlAttributePathLeaf)) { - parent = json; - json = token.navigateJsonElement(json); - if (token.child instanceof N5URL.N5UrlAttributePathLeaf) { - token.writeLeaf(json); - } - } else { - parent = json; - json = token.createJsonElement(json); - /* If the root is null, we need to set it based on the newly created element */ - if (root == null) { - if (token instanceof N5URL.N5UrlAttributePathLeaf ) { - /* if it's a leaf, we are the root*/ - root = json; - } else { - /* Otherwise, creating the element will have specified the root, and we can grab it now. */ - assert token.getRoot() != null; - root = token.getRoot(); - parent = root; - } - } + JsonElement parent = pathToken.setAndCreateParentElement(json); + + /* We may need to create or override the existing root if it is non-existent or incompatible. */ + final boolean rootOverriden = json == root && parent != json; + if (root == null || rootOverriden) { + root = parent; } + + json = pathToken.writeChild(gson, attribute); + + pathToken = pathToken.next(); } return root; } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/LinkedAttributePathToken.java b/src/main/java/org/janelia/saalfeldlab/n5/LinkedAttributePathToken.java new file mode 100644 index 00000000..398cb681 --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/LinkedAttributePathToken.java @@ -0,0 +1,257 @@ +package org.janelia.saalfeldlab.n5; + +import com.google.gson.Gson; +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonNull; +import com.google.gson.JsonObject; +import com.google.gson.JsonPrimitive; + +import java.util.Iterator; + +public abstract class LinkedAttributePathToken implements Iterator> { + + protected T parentJson; + protected LinkedAttributePathToken childToken; + + public void setChildToken(LinkedAttributePathToken childToken) { + + this.childToken = childToken; + } + + public T getParentJson() { + + return parentJson; + } + + public abstract T getJsonType(); + + public abstract boolean canNavigate(JsonElement parent); + + public abstract JsonElement getOrCreateChildElement(); + + public JsonElement writeChild(Gson gson, Object value) { + + if (childToken != null) + /* If we have a child, get/create it and set current json element to it */ + return getOrCreateChildElement(); + else + /* We are done, no token remaining */ + return writeJsonElement(gson, value); + } + + public JsonElement writeJsonElement(Gson gson, Object value) { + + if (hasNext()) { + writeChildElement(); + } else { + writeValue(gson, value); + } + return getChildElement(); + } + + /** + * Check if the provided {@code json} is compatible with this token. + *

+ * Compatibility means thath the provided {@code JsonElement} does not + * explicitly conflict with this token. That means that {@code null} is + * always compatible. The only real incompatibility is when the {@code JsonElement} + * that we receive is different that we expect. + * + * @param json element that we are testing for compatibility. + * @return false if {@code json} is not null and is not of type {@code T} + */ + @SuppressWarnings("BooleanMethodIsAlwaysInverted") + public boolean jsonCompatible(JsonElement json) { + + return json == null || json.getClass() == getJsonType().getClass(); + } + + protected abstract void writeValue(Gson gson, Object value); + + protected abstract void writeChildElement(); + + protected abstract JsonElement getChildElement(); + + protected JsonElement setAndCreateParentElement(JsonElement json) { + if (json == null || !jsonCompatible(json)) { + //noinspection unchecked + parentJson = (T) getJsonType().deepCopy(); + } else { + //noinspection unchecked + parentJson = (T) json; + } + return parentJson; + } + + @Override public boolean hasNext() { + + return childToken != null; + } + + @Override public LinkedAttributePathToken next() { + + return childToken; + } + + public static class ObjectAttributeToken extends LinkedAttributePathToken { + + private static final JsonObject JSON_OBJECT = new JsonObject(); + + public final String key; + + public ObjectAttributeToken(String key) { + + this.key = key; + } + + public String getKey() { + + return key; + } + + @Override public String toString() { + + return getKey(); + } + + @Override public JsonObject getJsonType() { + + return JSON_OBJECT; + } + + @Override public boolean canNavigate(JsonElement json) { + + return json != null && json.isJsonObject() && json.getAsJsonObject().has(key); + } + + @Override public JsonElement getOrCreateChildElement() { + + final JsonElement childElement = parentJson.getAsJsonObject().get(key); + if (!parentJson.has(key) || childToken != null && !childToken.jsonCompatible(childElement)) { + writeChildElement(); + } + return getChildElement(); + } + + @Override protected JsonElement getChildElement() { + + return parentJson.get(key); + } + + @Override protected void writeValue(Gson gson, Object value) { + + parentJson.add(key, gson.toJsonTree(value)); + } + + @Override protected void writeChildElement() { + + if (childToken != null) + parentJson.add(key, childToken.getJsonType().deepCopy()); + + } + + } + + public static class ArrayAttributeToken extends LinkedAttributePathToken { + + private static final JsonArray JSON_ARRAY = new JsonArray(); + + private final int index; + + public ArrayAttributeToken(int index) { + + this.index = index; + } + + public int getIndex() { + + return index; + } + + @Override public String toString() { + + return "[" + getIndex() + "]"; + } + + @Override public JsonArray getJsonType() { + + return JSON_ARRAY; + } + + @Override public boolean canNavigate(JsonElement parent) { + + return parent != null && parent.isJsonArray() && parent.getAsJsonArray().size() > index; + } + + @Override public JsonElement getOrCreateChildElement() { + + /* Two cases which required writing the child: + * - The child element doesn't exist (in this case, there is nothing at the specified index). + * - The child element is incompatible with the child token (replaces the existing child element). */ + if (index >= parentJson.size() || childToken != null && !childToken.jsonCompatible(parentJson.get(index))) + writeChildElement(); + + return parentJson.get(index); + } + + @Override protected void writeChildElement() { + + if (childToken != null) { + fillArrayToIndex(parentJson, index, null); + parentJson.set(index, childToken.getJsonType().deepCopy()); + } + } + + @Override protected JsonElement getChildElement() { + + return parentJson.get(index); + } + + @Override protected void writeValue(Gson gson, Object value) { + + fillArrayToIndex(parentJson, index, value); + parentJson.set(index, gson.toJsonTree(value)); + } + + /** + * Fill {@code array} up to, and including, {@code index} with a default value, determined by the type of {@code value}. + * Importantly, this does NOT set {@code array[index]=value}. + * + * @param array to fill + * @param index to fill the array to (inclusive) + * @param value used to determine the default array fill value + */ + private static void fillArrayToIndex(JsonArray array, int index, Object value) { + + final JsonElement fillValue; + if (valueRepresentsANumber(value)) { + fillValue = new JsonPrimitive(0); + } else { + fillValue = JsonNull.INSTANCE; + } + + for (int i = array.size(); i <= index; i++) { + array.add(fillValue); + } + } + + /** + * Check if {@value} represents a {@link Number}. + * True if {@code value} either: + *

    + *
  • is a {@code Number}, or;
    • + *
  • is a {@link JsonPrimitive} where {@link JsonPrimitive#isNumber()}
    • + *
+ * + * + * @param value we wish to check if represents a {@link Number} or not. + * @return true if {@code value} represents a {@link Number} + */ + private static boolean valueRepresentsANumber(Object value) { + + return value instanceof Number || (value instanceof JsonPrimitive && ((JsonPrimitive)value).isNumber()); + } + } +} + diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java b/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java index 42ae5ad5..6644a3be 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java @@ -1,17 +1,13 @@ package org.janelia.saalfeldlab.n5; -import com.google.gson.Gson; -import com.google.gson.JsonArray; -import com.google.gson.JsonElement; -import com.google.gson.JsonNull; -import com.google.gson.JsonObject; - -import com.google.gson.JsonPrimitive; import java.net.URI; import java.net.URISyntaxException; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.Objects; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Consumer; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -93,80 +89,55 @@ public String normalizeAttributePath() { /** * - * Parse this {@link N5URL} as a list of {@link N5UrlAttributePathToken}. + * Parse this {@link N5URL} as a {@link LinkedAttributePathToken}. * - * @see N5URL#getAttributePathTokens(Gson, String, Object) + * @see N5URL#getAttributePathTokens(String) */ - public ArrayList getAttributePathTokens(Gson gson) { - return getAttributePathTokens(gson, null); + public LinkedAttributePathToken getAttributePathTokens() { + return getAttributePathTokens(normalizeAttributePath()); } /** - * - * Parse this {@link N5URL} as a list of {@link N5UrlAttributePathToken}. - * - * @see N5URL#getAttributePathTokens(Gson, String, Object) - */ - public ArrayList getAttributePathTokens(Gson gson, T value) { - - return N5URL.getAttributePathTokens(gson, normalizeAttributePath(), value); - } - - /** - * Parses the {@link String normalizedAttributePath} to a list of {@link N5UrlAttributePathToken}. + * Parses the {@link String normalizedAttributePath} to a list of {@link LinkedAttributePathToken}. * This is useful for traversing or constructing a json representation of the provided {@link String normalizedAttributePath}. * Note that {@link String normalizedAttributePath} should be normalized prior to generating this list * - * @param gson used to parse {@link T value} - * @param normalizedAttributePath to parse into {@link N5UrlAttributePathToken N5UrlAttributePathTokens} - * @param value to be stored in the last {@link N5UrlAttributePathToken} - * @param type of the value representing the last {@link N5UrlAttributePathToken} - * @return a list of {@link N5UrlAttributePathToken N5AttributePathTokens} + * @param normalizedAttributePath to parse into {@link LinkedAttributePathToken}s + * @return the head of the {@link LinkedAttributePathToken}s */ - public static ArrayList getAttributePathTokens(Gson gson, String normalizedAttributePath, T value) { + public static LinkedAttributePathToken getAttributePathTokens(String normalizedAttributePath) { - final String[] attributePaths = normalizedAttributePath.replaceAll("^/", "").split("/"); + final String[] attributePathParts = normalizedAttributePath.replaceAll("^/", "").split("/"); - if (attributePaths.length == 0) { - return new ArrayList<>(); - } else if ( attributePaths.length == 1 && attributePaths[0].isEmpty()) { - final N5UrlAttributePathLeaf leaf = new N5UrlAttributePathLeaf<>(gson, value); - final ArrayList tokens = new ArrayList<>(); - tokens.add(leaf); - return tokens; - } - final ArrayList n5UrlAttributePathTokens = new ArrayList<>(); - for (int i = 0; i < attributePaths.length; i++) { - final String pathToken = attributePaths[i]; - final Matcher matcher = ARRAY_INDEX.matcher(pathToken); - final N5UrlAttributePathToken newToken; - if (matcher.matches()) { - final int index = Integer.parseInt(matcher.group().replace("[", "").replace("]", "")); - newToken = new N5UrlAttributePathArray(gson, index); - n5UrlAttributePathTokens.add(newToken); + if (attributePathParts.length == 0 || Arrays.stream(attributePathParts).allMatch(String::isEmpty)) + return null; + + final AtomicReference> firstTokenRef = new AtomicReference<>(); + final AtomicReference> currentTokenRef = new AtomicReference<>(); + final Consumer> updateCurrentToken = (newToken) -> { + if (firstTokenRef.get() == null) { + firstTokenRef.set(newToken); + currentTokenRef.set(firstTokenRef.get()); } else { - newToken = new N5UrlAttributePathObject(gson, pathToken); - n5UrlAttributePathTokens.add(newToken); + final LinkedAttributePathToken currentToken = currentTokenRef.get(); + currentToken.childToken = newToken; + currentTokenRef.set(newToken); } + }; - if (i > 0) { - final N5UrlAttributePathToken parent = n5UrlAttributePathTokens.get(i - 1); - parent.setChild(newToken); - newToken.setParent(parent); + for (final String pathPart : attributePathParts) { + final Matcher matcher = ARRAY_INDEX.matcher(pathPart); + final LinkedAttributePathToken newToken; + if (matcher.matches()) { + final int index = Integer.parseInt(matcher.group().replace("[", "").replace("]", "")); + newToken = new LinkedAttributePathToken.ArrayAttributeToken(index); + } else { + newToken = new LinkedAttributePathToken.ObjectAttributeToken(pathPart); } + updateCurrentToken.accept(newToken); } - if (value != null) { - final N5UrlAttributePathLeaf leaf = new N5UrlAttributePathLeaf<>(gson, value); - /* get last token, if present */ - if (n5UrlAttributePathTokens.size() >= 1) { - final N5UrlAttributePathToken lastToken = n5UrlAttributePathTokens.get(n5UrlAttributePathTokens.size() - 1); - lastToken.setChild(leaf); - leaf.setParent(lastToken); - } - n5UrlAttributePathTokens.add(leaf); - } - return n5UrlAttributePathTokens; + return firstTokenRef.get(); } private String getSchemePart() { @@ -196,6 +167,7 @@ private String getAttributePart() { private String getSchemeSpecificPartWithoutQuery() { + /* Why not substring "?"?*/ return uri.getSchemeSpecificPart().replace("?" + uri.getQuery(), ""); } @@ -443,7 +415,7 @@ public static String normalizeAttributePath(String attributePath) { curToken.append(character); } else if (character == ']') { /* If ']' add before terminating the token */ - curToken.append(character); + curToken.append(']'); } /* The current token is complete, add it to the list, if it isn't empty */ @@ -534,251 +506,6 @@ public static URI encodeAsUri(String uri) throws URISyntaxException { } return n5Uri; } - public static abstract class N5UrlAttributePathToken { - - protected final Gson gson; - - protected JsonElement root; - - protected N5UrlAttributePathToken parent; - protected N5UrlAttributePathToken child; - - public N5UrlAttributePathToken(Gson gson) { - - this.gson = gson; - } - - public void setChild(N5UrlAttributePathToken child) { - - this.child = child; - } - - public void setParent(N5UrlAttributePathToken parent) { - - this.parent = parent; - } - - public JsonElement getRoot() { - - return this.root; - } - - public abstract boolean canNavigate(JsonElement json); - - public abstract JsonElement navigateJsonElement(JsonElement json); - - public JsonElement replaceJsonElement(JsonElement jsonParent) { - - parent.removeJsonElement(jsonParent); - return parent.createJsonElement(jsonParent); - } - - public abstract JsonElement createJsonElement(JsonElement json); - - public abstract void writeLeaf(JsonElement json); - - protected abstract void removeJsonElement(JsonElement jsonParent); - - public abstract boolean jsonIncompatible(JsonElement json); - } - - public static class N5UrlAttributePathObject extends N5UrlAttributePathToken { - - public final String key; - - public N5UrlAttributePathObject(Gson gson, String key) { - - super(gson); - this.key = key; - } - - public String getKey() - { - return key; - } - - @Override public boolean jsonIncompatible(JsonElement json) { - - return json != null && !json.isJsonObject(); - } - - @Override public boolean canNavigate(JsonElement json) { - if (json == null) { - return false; - } - return json.isJsonObject() && json.getAsJsonObject().has(key); - } - - @Override public JsonElement navigateJsonElement(JsonElement json) { - - return json.getAsJsonObject().get(key); - } - - @Override protected void removeJsonElement(JsonElement jsonParent) { - jsonParent.getAsJsonObject().remove(key); - } - - @Override public JsonElement createJsonElement(JsonElement json) { - final JsonObject object; - if (json == null) { - object = new JsonObject(); - root = object; - } else { - object = json.getAsJsonObject(); - } - - final JsonElement newJsonElement; - if (child instanceof N5UrlAttributePathArray) { - newJsonElement = new JsonArray(); - object.add(key, newJsonElement); - } else if (child instanceof N5UrlAttributePathObject) { - newJsonElement = new JsonObject(); - object.add(key, newJsonElement); - } else { - newJsonElement = new JsonObject(); - writeLeaf(object); - } - return newJsonElement; - } - - @Override public void writeLeaf(JsonElement json) { - - final N5UrlAttributePathLeaf leaf = ((N5UrlAttributePathLeaf)child); - final JsonElement leafJson = leaf.createJsonElement(null); - json.getAsJsonObject().add(key, leafJson); - } - } - - public static class N5UrlAttributePathArray extends N5UrlAttributePathToken { - - @Override public boolean jsonIncompatible(JsonElement json) { - - return json!= null && !json.isJsonArray(); - } - - @Override protected void removeJsonElement(JsonElement jsonParent) { - jsonParent.getAsJsonArray().set(index, JsonNull.INSTANCE); - } - - private final int index; - - public N5UrlAttributePathArray(Gson gson, int index) { - - super(gson); - this.index = index; - } - - @Override public boolean canNavigate(JsonElement json) { - if (json == null) { - return false; - } - - return json.isJsonArray() && json.getAsJsonArray().size() > index; - } - - @Override public JsonElement navigateJsonElement(JsonElement json) { - - return json.getAsJsonArray().get(index); - } - - @Override public JsonElement createJsonElement(JsonElement json) { - - final JsonArray array; - if (json == null) { - array = new JsonArray(); - root = array; - } else { - array = json.getAsJsonArray(); - } - - JsonElement fillArrayValue = JsonNull.INSTANCE; - - if (child instanceof N5UrlAttributePathLeaf) { - final N5UrlAttributePathLeaf< ? > leaf = ( N5UrlAttributePathLeaf< ? > ) child; - if (leaf.value instanceof Number || (leaf.value instanceof JsonPrimitive && ((JsonPrimitive)leaf.value).isNumber())) { - fillArrayValue = new JsonPrimitive( 0 ); - } - } - - for (int i = array.size(); i <= index; i++) { - array.add(fillArrayValue); - } - final JsonElement newJsonElement; - if (child instanceof N5UrlAttributePathArray) { - newJsonElement = new JsonArray(); - array.set(index, newJsonElement); - } else if (child instanceof N5UrlAttributePathObject) { - newJsonElement = new JsonObject(); - array.set(index, newJsonElement); - } else { - newJsonElement = child.createJsonElement(null); - array.set(index, newJsonElement); - } - return newJsonElement; - } - - @Override public void writeLeaf(JsonElement json) { - - final N5UrlAttributePathLeaf leaf = ((N5UrlAttributePathLeaf)child); - final JsonElement leafJson = leaf.createJsonElement(null); - final JsonArray array = json.getAsJsonArray(); - - for (int i = array.size(); i <= index; i++) { - array.add(JsonNull.INSTANCE); - } - - array.set(index, leafJson); - } - } - - public static class N5UrlAttributePathLeaf extends N5UrlAttributePathToken { - - private final T value; - - public N5UrlAttributePathLeaf(Gson gson, T value) { - - super(gson); - this.value = value; - } - - @Override public boolean canNavigate(JsonElement json) { - - return false; - } - - @Override public boolean jsonIncompatible(JsonElement json) { - - return false; - } - - @Override public JsonElement navigateJsonElement(JsonElement json) { - - throw new UnsupportedOperationException("Cannot Navigate from a leaf "); - } - - @Override protected void removeJsonElement(JsonElement jsonParent) { - throw new UnsupportedOperationException("Leaf Removal Unnecessary, leaves should alway be overwritable"); - - } - - @Override public JsonElement createJsonElement(JsonElement json) { - /* Leaf is weird. It is never added manually, also via the previous path token. - * HOWEVER, when there is no path token, and the leaf is the root, than it is called manually, - * and we wish to be our own parent (i.e. the root of the json tree). */ - final JsonElement leaf = gson.toJsonTree(value); - if (json == null) { - root = leaf; - } - return leaf; - } - - @Override public void writeLeaf(JsonElement json) { - /* Practically, this method doesn't mean much when you are a leaf. - * It's primarily meant to have the parent write their children, if their child is a leaf. */ - } - - - } /** * Generate an {@link N5URL} from a container, group, and attribute From 8eea1ecbddec7ce03bac17d2729b48d427e2296c Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Thu, 9 Feb 2023 11:37:57 -0500 Subject: [PATCH 082/243] docs: document LinkedAttributePathToken --- .../n5/LinkedAttributePathToken.java | 115 +++++++++++------- 1 file changed, 74 insertions(+), 41 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/LinkedAttributePathToken.java b/src/main/java/org/janelia/saalfeldlab/n5/LinkedAttributePathToken.java index 398cb681..df803151 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/LinkedAttributePathToken.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/LinkedAttributePathToken.java @@ -11,48 +11,60 @@ public abstract class LinkedAttributePathToken implements Iterator> { + /** + * The JsonElement which contains the mapping that this token represents. + */ protected T parentJson; - protected LinkedAttributePathToken childToken; - - public void setChildToken(LinkedAttributePathToken childToken) { - - this.childToken = childToken; - } - - public T getParentJson() { - return parentJson; - } + /** + * The token representing the subsequent path element. + */ + protected LinkedAttributePathToken childToken; + /** + * @return a reference object of type {@link T} + */ public abstract T getJsonType(); - public abstract boolean canNavigate(JsonElement parent); - + /** + * This method will retgurn the child element, if the {@link #parentJson} contains a child that + * is valid for this token. If the {@link #parentJson} does not have a child, this method will + * create it. If the {@link #parentJson} does have a child, but it is not {@link #jsonCompatible(JsonElement)} + * with our {@link #childToken}, then this method will also create a new (compatible) child, and replace + * the existing incompatible child. + * + * @return the resulting child element + */ public abstract JsonElement getOrCreateChildElement(); + /** + * This method will write into {@link #parentJson} the subsequent {@link JsonElement}. + *
+ * The written JsonElement will EITHER be: + *
    + *
  • The result of serializing {@code value} ( if {@link #childToken} is {@code null} )
    • + *
  • The {@link JsonElement} which represents our {@link #childToken} (See {@link #getOrCreateChildElement()} )
    • + *
+ * + * @param gson instance used to serialize {@code value } + * @param value to write + * @return the object that was written. + */ public JsonElement writeChild(Gson gson, Object value) { if (childToken != null) /* If we have a child, get/create it and set current json element to it */ return getOrCreateChildElement(); - else + else { /* We are done, no token remaining */ - return writeJsonElement(gson, value); - } - - public JsonElement writeJsonElement(Gson gson, Object value) { - - if (hasNext()) { - writeChildElement(); - } else { writeValue(gson, value); + return getChildElement(); } - return getChildElement(); } /** * Check if the provided {@code json} is compatible with this token. - *

+ *
* Compatibility means thath the provided {@code JsonElement} does not * explicitly conflict with this token. That means that {@code null} is * always compatible. The only real incompatibility is when the {@code JsonElement} @@ -67,28 +79,58 @@ public boolean jsonCompatible(JsonElement json) { return json == null || json.getClass() == getJsonType().getClass(); } + /** + * Write {@code value} into {@link #parentJson}. + *
+ * Should only be called when {@link #childToken} is null (i.e. this is the last token in the attribute path). + * + * @param gson instance used to serialize {@code value} + * @param value to serialize + */ protected abstract void writeValue(Gson gson, Object value); + /** + * Write the {@link JsonElement} the corresponds to {@link #childToken} into {@link #parentJson}. + */ protected abstract void writeChildElement(); + /** + * @return the element that is represented by {@link #childToken} if present. + */ protected abstract JsonElement getChildElement(); + /** + * If {@code json} is compatible with the type or token that we are, then set {@link #parentJson} to {@code json}. + *
+ * However, if {@code json} is either {@code null} or not {@link #jsonCompatible(JsonElement)} then + * {@link #parentJson} will be set to a new instance of {@link T}. + * + * @param json to attempt to set to {@link #parentJson} + * @return the value set to {@link #parentJson}. + */ protected JsonElement setAndCreateParentElement(JsonElement json) { + if (json == null || !jsonCompatible(json)) { //noinspection unchecked - parentJson = (T) getJsonType().deepCopy(); + parentJson = (T)getJsonType().deepCopy(); } else { //noinspection unchecked - parentJson = (T) json; + parentJson = (T)json; } return parentJson; } + /** + * @return if we have a {@link #childToken}. + */ @Override public boolean hasNext() { return childToken != null; } + /** + * @return {@link #childToken} + */ @Override public LinkedAttributePathToken next() { return childToken; @@ -98,13 +140,16 @@ public static class ObjectAttributeToken extends LinkedAttributePathToken index; - } - @Override public JsonElement getOrCreateChildElement() { - - /* Two cases which required writing the child: - * - The child element doesn't exist (in this case, there is nothing at the specified index). - * - The child element is incompatible with the child token (replaces the existing child element). */ if (index >= parentJson.size() || childToken != null && !childToken.jsonCompatible(parentJson.get(index))) writeChildElement(); @@ -244,7 +278,6 @@ private static void fillArrayToIndex(JsonArray array, int index, Object value) { *

  • is a {@link JsonPrimitive} where {@link JsonPrimitive#isNumber()}
    • * * - * * @param value we wish to check if represents a {@link Number} or not. * @return true if {@code value} represents a {@link Number} */ From 3914a339abb7239b546a2d700e787deeab49b2d1 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Tue, 21 Feb 2023 13:55:18 -0500 Subject: [PATCH 083/243] test: when constructing a reader should fail * non-existant directory or incompatible n5 version --- .../saalfeldlab/n5/AbstractN5Test.java | 48 +++++++++++++++++-- 1 file changed, 44 insertions(+), 4 deletions(-) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java index 109230b9..3eb0f000 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java @@ -45,7 +45,9 @@ import com.google.gson.reflect.TypeToken; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertThrows; /** * Abstract base class for testing N5 functionality. @@ -53,6 +55,8 @@ * * @author Stephan Saalfeld <saalfelds@janelia.hhmi.org> * @author Igor Pisarev <pisarevi@janelia.hhmi.org> + * @author John Bogovic <bogovicj@janelia.hhmi.org> + * @author Caleb Hulbert <hulbertc@janelia.hhmi.org> */ public abstract class AbstractN5Test { @@ -76,6 +80,8 @@ public abstract class AbstractN5Test { protected abstract N5Writer createN5Writer(String location) throws IOException; + protected abstract N5Reader createN5Reader(String location) throws IOException; + protected Compression[] getCompressions() { return new Compression[] { @@ -721,12 +727,9 @@ public void testVersion() throws NumberFormatException, IOException { final Version n5Version = n5.getVersion(); - System.out.println(n5Version); - Assert.assertTrue(n5Version.equals(N5Reader.VERSION)); - - final File tmpFile = Files.createTempDirectory("aws-version-test-").toFile(); + final File tmpFile = Files.createTempDirectory("version-test-").toFile(); tmpFile.deleteOnExit(); final String canonicalPath = tmpFile.getCanonicalPath(); try (N5Writer writer = createN5Writer(canonicalPath)) { @@ -735,6 +738,43 @@ public void testVersion() throws NumberFormatException, IOException { } } + @Test + public void testReaderCreation() throws IOException + { + final File tmpFile = Files.createTempDirectory("reader-create-test-").toFile(); + tmpFile.deleteOnExit(); + final String canonicalPath = tmpFile.getCanonicalPath(); + try (N5Writer writer = createN5Writer(canonicalPath)) { + + final N5Reader n5r = createN5Reader( canonicalPath ); + assertNotNull( n5r ); + + // existing directory without attributes is okay + writer.createGroup( "noAttrs" ); + final N5Reader na = createN5Reader( canonicalPath + "/noAttrs" ); + assertNotNull( na ); + + // existing directory without attributes is okay + writer.createGroup( "withAttrs" ); + writer.setAttribute( "withAttrs", "mystring", "ms" ); + final N5Reader wa = createN5Reader( canonicalPath + "/withAttrs" ); + assertNotNull( wa ); + + // non-existent directory should fail + assertThrows( "Non-existant directory throws error", IOException.class, + () -> { + createN5Reader( canonicalPath + "/nonExistant" ); + } ); + + // existing directory with incompatible version should fail + writer.setAttribute("/", N5Reader.VERSION_KEY, new Version(N5Reader.VERSION.getMajor() + 1, N5Reader.VERSION.getMinor(), N5Reader.VERSION.getPatch()).toString()); + assertThrows( "Incompatible version throws error", IOException.class, + () -> { + createN5Reader(canonicalPath); + } ); + } + } + @Test public void testDelete() throws IOException { final String datasetName = AbstractN5Test.datasetName + "-test-delete"; From 3adb677bcc416453e9fec2293204c89ce327a30b Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Tue, 21 Feb 2023 13:56:33 -0500 Subject: [PATCH 084/243] test: setting attrs with escaping special chars for keys --- .../org/janelia/saalfeldlab/n5/N5FSTest.java | 105 +++++++++++++++++- 1 file changed, 104 insertions(+), 1 deletion(-) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java b/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java index c0cd8a81..56c11139 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java @@ -19,8 +19,13 @@ import org.janelia.saalfeldlab.n5.url.UrlAttributeTest; import org.junit.Test; +import com.google.gson.JsonObject; + +import static org.junit.Assert.assertEquals; + import java.io.IOException; import java.nio.file.Files; +import java.nio.file.Paths; import java.util.ArrayList; /** @@ -50,6 +55,11 @@ protected N5Writer createN5Writer(String location) throws IOException { return new N5FSWriter(location); } + @Override + protected N5Reader createN5Reader(String location) throws IOException { + return new N5FSReader(location); + } + @Test public void customObjectTest() { @@ -69,4 +79,97 @@ public void customObjectTest() { addAndTest(existingTests, new TestData<>(testGroup, "/doubles[1]", doubles4)); } -} + @Test + public void testAttributePathEscaping() throws IOException { + + final JsonObject emptyObj = new JsonObject(); + final String empty = "{}"; + + final String slashKey = "/"; + final String abcdefKey = "abc/def"; + final String zeroKey = "[0]"; + final String bracketsKey = "]] [] [["; + final String doubleBracketsKey = "[[2][33]]"; + final String doubleBackslashKey = "\\\\"; + + final String dataString = "dataString"; + final String rootSlash = jsonKeyVal( slashKey, dataString ); + final String abcdef = jsonKeyVal( abcdefKey, dataString ); + final String zero = jsonKeyVal( zeroKey, dataString ); + final String brackets = jsonKeyVal( bracketsKey, dataString ); + final String doubleBrackets = jsonKeyVal( doubleBracketsKey, dataString ); + final String doubleBackslash = jsonKeyVal( doubleBackslashKey, dataString ); + + // "/" as key + String grp = "a"; + n5.createGroup( grp ); + n5.setAttribute( grp, "\\/", dataString ); + assertEquals( dataString, n5.getAttribute( grp, "\\/", String.class ) ); + String jsonContents = readAttributesAsString( grp ); + assertEquals( rootSlash, jsonContents ); + + // "abc/def" as key + grp = "b"; + n5.createGroup( grp ); + n5.setAttribute( grp, "abc\\/def", dataString ); + assertEquals( dataString, n5.getAttribute( grp, "abc\\/def", String.class ) ); + jsonContents = readAttributesAsString( grp ); + assertEquals( abcdef, jsonContents ); + + // "[0]" as a key + grp = "c"; + n5.createGroup( grp ); + n5.setAttribute( grp, "\\[0]", dataString ); + assertEquals( dataString, n5.getAttribute( grp, "\\[0]", String.class ) ); + jsonContents = readAttributesAsString( grp ); + assertEquals( zero, jsonContents ); + + // "]] [] [[" as a key + grp = "d"; + n5.createGroup( grp ); + n5.setAttribute( grp, bracketsKey, dataString ); + assertEquals( dataString, n5.getAttribute( grp, bracketsKey, String.class ) ); + jsonContents = readAttributesAsString( grp ); + assertEquals( brackets, jsonContents ); + + // "[[2][33]]" as a key + grp = "e"; + n5.createGroup( grp ); + n5.setAttribute( grp, "[\\[2]\\[33]]", dataString ); + assertEquals( dataString, n5.getAttribute( grp, "[\\[2]\\[33]]", String.class ) ); + jsonContents = readAttributesAsString( grp ); + assertEquals( doubleBrackets, jsonContents ); + + // "\\" as key + grp = "f"; + n5.createGroup( grp ); + n5.setAttribute( grp, "\\\\\\\\", dataString ); + assertEquals( dataString, n5.getAttribute( grp, "\\\\\\\\", String.class ) ); + jsonContents = readAttributesAsString( grp ); + assertEquals( doubleBackslash, jsonContents ); + + // clear + n5.setAttribute( grp, "/", emptyObj ); + jsonContents = readAttributesAsString( grp ); + assertEquals( empty, jsonContents ); + } + + /* + * For readability above + */ + private String jsonKeyVal( String key, String val ) { + + return String.format( "{\"%s\":\"%s\"}", key, val ); + } + + private String readAttributesAsString( final String group ) { + + final String basePath = ((N5FSWriter)n5).getBasePath(); + try + { + return new String( Files.readAllBytes( Paths.get( basePath, group, "attributes.json" ))); + } catch ( IOException e ) { } + return null; + } + +} \ No newline at end of file From a335b0797e39e9a4b685ace33d12f6d6aa6f1968 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Wed, 22 Feb 2023 11:44:32 -0500 Subject: [PATCH 085/243] feat!: replace Map variants with root JsonElement --- .../saalfeldlab/n5/AbstractGsonReader.java | 33 +-- .../saalfeldlab/n5/GsonAttributesParser.java | 257 +++++++----------- .../janelia/saalfeldlab/n5/N5FSReader.java | 20 +- .../janelia/saalfeldlab/n5/N5FSWriter.java | 22 +- .../saalfeldlab/n5/AbstractN5Test.java | 21 +- 5 files changed, 138 insertions(+), 215 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/AbstractGsonReader.java b/src/main/java/org/janelia/saalfeldlab/n5/AbstractGsonReader.java index ea1f8da0..962d8eae 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/AbstractGsonReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/AbstractGsonReader.java @@ -28,14 +28,13 @@ import com.google.gson.Gson; import com.google.gson.GsonBuilder; import com.google.gson.JsonElement; -import com.google.gson.JsonSyntaxException; +import com.google.gson.JsonObject; import com.google.gson.reflect.TypeToken; import java.io.IOException; import java.lang.reflect.Type; import java.net.URISyntaxException; import java.util.Arrays; -import java.util.HashMap; /** * Abstract base class implementing {@link N5Reader} with JSON attributes @@ -81,26 +80,31 @@ public Gson getGson() { @Override public DatasetAttributes getDatasetAttributes(final String pathName) throws IOException { - final HashMap map = getAttributes(pathName); + final JsonElement root = getAttributes(pathName); final Gson gson = getGson(); - final long[] dimensions = GsonAttributesParser.parseAttribute(map, DatasetAttributes.dimensionsKey, long[].class, gson); + if (root == null || !root.isJsonObject()) + return null; + + final JsonObject rootObject = root.getAsJsonObject(); + + final long[] dimensions = GsonAttributesParser.readAttribute(rootObject, DatasetAttributes.dimensionsKey, long[].class, gson); if (dimensions == null) return null; - final DataType dataType = GsonAttributesParser.parseAttribute(map, DatasetAttributes.dataTypeKey, DataType.class, gson); + final DataType dataType = GsonAttributesParser.readAttribute(rootObject, DatasetAttributes.dataTypeKey, DataType.class, gson); if (dataType == null) return null; - int[] blockSize = GsonAttributesParser.parseAttribute(map, DatasetAttributes.blockSizeKey, int[].class, gson); + int[] blockSize = GsonAttributesParser.readAttribute(rootObject, DatasetAttributes.blockSizeKey, int[].class, gson); if (blockSize == null) blockSize = Arrays.stream(dimensions).mapToInt(a -> (int)a).toArray(); - Compression compression = GsonAttributesParser.parseAttribute(map, DatasetAttributes.compressionKey, Compression.class, gson); + Compression compression = GsonAttributesParser.readAttribute(rootObject, DatasetAttributes.compressionKey, Compression.class, gson); /* version 0 */ if (compression == null) { - switch (GsonAttributesParser.parseAttribute(map, DatasetAttributes.compressionTypeKey, String.class, gson)) { + switch (GsonAttributesParser.readAttribute(rootObject, DatasetAttributes.compressionTypeKey, String.class, gson)) { case "raw": compression = new RawCompression(); break; @@ -136,17 +140,6 @@ public T getAttribute( final String pathName, final String key, final Type type) throws IOException { - /* Short Circuit if the key exists exactly as is */ - final HashMap attributes = getAttributes(pathName); - if (attributes.containsKey(key)) { - try { - return gson.fromJson(attributes.get(key), type); - } catch (JsonSyntaxException e) { - /* This can happen for example when you have an attribute object with the exact key, but ALSO - * have a structured json path that you want to query. In this case, it will fail here if the types are different - * but we may want it to continue to try below. */ - } - } final String normalizedGroupPath; final String normalizedAttributePath; @@ -157,7 +150,7 @@ public T getAttribute( } catch (URISyntaxException e) { throw new IOException(e); } - return GsonAttributesParser.readAttribute( getAttributesJson( normalizedGroupPath ), normalizedAttributePath, type, gson); + return GsonAttributesParser.readAttribute( getAttributes( normalizedGroupPath ), normalizedAttributePath, type, gson); } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonAttributesParser.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonAttributesParser.java index eab8ef58..9bb97c12 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonAttributesParser.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonAttributesParser.java @@ -1,16 +1,16 @@ /** * Copyright (c) 2017, Stephan Saalfeld * All rights reserved. - * + *

    * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: - * + *

    * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. + * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + *

    * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE @@ -25,8 +25,14 @@ */ package org.janelia.saalfeldlab.n5; +import com.google.gson.Gson; +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; import com.google.gson.JsonObject; +import com.google.gson.JsonPrimitive; import com.google.gson.JsonSyntaxException; +import com.google.gson.reflect.TypeToken; + import java.io.IOException; import java.io.Reader; import java.io.Writer; @@ -35,12 +41,6 @@ import java.util.HashMap; import java.util.Map; import java.util.Map.Entry; - -import com.google.gson.Gson; -import com.google.gson.JsonArray; -import com.google.gson.JsonElement; -import com.google.gson.JsonPrimitive; -import com.google.gson.reflect.TypeToken; import java.util.regex.Matcher; /** @@ -52,15 +52,6 @@ public interface GsonAttributesParser extends N5Reader { public Gson getGson(); - /** - * Reads or creates the attributes map of a group or dataset. - * - * @param pathName group path - * @return - * @throws IOException - */ - public HashMap getAttributes(final String pathName) throws IOException; - /** * Reads the attributes a group or dataset. * @@ -68,68 +59,7 @@ public interface GsonAttributesParser extends N5Reader { * @return the root {@link JsonElement} of the attributes * @throws IOException */ - public JsonElement getAttributesJson(final String pathName) throws IOException; - - /** - * Parses an attribute from the given attributes map. - * - * @param map - * @param key - * @param clazz - * @return - * @throws IOException - */ - public static T parseAttribute( - final HashMap map, - final String key, - final Class clazz, - final Gson gson) throws IOException { - - final JsonElement attribute = map.get(key); - if (attribute != null) - return gson.fromJson(attribute, clazz); - else - return null; - } - - /** - * Parses an attribute from the given attributes map. - * - * @param map - * @param key - * @param type - * @return - * @throws IOException - */ - public static T parseAttribute( - final HashMap map, - final String key, - final Type type, - final Gson gson) throws IOException { - - final JsonElement attribute = map.get(key); - if (attribute != null) - return gson.fromJson(attribute, type); - else - return null; - } - - /** - * Reads the attributes map from a given {@link Reader}. - * - * @param reader - * @return - * @throws IOException - */ - public static HashMap readAttributes(final Reader reader, final Gson gson) throws IOException { - - /* Handle that case where the attributes.json file is valid json, but not a JsonObject, but returning an empty map. */ - final JsonElement attributes = readAttributesJson(reader, gson); - if (attributes == null || !attributes.isJsonObject()) return new HashMap<>(); - final Type mapType = new TypeToken>(){}.getType(); - final HashMap map = gson.fromJson(attributes, mapType); - return map == null ? new HashMap<>() : map; - } + public JsonElement getAttributes(final String pathName) throws IOException; /** * Reads the attributes json from a given {@link Reader}. @@ -138,24 +68,27 @@ public static HashMap readAttributes(final Reader reader, f * @return the root {@link JsonObject} of the attributes * @throws IOException */ - public static JsonElement readAttributesJson(final Reader reader, final Gson gson) throws IOException { + public static JsonElement readAttributes(final Reader reader, final Gson gson) throws IOException { final JsonElement json = gson.fromJson(reader, JsonElement.class); return json; } - static < T > T readAttribute( final JsonElement root, final String normalizedAttributePath, final Class cls, final Gson gson ) { + static T readAttribute(final JsonElement root, final String normalizedAttributePath, final Class cls, final Gson gson) { - return readAttribute(root, normalizedAttributePath, TypeToken.get( cls ).getType(), gson ); + return readAttribute(root, normalizedAttributePath, TypeToken.get(cls).getType(), gson); } - static < T > T readAttribute( final JsonElement root, final String normalizedAttributePath, final Type type, final Gson gson ) { - final Class clazz = ( type instanceof Class) ? ((Class) type ) : null; + + static T readAttribute(final JsonElement root, final String normalizedAttributePath, final Type type, final Gson gson) { + + final Class clazz = (type instanceof Class) ? ((Class)type) : null; JsonElement json = root; - for (final String pathPart : normalizedAttributePath.split("/")) { + for (final String pathPart : normalizedAttributePath.split("(? T readAttribute( final JsonElement root, final String normalizedAtt } } if (clazz != null && clazz.isAssignableFrom(HashMap.class)) { - Type mapType = new TypeToken>() {}.getType(); + Type mapType = new TypeToken>() { + + }.getType(); Map retMap = gson.fromJson(json, mapType); //noinspection unchecked - return ( T ) retMap; + return (T)retMap; } if (json instanceof JsonArray) { final JsonArray array = json.getAsJsonArray(); - T retArray = GsonAttributesParser.getJsonAsArray(gson, array, type ); + T retArray = GsonAttributesParser.getJsonAsArray(gson, array, type); if (retArray != null) return retArray; } try { - return gson.fromJson( json, type ); - } catch ( JsonSyntaxException e) { + return gson.fromJson(json, type); + } catch (JsonSyntaxException e) { if (type == String.class) - return (T) gson.toJson(json); + return (T)gson.toJson(json); return null; } } /** - * Inserts new the JSON export of attributes into the given attributes map. + * Inserts {@code attribute} into {@code root} at location {@code normalizedAttributePath} and write the resulting {@code root}. + * + * If {@code root} is not a {@link JsonObject}, then it is overwritten with an object containing {@code "normalizedAttributePath": attribute } * - * @param map - * @param attributes + * @param writer + * @param root + * @param normalizedAttributePath + * @param attribute * @param gson * @throws IOException */ - public static void insertAttributes( - final HashMap map, - final Map attributes, + public static void writeAttribute( + final Writer writer, + JsonElement root, + final String normalizedAttributePath, + final T attribute, final Gson gson) throws IOException { - for (final Entry entry : attributes.entrySet()) - map.put(entry.getKey(), gson.toJsonTree(entry.getValue())); + root = insertAttribute(root, normalizedAttributePath, attribute, gson); + writeAttributes(writer, root, gson); } /** - * Writes the attributes map to a given {@link Writer}. + * Writes the attributes JsonElemnt to a given {@link Writer}. + * This will overwrite any existing attributes. * * @param writer - * @param map + * @param root * @throws IOException */ - public static void writeAttributes( - final Writer writer, - final HashMap map, - final Gson gson) throws IOException { - - final Type mapType = new TypeToken>(){}.getType(); - gson.toJson(map, mapType, writer); - writer.flush(); - } - - public static void writeAttribute( + public static void writeAttributes( final Writer writer, JsonElement root, - final String normalizedAttributePath, - final T attribute, final Gson gson) throws IOException { - root = insertAttribute(root, normalizedAttributePath, attribute, gson); gson.toJson(root, writer); writer.flush(); } @@ -262,10 +191,12 @@ public static JsonElement insertAttribute(JsonElement root, String normalize return root; } - static T getJsonAsArray(Gson gson, JsonArray array, Class cls) { - return getJsonAsArray( gson, array, TypeToken.get( cls ).getType() ); + static T getJsonAsArray(Gson gson, JsonArray array, Class cls) { + + return getJsonAsArray(gson, array, TypeToken.get(cls).getType()); } - static T getJsonAsArray(Gson gson, JsonArray array, Type type) { + + static T getJsonAsArray(Gson gson, JsonArray array, Type type) { final Class clazz = (type instanceof Class) ? ((Class)type) : null; @@ -362,7 +293,8 @@ else if (jsonPrimitive.isNumber()) { return double.class; } else if (jsonPrimitive.isString()) return String.class; - else return Object.class; + else + return Object.class; } /** @@ -383,47 +315,52 @@ else if (jsonPrimitive.isNumber()) { @Override public default Map> listAttributes(final String pathName) throws IOException { - final HashMap jsonElementMap = getAttributes(pathName); + final JsonElement root = getAttributes(pathName); + if (!root.isJsonObject()) + return new HashMap<>(); + + final JsonObject rootObj = root.getAsJsonObject(); final HashMap> attributes = new HashMap<>(); - jsonElementMap.forEach( - (key, jsonElement) -> { - final Class clazz; - if (jsonElement.isJsonNull()) - clazz = null; - else if (jsonElement.isJsonPrimitive()) - clazz = classForJsonPrimitive((JsonPrimitive)jsonElement); - else if (jsonElement.isJsonArray()) { - final JsonArray jsonArray = (JsonArray)jsonElement; - Class arrayElementClass = Object.class; - if (jsonArray.size() > 0) { - final JsonElement firstElement = jsonArray.get(0); - if (firstElement.isJsonPrimitive()) { - arrayElementClass = classForJsonPrimitive(firstElement.getAsJsonPrimitive()); - for (int i = 1; i < jsonArray.size() && arrayElementClass != Object.class; ++i) { - final JsonElement element = jsonArray.get(i); - if (element.isJsonPrimitive()) { - final Class nextArrayElementClass = classForJsonPrimitive(element.getAsJsonPrimitive()); - if (nextArrayElementClass != arrayElementClass) - if (nextArrayElementClass == double.class && arrayElementClass == long.class) - arrayElementClass = double.class; - else { - arrayElementClass = Object.class; - break; - } - } else { + for (final Entry entry : rootObj.entrySet()) { + final String key = entry.getKey(); + final JsonElement jsonElement = entry.getValue(); + + final Class clazz; + if (jsonElement.isJsonNull()) + clazz = null; + else if (jsonElement.isJsonPrimitive()) + clazz = classForJsonPrimitive((JsonPrimitive)jsonElement); + else if (jsonElement.isJsonArray()) { + final JsonArray jsonArray = (JsonArray)jsonElement; + Class arrayElementClass = Object.class; + if (jsonArray.size() > 0) { + final JsonElement firstElement = jsonArray.get(0); + if (firstElement.isJsonPrimitive()) { + arrayElementClass = classForJsonPrimitive(firstElement.getAsJsonPrimitive()); + for (int i = 1; i < jsonArray.size() && arrayElementClass != Object.class; ++i) { + final JsonElement element = jsonArray.get(i); + if (element.isJsonPrimitive()) { + final Class nextArrayElementClass = classForJsonPrimitive(element.getAsJsonPrimitive()); + if (nextArrayElementClass != arrayElementClass) + if (nextArrayElementClass == double.class && arrayElementClass == long.class) + arrayElementClass = double.class; + else { arrayElementClass = Object.class; break; } - } + } else { + arrayElementClass = Object.class; + break; } - clazz = Array.newInstance(arrayElementClass, 0).getClass(); - } else - clazz = Object[].class; + } } - else - clazz = Object.class; - attributes.put(key, clazz); - }); + clazz = Array.newInstance(arrayElementClass, 0).getClass(); + } else + clazz = Object[].class; + } else + clazz = Object.class; + attributes.put(key, clazz); + } return attributes; } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5FSReader.java b/src/main/java/org/janelia/saalfeldlab/n5/N5FSReader.java index 8ad58bf2..4cd29852 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5FSReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5FSReader.java @@ -25,11 +25,8 @@ */ package org.janelia.saalfeldlab.n5; -import com.google.gson.JsonObject; import java.io.Closeable; import java.io.IOException; -import java.net.URI; -import java.net.URISyntaxException; import java.nio.channels.Channels; import java.nio.channels.FileChannel; import java.nio.channels.OverlappingFileLockException; @@ -39,7 +36,6 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.nio.file.StandardOpenOption; -import java.util.HashMap; import java.util.stream.Stream; import com.google.gson.GsonBuilder; @@ -157,26 +153,14 @@ public boolean exists(final String pathName) { } @Override - public HashMap getAttributes(final String pathName) throws IOException { - - final Path path = Paths.get(basePath, getAttributesPath(pathName).toString()); - if (exists(pathName) && !Files.exists(path)) - return new HashMap<>(); - - try (final LockedFileChannel lockedFileChannel = LockedFileChannel.openForReading(path)) { - return GsonAttributesParser.readAttributes(Channels.newReader(lockedFileChannel.getFileChannel(), StandardCharsets.UTF_8.name()), getGson()); - } - } - - @Override - public JsonElement getAttributesJson(final String pathName) throws IOException { + public JsonElement getAttributes(final String pathName) throws IOException { final Path path = Paths.get(basePath, getAttributesPath(pathName).toString()); if (exists(pathName) && !Files.exists(path)) return null; try (final LockedFileChannel lockedFileChannel = LockedFileChannel.openForReading(path)) { - return GsonAttributesParser.readAttributesJson(Channels.newReader(lockedFileChannel.getFileChannel(), StandardCharsets.UTF_8.name()), getGson()); + return GsonAttributesParser.readAttributes(Channels.newReader(lockedFileChannel.getFileChannel(), StandardCharsets.UTF_8.name()), getGson()); } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java b/src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java index 38908a44..a1fce34d 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java @@ -39,11 +39,11 @@ import java.nio.file.Paths; import java.nio.file.attribute.FileAttribute; import java.util.Comparator; -import java.util.HashMap; import java.util.Map; import java.util.stream.Stream; import com.google.gson.GsonBuilder; +import com.google.gson.JsonObject; /** * Filesystem {@link N5Writer} implementation with version compatibility check. @@ -111,7 +111,7 @@ public void createGroup(final String pathName) throws IOException { final Path path = Paths.get(basePath, getAttributesPath(pathName).toString()); createDirectories(path.getParent()); - JsonElement attributesRoot = getAttributesJson(pathName); + JsonElement attributesRoot = getAttributes(pathName); try (final LockedFileChannel lockedFileChannel = LockedFileChannel.openForWriting(path)) { final String attributePath = N5URL.normalizeAttributePath(key); lockedFileChannel.getFileChannel().truncate(0); @@ -121,18 +121,26 @@ public void createGroup(final String pathName) throws IOException { @Override public void setAttributes( - final String pathName, + final String pathName, final Map attributes) throws IOException { final Path path = Paths.get(basePath, getAttributesPath(pathName).toString()); - final HashMap map = new HashMap<>(); + JsonObject newRoot = new JsonObject(); try (final LockedFileChannel lockedFileChannel = LockedFileChannel.openForWriting(path)) { - map.putAll(GsonAttributesParser.readAttributes(Channels.newReader(lockedFileChannel.getFileChannel(), StandardCharsets.UTF_8.name()), getGson())); - GsonAttributesParser.insertAttributes(map, attributes, gson); + final JsonElement oldRoot = GsonAttributesParser.readAttributes(Channels.newReader(lockedFileChannel.getFileChannel(), StandardCharsets.UTF_8.name()), getGson()); + if (oldRoot != null && oldRoot.isJsonObject()) + newRoot = oldRoot.getAsJsonObject(); + + + for (Map.Entry attrPathValue : attributes.entrySet()) { + final String attributePath = N5URL.normalizeAttributePath(attrPathValue.getKey()); + final Object attributeValue = attrPathValue.getValue(); + GsonAttributesParser.insertAttribute(newRoot, attributePath, attributeValue, gson); + } lockedFileChannel.getFileChannel().truncate(0); - GsonAttributesParser.writeAttributes(Channels.newWriter(lockedFileChannel.getFileChannel(), StandardCharsets.UTF_8.name()), map, getGson()); + GsonAttributesParser.writeAttributes(Channels.newWriter(lockedFileChannel.getFileChannel(), StandardCharsets.UTF_8.name()), newRoot, getGson()); } } diff --git a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java index 3eb0f000..cd58328d 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java @@ -917,20 +917,21 @@ public void testAttributePaths() throws IOException { /* We intentionally skipped index 3, it should be null */ Assert.assertNull( n5.getAttribute( testGroup, "/filled/double_array[3]", JsonNull.class ) ); - /* Lastly, we wish to ensure that adding via a map does NOT interpret the json path structure, but rather it adds the keys opaquely*/ + /* Lastly, we wish to ensure that escaping does NOT interpret the json path structure, but rather it adds the keys opaquely*/ final HashMap testAttributes = new HashMap<>(); - testAttributes.put("/z/y/x", 10); - testAttributes.put("q/r/t", 11); - testAttributes.put("/l/m[10]/n", 12); - testAttributes.put("/", 13); + testAttributes.put("\\/z\\/y\\/x", 10); + testAttributes.put("q\\/r\\/t", 11); + testAttributes.put("\\/l\\/m\\[10]\\/n", 12); + testAttributes.put("\\/", 13); /* intentionally the same as above, but this time it should be added as an opaque key*/ - testAttributes.put("/a/b/key2", "value2"); + testAttributes.put("\\/a\\/b/key2", "value2"); n5.setAttributes(testGroup, testAttributes); - assertEquals((Integer)10, n5.getAttribute(testGroup, "/z/y/x", Integer.class)); - assertEquals((Integer)11, n5.getAttribute(testGroup, "q/r/t", Integer.class)); - assertEquals((Integer)12, n5.getAttribute(testGroup, "/l/m[10]/n", Integer.class)); - assertEquals((Integer)13, n5.getAttribute(testGroup, "/", Integer.class)); + assertEquals((Integer)10, n5.getAttribute(testGroup, "\\/z\\/y\\/x", Integer.class)); + assertEquals((Integer)11, n5.getAttribute(testGroup, "q\\/r\\/t", Integer.class)); + assertEquals((Integer)12, n5.getAttribute(testGroup, "\\/l\\/m\\[10]\\/n", Integer.class)); + assertEquals((Integer)13, n5.getAttribute(testGroup, "\\/", Integer.class)); + /* We are passing a different type for the same key ("/"). * This means it will try ot grab the exact match first, but then fail, and continuat on * to try and grab the value as a json structure. I should grab the root, and match the empty string case */ From 4d36c2a8cdb7c4bbd7847ff8bc02995babe1ca67 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Wed, 22 Feb 2023 11:45:01 -0500 Subject: [PATCH 086/243] feat,test: support escape characters in attribute paths --- .../org/janelia/saalfeldlab/n5/N5URL.java | 99 ++++++------------- .../org/janelia/saalfeldlab/n5/N5URLTest.java | 26 ++++- 2 files changed, 51 insertions(+), 74 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java b/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java index 6644a3be..43dc323a 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java @@ -88,12 +88,12 @@ public String normalizeAttributePath() { } /** - * * Parse this {@link N5URL} as a {@link LinkedAttributePathToken}. * * @see N5URL#getAttributePathTokens(String) */ public LinkedAttributePathToken getAttributePathTokens() { + return getAttributePathTokens(normalizeAttributePath()); } @@ -107,7 +107,11 @@ public LinkedAttributePathToken getAttributePathTokens() { */ public static LinkedAttributePathToken getAttributePathTokens(String normalizedAttributePath) { - final String[] attributePathParts = normalizedAttributePath.replaceAll("^/", "").split("/"); + final String[] attributePathParts = Arrays.stream(normalizedAttributePath + .replaceAll("^/", "") + .split("(? it.replaceAll("\\\\/", "/").replaceAll("\\\\\\[", "[")) + .toArray(String[]::new); if (attributePathParts.length == 0 || Arrays.stream(attributePathParts).allMatch(String::isEmpty)) return null; @@ -179,7 +183,8 @@ private String getSchemeSpecificPartWithoutQuery() { */ public boolean isAbsolute() { - if (scheme != null) return true; + if (scheme != null) + return true; final String path = uri.getPath(); if (!path.isEmpty()) { final char char0 = path.charAt(0); @@ -360,7 +365,7 @@ public static String normalizePath(String path) { /** * Normalize the {@link String attributePath}. - *

    + *

    * Attribute paths have a few of special characters: *

      *
    • "." which represents the current element
      • @@ -368,6 +373,9 @@ public static String normalizePath(String path) { *
    • "/" which is used to separate elements in the json tree
      • *
    • [N] where N is an integer, refer to an index in the previous element in the tree; the previous element must be an array.
      • * Note: [N] also separates the previous and following elements, regardless of whether it is preceded by "/" or not. + *
    • "\" which is an escape character, which indicates the subquent '/' or '[N]' should not be interpreted as a path delimeter, + * but as part of the current path name.
      • + * *
    *

    * When normalizing: @@ -376,7 +384,7 @@ public static String normalizePath(String path) { *

  • any redundant "/" are removed
    • *
  • any relative ".." and "." are resolved
    • * - * + *

    * Examples of valid attribute paths, and their normalizations *

      *
    • /a/b/c -> /a/b/c
      • @@ -394,79 +402,28 @@ public static String normalizePath(String path) { */ public static String normalizeAttributePath(String attributePath) { - final char[] pathChars = attributePath.toCharArray(); - - final List tokens = new ArrayList<>(); - StringBuilder curToken = new StringBuilder(); - boolean escape = false; - for (final char character : pathChars) { - /* Skip if we last saw escape*/ - if (escape) { - escape = false; - curToken.append(character); - continue; - } - /* Check if we are escape character */ - if (character == '\\') { - escape = true; - } else if (character == '/' || character == '[' || character == ']') { - if (character == '/' && tokens.isEmpty() && curToken.length() == 0) { - /* If we are root, and the first token, then add the '/' */ - curToken.append(character); - } else if (character == ']') { - /* If ']' add before terminating the token */ - curToken.append(']'); - } - - /* The current token is complete, add it to the list, if it isn't empty */ - final String newToken = curToken.toString(); - if (!newToken.isEmpty()) { - /* If our token is '..' then remove the last token instead of adding a new one */ - if (newToken.equals("..")) { - tokens.remove(tokens.size() - 1); - } else { - tokens.add(newToken); - } - } - /* reset for the next token */ - curToken.setLength(0); - - /* if '[' add to the start of the next token */ - if (character == '[') { - curToken.append('['); - } - } else { - curToken.append(character); - } - } - final String lastToken = curToken.toString(); - if (!lastToken.isEmpty()) { - if (lastToken.equals("..")) { - tokens.remove(tokens.size() - 1); - } else { - tokens.add(lastToken); - } + final String attrPathPlusFirstIndexSeparator = attributePath.replaceAll("^(?\\[[0-9]+])", "${array}/"); + final String attrPathPlusIndexSeparators = attrPathPlusFirstIndexSeparator.replaceAll("((?[^\\\\])(?\\[[0-9]+]))", + "${prev}/${array}/"); + final String attrPathRemoveMultipleSeparators = attrPathPlusIndexSeparators.replaceAll("(?/)/+", "${slash}"); + final String attrPathNoDot = attrPathRemoveMultipleSeparators.replaceAll("((?[^(^|\\\\)])/$", "${nonSlash}"); + + final Pattern relativePathPattern = Pattern.compile("[^(/|\\.\\.)]+/\\.\\./?"); + int prevStringLenth = 0; + String resolvedAttributePath = normalizedAttributePath; + while (prevStringLenth != resolvedAttributePath.length()) { + prevStringLenth = resolvedAttributePath.length(); + resolvedAttributePath = relativePathPattern.matcher(resolvedAttributePath).replaceAll(""); } - if (tokens.isEmpty()) - return ""; - String root = ""; - if (tokens.get(0).equals("/")) { - tokens.remove(0); - root = "/"; - } - return root + tokens.stream() - .filter(it -> !it.equals(".")) - .filter(it -> !it.isEmpty()) - .reduce((l, r) -> l + "/" + r).orElse(""); + return resolvedAttributePath; } /** * Encode the inpurt {@link String uri} so that illegal characters are properly escaped prior to generating the resulting {@link URI}. * * @param uri to encode - * * @return the {@link URI} created from encoding the {@link String uri} - * * @throws URISyntaxException if {@link String uri} is not valid */ public static URI encodeAsUri(String uri) throws URISyntaxException { @@ -511,7 +468,7 @@ public static URI encodeAsUri(String uri) throws URISyntaxException { * Generate an {@link N5URL} from a container, group, and attribute * * @param container of the N5Url - * @param group of the N5Url + * @param group of the N5Url * @param attribute of the N5Url * @return the {@link N5URL} * @throws URISyntaxException diff --git a/src/test/java/org/janelia/saalfeldlab/n5/N5URLTest.java b/src/test/java/org/janelia/saalfeldlab/n5/N5URLTest.java index 02a6612c..b81633fc 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/N5URLTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/N5URLTest.java @@ -6,7 +6,6 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertThrows; import static org.junit.Assert.assertTrue; public class N5URLTest { @@ -21,6 +20,7 @@ public void testAttributePath() { assertEquals("/", N5URL.normalizeAttributePath("/a/../b/../c/d/../..")); assertEquals("/", N5URL.normalizeAttributePath("/a/../b/../c/d/../..")); assertEquals("/", N5URL.normalizeAttributePath("/./././././")); + assertEquals("/", N5URL.normalizeAttributePath("/./././././.")); assertEquals("", N5URL.normalizeAttributePath("./././././")); assertEquals("/a/[0]/b/[0]", N5URL.normalizeAttributePath("/a/[0]/b[0]/")); @@ -29,14 +29,34 @@ public void testAttributePath() { assertEquals("/a/[0]", N5URL.normalizeAttributePath("/a[0]/")); assertEquals("/[0]/b", N5URL.normalizeAttributePath("/[0]b/")); + assertEquals("[b]", N5URL.normalizeAttributePath("[b]")); + assertEquals("a[b]c", N5URL.normalizeAttributePath("a[b]c")); + assertEquals("a[bc", N5URL.normalizeAttributePath("a[bc")); + assertEquals("ab]c", N5URL.normalizeAttributePath("ab]c")); + assertEquals("a[b00]c", N5URL.normalizeAttributePath("a[b00]c")); + assertEquals("let's/try/a/real/case/with spaces", N5URL.normalizeAttributePath("let's/try/a/real/case/with spaces/")); assertEquals("let's/try/a/real/case/with spaces", N5URL.normalizeAttributePath("let's/try/a/real/////case////with spaces/")); - assertThrows( IndexOutOfBoundsException.class, () -> N5URL.normalizeAttributePath("../first/relative/../not/allowed")); + assertEquals( "../first/relative/a/wd/.w/asd", N5URL.normalizeAttributePath("../first/relative/test/../a/b/.././wd///.w/asd")); + assertEquals( "../", N5URL.normalizeAttributePath("../result/../only/../single/..")); + assertEquals( "../..", N5URL.normalizeAttributePath("../result/../multiple/../..")); - String normalizedPath = N5URL.normalizeAttributePath("let's/try/a/real/////case////with spaces/"); + String normalizedPath = N5URL.normalizeAttributePath("let's/try/a/some/////with/ / //white spaces/"); assertEquals("Normalizing a normal path should be the identity", normalizedPath, N5URL.normalizeAttributePath(normalizedPath)); } + @Test + public void testEscapedAttributePaths() { + assertEquals("\\/a\\/b\\/c\\/d\\/e", N5URL.normalizeAttributePath("\\/a\\/b\\/c\\/d\\/e")); + assertEquals("/a\\\\/b/c", N5URL.normalizeAttributePath("/a\\\\/b/c")); + assertEquals("a[b]\\[10]", N5URL.normalizeAttributePath("a[b]\\[10]")); + assertEquals("\\[10]", N5URL.normalizeAttributePath("\\[10]")); + assertEquals("a/[0]/\\[10]b", N5URL.normalizeAttributePath("a[0]\\[10]b")); + assertEquals("[\\[10]\\[20]]", N5URL.normalizeAttributePath("[\\[10]\\[20]]")); + assertEquals("[\\[10]/[20]/]", N5URL.normalizeAttributePath("[\\[10][20]]")); + assertEquals("\\/", N5URL.normalizeAttributePath("\\/")); + } + @Test public void testIsAbsolute() throws URISyntaxException { /* Always true if scheme provided */ From 536dc40aec7ca35fb477c6a6b801cc193ad94dc7 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Wed, 22 Feb 2023 13:20:07 -0500 Subject: [PATCH 087/243] fix: improper handling of escape characters --- src/main/java/org/janelia/saalfeldlab/n5/N5URL.java | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java b/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java index 43dc323a..8a709459 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java @@ -107,11 +107,7 @@ public LinkedAttributePathToken getAttributePathTokens() { */ public static LinkedAttributePathToken getAttributePathTokens(String normalizedAttributePath) { - final String[] attributePathParts = Arrays.stream(normalizedAttributePath - .replaceAll("^/", "") - .split("(? it.replaceAll("\\\\/", "/").replaceAll("\\\\\\[", "[")) - .toArray(String[]::new); + final String[] attributePathParts = normalizedAttributePath.replaceAll("^/", "").split("(? getAttributePathTokens(String normaliz final int index = Integer.parseInt(matcher.group().replace("[", "").replace("]", "")); newToken = new LinkedAttributePathToken.ArrayAttributeToken(index); } else { - newToken = new LinkedAttributePathToken.ObjectAttributeToken(pathPart); + final String pathPartUnEscaped = pathPart.replaceAll("\\\\/", "/").replaceAll("\\\\\\[", "["); + newToken = new LinkedAttributePathToken.ObjectAttributeToken(pathPartUnEscaped); } updateCurrentToken.accept(newToken); } From 13f535a8ac795af82d672196baacee8afdf0eee7 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Wed, 22 Feb 2023 13:23:54 -0500 Subject: [PATCH 088/243] test: move to abstract, comment some currently non-functional tests --- .../saalfeldlab/n5/AbstractN5Test.java | 106 +++++++++++++++++- .../org/janelia/saalfeldlab/n5/N5FSTest.java | 98 ---------------- 2 files changed, 100 insertions(+), 104 deletions(-) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java index cd58328d..787ccbe6 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java @@ -484,19 +484,19 @@ public void testRemove() { @Test public void testList() { - try { - n5.createGroup(groupName); + try( final N5Writer listN5 = createN5Writer()) { + listN5.createGroup(groupName); for (final String subGroup : subGroupNames) - n5.createGroup(groupName + "/" + subGroup); + listN5.createGroup(groupName + "/" + subGroup); - final String[] groupsList = n5.list(groupName); + final String[] groupsList = listN5.list(groupName); Arrays.sort(groupsList); Assert.assertArrayEquals(subGroupNames, groupsList); // test listing the root group ("" and "/" should give identical results) - Assert.assertArrayEquals(new String[] {"test"}, n5.list("")); - Assert.assertArrayEquals(new String[] {"test"}, n5.list("/")); + Assert.assertArrayEquals(new String[] {"test"}, listN5.list("")); + Assert.assertArrayEquals(new String[] {"test"}, listN5.list("/")); } catch (final IOException e) { @@ -943,6 +943,100 @@ public void testAttributePaths() throws IOException { n5.remove(testGroup); } + + @Test + public void testAttributePathEscaping() throws IOException { + + final JsonObject emptyObj = new JsonObject(); + final String empty = "{}"; + + final String slashKey = "/"; + final String abcdefKey = "abc/def"; + final String zeroKey = "[0]"; + final String bracketsKey = "]] [] [["; + final String doubleBracketsKey = "[[2][33]]"; + final String doubleBackslashKey = "\\\\"; + + final String dataString = "dataString"; + final String rootSlash = jsonKeyVal( slashKey, dataString ); + final String abcdef = jsonKeyVal( abcdefKey, dataString ); + final String zero = jsonKeyVal( zeroKey, dataString ); + final String brackets = jsonKeyVal( bracketsKey, dataString ); + final String doubleBrackets = jsonKeyVal( doubleBracketsKey, dataString ); + final String doubleBackslash = jsonKeyVal( doubleBackslashKey, dataString ); + + // "/" as key + String grp = "a"; + n5.createGroup( grp ); + n5.setAttribute( grp, "\\/", dataString ); + assertEquals( dataString, n5.getAttribute( grp, "\\/", String.class ) ); + String jsonContents = readAttributesAsString( grp ); + assertEquals( rootSlash, jsonContents ); + + // "abc/def" as key + grp = "b"; + n5.createGroup( grp ); + n5.setAttribute( grp, "abc\\/def", dataString ); + assertEquals( dataString, n5.getAttribute( grp, "abc\\/def", String.class ) ); + jsonContents = readAttributesAsString( grp ); + assertEquals( abcdef, jsonContents ); + + // "[0]" as a key + grp = "c"; + n5.createGroup( grp ); + n5.setAttribute( grp, "\\[0]", dataString ); + assertEquals( dataString, n5.getAttribute( grp, "\\[0]", String.class ) ); + jsonContents = readAttributesAsString( grp ); + assertEquals( zero, jsonContents ); + + // "]] [] [[" as a key + grp = "d"; + n5.createGroup( grp ); + n5.setAttribute( grp, bracketsKey, dataString ); + assertEquals( dataString, n5.getAttribute( grp, bracketsKey, String.class ) ); + jsonContents = readAttributesAsString( grp ); + assertEquals( brackets, jsonContents ); + + // "[[2][33]]" as a key FIXME: What to do about escaping inside square brackets? + grp = "e"; + n5.createGroup( grp ); + n5.setAttribute( grp, "[\\[2]\\[33]]", dataString ); + assertEquals( dataString, n5.getAttribute( grp, "[\\[2]\\[33]]", String.class ) ); + jsonContents = readAttributesAsString( grp ); + assertEquals( doubleBrackets, jsonContents ); + + // "\\" as key FIXME: Do we allow escaping escape characters? They shouldn't need to be, can just add normally if not escaping anything + grp = "f"; + n5.createGroup( grp ); + n5.setAttribute( grp, "\\\\\\\\", dataString ); + assertEquals( dataString, n5.getAttribute( grp, "\\\\\\\\", String.class ) ); + jsonContents = readAttributesAsString( grp ); + assertEquals( doubleBackslash, jsonContents ); + + // clear + n5.setAttribute( grp, "/", emptyObj ); + jsonContents = readAttributesAsString( grp ); + assertEquals( empty, jsonContents ); + } + + /* + * For readability above + */ + private String jsonKeyVal( String key, String val ) { + + return String.format( "{\"%s\":\"%s\"}", key, val ); + } + + private String readAttributesAsString( final String group ) { + + final String basePath = ((N5FSWriter)n5).getBasePath(); + try + { + return new String( Files.readAllBytes( Paths.get( basePath, group, "attributes.json" ))); + } catch ( IOException e ) { } + return null; + } + @Test public void testRootLeaves() throws IOException { diff --git a/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java b/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java index 56c11139..9d995c27 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java @@ -19,13 +19,8 @@ import org.janelia.saalfeldlab.n5.url.UrlAttributeTest; import org.junit.Test; -import com.google.gson.JsonObject; - -import static org.junit.Assert.assertEquals; - import java.io.IOException; import java.nio.file.Files; -import java.nio.file.Paths; import java.util.ArrayList; /** @@ -79,97 +74,4 @@ public void customObjectTest() { addAndTest(existingTests, new TestData<>(testGroup, "/doubles[1]", doubles4)); } - @Test - public void testAttributePathEscaping() throws IOException { - - final JsonObject emptyObj = new JsonObject(); - final String empty = "{}"; - - final String slashKey = "/"; - final String abcdefKey = "abc/def"; - final String zeroKey = "[0]"; - final String bracketsKey = "]] [] [["; - final String doubleBracketsKey = "[[2][33]]"; - final String doubleBackslashKey = "\\\\"; - - final String dataString = "dataString"; - final String rootSlash = jsonKeyVal( slashKey, dataString ); - final String abcdef = jsonKeyVal( abcdefKey, dataString ); - final String zero = jsonKeyVal( zeroKey, dataString ); - final String brackets = jsonKeyVal( bracketsKey, dataString ); - final String doubleBrackets = jsonKeyVal( doubleBracketsKey, dataString ); - final String doubleBackslash = jsonKeyVal( doubleBackslashKey, dataString ); - - // "/" as key - String grp = "a"; - n5.createGroup( grp ); - n5.setAttribute( grp, "\\/", dataString ); - assertEquals( dataString, n5.getAttribute( grp, "\\/", String.class ) ); - String jsonContents = readAttributesAsString( grp ); - assertEquals( rootSlash, jsonContents ); - - // "abc/def" as key - grp = "b"; - n5.createGroup( grp ); - n5.setAttribute( grp, "abc\\/def", dataString ); - assertEquals( dataString, n5.getAttribute( grp, "abc\\/def", String.class ) ); - jsonContents = readAttributesAsString( grp ); - assertEquals( abcdef, jsonContents ); - - // "[0]" as a key - grp = "c"; - n5.createGroup( grp ); - n5.setAttribute( grp, "\\[0]", dataString ); - assertEquals( dataString, n5.getAttribute( grp, "\\[0]", String.class ) ); - jsonContents = readAttributesAsString( grp ); - assertEquals( zero, jsonContents ); - - // "]] [] [[" as a key - grp = "d"; - n5.createGroup( grp ); - n5.setAttribute( grp, bracketsKey, dataString ); - assertEquals( dataString, n5.getAttribute( grp, bracketsKey, String.class ) ); - jsonContents = readAttributesAsString( grp ); - assertEquals( brackets, jsonContents ); - - // "[[2][33]]" as a key - grp = "e"; - n5.createGroup( grp ); - n5.setAttribute( grp, "[\\[2]\\[33]]", dataString ); - assertEquals( dataString, n5.getAttribute( grp, "[\\[2]\\[33]]", String.class ) ); - jsonContents = readAttributesAsString( grp ); - assertEquals( doubleBrackets, jsonContents ); - - // "\\" as key - grp = "f"; - n5.createGroup( grp ); - n5.setAttribute( grp, "\\\\\\\\", dataString ); - assertEquals( dataString, n5.getAttribute( grp, "\\\\\\\\", String.class ) ); - jsonContents = readAttributesAsString( grp ); - assertEquals( doubleBackslash, jsonContents ); - - // clear - n5.setAttribute( grp, "/", emptyObj ); - jsonContents = readAttributesAsString( grp ); - assertEquals( empty, jsonContents ); - } - - /* - * For readability above - */ - private String jsonKeyVal( String key, String val ) { - - return String.format( "{\"%s\":\"%s\"}", key, val ); - } - - private String readAttributesAsString( final String group ) { - - final String basePath = ((N5FSWriter)n5).getBasePath(); - try - { - return new String( Files.readAllBytes( Paths.get( basePath, group, "attributes.json" ))); - } catch ( IOException e ) { } - return null; - } - } \ No newline at end of file From 607314d0703d4463119da7f094dd64a1330099f8 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Thu, 23 Feb 2023 16:04:52 -0500 Subject: [PATCH 089/243] feat(test): add whitespace and escaped-character tests --- .../saalfeldlab/n5/AbstractN5Test.java | 14 ++++ .../org/janelia/saalfeldlab/n5/N5URLTest.java | 77 +++++++++++-------- 2 files changed, 59 insertions(+), 32 deletions(-) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java index 787ccbe6..870571fd 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java @@ -882,6 +882,20 @@ public void testAttributePaths() throws IOException { addAndTest(existingTests, new TestData<>(testGroup, "/array[1][2]/[3]key2", "array3_value3")); addAndTest(existingTests, new TestData<>(testGroup, "array[1]/[2][3]/key2", "array3_value4")); addAndTest(existingTests, new TestData<>(testGroup, "/array/[1]/[2]/[3]/key2", "array3_value5")); + /* test with whitespace*/ + addAndTest(existingTests, new TestData<>(testGroup, " ", "space")); + addAndTest(existingTests, new TestData<>(testGroup, "\n", "newline")); + addAndTest(existingTests, new TestData<>(testGroup, "\t", "tab")); + addAndTest(existingTests, new TestData<>(testGroup, "\r\n", "windows_newline")); + addAndTest(existingTests, new TestData<>(testGroup, " \n\t \t \n \r\n\r\n", "mixed")); + /* test URI encoded characters inside square braces */ + addAndTest(existingTests, new TestData<>(testGroup, "[ ]", "space")); + addAndTest(existingTests, new TestData<>(testGroup, "[\n]", "newline")); + addAndTest(existingTests, new TestData<>(testGroup, "[\t]", "tab")); + addAndTest(existingTests, new TestData<>(testGroup, "[\r\n]", "windows_newline")); + addAndTest(existingTests, new TestData<>(testGroup, "[ ][\n][\t][ \t \n \r\n][\r\n]", "mixed")); + addAndTest(existingTests, new TestData<>(testGroup, "[ ][\\n][\\t][ \\t \\n \\r\\n][\\r\\n]", "mixed")); + addAndTest(existingTests, new TestData<>(testGroup, "[\\]", "backslash")); /* Non String tests */ diff --git a/src/test/java/org/janelia/saalfeldlab/n5/N5URLTest.java b/src/test/java/org/janelia/saalfeldlab/n5/N5URLTest.java index b81633fc..74b91b54 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/N5URLTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/N5URLTest.java @@ -11,42 +11,55 @@ public class N5URLTest { @Test public void testAttributePath() { - assertEquals("/a/b/c/d/e", N5URL.normalizeAttributePath("/a/b/c/d/e")); - assertEquals("/a/b/c/d/e", N5URL.normalizeAttributePath("/a/b/c/d/e/")); - assertEquals("/", N5URL.normalizeAttributePath("/")); - assertEquals("", N5URL.normalizeAttributePath("")); - assertEquals("/", N5URL.normalizeAttributePath("/a/..")); - assertEquals("/", N5URL.normalizeAttributePath("/a/../b/../c/d/../..")); - assertEquals("/", N5URL.normalizeAttributePath("/a/../b/../c/d/../..")); - assertEquals("/", N5URL.normalizeAttributePath("/a/../b/../c/d/../..")); - assertEquals("/", N5URL.normalizeAttributePath("/./././././")); - assertEquals("/", N5URL.normalizeAttributePath("/./././././.")); - assertEquals("", N5URL.normalizeAttributePath("./././././")); - - assertEquals("/a/[0]/b/[0]", N5URL.normalizeAttributePath("/a/[0]/b[0]/")); - assertEquals("[0]", N5URL.normalizeAttributePath("[0]")); - assertEquals("/[0]", N5URL.normalizeAttributePath("/[0]/")); - assertEquals("/a/[0]", N5URL.normalizeAttributePath("/a[0]/")); - assertEquals("/[0]/b", N5URL.normalizeAttributePath("/[0]b/")); - - assertEquals("[b]", N5URL.normalizeAttributePath("[b]")); - assertEquals("a[b]c", N5URL.normalizeAttributePath("a[b]c")); - assertEquals("a[bc", N5URL.normalizeAttributePath("a[bc")); - assertEquals("ab]c", N5URL.normalizeAttributePath("ab]c")); - assertEquals("a[b00]c", N5URL.normalizeAttributePath("a[b00]c")); - - assertEquals("let's/try/a/real/case/with spaces", N5URL.normalizeAttributePath("let's/try/a/real/case/with spaces/")); - assertEquals("let's/try/a/real/case/with spaces", N5URL.normalizeAttributePath("let's/try/a/real/////case////with spaces/")); - assertEquals( "../first/relative/a/wd/.w/asd", N5URL.normalizeAttributePath("../first/relative/test/../a/b/.././wd///.w/asd")); - assertEquals( "../", N5URL.normalizeAttributePath("../result/../only/../single/..")); - assertEquals( "../..", N5URL.normalizeAttributePath("../result/../multiple/../..")); - - String normalizedPath = N5URL.normalizeAttributePath("let's/try/a/some/////with/ / //white spaces/"); - assertEquals("Normalizing a normal path should be the identity", normalizedPath, N5URL.normalizeAttributePath(normalizedPath)); + + assertEquals("/a/b/c/d/e", N5URL.normalizeAttributePath("/a/b/c/d/e")); + assertEquals("/a/b/c/d/e", N5URL.normalizeAttributePath("/a/b/c/d/e/")); + assertEquals("/", N5URL.normalizeAttributePath("/")); + assertEquals("", N5URL.normalizeAttributePath("")); + assertEquals("/", N5URL.normalizeAttributePath("/a/..")); + assertEquals("/", N5URL.normalizeAttributePath("/a/../b/../c/d/../..")); + assertEquals("/", N5URL.normalizeAttributePath("/a/../b/../c/d/../..")); + assertEquals("/", N5URL.normalizeAttributePath("/a/../b/../c/d/../..")); + assertEquals("/", N5URL.normalizeAttributePath("/./././././")); + assertEquals("/", N5URL.normalizeAttributePath("/./././././.")); + assertEquals("", N5URL.normalizeAttributePath("./././././")); + assertEquals("a.", N5URL.normalizeAttributePath("./a././././")); + assertEquals("\\\\.", N5URL.normalizeAttributePath("./a./../\\\\././.")); + assertEquals("a./\\\\.", N5URL.normalizeAttributePath("./a./\\\\././.")); + assertEquals("/a./\\\\.", N5URL.normalizeAttributePath("/./a./\\\\././.")); + + assertEquals("/a/[0]/b/[0]", N5URL.normalizeAttributePath("/a/[0]/b[0]/")); + assertEquals("[0]", N5URL.normalizeAttributePath("[0]")); + assertEquals("/[0]", N5URL.normalizeAttributePath("/[0]/")); + assertEquals("/a/[0]", N5URL.normalizeAttributePath("/a[0]/")); + assertEquals("/[0]/b", N5URL.normalizeAttributePath("/[0]b/")); + + assertEquals("[b]", N5URL.normalizeAttributePath("[b]")); + assertEquals("a[b]c", N5URL.normalizeAttributePath("a[b]c")); + assertEquals("a[bc", N5URL.normalizeAttributePath("a[bc")); + assertEquals("ab]c", N5URL.normalizeAttributePath("ab]c")); + assertEquals("a[b00]c", N5URL.normalizeAttributePath("a[b00]c")); + + assertEquals("[ ]", N5URL.normalizeAttributePath("[ ]")); + assertEquals("[\n]", N5URL.normalizeAttributePath("[\n]")); + assertEquals("[\t]", N5URL.normalizeAttributePath("[\t]")); + assertEquals("[\r\n]", N5URL.normalizeAttributePath("[\r\n]")); + assertEquals("[ ][\n][\t][ \t \n \r\n][\r\n]", N5URL.normalizeAttributePath("[ ][\n][\t][ \t \n \r\n][\r\n]")); + assertEquals("[\\]", N5URL.normalizeAttributePath("[\\]")); + + assertEquals("let's/try/a/real/case/with spaces", N5URL.normalizeAttributePath("let's/try/a/real/case/with spaces/")); + assertEquals("let's/try/a/real/case/with spaces", N5URL.normalizeAttributePath("let's/try/a/real/////case////with spaces/")); + assertEquals("../first/relative/a/wd/.w/asd", N5URL.normalizeAttributePath("../first/relative/test/../a/b/.././wd///.w/asd")); + assertEquals("../", N5URL.normalizeAttributePath("../result/../only/../single/..")); + assertEquals("../..", N5URL.normalizeAttributePath("../result/../multiple/../..")); + + String normalizedPath = N5URL.normalizeAttributePath("let's/try/a/some/////with/ / //white spaces/"); + assertEquals("Normalizing a normal path should be the identity", normalizedPath, N5URL.normalizeAttributePath(normalizedPath)); } @Test public void testEscapedAttributePaths() { + assertEquals("\\/a\\/b\\/c\\/d\\/e", N5URL.normalizeAttributePath("\\/a\\/b\\/c\\/d\\/e")); assertEquals("/a\\\\/b/c", N5URL.normalizeAttributePath("/a\\\\/b/c")); assertEquals("a[b]\\[10]", N5URL.normalizeAttributePath("a[b]\\[10]")); From bfa7dd4ad0c85d8d60e08d739fc9bf60bfe4c1a9 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Thu, 23 Feb 2023 16:05:42 -0500 Subject: [PATCH 090/243] fix(test): evaluate `\` through json properly --- .../java/org/janelia/saalfeldlab/n5/AbstractN5Test.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java index 870571fd..98ec180b 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java @@ -969,7 +969,7 @@ public void testAttributePathEscaping() throws IOException { final String zeroKey = "[0]"; final String bracketsKey = "]] [] [["; final String doubleBracketsKey = "[[2][33]]"; - final String doubleBackslashKey = "\\\\"; + final String doubleBackslashKey = "\\\\\\\\"; //Evaluates to `\\` through java and json final String dataString = "dataString"; final String rootSlash = jsonKeyVal( slashKey, dataString ); @@ -1019,11 +1019,11 @@ public void testAttributePathEscaping() throws IOException { jsonContents = readAttributesAsString( grp ); assertEquals( doubleBrackets, jsonContents ); - // "\\" as key FIXME: Do we allow escaping escape characters? They shouldn't need to be, can just add normally if not escaping anything + // "\\" as key grp = "f"; n5.createGroup( grp ); - n5.setAttribute( grp, "\\\\\\\\", dataString ); - assertEquals( dataString, n5.getAttribute( grp, "\\\\\\\\", String.class ) ); + n5.setAttribute( grp, "\\\\", dataString ); + assertEquals( dataString, n5.getAttribute( grp, "\\\\", String.class ) ); jsonContents = readAttributesAsString( grp ); assertEquals( doubleBackslash, jsonContents ); From b92ecfb4018cd4df276bc32a7368ad00903b0bbd Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Thu, 23 Feb 2023 16:08:24 -0500 Subject: [PATCH 091/243] fix: issue where attribute was not fully decoding encoded characters This is intentional in the URI spec, but not what we want for this use case. Unfortunately it requires us to copy `private static` methods, with minor modifications, for internal use, --- .../org/janelia/saalfeldlab/n5/N5URL.java | 103 +++++++++++++++++- 1 file changed, 102 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java b/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java index 8a709459..e54b53df 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java @@ -1,7 +1,14 @@ package org.janelia.saalfeldlab.n5; +import sun.nio.cs.ThreadLocalCoders; + import java.net.URI; import java.net.URISyntaxException; +import java.nio.ByteBuffer; +import java.nio.CharBuffer; +import java.nio.charset.CharsetDecoder; +import java.nio.charset.CoderResult; +import java.nio.charset.CodingErrorAction; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -36,7 +43,7 @@ public N5URL(URI uri) { container = uri.getScheme() + ":" + schemeSpecificPartWithoutQuery; } group = uri.getQuery(); - attribute = uri.getFragment(); + attribute = decodeFragment(uri.getRawFragment()); } /** @@ -477,4 +484,98 @@ public static N5URL from(String container, String group, String attribute) throw final String attributePart = attribute != null ? "#" + attribute : "#"; return new N5URL(containerPart + groupPart + attributePart); } + + /** + * Intentionally copied from {@link URI} for internal use + * + * @see URI#decode(char) + */ + private static int decode(char c) { + + if ((c >= '0') && (c <= '9')) + return c - '0'; + if ((c >= 'a') && (c <= 'f')) + return c - 'a' + 10; + if ((c >= 'A') && (c <= 'F')) + return c - 'A' + 10; + assert false; + return -1; + } + + /** + * Intentionally copied from {@link URI} for internal use + * + * @see URI#decode(char, char) + */ + private static byte decode(char c1, char c2) { + + return (byte)(((decode(c1) & 0xf) << 4) + | ((decode(c2) & 0xf) << 0)); + } + + /** + * Modifier from {@link URI#decode(String)} to ignore the listed Exception, where it doesn't decode escape values inside square braces. + *

      + * As an example of the origin implement, a backslash inside a square brace would be encoded to "[%5C]", and + * when calling {@code decode("[%5C]")} it would not decode to "[\]" since the encode escape sequence is inside square braces. + *

      + * We keep all the decoding logic in this modified version, EXCEPT, that we don't check for and ignore encoded sequences inside square braces. + *

      + * Thus, {@code decode("[%5C]")} -> "[\]". + * + * @see URI#decode(char, char) + */ + private static String decodeFragment(String rawFragment) { + + if (rawFragment == null) + return rawFragment; + int n = rawFragment.length(); + if (n == 0) + return rawFragment; + if (rawFragment.indexOf('%') < 0) + return rawFragment; + + StringBuffer sb = new StringBuffer(n); + ByteBuffer bb = ByteBuffer.allocate(n); + CharBuffer cb = CharBuffer.allocate(n); + CharsetDecoder dec = ThreadLocalCoders.decoderFor("UTF-8") + .onMalformedInput(CodingErrorAction.REPLACE) + .onUnmappableCharacter(CodingErrorAction.REPLACE); + + // This is not horribly efficient, but it will do for now + char c = rawFragment.charAt(0); + boolean betweenBrackets = false; + + for (int i = 0; i < n; ) { + assert c == rawFragment.charAt(i); // Loop invariant + if (c != '%') { + sb.append(c); + if (++i >= n) + break; + c = rawFragment.charAt(i); + continue; + } + bb.clear(); + int ui = i; + for (; ; ) { + assert (n - i >= 2); + bb.put(decode(rawFragment.charAt(++i), rawFragment.charAt(++i))); + if (++i >= n) + break; + c = rawFragment.charAt(i); + if (c != '%') + break; + } + bb.flip(); + cb.clear(); + dec.reset(); + CoderResult cr = dec.decode(bb, cb, true); + assert cr.isUnderflow(); + cr = dec.flush(cb); + assert cr.isUnderflow(); + sb.append(cb.flip().toString()); + } + + return sb.toString(); + } } From ea203352e93ce7e304fd9bceefb10095f4aaa3c3 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Thu, 23 Feb 2023 16:13:29 -0500 Subject: [PATCH 092/243] feat!: initial `removeAttribute` support, separate get/parse/read attribute methods --- .../saalfeldlab/n5/GsonAttributesParser.java | 168 +++++++++++++++--- 1 file changed, 142 insertions(+), 26 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonAttributesParser.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonAttributesParser.java index 9bb97c12..3fbf9d34 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonAttributesParser.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonAttributesParser.java @@ -81,51 +81,167 @@ static T readAttribute(final JsonElement root, final String normalizedAttrib static T readAttribute(final JsonElement root, final String normalizedAttributePath, final Type type, final Gson gson) { + final JsonElement attribute = getAttribute(root, normalizedAttributePath); + return parseAttributeElement(attribute, gson, type); + } + + /** + * Deserialize the {@code attribute} as {@link Type type} {@link T}. + * + * @param attribute to deserialize as {@link Type type} + * @param gson used to deserialize {@code attribute} + * @param type to desrialize {@code attribute} as + * @param return type represented by {@link Type type} + * @return the deserialized attribute object, or {@code null} if {@code attribute} cannot deserialize to {@link T} + */ + static T parseAttributeElement(JsonElement attribute, Gson gson, Type type) { + + if (attribute == null) + return null; + final Class clazz = (type instanceof Class) ? ((Class)type) : null; - JsonElement json = root; - for (final String pathPart : normalizedAttributePath.split("(?= jsonArray.size()) { - return null; - } - json = jsonArray.get(index); - } else { - return null; - } - } - } if (clazz != null && clazz.isAssignableFrom(HashMap.class)) { Type mapType = new TypeToken>() { }.getType(); - Map retMap = gson.fromJson(json, mapType); + Map retMap = gson.fromJson(attribute, mapType); //noinspection unchecked return (T)retMap; } - if (json instanceof JsonArray) { - final JsonArray array = json.getAsJsonArray(); + if (attribute instanceof JsonArray) { + final JsonArray array = attribute.getAsJsonArray(); T retArray = GsonAttributesParser.getJsonAsArray(gson, array, type); if (retArray != null) return retArray; } try { - return gson.fromJson(json, type); + return gson.fromJson(attribute, type); } catch (JsonSyntaxException e) { if (type == String.class) - return (T)gson.toJson(json); + return (T)gson.toJson(attribute); return null; } } + /** + * If there is an attribute in {@code root} such that it can be parsed and desrialized as {@link T}, + * then remove it from {@code root}, write {@code root} to the {@code writer}, and return the removed attribute. + *

      + * If there is an attribute at the location specified by {@code normalizedAttributePath} but it cannot be deserialized to {@link T}, then it is not removed. + * + * If nothing is removed, then {@ecode root} is not writen to the {@code writer}. + * + * @param writer to write the modified {@code root} to after removal of the attribute + * @param root to remove the attribute from + * @param normalizedAttributePath to the attribute location + * @param cls of the attribute to remove + * @param gson to deserialize the attribute with + * @param of the removed attribute + * @return the removed attribute, or null if nothing removed + * @throws IOException + */ + static T removeAttribute( + final Writer writer, + JsonElement root, + final String normalizedAttributePath, + final Class cls, + final Gson gson) throws IOException { + + final T removed = removeAttribute(root, normalizedAttributePath, cls, gson); + //TODO: test how to remove `null` attribute + if (removed != null ) { + writeAttributes(writer, root, gson); + } + return removed; + } + + /** + * If there is an attribute in {@code root} such that it can be parsed and desrialized as {@link T}, + * then remove it from {@code root} and return the removed attribute. + *

      + * If there is an attribute at the location specified by {@code normalizedAttributePath} but it cannot be deserialized to {@link T}, then it is not removed. + * + * @param root to remove the attribute from + * @param normalizedAttributePath to the attribute location + * @param cls of the attribute to remove + * @param gson to deserialize the attribute with + * @param of the removed attribute + * @return the removed attribute, or null if nothing removed + */ + static T removeAttribute(final JsonElement root, final String normalizedAttributePath, final Class cls, final Gson gson) { + final T attribute = readAttribute(root, normalizedAttributePath, cls, gson); + if (attribute != null) { + removeAttribute(root, normalizedAttributePath); + } + return attribute; + } + + /** + * Removes the JsonElement at {@code normalizedAttributePath} from {@code root} if present. + * + * @param root to remove the attribute from + * @param normalizedAttributePath to the attribute location + * @return the removed {@link JsonElement}, or null if nothing removed. + */ + static JsonElement removeAttribute(JsonElement root, String normalizedAttributePath) { + return getAttribute(root, normalizedAttributePath, true); + } + + /** + * Return the attribute at {@code normalizedAttributePath} as a {@link JsonElement}. + * Does not attempt to parse the attribute. + * + * @param root to search for the {@link JsonElement} at location {@code normalizedAttributePath} + * @param normalizedAttributePath to the attribute + * @return the attribute as a {@link JsonElement}. + */ + static JsonElement getAttribute(JsonElement root, String normalizedAttributePath) { + return getAttribute(root, normalizedAttributePath, false); + } + + /** + * Return the attribute at {@code normalizedAttributePath} as a {@link JsonElement}. If {@code remove}, remove the attribute from {@code root}. + * Does not attempt to parse the attribute. + * + * @param root to search for the {@link JsonElement} at location {@code normalizedAttributePath} + * @param normalizedAttributePath to the attribute + * @param remove the attribute {code @JsonElement} after finding it. + * @return the attribute as a {@link JsonElement}. + */ + static JsonElement getAttribute(JsonElement root, String normalizedAttributePath, boolean remove) { + + for (final String pathPart : normalizedAttributePath.split("(?= jsonArray.size()) { + return null; + } + root = jsonArray.get(index); + if (remove) { + jsonArray.remove(index); + } + } else { + return null; + } + } + } + return root; + } + /** * Inserts {@code attribute} into {@code root} at location {@code normalizedAttributePath} and write the resulting {@code root}. * From 62ed447503e06c2b40a2834bba4064f165ba1426 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Mon, 27 Feb 2023 15:21:19 -0500 Subject: [PATCH 093/243] feat: properly normalize path, respecting escape sequences --- .../org/janelia/saalfeldlab/n5/N5URL.java | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java b/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java index e54b53df..c8ae0df2 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java @@ -406,14 +406,19 @@ public static String normalizePath(String path) { */ public static String normalizeAttributePath(String attributePath) { + /* Add separator after arrays at the beginning `[10]b` -> `[10]/b`*/ final String attrPathPlusFirstIndexSeparator = attributePath.replaceAll("^(?\\[[0-9]+])", "${array}/"); - final String attrPathPlusIndexSeparators = attrPathPlusFirstIndexSeparator.replaceAll("((?[^\\\\])(?\\[[0-9]+]))", - "${prev}/${array}/"); - final String attrPathRemoveMultipleSeparators = attrPathPlusIndexSeparators.replaceAll("(?/)/+", "${slash}"); - final String attrPathNoDot = attrPathRemoveMultipleSeparators.replaceAll("((?[^(^|\\\\)])/$", "${nonSlash}"); - - final Pattern relativePathPattern = Pattern.compile("[^(/|\\.\\.)]+/\\.\\./?"); + /* Add separator before and after arrays not at the beginning `a[10]b` -> `a/[10]/b`*/ + final String attrPathPlusIndexSeparators = attrPathPlusFirstIndexSeparator.replaceAll("((?\\[[0-9]+]))", "/${array}/"); + /* remove redundant separators `a///b` -> `a/b`*/ + final String attrPathRemoveMultipleSeparators = attrPathPlusIndexSeparators.replaceAll("(?<=/)/+", ""); + /* remove redundant paths `a/./b` -> `a/b``*/ + final String attrPathNoDot = attrPathRemoveMultipleSeparators.replaceAll("(?<=(/|^))(\\./)+|((/|(?<=/))\\.)$", ""); + /* remove trailing separator `a/b/c/` -> `a/b/c` */ + final String normalizedAttributePath = attrPathNoDot.replaceAll("(? b*/ + final Pattern relativePathPattern = Pattern.compile("(?<=(/|^))[^/]+(? Date: Mon, 27 Feb 2023 15:21:30 -0500 Subject: [PATCH 094/243] feat!: add explicit `removeAttribute` methods --- .../saalfeldlab/n5/GsonAttributesParser.java | 32 +++++++-- .../janelia/saalfeldlab/n5/N5FSWriter.java | 69 +++++++++++++++++++ .../org/janelia/saalfeldlab/n5/N5Writer.java | 44 +++++++++++- 3 files changed, 138 insertions(+), 7 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonAttributesParser.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonAttributesParser.java index 3fbf9d34..c5e4a9fb 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonAttributesParser.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonAttributesParser.java @@ -148,13 +148,35 @@ static T removeAttribute( final Gson gson) throws IOException { final T removed = removeAttribute(root, normalizedAttributePath, cls, gson); - //TODO: test how to remove `null` attribute if (removed != null ) { writeAttributes(writer, root, gson); } return removed; } + /** + * If there is an attribute in {@code root} at location {@code normalizedAttributePath} then remove it from {@code root}.. + * + * @param writer to write the modified {@code root} to after removal of the attribute + * @param root to remove the attribute from + * @param normalizedAttributePath to the attribute location + * @param gson to deserialize the attribute with + * @return if the attribute was removed or not + */ + static boolean removeAttribute( + final Writer writer, + final JsonElement root, + final String normalizedAttributePath, + final Gson gson) throws IOException { + final JsonElement removed = removeAttribute(root, normalizedAttributePath, JsonElement.class, gson); + //TODO: test how to remove `null` attribute + if (removed != null ) { + writeAttributes(writer, root, gson); + return true; + } + return false; + } + /** * If there is an attribute in {@code root} such that it can be parsed and desrialized as {@link T}, * then remove it from {@code root} and return the removed attribute. @@ -210,7 +232,9 @@ static JsonElement getAttribute(JsonElement root, String normalizedAttributePath */ static JsonElement getAttribute(JsonElement root, String normalizedAttributePath, boolean remove) { - for (final String pathPart : normalizedAttributePath.split("(? T removeAttribute(String pathName, String key, Class cls) throws IOException { + + if (!exists(pathName)) + return null; + + final Path path = Paths.get(basePath, getAttributesPath(pathName).toString()); + JsonElement attributesRoot = getAttributes(pathName); + try (final LockedFileChannel lockedFileChannel = LockedFileChannel.openForWriting(path)) { + final String attributePath = N5URL.normalizeAttributePath(key); + + final T removed = GsonAttributesParser.removeAttribute(attributesRoot, attributePath, cls, gson); + if (removed != null ) { + lockedFileChannel.getFileChannel().truncate(0); + final Writer writer = Channels.newWriter(lockedFileChannel.getFileChannel(), StandardCharsets.UTF_8.name()); + GsonAttributesParser.writeAttributes(writer, attributesRoot, gson); + } + return removed; + } + } + + @Override + public boolean removeAttributes( + final String pathName, + final List attributes) throws IOException { + + if(!exists(pathName)) + return false; + + final Path path = Paths.get(basePath, getAttributesPath(pathName).toString()); + JsonObject newRoot = new JsonObject(); + + try (final LockedFileChannel lockedFileChannel = LockedFileChannel.openForWriting(path)) { + final JsonElement oldRoot = GsonAttributesParser.readAttributes(Channels.newReader(lockedFileChannel.getFileChannel(), StandardCharsets.UTF_8.name()), getGson()); + if (oldRoot != null && oldRoot.isJsonObject()) + newRoot = oldRoot.getAsJsonObject(); + + + boolean removed = false; + for (String attribute : attributes) { + final String attributePath = N5URL.normalizeAttributePath(attribute); + removed |= GsonAttributesParser.removeAttribute(newRoot, attributePath) != null; + } + if (removed) { + lockedFileChannel.getFileChannel().truncate(0); + final Writer writer = Channels.newWriter(lockedFileChannel.getFileChannel(), StandardCharsets.UTF_8.name()); + GsonAttributesParser.writeAttributes(writer, newRoot, getGson()); + } + + return removed; + } + } + @Override public void writeBlock( final String pathName, diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java index 47a2db65..9f6fba0d 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java @@ -30,6 +30,7 @@ import java.io.ObjectOutputStream; import java.io.Serializable; import java.util.Collections; +import java.util.List; import java.util.Map; /** @@ -42,7 +43,7 @@ public interface N5Writer extends N5Reader { /** - * Sets an attribute. Setting an attribute to null removes the attribute. + * Sets an attribute. * * @param pathName group path * @param key @@ -58,8 +59,7 @@ public default void setAttribute( } /** - * Sets a map of attributes. Any attribute with a value null will will be - * removed. + * Sets a map of attributes. * * @param pathName group path * @param attributes @@ -69,6 +69,44 @@ public void setAttributes( final String pathName, final Map attributes) throws IOException; + /** + * Remove the attribute from group {@code pathName} with key {@code key}. + * + * @param pathName group path + * @param key of attribute to remove + * @return true if attribute removed, else false + * @throws IOException + */ + boolean removeAttribute(String pathName, String key) throws IOException; + + /** + * Remove the attribute from group {@code pathName} with key {@code key} and type {@link T}. + *

      + * If an attribute at {@code pathName} and {@code key} exists, but is not of type {@link T}, it is not removed. + * + * @param pathName group path + * @param key of attribute to remove + * @param cls of the attribute to remove + * @param of the attribute + * @return the removed attribute, as {@link T}, or {@code null} if no matching attribute + * @throws IOException + */ + T removeAttribute(String pathName, String key, Class cls) throws IOException; + + /** + * Remove attributes as provided by {@code attributes}. + *

      + * If any element of {@code attributes} does not exist, it wil be ignored. + * If at least one attribute from {@code attributes} is removed, this will return {@code true}. + * + * + * @param pathName group path + * @param attributes to remove + * @return true if any of the listed attributes were removed + * @throws IOException + */ + boolean removeAttributes( String pathName, List attributes) throws IOException; + /** * Sets mandatory dataset attributes. * From 233c30cd134629ff7977180751f6757cba7ae81e Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Mon, 27 Feb 2023 15:22:22 -0500 Subject: [PATCH 095/243] feat(test): add tests for `removeAttribute(s)` --- .../saalfeldlab/n5/AbstractN5Test.java | 235 +++++++++++++++++- .../org/janelia/saalfeldlab/n5/N5FSTest.java | 9 +- 2 files changed, 228 insertions(+), 16 deletions(-) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java index 98ec180b..ab1b03a2 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java @@ -16,8 +16,13 @@ */ package org.janelia.saalfeldlab.n5; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; +import com.google.gson.GsonBuilder; +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; import com.google.gson.JsonNull; import com.google.gson.JsonObject; import java.io.File; @@ -78,9 +83,16 @@ public abstract class AbstractN5Test { protected abstract N5Writer createN5Writer() throws IOException; - protected abstract N5Writer createN5Writer(String location) throws IOException; + protected N5Writer createN5Writer(String location) throws IOException { - protected abstract N5Reader createN5Reader(String location) throws IOException; + return createN5Writer(location, new GsonBuilder()); + } + protected abstract N5Writer createN5Writer(String location, GsonBuilder gson) throws IOException; + + protected N5Reader createN5Reader(String location) throws IOException { + return createN5Reader(location, new GsonBuilder()); + } + protected abstract N5Reader createN5Reader(String location, GsonBuilder gson) throws IOException; protected Compression[] getCompressions() { @@ -461,12 +473,211 @@ public void testAttributes() { Assert.assertEquals(new Integer(1), n5.getAttribute(groupName, "key1", new TypeToken(){}.getType())); Assert.assertEquals(new Integer(2), n5.getAttribute(groupName, "key2", new TypeToken(){}.getType())); Assert.assertEquals("value3", n5.getAttribute(groupName, "key3", new TypeToken(){}.getType())); - } catch (final IOException e) { fail(e.getMessage()); } } + @Test + public void testNullAttributes() throws IOException { + File tmpFile = Files.createTempDirectory("nulls-test-").toFile(); + tmpFile.deleteOnExit(); + String canonicalPath = tmpFile.getCanonicalPath(); + /* serializeNulls*/ + try (N5Writer writer = createN5Writer(canonicalPath, new GsonBuilder().serializeNulls())) { + + writer.setAttribute(groupName, "nullValue", null); + assertEquals(null, writer.getAttribute(groupName, "nullValue", Object.class)); + assertEquals(JsonNull.INSTANCE, writer.getAttribute(groupName, "nullValue", JsonElement.class)); + final HashMap nulls = new HashMap<>(); + nulls.put("anotherNullValue", null); + nulls.put("structured/nullValue", null); + nulls.put("implicitNulls[3]", null); + writer.setAttributes(groupName, nulls); + + assertEquals(null, writer.getAttribute(groupName, "anotherNullValue", Object.class)); + assertEquals(JsonNull.INSTANCE, writer.getAttribute(groupName, "anotherNullValue", JsonElement.class)); + + assertEquals(null, writer.getAttribute(groupName, "structured/nullValue", Object.class)); + assertEquals(JsonNull.INSTANCE, writer.getAttribute(groupName, "structured/nullValue", JsonElement.class)); + + assertEquals(null, writer.getAttribute(groupName, "implicitNulls[3]", Object.class)); + assertEquals(JsonNull.INSTANCE, writer.getAttribute(groupName, "implicitNulls[3]", JsonElement.class)); + + assertEquals(null, writer.getAttribute(groupName, "implicitNulls[1]", Object.class)); + assertEquals(JsonNull.INSTANCE, writer.getAttribute(groupName, "implicitNulls[1]", JsonElement.class)); + + /* Negative test; a value that truly doesn't exist will still return `null` but will also return `null` when querying as a `JsonElement` */ + assertEquals(null, writer.getAttribute(groupName, "implicitNulls[10]", Object.class)); + assertEquals(null, writer.getAttribute(groupName, "implicitNulls[10]", JsonElement.class)); + + assertEquals(null, writer.getAttribute(groupName, "keyDoesn'tExist", Object.class)); + assertEquals(null, writer.getAttribute(groupName, "keyDoesn'tExist", JsonElement.class)); + + /* check existing value gets overwritten */ + writer.setAttribute(groupName, "existingValue", 1); + assertEquals((Integer)1, writer.getAttribute(groupName, "existingValue", Integer.class)); + writer.setAttribute(groupName, "existingValue", null); + assertEquals(null, writer.getAttribute(groupName, "existingValue", Integer.class)); + assertEquals(JsonNull.INSTANCE, writer.getAttribute(groupName, "existingValue", JsonElement.class)); + } + + + tmpFile = Files.createTempDirectory("nulls-test-").toFile(); + tmpFile.deleteOnExit(); + canonicalPath = tmpFile.getCanonicalPath(); + /* without serializeNulls*/ + try (N5Writer writer = createN5Writer(canonicalPath, new GsonBuilder())) { + + writer.setAttribute(groupName, "nullValue", null); + assertEquals(null, writer.getAttribute(groupName, "nullValue", Object.class)); + assertEquals(null, writer.getAttribute(groupName, "nullValue", JsonElement.class)); + final HashMap nulls = new HashMap<>(); + nulls.put("anotherNullValue", null); + nulls.put("structured/nullValue", null); + nulls.put("implicitNulls[3]", null); + writer.setAttributes(groupName, nulls); + + assertEquals(null, writer.getAttribute(groupName, "anotherNullValue", Object.class)); + assertEquals(null, writer.getAttribute(groupName, "anotherNullValue", JsonElement.class)); + + assertEquals(null, writer.getAttribute(groupName, "structured/nullValue", Object.class)); + assertEquals(null, writer.getAttribute(groupName, "structured/nullValue", JsonElement.class)); + + /* Arrays are still filled with `null`, regardless of `serializeNulls()`*/ + assertEquals(null, writer.getAttribute(groupName, "implicitNulls[3]", Object.class)); + assertEquals(JsonNull.INSTANCE, writer.getAttribute(groupName, "implicitNulls[3]", JsonElement.class)); + + assertEquals(null, writer.getAttribute(groupName, "implicitNulls[1]", Object.class)); + assertEquals(JsonNull.INSTANCE, writer.getAttribute(groupName, "implicitNulls[1]", JsonElement.class)); + + /* Negative test; a value that truly doesn't exist will still return `null` but will also return `null` when querying as a `JsonElement` */ + assertEquals(null, writer.getAttribute(groupName, "implicitNulls[10]", Object.class)); + assertEquals(null, writer.getAttribute(groupName, "implicitNulls[10]", JsonElement.class)); + + assertEquals(null, writer.getAttribute(groupName, "keyDoesn'tExist", Object.class)); + assertEquals(null, writer.getAttribute(groupName, "keyDoesn'tExist", JsonElement.class)); + + /* check existing value gets overwritten */ + writer.setAttribute(groupName, "existingValue", 1); + assertEquals((Integer)1, writer.getAttribute(groupName, "existingValue", Integer.class)); + writer.setAttribute(groupName, "existingValue", null); + assertEquals(null, writer.getAttribute(groupName, "existingValue", Integer.class)); + assertEquals(null, writer.getAttribute(groupName, "existingValue", JsonElement.class)); + } + } + + @Test + public void testRemoveAttributes() throws IOException { + final File tmpFile = Files.createTempDirectory("nulls-test-").toFile(); + tmpFile.deleteOnExit(); + final String canonicalPath = tmpFile.getCanonicalPath(); + try (N5Writer writer = createN5Writer(canonicalPath, new GsonBuilder().serializeNulls())) { + + writer.setAttribute("", "a/b/c", 100); + assertEquals((Integer)100, writer.getAttribute("", "a/b/c", Integer.class)); + /* Remove Test without Type */ + assertTrue(writer.removeAttribute("", "a/b/c")); + assertNull(writer.getAttribute("", "a/b/c", Integer.class)); + + writer.setAttribute("", "a/b/c", 100); + assertEquals((Integer)100, writer.getAttribute("", "a/b/c", Integer.class)); + /* Remove Test with correct Type */ + assertEquals((Integer)100, writer.removeAttribute("", "a/b/c", Integer.class)); + assertNull(writer.getAttribute("", "a/b/c", Integer.class)); + + writer.setAttribute("", "a/b/c", 100); + assertEquals((Integer)100, writer.getAttribute("", "a/b/c", Integer.class)); + /* Remove Test with incorrect Type */ + assertNull(writer.removeAttribute("", "a/b/c", Boolean.class)); + final Integer abcInteger = writer.removeAttribute("", "a/b/c", Integer.class); + assertEquals((Integer)100, abcInteger); + assertNull(writer.getAttribute("", "a/b/c", Integer.class)); + + writer.setAttribute("", "a/b/c", 100); + assertEquals((Integer)100, writer.getAttribute("", "a/b/c", Integer.class)); + /* Remove Test with non-leaf */ + assertTrue(writer.removeAttribute("", "a/b")); + assertNull(writer.getAttribute("", "a/b/c", Integer.class)); + assertNull(writer.getAttribute("", "a/b", JsonObject.class)); + + writer.setAttribute("", "a\\b\\c/b\\[10]\\c/c", 100); + assertEquals((Integer)100, writer.getAttribute("", "a\\b\\c/b\\[10]\\c/c", Integer.class)); + /* Remove Test with escape-requiring key */ + assertTrue(writer.removeAttribute("", "a\\b\\c/b\\[10]\\c/c")); + assertNull(writer.getAttribute("", "a\\b\\c/b\\[10]\\c/c", Integer.class)); + + writer.setAttribute("", "a/b[9]", 10); + assertEquals((Integer)10, writer.getAttribute("", "a/b[9]", Integer.class)); + assertEquals((Integer)0, writer.getAttribute("", "a/b[8]", Integer.class)); + /*Remove test with arrays */ + assertTrue(writer.removeAttribute("", "a/b[5]")); + assertEquals(9, writer.getAttribute("", "a/b", JsonArray.class).size()); + assertEquals((Integer)0, writer.getAttribute("", "a/b[5]", Integer.class)); + assertEquals((Integer)10, writer.getAttribute("", "a/b[8]", Integer.class)); + assertTrue(writer.removeAttribute("", "a/b[8]")); + assertEquals(8, writer.getAttribute("", "a/b", JsonArray.class).size()); + assertNull(writer.getAttribute("", "a/b[8]", Integer.class)); + assertTrue(writer.removeAttribute("", "a/b")); + assertNull(writer.getAttribute("", "a/b[9]", Integer.class)); + assertNull(writer.getAttribute("", "a/b", Integer.class)); + + /* ensure old remove behavior no longer works (i.e. set to null no longer should remove) */ + writer.setAttribute("", "a/b/c", 100); + assertEquals((Integer)100, writer.getAttribute("", "a/b/c", Integer.class)); + writer.setAttribute("", "a/b/c", null); + assertEquals(JsonNull.INSTANCE, writer.getAttribute("", "a/b/c", JsonNull.class)); + writer.removeAttribute("", "a/b/c"); + assertNull(writer.getAttribute("", "a/b/c", JsonNull.class)); + + /* remove multiple, all present */ + writer.setAttribute("", "a/b/c", 100); + writer.setAttribute("", "a/b/d", "test"); + writer.setAttribute("", "a/c[9]", 10); + writer.setAttribute("", "a/c[5]", 5); + + assertTrue(writer.removeAttributes("", Arrays.asList("a/b/c", "a/b/d", "a/c[5]"))); + assertNull(writer.getAttribute("", "a/b/c", Integer.class)); + assertNull(writer.getAttribute("", "a/b/d", String.class)); + assertEquals(9, writer.getAttribute("", "a/c", JsonArray.class).size()); + assertEquals((Integer)10, writer.getAttribute("", "a/c[8]", Integer.class)); + assertEquals((Integer)0, writer.getAttribute("", "a/c[5]", Integer.class)); + + /* remove multiple, any present */ + writer.setAttribute("", "a/b/c", 100); + writer.setAttribute("", "a/b/d", "test"); + writer.setAttribute("", "a/c[9]", 10); + writer.setAttribute("", "a/c[5]", 5); + + assertTrue(writer.removeAttributes("", Arrays.asList("a/b/c", "a/b/d", "a/x[5]"))); + assertNull(writer.getAttribute("", "a/b/c", Integer.class)); + assertNull(writer.getAttribute("", "a/b/d", String.class)); + assertEquals(10, writer.getAttribute("", "a/c", JsonArray.class).size()); + assertEquals((Integer)10, writer.getAttribute("", "a/c[9]", Integer.class)); + assertEquals((Integer)5, writer.getAttribute("", "a/c[5]", Integer.class)); + + /* remove multiple, none present */ + writer.setAttribute("", "a/b/c", 100); + writer.setAttribute("", "a/b/d", "test"); + writer.setAttribute("", "a/c[9]", 10); + writer.setAttribute("", "a/c[5]", 5); + + assertFalse(writer.removeAttributes("", Arrays.asList("X/b/c", "Z/b/d", "a/x[5]"))); + assertEquals((Integer)100, writer.getAttribute("", "a/b/c", Integer.class)); + assertEquals("test", writer.getAttribute("", "a/b/d", String.class)); + assertEquals(10, writer.getAttribute("", "a/c", JsonArray.class).size()); + assertEquals((Integer)10, writer.getAttribute("", "a/c[9]", Integer.class)); + assertEquals((Integer)5, writer.getAttribute("", "a/c[5]", Integer.class)); + + /* Test path normalization */ + writer.setAttribute("", "a/b/c", 100); + assertEquals((Integer)100, writer.getAttribute("", "a/b/c", Integer.class)); + assertEquals((Integer)100, writer.removeAttribute("", "a////b/x/../c", Integer.class)); + assertNull(writer.getAttribute("", "a/b/c", Integer.class)); + + } + } + @Test public void testRemove() { @@ -531,13 +742,13 @@ public void testDeepList() throws IOException, ExecutionException, InterruptedEx final List datasetList = Arrays.asList(writer.deepList("/")); for (final String subGroup : subGroupNames) Assert.assertTrue("deepList contents", datasetList.contains(groupName.replaceFirst("/", "") + "/" + subGroup)); - Assert.assertFalse("deepList stops at datasets", datasetList.contains(datasetName + "/0")); + assertFalse("deepList stops at datasets", datasetList.contains(datasetName + "/0")); final List datasetList2 = Arrays.asList(writer.deepList("")); for (final String subGroup : subGroupNames) Assert.assertTrue("deepList contents", datasetList2.contains(groupName.replaceFirst("/", "") + "/" + subGroup)); Assert.assertTrue("deepList contents", datasetList2.contains(datasetName.replaceFirst("/", ""))); - Assert.assertFalse("deepList stops at datasets", datasetList2.contains(datasetName + "/0")); + assertFalse("deepList stops at datasets", datasetList2.contains(datasetName + "/0")); final String prefix = "/test"; final String datasetSuffix = "group/dataset"; @@ -551,19 +762,19 @@ public void testDeepList() throws IOException, ExecutionException, InterruptedEx for (final String subGroup : subGroupNames) Assert.assertTrue("deepList contents", datasetListP.contains(groupName.replaceFirst("/", "") + "/" + subGroup)); Assert.assertTrue("deepList contents", datasetListP.contains(datasetName.replaceFirst("/", ""))); - Assert.assertFalse("deepList stops at datasets", datasetListP.contains(datasetName + "/0")); + assertFalse("deepList stops at datasets", datasetListP.contains(datasetName + "/0")); final List datasetListP2 = Arrays.asList(writer.deepList("", Executors.newFixedThreadPool(2))); for (final String subGroup : subGroupNames) Assert.assertTrue("deepList contents", datasetListP2.contains(groupName.replaceFirst("/", "") + "/" + subGroup)); Assert.assertTrue("deepList contents", datasetListP2.contains(datasetName.replaceFirst("/", ""))); - Assert.assertFalse("deepList stops at datasets", datasetListP2.contains(datasetName + "/0")); + assertFalse("deepList stops at datasets", datasetListP2.contains(datasetName + "/0")); final List datasetListP3 = Arrays.asList(writer.deepList(prefix, Executors.newFixedThreadPool(2))); for (final String subGroup : subGroupNames) Assert.assertTrue("deepList contents", datasetListP3.contains("group/" + subGroup)); Assert.assertTrue("deepList contents", datasetListP3.contains(datasetName.replaceFirst(prefix + "/", ""))); - Assert.assertFalse("deepList stops at datasets", datasetListP3.contains(datasetName + "/0")); + assertFalse("deepList stops at datasets", datasetListP3.contains(datasetName + "/0")); // test filtering final Predicate isCalledDataset = d -> { @@ -663,10 +874,10 @@ public void testExists() { n5.createGroup(groupName2); Assert.assertTrue(n5.exists(groupName2)); - Assert.assertFalse(n5.datasetExists(groupName2)); + assertFalse(n5.datasetExists(groupName2)); - Assert.assertFalse(n5.exists(notExists)); - Assert.assertFalse(n5.datasetExists(notExists)); + assertFalse(n5.exists(notExists)); + assertFalse(n5.datasetExists(notExists)); } catch (final IOException e) { fail(e.getMessage()); } @@ -734,7 +945,7 @@ public void testVersion() throws NumberFormatException, IOException { final String canonicalPath = tmpFile.getCanonicalPath(); try (N5Writer writer = createN5Writer(canonicalPath)) { writer.setAttribute("/", N5Reader.VERSION_KEY, new Version(N5Reader.VERSION.getMajor() + 1, N5Reader.VERSION.getMinor(), N5Reader.VERSION.getPatch()).toString()); - Assert.assertFalse(N5Reader.VERSION.isCompatible(writer.getVersion())); + assertFalse(N5Reader.VERSION.isCompatible(writer.getVersion())); } } diff --git a/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java b/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java index 9d995c27..4d6f6252 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java @@ -16,6 +16,7 @@ */ package org.janelia.saalfeldlab.n5; +import com.google.gson.GsonBuilder; import org.janelia.saalfeldlab.n5.url.UrlAttributeTest; import org.junit.Test; @@ -46,13 +47,13 @@ protected N5Writer createN5Writer() throws IOException { } @Override - protected N5Writer createN5Writer(String location) throws IOException { - return new N5FSWriter(location); + protected N5Writer createN5Writer(String location, GsonBuilder gson) throws IOException { + return new N5FSWriter(location, gson); } @Override - protected N5Reader createN5Reader(String location) throws IOException { - return new N5FSReader(location); + protected N5Reader createN5Reader(String location, GsonBuilder gson) throws IOException { + return new N5FSReader(location, gson); } @Test From 69e3c8fd8b8b95f01659b3f6d128bcdf120be330 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Fri, 3 Mar 2023 13:45:56 -0500 Subject: [PATCH 096/243] feat: add GsonN5Reader/Writer --- .../janelia/saalfeldlab/n5/GsonN5Reader.java | 425 ++++++++++++++++++ .../janelia/saalfeldlab/n5/GsonN5Writer.java | 238 ++++++++++ 2 files changed, 663 insertions(+) create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/GsonN5Reader.java create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/GsonN5Writer.java diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonN5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonN5Reader.java new file mode 100644 index 00000000..855c8274 --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonN5Reader.java @@ -0,0 +1,425 @@ +/** + * Copyright (c) 2017, Stephan Saalfeld + * All rights reserved. + *

      + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + *

      + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + *

      + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +package org.janelia.saalfeldlab.n5; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonPrimitive; +import com.google.gson.JsonSyntaxException; +import com.google.gson.reflect.TypeToken; + +import java.io.IOException; +import java.io.Reader; +import java.lang.reflect.Array; +import java.lang.reflect.Type; +import java.net.URISyntaxException; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; +import java.util.regex.Matcher; + +/** + * {@link N5Reader} for JSON attributes parsed by {@link Gson}. + * + * @author Stephan Saalfeld + */ +public interface GsonN5Reader extends N5Reader { + + Gson getGson(); + + /** + * Reads the attributes a group or dataset. + * + * @param pathName group path + * @return the root {@link JsonElement} of the attributes + * @throws IOException + */ + JsonElement getAttributes(final String pathName) throws IOException; + + @Override + default T getAttribute( + final String pathName, + final String key, + final Class clazz) throws IOException { + + return getAttribute(pathName, key, TypeToken.get(clazz).getType()); + } + + @Override + default T getAttribute( + final String pathName, + final String key, + final Type type) throws IOException { + + final String normalizedGroupPath; + final String normalizedAttributePath; + try { + final N5URL n5url = N5URL.from(null, pathName, key ); + normalizedGroupPath = n5url.normalizeGroupPath(); + normalizedAttributePath = n5url.normalizeAttributePath(); + } catch (URISyntaxException e) { + throw new IOException(e); + } + return GsonN5Reader.readAttribute( getAttributes( normalizedGroupPath ), normalizedAttributePath, type, getGson()); + } + + @Override + default Map> listAttributes(String pathName) throws IOException { + + return listAttributes(getAttributes(pathName)); + } + + + @Override + default DatasetAttributes getDatasetAttributes(final String pathName) throws IOException { + + final JsonElement root = getAttributes(pathName); + final Gson gson = getGson(); + + if (root == null || !root.isJsonObject()) + return null; + + final JsonObject rootObject = root.getAsJsonObject(); + + final long[] dimensions = GsonN5Reader.readAttribute(rootObject, DatasetAttributes.dimensionsKey, long[].class, gson); + if (dimensions == null) + return null; + + final DataType dataType = GsonN5Reader.readAttribute(rootObject, DatasetAttributes.dataTypeKey, DataType.class, gson); + if (dataType == null) + return null; + + int[] blockSize = GsonN5Reader.readAttribute(rootObject, DatasetAttributes.blockSizeKey, int[].class, gson); + if (blockSize == null) + blockSize = Arrays.stream(dimensions).mapToInt(a -> (int)a).toArray(); + + Compression compression = GsonN5Reader.readAttribute(rootObject, DatasetAttributes.compressionKey, Compression.class, gson); + + /* version 0 */ + if (compression == null) { + switch (GsonN5Reader.readAttribute(rootObject, DatasetAttributes.compressionTypeKey, String.class, gson)) { + case "raw": + compression = new RawCompression(); + break; + case "gzip": + compression = new GzipCompression(); + break; + case "bzip2": + compression = new Bzip2Compression(); + break; + case "lz4": + compression = new Lz4Compression(); + break; + case "xz": + compression = new XzCompression(); + break; + } + } + + return new DatasetAttributes(dimensions, blockSize, dataType, compression); + } + + + static Gson registerGson(final GsonBuilder gsonBuilder) { + gsonBuilder.registerTypeAdapter(DataType.class, new DataType.JsonAdapter()); + gsonBuilder.registerTypeHierarchyAdapter(Compression.class, CompressionAdapter.getJsonAdapter()); + gsonBuilder.disableHtmlEscaping(); + return gsonBuilder.create(); + } + + /** + * Reads the attributes json from a given {@link Reader}. + * + * @param reader + * @return the root {@link JsonObject} of the attributes + * @throws IOException + */ + static JsonElement readAttributes(final Reader reader, final Gson gson) throws IOException { + + final JsonElement json = gson.fromJson(reader, JsonElement.class); + return json; + } + + static T readAttribute(final JsonElement root, final String normalizedAttributePath, final Class cls, final Gson gson) { + + return readAttribute(root, normalizedAttributePath, TypeToken.get(cls).getType(), gson); + } + + static T readAttribute(final JsonElement root, final String normalizedAttributePath, final Type type, final Gson gson) { + + final JsonElement attribute = getAttribute(root, normalizedAttributePath); + return parseAttributeElement(attribute, gson, type); + } + + /** + * Deserialize the {@code attribute} as {@link Type type} {@link T}. + * + * @param attribute to deserialize as {@link Type type} + * @param gson used to deserialize {@code attribute} + * @param type to desrialize {@code attribute} as + * @param return type represented by {@link Type type} + * @return the deserialized attribute object, or {@code null} if {@code attribute} cannot deserialize to {@link T} + */ + static T parseAttributeElement(JsonElement attribute, Gson gson, Type type) { + + if (attribute == null) + return null; + + final Class clazz = (type instanceof Class) ? ((Class)type) : null; + if (clazz != null && clazz.isAssignableFrom(HashMap.class)) { + Type mapType = new TypeToken>() { + + }.getType(); + Map retMap = gson.fromJson(attribute, mapType); + //noinspection unchecked + return (T)retMap; + } + if (attribute instanceof JsonArray) { + final JsonArray array = attribute.getAsJsonArray(); + T retArray = GsonN5Reader.getJsonAsArray(gson, array, type); + if (retArray != null) + return retArray; + } + try { + return gson.fromJson(attribute, type); + } catch (JsonSyntaxException e) { + if (type == String.class) + return (T)gson.toJson(attribute); + return null; + } + } + + /** + * Return the attribute at {@code normalizedAttributePath} as a {@link JsonElement}. + * Does not attempt to parse the attribute. + * + * @param root to search for the {@link JsonElement} at location {@code normalizedAttributePath} + * @param normalizedAttributePath to the attribute + * @return the attribute as a {@link JsonElement}. + */ + static JsonElement getAttribute(JsonElement root, String normalizedAttributePath) { + + final String[] pathParts = normalizedAttributePath.split("(?= jsonArray.size()) { + return null; + } + root = jsonArray.get(index); + } else { + return null; + } + } + } + return root; + } + + /** + * Best effort implementation of {@link N5Reader#listAttributes(String)} + * with limited type resolution. Possible return types are + *

        + *
      • null
        • + *
      • boolean
        • + *
      • double
        • + *
      • String
        • + *
      • Object
        • + *
      • boolean[]
        • + *
      • double[]
        • + *
      • String[]
        • + *
      • Object[]
        • + *
      + */ + static Map> listAttributes(JsonElement root) throws IOException { + + if (root != null && !root.isJsonObject()) { + return null; + } + + final HashMap> attributes = new HashMap<>(); + root.getAsJsonObject().entrySet().forEach(entry -> { + final Class clazz; + final String key = entry.getKey(); + final JsonElement jsonElement = entry.getValue(); + if (jsonElement.isJsonNull()) + clazz = null; + else if (jsonElement.isJsonPrimitive()) + clazz = classForJsonPrimitive((JsonPrimitive)jsonElement); + else if (jsonElement.isJsonArray()) { + final JsonArray jsonArray = (JsonArray)jsonElement; + Class arrayElementClass = Object.class; + if (jsonArray.size() > 0) { + final JsonElement firstElement = jsonArray.get(0); + if (firstElement.isJsonPrimitive()) { + arrayElementClass = classForJsonPrimitive(firstElement.getAsJsonPrimitive()); + for (int i = 1; i < jsonArray.size() && arrayElementClass != Object.class; ++i) { + final JsonElement element = jsonArray.get(i); + if (element.isJsonPrimitive()) { + final Class nextArrayElementClass = classForJsonPrimitive(element.getAsJsonPrimitive()); + if (nextArrayElementClass != arrayElementClass) + if (nextArrayElementClass == double.class && arrayElementClass == long.class) + arrayElementClass = double.class; + else { + arrayElementClass = Object.class; + break; + } + } else { + arrayElementClass = Object.class; + break; + } + } + } + clazz = Array.newInstance(arrayElementClass, 0).getClass(); + } else + clazz = Object[].class; + } else + clazz = Object.class; + attributes.put(key, clazz); + }); + return attributes; + } + + static T getJsonAsArray(Gson gson, JsonArray array, Class cls) { + + return getJsonAsArray(gson, array, TypeToken.get(cls).getType()); + } + + static T getJsonAsArray(Gson gson, JsonArray array, Type type) { + + final Class clazz = (type instanceof Class) ? ((Class)type) : null; + + if (type == boolean[].class) { + final boolean[] retArray = new boolean[array.size()]; + for (int i = 0; i < array.size(); i++) { + final Boolean value = gson.fromJson(array.get(i), boolean.class); + retArray[i] = value; + } + return (T)retArray; + } else if (type == double[].class) { + final double[] retArray = new double[array.size()]; + for (int i = 0; i < array.size(); i++) { + final double value = gson.fromJson(array.get(i), double.class); + retArray[i] = value; + } + return (T)retArray; + } else if (type == float[].class) { + final float[] retArray = new float[array.size()]; + for (int i = 0; i < array.size(); i++) { + final float value = gson.fromJson(array.get(i), float.class); + retArray[i] = value; + } + return (T)retArray; + } else if (type == long[].class) { + final long[] retArray = new long[array.size()]; + for (int i = 0; i < array.size(); i++) { + final long value = gson.fromJson(array.get(i), long.class); + retArray[i] = value; + } + return (T)retArray; + } else if (type == short[].class) { + final short[] retArray = new short[array.size()]; + for (int i = 0; i < array.size(); i++) { + final short value = gson.fromJson(array.get(i), short.class); + retArray[i] = value; + } + return (T)retArray; + } else if (type == int[].class) { + final int[] retArray = new int[array.size()]; + for (int i = 0; i < array.size(); i++) { + final int value = gson.fromJson(array.get(i), int.class); + retArray[i] = value; + } + return (T)retArray; + } else if (type == byte[].class) { + final byte[] retArray = new byte[array.size()]; + for (int i = 0; i < array.size(); i++) { + final byte value = gson.fromJson(array.get(i), byte.class); + retArray[i] = value; + } + return (T)retArray; + } else if (type == char[].class) { + final char[] retArray = new char[array.size()]; + for (int i = 0; i < array.size(); i++) { + final char value = gson.fromJson(array.get(i), char.class); + retArray[i] = value; + } + return (T)retArray; + } else if (clazz != null && clazz.isArray()) { + final Class componentCls = clazz.getComponentType(); + final Object[] clsArray = (Object[])Array.newInstance(componentCls, array.size()); + for (int i = 0; i < array.size(); i++) { + clsArray[i] = gson.fromJson(array.get(i), componentCls); + } + //noinspection unchecked + return (T)clsArray; + } + return null; + } + + /** + * Return a reasonable class for a {@link JsonPrimitive}. Possible return + * types are + *
        + *
      • boolean
        • + *
      • double
        • + *
      • String
        • + *
      • Object
        • + *
      + * + * @param jsonPrimitive + * @return + */ + static Class classForJsonPrimitive(final JsonPrimitive jsonPrimitive) { + + if (jsonPrimitive.isBoolean()) + return boolean.class; + else if (jsonPrimitive.isNumber()) { + final Number number = jsonPrimitive.getAsNumber(); + if (number.longValue() == number.doubleValue()) + return long.class; + else + return double.class; + } else if (jsonPrimitive.isString()) + return String.class; + else + return Object.class; + } +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonN5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonN5Writer.java new file mode 100644 index 00000000..9d1ca1ed --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonN5Writer.java @@ -0,0 +1,238 @@ +/** + * Copyright (c) 2017, Stephan Saalfeld + * All rights reserved. + *

      + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + *

      + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + *

      + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +package org.janelia.saalfeldlab.n5; + +import com.google.gson.Gson; +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; + +import java.io.IOException; +import java.io.Writer; +import java.util.Map; +import java.util.regex.Matcher; + +/** + * {@link N5Reader} for JSON attributes parsed by {@link Gson}. + * + * @author Stephan Saalfeld + */ +public interface GsonN5Writer extends GsonN5Reader, N5Writer { + + + /** + * If there is an attribute in {@code root} such that it can be parsed and desrialized as {@link T}, + * then remove it from {@code root}, write {@code root} to the {@code writer}, and return the removed attribute. + *

      + * If there is an attribute at the location specified by {@code normalizedAttributePath} but it cannot be deserialized to {@link T}, then it is not removed. + *

      + * If nothing is removed, then {@ecode root} is not writen to the {@code writer}. + * + * @param writer to write the modified {@code root} to after removal of the attribute + * @param root to remove the attribute from + * @param normalizedAttributePath to the attribute location + * @param cls of the attribute to remove + * @param gson to deserialize the attribute with + * @param of the removed attribute + * @return the removed attribute, or null if nothing removed + * @throws IOException + */ + static T removeAttribute( + final Writer writer, + JsonElement root, + final String normalizedAttributePath, + final Class cls, + final Gson gson) throws IOException { + + final T removed = removeAttribute(root, normalizedAttributePath, cls, gson); + if (removed != null) { + writeAttributes(writer, root, gson); + } + return removed; + } + + /** + * If there is an attribute in {@code root} at location {@code normalizedAttributePath} then remove it from {@code root}.. + * + * @param writer to write the modified {@code root} to after removal of the attribute + * @param root to remove the attribute from + * @param normalizedAttributePath to the attribute location + * @param gson to deserialize the attribute with + * @return if the attribute was removed or not + */ + static boolean removeAttribute( + final Writer writer, + final JsonElement root, + final String normalizedAttributePath, + final Gson gson) throws IOException { + + final JsonElement removed = removeAttribute(root, normalizedAttributePath, JsonElement.class, gson); + if (removed != null) { + writeAttributes(writer, root, gson); + return true; + } + return false; + } + + /** + * If there is an attribute in {@code root} such that it can be parsed and desrialized as {@link T}, + * then remove it from {@code root} and return the removed attribute. + *

      + * If there is an attribute at the location specified by {@code normalizedAttributePath} but it cannot be deserialized to {@link T}, then it is not removed. + * + * @param root to remove the attribute from + * @param normalizedAttributePath to the attribute location + * @param cls of the attribute to remove + * @param gson to deserialize the attribute with + * @param of the removed attribute + * @return the removed attribute, or null if nothing removed + */ + static T removeAttribute(final JsonElement root, final String normalizedAttributePath, final Class cls, final Gson gson) { + + final T attribute = GsonN5Reader.readAttribute(root, normalizedAttributePath, cls, gson); + if (attribute != null) { + removeAttribute(root, normalizedAttributePath); + } + return attribute; + } + + /** + * Remove and return the attribute at {@code normalizedAttributePath} as a {@link JsonElement}. + * Does not attempt to parse the attribute. + * + * @param root to search for the {@link JsonElement} at location {@code normalizedAttributePath} + * @param normalizedAttributePath to the attribute + * @return the attribute as a {@link JsonElement}. + */ + static JsonElement removeAttribute(JsonElement root, String normalizedAttributePath) { + + final String[] pathParts = normalizedAttributePath.split("(?= jsonArray.size()) { + return null; + } + root = jsonArray.get(index); + if (i == pathParts.length - 1) { + jsonArray.remove(index); + } + } else { + return null; + } + } + } + return root; + } + + /** + * Inserts {@code attribute} into {@code root} at location {@code normalizedAttributePath} and write the resulting {@code root}. + *

      + * If {@code root} is not a {@link JsonObject}, then it is overwritten with an object containing {@code "normalizedAttributePath": attribute } + * + * @param writer + * @param root + * @param normalizedAttributePath + * @param attribute + * @param gson + * @throws IOException + */ + static void writeAttribute( + final Writer writer, + JsonElement root, + final String normalizedAttributePath, + final T attribute, + final Gson gson) throws IOException { + + root = insertAttribute(root, normalizedAttributePath, attribute, gson); + writeAttributes(writer, root, gson); + } + + /** + * Writes the attributes JsonElemnt to a given {@link Writer}. + * This will overwrite any existing attributes. + * + * @param writer + * @param root + * @throws IOException + */ + static void writeAttributes( + final Writer writer, + JsonElement root, + final Gson gson) throws IOException { + + gson.toJson(root, writer); + writer.flush(); + } + + static JsonElement insertAttributes(JsonElement root, Map attributes, Gson gson) { + + for (Map.Entry attribute : attributes.entrySet()) { + root = insertAttribute(root, N5URL.normalizeAttributePath(attribute.getKey()), attribute.getValue(), gson); + } + return root; + } + + static JsonElement insertAttribute(JsonElement root, String normalizedAttributePath, T attribute, Gson gson) { + + LinkedAttributePathToken pathToken = N5URL.getAttributePathTokens(normalizedAttributePath); + /* No path to traverse or build; just write the value */ + if (pathToken == null) + return gson.toJsonTree(attribute); + + JsonElement json = root; + while (pathToken != null) { + + JsonElement parent = pathToken.setAndCreateParentElement(json); + + /* We may need to create or override the existing root if it is non-existent or incompatible. */ + final boolean rootOverriden = json == root && parent != json; + if (root == null || rootOverriden) { + root = parent; + } + + json = pathToken.writeChild(gson, attribute); + + pathToken = pathToken.next(); + } + return root; + } +} From 1f4b13d79809b22535fe447a08f9d532803128c6 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Fri, 3 Mar 2023 13:35:58 -0500 Subject: [PATCH 097/243] feat!: migrate away from AbstractGsonParser to GsonN5Reader/Writer --- .../janelia/saalfeldlab/n5/N5FSReader.java | 28 +++++++++++++++++-- .../janelia/saalfeldlab/n5/N5FSWriter.java | 22 +++++++-------- 2 files changed, 36 insertions(+), 14 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5FSReader.java b/src/main/java/org/janelia/saalfeldlab/n5/N5FSReader.java index 4cd29852..5cd9e300 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5FSReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5FSReader.java @@ -36,8 +36,11 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.nio.file.StandardOpenOption; +import java.util.HashMap; +import java.util.Map; import java.util.stream.Stream; +import com.google.gson.Gson; import com.google.gson.GsonBuilder; import com.google.gson.JsonElement; @@ -46,7 +49,7 @@ * * @author Stephan Saalfeld */ -public class N5FSReader extends AbstractGsonReader { +public class N5FSReader implements GsonN5Reader, GsonAttributesParser { protected static class LockedFileChannel implements Closeable { @@ -99,6 +102,8 @@ public void close() throws IOException { protected final String basePath; + protected final Gson gson; + /** * Opens an {@link N5FSReader} at a given base path with a custom * {@link GsonBuilder} to support custom attributes. @@ -112,7 +117,7 @@ public void close() throws IOException { */ public N5FSReader(final String basePath, final GsonBuilder gsonBuilder) throws IOException { - super(gsonBuilder); + this.gson = GsonN5Reader.registerGson(gsonBuilder); this.basePath = basePath; if (exists("/")) { final Version version = getVersion(); @@ -137,6 +142,17 @@ public N5FSReader(final String basePath) throws IOException { this(basePath, new GsonBuilder()); } + @Override public Gson getGson() { + + return gson; + } + + @Deprecated + public HashMap getAttributesMap(String pathName) throws IOException { + + return GsonAttributesParser.super.getAttributesMap(getAttributes(pathName)); + } + /** * * @return N5 base path @@ -160,10 +176,16 @@ public JsonElement getAttributes(final String pathName) throws IOException { return null; try (final LockedFileChannel lockedFileChannel = LockedFileChannel.openForReading(path)) { - return GsonAttributesParser.readAttributes(Channels.newReader(lockedFileChannel.getFileChannel(), StandardCharsets.UTF_8.name()), getGson()); + return GsonN5Reader.readAttributes(Channels.newReader(lockedFileChannel.getFileChannel(), StandardCharsets.UTF_8.name()), getGson()); } } + @Override + public Map> listAttributes(String pathName) throws IOException { + + return GsonN5Reader.listAttributes(getAttributes(pathName)); + } + @Override public DataBlock readBlock( final String pathName, diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java b/src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java index 84318285..dc8bc8b1 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java @@ -52,7 +52,7 @@ * * @author Stephan Saalfeld */ -public class N5FSWriter extends N5FSReader implements N5Writer { +public class N5FSWriter extends N5FSReader implements GsonN5Writer { /** * Opens an {@link N5FSWriter} at a given base path with a custom @@ -117,7 +117,7 @@ public void createGroup(final String pathName) throws IOException { try (final LockedFileChannel lockedFileChannel = LockedFileChannel.openForWriting(path)) { final String attributePath = N5URL.normalizeAttributePath(key); lockedFileChannel.getFileChannel().truncate(0); - GsonAttributesParser.writeAttribute(Channels.newWriter(lockedFileChannel.getFileChannel(), StandardCharsets.UTF_8.name()), attributesRoot, attributePath, attribute, getGson()); + GsonN5Writer.writeAttribute(Channels.newWriter(lockedFileChannel.getFileChannel(), StandardCharsets.UTF_8.name()), attributesRoot, attributePath, attribute, getGson()); } } @@ -130,7 +130,7 @@ public void setAttributes( JsonObject newRoot = new JsonObject(); try (final LockedFileChannel lockedFileChannel = LockedFileChannel.openForWriting(path)) { - final JsonElement oldRoot = GsonAttributesParser.readAttributes(Channels.newReader(lockedFileChannel.getFileChannel(), StandardCharsets.UTF_8.name()), getGson()); + final JsonElement oldRoot = GsonN5Reader.readAttributes(Channels.newReader(lockedFileChannel.getFileChannel(), StandardCharsets.UTF_8.name()), getGson()); if (oldRoot != null && oldRoot.isJsonObject()) newRoot = oldRoot.getAsJsonObject(); @@ -138,11 +138,11 @@ public void setAttributes( for (Map.Entry attrPathValue : attributes.entrySet()) { final String attributePath = N5URL.normalizeAttributePath(attrPathValue.getKey()); final Object attributeValue = attrPathValue.getValue(); - GsonAttributesParser.insertAttribute(newRoot, attributePath, attributeValue, gson); + GsonN5Writer.insertAttribute(newRoot, attributePath, attributeValue, gson); } lockedFileChannel.getFileChannel().truncate(0); - GsonAttributesParser.writeAttributes(Channels.newWriter(lockedFileChannel.getFileChannel(), StandardCharsets.UTF_8.name()), newRoot, getGson()); + GsonN5Writer.writeAttributes(Channels.newWriter(lockedFileChannel.getFileChannel(), StandardCharsets.UTF_8.name()), newRoot, getGson()); } } @@ -157,7 +157,7 @@ public void setAttributes( try (final LockedFileChannel lockedFileChannel = LockedFileChannel.openForWriting(path)) { final String attributePath = N5URL.normalizeAttributePath(key); lockedFileChannel.getFileChannel().truncate(0); - return GsonAttributesParser.removeAttribute(Channels.newWriter(lockedFileChannel.getFileChannel(), StandardCharsets.UTF_8.name()), attributesRoot, attributePath, getGson()); + return GsonN5Writer.removeAttribute(Channels.newWriter(lockedFileChannel.getFileChannel(), StandardCharsets.UTF_8.name()), attributesRoot, attributePath, getGson()); } } @@ -171,11 +171,11 @@ public void setAttributes( try (final LockedFileChannel lockedFileChannel = LockedFileChannel.openForWriting(path)) { final String attributePath = N5URL.normalizeAttributePath(key); - final T removed = GsonAttributesParser.removeAttribute(attributesRoot, attributePath, cls, gson); + final T removed = GsonN5Writer.removeAttribute(attributesRoot, attributePath, cls, gson); if (removed != null ) { lockedFileChannel.getFileChannel().truncate(0); final Writer writer = Channels.newWriter(lockedFileChannel.getFileChannel(), StandardCharsets.UTF_8.name()); - GsonAttributesParser.writeAttributes(writer, attributesRoot, gson); + GsonN5Writer.writeAttributes(writer, attributesRoot, gson); } return removed; } @@ -193,7 +193,7 @@ public boolean removeAttributes( JsonObject newRoot = new JsonObject(); try (final LockedFileChannel lockedFileChannel = LockedFileChannel.openForWriting(path)) { - final JsonElement oldRoot = GsonAttributesParser.readAttributes(Channels.newReader(lockedFileChannel.getFileChannel(), StandardCharsets.UTF_8.name()), getGson()); + final JsonElement oldRoot = GsonN5Reader.readAttributes(Channels.newReader(lockedFileChannel.getFileChannel(), StandardCharsets.UTF_8.name()), getGson()); if (oldRoot != null && oldRoot.isJsonObject()) newRoot = oldRoot.getAsJsonObject(); @@ -201,12 +201,12 @@ public boolean removeAttributes( boolean removed = false; for (String attribute : attributes) { final String attributePath = N5URL.normalizeAttributePath(attribute); - removed |= GsonAttributesParser.removeAttribute(newRoot, attributePath) != null; + removed |= GsonN5Writer.removeAttribute(newRoot, attributePath) != null; } if (removed) { lockedFileChannel.getFileChannel().truncate(0); final Writer writer = Channels.newWriter(lockedFileChannel.getFileChannel(), StandardCharsets.UTF_8.name()); - GsonAttributesParser.writeAttributes(writer, newRoot, getGson()); + GsonN5Writer.writeAttributes(writer, newRoot, getGson()); } return removed; From 63e8b5db24c91a44f31364f7af711f8f183cdabf Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Fri, 3 Mar 2023 13:36:14 -0500 Subject: [PATCH 098/243] feat!: deprecate for future removal --- .../saalfeldlab/n5/GsonAttributesParser.java | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonAttributesParser.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonAttributesParser.java index c5e4a9fb..96eccc25 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonAttributesParser.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonAttributesParser.java @@ -50,6 +50,19 @@ */ public interface GsonAttributesParser extends N5Reader { + default HashMap getAttributesMap(JsonElement root) { + + final HashMap attributeMap = new HashMap<>(); + if (root == null || !root.isJsonObject()) { + return attributeMap; + } + + for (Map.Entry entry : root.getAsJsonObject().entrySet()) { + attributeMap.put(entry.getKey(), entry.getValue()); + } + return attributeMap; + } + public Gson getGson(); /** From a6d92f625c4e26d15e8fbae1b84d0cf404c56508 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Fri, 3 Mar 2023 13:36:40 -0500 Subject: [PATCH 099/243] feat: finalize and document attribute path regex normalization --- .../org/janelia/saalfeldlab/n5/N5URL.java | 34 +++++++++++++------ 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java b/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java index c8ae0df2..88ddf9af 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java @@ -406,21 +406,33 @@ public static String normalizePath(String path) { */ public static String normalizeAttributePath(String attributePath) { + /* Short circuit if there are no non-escaped `/` or array indices (e.g. [N] where N is a non-negative integer) */ + if (!attributePath.matches(".*((? `[10]/b`*/ final String attrPathPlusFirstIndexSeparator = attributePath.replaceAll("^(?\\[[0-9]+])", "${array}/"); /* Add separator before and after arrays not at the beginning `a[10]b` -> `a/[10]/b`*/ - final String attrPathPlusIndexSeparators = attrPathPlusFirstIndexSeparator.replaceAll("((?\\[[0-9]+]))", "/${array}/"); - /* remove redundant separators `a///b` -> `a/b`*/ - final String attrPathRemoveMultipleSeparators = attrPathPlusIndexSeparators.replaceAll("(?<=/)/+", ""); - /* remove redundant paths `a/./b` -> `a/b``*/ - final String attrPathNoDot = attrPathRemoveMultipleSeparators.replaceAll("(?<=(/|^))(\\./)+|((/|(?<=/))\\.)$", ""); - /* remove trailing separator `a/b/c/` -> `a/b/c` */ - final String normalizedAttributePath = attrPathNoDot.replaceAll("(? b*/ - final Pattern relativePathPattern = Pattern.compile("(?<=(/|^))[^/]+(?\\[[0-9]+]))", "/${array}/"); + + /* + * The following has 4 possible matches, in each case it removes the match: + * The first 3 remove redundant separators of the form: + * 1.`a///b` -> `a/b` : (?<=/)/+ + * 2.`a/./b` -> `a/b` : (?<=(/|^))(\./)+ + * 3.`a/b/` -> `a/b` : ((/|(?<=/))\.)$ + * The last resolves relative paths: + * 4. `a/../b` -> `a/b` : (? Date: Fri, 3 Mar 2023 13:37:29 -0500 Subject: [PATCH 100/243] feat!: deprecate for future removal, change `getAttributes` to `getAttributesMap` due to collision with new `N5GsonReader` --- .../saalfeldlab/n5/AbstractGsonReader.java | 35 +++++++++++++++---- 1 file changed, 28 insertions(+), 7 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/AbstractGsonReader.java b/src/main/java/org/janelia/saalfeldlab/n5/AbstractGsonReader.java index 962d8eae..91af7e07 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/AbstractGsonReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/AbstractGsonReader.java @@ -35,6 +35,8 @@ import java.lang.reflect.Type; import java.net.URISyntaxException; import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; /** * Abstract base class implementing {@link N5Reader} with JSON attributes @@ -43,7 +45,9 @@ * @author Stephan Saalfeld * @author Igor Pisarev * @author Philipp Hanslovsky + * @author Caleb Hulbert */ +@Deprecated public abstract class AbstractGsonReader implements GsonAttributesParser, N5Reader { protected final Gson gson; @@ -54,6 +58,7 @@ public abstract class AbstractGsonReader implements GsonAttributesParser, N5Read * * @param gsonBuilder */ + @Deprecated public AbstractGsonReader(final GsonBuilder gsonBuilder) { gsonBuilder.registerTypeAdapter(DataType.class, new DataType.JsonAdapter()); @@ -66,18 +71,21 @@ public AbstractGsonReader(final GsonBuilder gsonBuilder) { * Constructs an {@link AbstractGsonReader} with a default * {@link GsonBuilder}. */ + @Deprecated public AbstractGsonReader() { this(new GsonBuilder()); } @Override + @Deprecated public Gson getGson() { return gson; } @Override + @Deprecated public DatasetAttributes getDatasetAttributes(final String pathName) throws IOException { final JsonElement root = getAttributes(pathName); @@ -88,23 +96,23 @@ public DatasetAttributes getDatasetAttributes(final String pathName) throws IOEx final JsonObject rootObject = root.getAsJsonObject(); - final long[] dimensions = GsonAttributesParser.readAttribute(rootObject, DatasetAttributes.dimensionsKey, long[].class, gson); + final long[] dimensions = GsonN5Reader.readAttribute(rootObject, DatasetAttributes.dimensionsKey, long[].class, gson); if (dimensions == null) return null; - final DataType dataType = GsonAttributesParser.readAttribute(rootObject, DatasetAttributes.dataTypeKey, DataType.class, gson); + final DataType dataType = GsonN5Reader.readAttribute(rootObject, DatasetAttributes.dataTypeKey, DataType.class, gson); if (dataType == null) return null; - int[] blockSize = GsonAttributesParser.readAttribute(rootObject, DatasetAttributes.blockSizeKey, int[].class, gson); + int[] blockSize = GsonN5Reader.readAttribute(rootObject, DatasetAttributes.blockSizeKey, int[].class, gson); if (blockSize == null) blockSize = Arrays.stream(dimensions).mapToInt(a -> (int)a).toArray(); - Compression compression = GsonAttributesParser.readAttribute(rootObject, DatasetAttributes.compressionKey, Compression.class, gson); + Compression compression = GsonN5Reader.readAttribute(rootObject, DatasetAttributes.compressionKey, Compression.class, gson); /* version 0 */ if (compression == null) { - switch (GsonAttributesParser.readAttribute(rootObject, DatasetAttributes.compressionTypeKey, String.class, gson)) { + switch (GsonN5Reader.readAttribute(rootObject, DatasetAttributes.compressionTypeKey, String.class, gson)) { case "raw": compression = new RawCompression(); break; @@ -127,6 +135,7 @@ public DatasetAttributes getDatasetAttributes(final String pathName) throws IOEx } @Override + @Deprecated public T getAttribute( final String pathName, final String key, @@ -136,6 +145,7 @@ public T getAttribute( } @Override + @Deprecated public T getAttribute( final String pathName, final String key, @@ -144,13 +154,24 @@ public T getAttribute( final String normalizedGroupPath; final String normalizedAttributePath; try { - final N5URL n5url = N5URL.from(null, pathName, key ); + final N5URL n5url = N5URL.from(null, pathName, key); normalizedGroupPath = n5url.normalizeGroupPath(); normalizedAttributePath = n5url.normalizeAttributePath(); } catch (URISyntaxException e) { throw new IOException(e); } - return GsonAttributesParser.readAttribute( getAttributes( normalizedGroupPath ), normalizedAttributePath, type, gson); + return GsonN5Reader.readAttribute(getAttributes(normalizedGroupPath), normalizedAttributePath, type, gson); } + @Deprecated + public HashMap getAttributesMap(String pathName) throws IOException { + return GsonAttributesParser.super.getAttributesMap(getAttributes(pathName)); + } + + @Override + @Deprecated + public Map> listAttributes(String pathName) throws IOException { + + return GsonN5Reader.listAttributes(getAttributes(pathName)); + } } From 7fe679901ec3929f2b8053a90cbec0896d4532d5 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Fri, 3 Mar 2023 13:38:17 -0500 Subject: [PATCH 101/243] fix(test): fix some incorrect tests --- .../saalfeldlab/n5/AbstractN5Test.java | 64 +++++++++++-------- .../org/janelia/saalfeldlab/n5/N5URLTest.java | 2 +- 2 files changed, 38 insertions(+), 28 deletions(-) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java index ab1b03a2..c208d260 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java @@ -693,9 +693,12 @@ public void testRemove() { } @Test - public void testList() { + public void testList() throws IOException { - try( final N5Writer listN5 = createN5Writer()) { + final File tmpFile = Files.createTempDirectory("list-test-").toFile(); + tmpFile.deleteOnExit(); + final String canonicalPath = tmpFile.getCanonicalPath(); + try (final N5Writer listN5 = createN5Writer(canonicalPath)) { listN5.createGroup(groupName); for (final String subGroup : subGroupNames) listN5.createGroup(groupName + "/" + subGroup); @@ -957,32 +960,39 @@ public void testReaderCreation() throws IOException final String canonicalPath = tmpFile.getCanonicalPath(); try (N5Writer writer = createN5Writer(canonicalPath)) { - final N5Reader n5r = createN5Reader( canonicalPath ); - assertNotNull( n5r ); - - // existing directory without attributes is okay - writer.createGroup( "noAttrs" ); - final N5Reader na = createN5Reader( canonicalPath + "/noAttrs" ); - assertNotNull( na ); - - // existing directory without attributes is okay - writer.createGroup( "withAttrs" ); - writer.setAttribute( "withAttrs", "mystring", "ms" ); - final N5Reader wa = createN5Reader( canonicalPath + "/withAttrs" ); + final N5Reader n5r = createN5Reader(canonicalPath); + assertNotNull(n5r); + + // existing directory without attributes is okay; + // Remove and create to remove attributes store + writer.remove("/"); + writer.createGroup("/"); + final N5Reader na = createN5Reader(canonicalPath); + assertNotNull(na); + + // existing location with attributes, but no version + writer.remove("/"); + writer.createGroup("/"); + writer.setAttribute( "/", "mystring", "ms" ); + final N5Reader wa = createN5Reader( canonicalPath); assertNotNull( wa ); - // non-existent directory should fail - assertThrows( "Non-existant directory throws error", IOException.class, - () -> { - createN5Reader( canonicalPath + "/nonExistant" ); - } ); - // existing directory with incompatible version should fail + writer.remove("/"); + writer.createGroup("/"); writer.setAttribute("/", N5Reader.VERSION_KEY, new Version(N5Reader.VERSION.getMajor() + 1, N5Reader.VERSION.getMinor(), N5Reader.VERSION.getPatch()).toString()); assertThrows( "Incompatible version throws error", IOException.class, () -> { createN5Reader(canonicalPath); } ); + + // non-existent directory should fail + writer.remove("/"); + assertThrows( "Non-existant location throws error", IOException.class, + () -> { + final N5Reader test = createN5Reader(canonicalPath); + test.list("/"); + } ); } } @@ -1195,7 +1205,7 @@ public void testAttributePathEscaping() throws IOException { n5.createGroup( grp ); n5.setAttribute( grp, "\\/", dataString ); assertEquals( dataString, n5.getAttribute( grp, "\\/", String.class ) ); - String jsonContents = readAttributesAsString( grp ); + String jsonContents = n5.getAttribute( grp , "/", String.class); assertEquals( rootSlash, jsonContents ); // "abc/def" as key @@ -1203,7 +1213,7 @@ public void testAttributePathEscaping() throws IOException { n5.createGroup( grp ); n5.setAttribute( grp, "abc\\/def", dataString ); assertEquals( dataString, n5.getAttribute( grp, "abc\\/def", String.class ) ); - jsonContents = readAttributesAsString( grp ); + jsonContents = n5.getAttribute( grp , "/", String.class); assertEquals( abcdef, jsonContents ); // "[0]" as a key @@ -1211,7 +1221,7 @@ public void testAttributePathEscaping() throws IOException { n5.createGroup( grp ); n5.setAttribute( grp, "\\[0]", dataString ); assertEquals( dataString, n5.getAttribute( grp, "\\[0]", String.class ) ); - jsonContents = readAttributesAsString( grp ); + jsonContents = n5.getAttribute( grp , "/", String.class); assertEquals( zero, jsonContents ); // "]] [] [[" as a key @@ -1219,7 +1229,7 @@ public void testAttributePathEscaping() throws IOException { n5.createGroup( grp ); n5.setAttribute( grp, bracketsKey, dataString ); assertEquals( dataString, n5.getAttribute( grp, bracketsKey, String.class ) ); - jsonContents = readAttributesAsString( grp ); + jsonContents = n5.getAttribute( grp , "/", String.class); assertEquals( brackets, jsonContents ); // "[[2][33]]" as a key FIXME: What to do about escaping inside square brackets? @@ -1227,7 +1237,7 @@ public void testAttributePathEscaping() throws IOException { n5.createGroup( grp ); n5.setAttribute( grp, "[\\[2]\\[33]]", dataString ); assertEquals( dataString, n5.getAttribute( grp, "[\\[2]\\[33]]", String.class ) ); - jsonContents = readAttributesAsString( grp ); + jsonContents = n5.getAttribute( grp , "/", String.class); assertEquals( doubleBrackets, jsonContents ); // "\\" as key @@ -1235,12 +1245,12 @@ public void testAttributePathEscaping() throws IOException { n5.createGroup( grp ); n5.setAttribute( grp, "\\\\", dataString ); assertEquals( dataString, n5.getAttribute( grp, "\\\\", String.class ) ); - jsonContents = readAttributesAsString( grp ); + jsonContents = n5.getAttribute( grp , "/", String.class); assertEquals( doubleBackslash, jsonContents ); // clear n5.setAttribute( grp, "/", emptyObj ); - jsonContents = readAttributesAsString( grp ); + jsonContents = n5.getAttribute( grp , "/", String.class); assertEquals( empty, jsonContents ); } diff --git a/src/test/java/org/janelia/saalfeldlab/n5/N5URLTest.java b/src/test/java/org/janelia/saalfeldlab/n5/N5URLTest.java index 74b91b54..1aa8f163 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/N5URLTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/N5URLTest.java @@ -50,7 +50,7 @@ public void testAttributePath() { assertEquals("let's/try/a/real/case/with spaces", N5URL.normalizeAttributePath("let's/try/a/real/case/with spaces/")); assertEquals("let's/try/a/real/case/with spaces", N5URL.normalizeAttributePath("let's/try/a/real/////case////with spaces/")); assertEquals("../first/relative/a/wd/.w/asd", N5URL.normalizeAttributePath("../first/relative/test/../a/b/.././wd///.w/asd")); - assertEquals("../", N5URL.normalizeAttributePath("../result/../only/../single/..")); + assertEquals("..", N5URL.normalizeAttributePath("../result/../only/../single/..")); assertEquals("../..", N5URL.normalizeAttributePath("../result/../multiple/../..")); String normalizedPath = N5URL.normalizeAttributePath("let's/try/a/some/////with/ / //white spaces/"); From ca91baea3260564de336a6fa580e54b158013afb Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Tue, 7 Mar 2023 13:43:39 -0500 Subject: [PATCH 102/243] feat!: revert changes and deprecate --- .../saalfeldlab/n5/AbstractGsonReader.java | 69 +-- .../saalfeldlab/n5/GsonAttributesParser.java | 488 ++++-------------- 2 files changed, 131 insertions(+), 426 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/AbstractGsonReader.java b/src/main/java/org/janelia/saalfeldlab/n5/AbstractGsonReader.java index 91af7e07..85b9a083 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/AbstractGsonReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/AbstractGsonReader.java @@ -1,16 +1,16 @@ /** * Copyright (c) 2017, Stephan Saalfeld * All rights reserved. - *

      + * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: - *

      + * * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. + * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - *

      + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE @@ -25,18 +25,14 @@ */ package org.janelia.saalfeldlab.n5; -import com.google.gson.Gson; -import com.google.gson.GsonBuilder; -import com.google.gson.JsonElement; -import com.google.gson.JsonObject; -import com.google.gson.reflect.TypeToken; - import java.io.IOException; import java.lang.reflect.Type; -import java.net.URISyntaxException; import java.util.Arrays; import java.util.HashMap; -import java.util.Map; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.JsonElement; /** * Abstract base class implementing {@link N5Reader} with JSON attributes @@ -45,7 +41,6 @@ * @author Stephan Saalfeld * @author Igor Pisarev * @author Philipp Hanslovsky - * @author Caleb Hulbert */ @Deprecated public abstract class AbstractGsonReader implements GsonAttributesParser, N5Reader { @@ -88,31 +83,26 @@ public Gson getGson() { @Deprecated public DatasetAttributes getDatasetAttributes(final String pathName) throws IOException { - final JsonElement root = getAttributes(pathName); + final HashMap map = getAttributes(pathName); final Gson gson = getGson(); - if (root == null || !root.isJsonObject()) - return null; - - final JsonObject rootObject = root.getAsJsonObject(); - - final long[] dimensions = GsonN5Reader.readAttribute(rootObject, DatasetAttributes.dimensionsKey, long[].class, gson); + final long[] dimensions = GsonAttributesParser.parseAttribute(map, DatasetAttributes.dimensionsKey, long[].class, gson); if (dimensions == null) return null; - final DataType dataType = GsonN5Reader.readAttribute(rootObject, DatasetAttributes.dataTypeKey, DataType.class, gson); + final DataType dataType = GsonAttributesParser.parseAttribute(map, DatasetAttributes.dataTypeKey, DataType.class, gson); if (dataType == null) return null; - int[] blockSize = GsonN5Reader.readAttribute(rootObject, DatasetAttributes.blockSizeKey, int[].class, gson); + int[] blockSize = GsonAttributesParser.parseAttribute(map, DatasetAttributes.blockSizeKey, int[].class, gson); if (blockSize == null) blockSize = Arrays.stream(dimensions).mapToInt(a -> (int)a).toArray(); - Compression compression = GsonN5Reader.readAttribute(rootObject, DatasetAttributes.compressionKey, Compression.class, gson); + Compression compression = GsonAttributesParser.parseAttribute(map, DatasetAttributes.compressionKey, Compression.class, gson); /* version 0 */ if (compression == null) { - switch (GsonN5Reader.readAttribute(rootObject, DatasetAttributes.compressionTypeKey, String.class, gson)) { + switch (GsonAttributesParser.parseAttribute(map, DatasetAttributes.compressionTypeKey, String.class, gson)) { case "raw": compression = new RawCompression(); break; @@ -141,7 +131,8 @@ public T getAttribute( final String key, final Class clazz) throws IOException { - return getAttribute(pathName, key, TypeToken.get(clazz).getType()); + final HashMap map = getAttributes(pathName); + return GsonAttributesParser.parseAttribute(map, key, clazz, getGson()); } @Override @@ -151,27 +142,7 @@ public T getAttribute( final String key, final Type type) throws IOException { - final String normalizedGroupPath; - final String normalizedAttributePath; - try { - final N5URL n5url = N5URL.from(null, pathName, key); - normalizedGroupPath = n5url.normalizeGroupPath(); - normalizedAttributePath = n5url.normalizeAttributePath(); - } catch (URISyntaxException e) { - throw new IOException(e); - } - return GsonN5Reader.readAttribute(getAttributes(normalizedGroupPath), normalizedAttributePath, type, gson); - } - @Deprecated - public HashMap getAttributesMap(String pathName) throws IOException { - - return GsonAttributesParser.super.getAttributesMap(getAttributes(pathName)); - } - - @Override - @Deprecated - public Map> listAttributes(String pathName) throws IOException { - - return GsonN5Reader.listAttributes(getAttributes(pathName)); + final HashMap map = getAttributes(pathName); + return GsonAttributesParser.parseAttribute(map, key, type, getGson()); } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonAttributesParser.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonAttributesParser.java index bc45e474..474e93c5 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonAttributesParser.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonAttributesParser.java @@ -1,16 +1,16 @@ /** * Copyright (c) 2017, Stephan Saalfeld * All rights reserved. - *

      + * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: - *

      + * * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. + * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - *

      + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE @@ -25,14 +25,6 @@ */ package org.janelia.saalfeldlab.n5; -import com.google.gson.Gson; -import com.google.gson.JsonArray; -import com.google.gson.JsonElement; -import com.google.gson.JsonObject; -import com.google.gson.JsonPrimitive; -import com.google.gson.JsonSyntaxException; -import com.google.gson.reflect.TypeToken; - import java.io.IOException; import java.io.Reader; import java.io.Writer; @@ -41,385 +33,131 @@ import java.util.HashMap; import java.util.Map; import java.util.Map.Entry; -import java.util.regex.Matcher; + +import com.google.gson.Gson; +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonPrimitive; +import com.google.gson.reflect.TypeToken; /** * {@link N5Reader} for JSON attributes parsed by {@link Gson}. * * @author Stephan Saalfeld */ +@Deprecated public interface GsonAttributesParser extends N5Reader { - default HashMap getAttributesMap(JsonElement root) { - - final HashMap attributeMap = new HashMap<>(); - if (root == null || !root.isJsonObject()) { - return attributeMap; - } - - for (Map.Entry entry : root.getAsJsonObject().entrySet()) { - attributeMap.put(entry.getKey(), entry.getValue()); - } - return attributeMap; - } - + @Deprecated public Gson getGson(); /** - * Reads the attributes a group or dataset. + * Reads or creates the attributes map of a group or dataset. * * @param pathName group path - * @return the root {@link JsonElement} of the attributes + * @return * @throws IOException */ - public JsonElement getAttributes(final String pathName) throws IOException; + @Deprecated + public HashMap getAttributes(final String pathName) throws IOException; /** - * Reads the attributes json from a given {@link Reader}. + * Parses an attribute from the given attributes map. * - * @param reader - * @return the root {@link JsonObject} of the attributes + * @param map + * @param key + * @param clazz + * @return * @throws IOException */ - public static JsonElement readAttributes(final Reader reader, final Gson gson) throws IOException { - - final JsonElement json = gson.fromJson(reader, JsonElement.class); - return json; - } - - static T readAttribute(final JsonElement root, final String normalizedAttributePath, final Class cls, final Gson gson) { - - return readAttribute(root, normalizedAttributePath, TypeToken.get(cls).getType(), gson); - } - - static T readAttribute(final JsonElement root, final String normalizedAttributePath, final Type type, final Gson gson) { - - final JsonElement attribute = getAttribute(root, normalizedAttributePath); - return parseAttributeElement(attribute, gson, type); - } - - /** - * Deserialize the {@code attribute} as {@link Type type} {@link T}. - * - * @param attribute to deserialize as {@link Type type} - * @param gson used to deserialize {@code attribute} - * @param type to desrialize {@code attribute} as - * @param return type represented by {@link Type type} - * @return the deserialized attribute object, or {@code null} if {@code attribute} cannot deserialize to {@link T} - */ - static T parseAttributeElement(JsonElement attribute, Gson gson, Type type) { - - if (attribute == null) - return null; - - final Class clazz = (type instanceof Class) ? ((Class)type) : null; - if (clazz != null && clazz.isAssignableFrom(HashMap.class)) { - Type mapType = new TypeToken>() { + @Deprecated + public static T parseAttribute( + final HashMap map, + final String key, + final Class clazz, + final Gson gson) throws IOException { - }.getType(); - Map retMap = gson.fromJson(attribute, mapType); - //noinspection unchecked - return (T)retMap; - } - if (attribute instanceof JsonArray) { - final JsonArray array = attribute.getAsJsonArray(); - T retArray = GsonAttributesParser.getJsonAsArray(gson, array, type); - if (retArray != null) - return retArray; - } - try { - return gson.fromJson(attribute, type); - } catch (JsonSyntaxException e) { - if (type == String.class) - return (T)gson.toJson(attribute); + final JsonElement attribute = map.get(key); + if (attribute != null) + return gson.fromJson(attribute, clazz); + else return null; - } } /** - * If there is an attribute in {@code root} such that it can be parsed and desrialized as {@link T}, - * then remove it from {@code root}, write {@code root} to the {@code writer}, and return the removed attribute. - *

      - * If there is an attribute at the location specified by {@code normalizedAttributePath} but it cannot be deserialized to {@link T}, then it is not removed. - * - * If nothing is removed, then {@ecode root} is not writen to the {@code writer}. + * Parses an attribute from the given attributes map. * - * @param writer to write the modified {@code root} to after removal of the attribute - * @param root to remove the attribute from - * @param normalizedAttributePath to the attribute location - * @param cls of the attribute to remove - * @param gson to deserialize the attribute with - * @param of the removed attribute - * @return the removed attribute, or null if nothing removed + * @param map + * @param key + * @param type + * @return * @throws IOException */ - static T removeAttribute( - final Writer writer, - JsonElement root, - final String normalizedAttributePath, - final Class cls, - final Gson gson) throws IOException { - - final T removed = removeAttribute(root, normalizedAttributePath, cls, gson); - if (removed != null ) { - writeAttributes(writer, root, gson); - } - return removed; - } - - /** - * If there is an attribute in {@code root} at location {@code normalizedAttributePath} then remove it from {@code root}.. - * - * @param writer to write the modified {@code root} to after removal of the attribute - * @param root to remove the attribute from - * @param normalizedAttributePath to the attribute location - * @param gson to deserialize the attribute with - * @return if the attribute was removed or not - */ - static boolean removeAttribute( - final Writer writer, - final JsonElement root, - final String normalizedAttributePath, + @Deprecated + public static T parseAttribute( + final HashMap map, + final String key, + final Type type, final Gson gson) throws IOException { - final JsonElement removed = removeAttribute(root, normalizedAttributePath, JsonElement.class, gson); - if (removed != null ) { - writeAttributes(writer, root, gson); - return true; - } - return false; - } - /** - * If there is an attribute in {@code root} such that it can be parsed and desrialized as {@link T}, - * then remove it from {@code root} and return the removed attribute. - *

      - * If there is an attribute at the location specified by {@code normalizedAttributePath} but it cannot be deserialized to {@link T}, then it is not removed. - * - * @param root to remove the attribute from - * @param normalizedAttributePath to the attribute location - * @param cls of the attribute to remove - * @param gson to deserialize the attribute with - * @param of the removed attribute - * @return the removed attribute, or null if nothing removed - */ - static T removeAttribute(final JsonElement root, final String normalizedAttributePath, final Class cls, final Gson gson) { - final T attribute = readAttribute(root, normalizedAttributePath, cls, gson); - if (attribute != null) { - removeAttribute(root, normalizedAttributePath); - } - return attribute; + final JsonElement attribute = map.get(key); + if (attribute != null) + return gson.fromJson(attribute, type); + else + return null; } /** - * Removes the JsonElement at {@code normalizedAttributePath} from {@code root} if present. + * Reads the attributes map from a given {@link Reader}. * - * @param root to remove the attribute from - * @param normalizedAttributePath to the attribute location - * @return the removed {@link JsonElement}, or null if nothing removed. + * @param reader + * @return + * @throws IOException */ - static JsonElement removeAttribute(JsonElement root, String normalizedAttributePath) { - return getAttribute(root, normalizedAttributePath, true); - } + @Deprecated + public static HashMap readAttributes(final Reader reader, final Gson gson) throws IOException { - /** - * Return the attribute at {@code normalizedAttributePath} as a {@link JsonElement}. - * Does not attempt to parse the attribute. - * - * @param root to search for the {@link JsonElement} at location {@code normalizedAttributePath} - * @param normalizedAttributePath to the attribute - * @return the attribute as a {@link JsonElement}. - */ - static JsonElement getAttribute(JsonElement root, String normalizedAttributePath) { - return getAttribute(root, normalizedAttributePath, false); + final Type mapType = new TypeToken>(){}.getType(); + final HashMap map = gson.fromJson(reader, mapType); + return map == null ? new HashMap<>() : map; } /** - * Return the attribute at {@code normalizedAttributePath} as a {@link JsonElement}. If {@code remove}, remove the attribute from {@code root}. - * Does not attempt to parse the attribute. + * Inserts new the JSON export of attributes into the given attributes map. * - * @param root to search for the {@link JsonElement} at location {@code normalizedAttributePath} - * @param normalizedAttributePath to the attribute - * @param remove the attribute {code @JsonElement} after finding it. - * @return the attribute as a {@link JsonElement}. - */ - static JsonElement getAttribute(JsonElement root, String normalizedAttributePath, boolean remove) { - - final String[] pathParts = normalizedAttributePath.split("(?= jsonArray.size()) { - return null; - } - root = jsonArray.get(index); - if (remove && i == pathParts.length - 1) { - jsonArray.remove(index); - } - } else { - return null; - } - } - } - return root; - } - - /** - * Inserts {@code attribute} into {@code root} at location {@code normalizedAttributePath} and write the resulting {@code root}. - * - * If {@code root} is not a {@link JsonObject}, then it is overwritten with an object containing {@code "normalizedAttributePath": attribute } - * - * @param writer - * @param root - * @param normalizedAttributePath - * @param attribute + * @param map + * @param attributes * @param gson * @throws IOException */ - public static void writeAttribute( - final Writer writer, - JsonElement root, - final String normalizedAttributePath, - final T attribute, + @Deprecated + public static void insertAttributes( + final HashMap map, + final Map attributes, final Gson gson) throws IOException { - root = insertAttribute(root, normalizedAttributePath, attribute, gson); - writeAttributes(writer, root, gson); + for (final Entry entry : attributes.entrySet()) + map.put(entry.getKey(), gson.toJsonTree(entry.getValue())); } /** - * Writes the attributes JsonElemnt to a given {@link Writer}. - * This will overwrite any existing attributes. + * Writes the attributes map to a given {@link Writer}. * * @param writer - * @param root + * @param map * @throws IOException */ - public static void writeAttributes( + @Deprecated + public static void writeAttributes( final Writer writer, - JsonElement root, + final HashMap map, final Gson gson) throws IOException { - gson.toJson(root, writer); + final Type mapType = new TypeToken>(){}.getType(); + gson.toJson(map, mapType, writer); writer.flush(); } - public static JsonElement insertAttribute(JsonElement root, String normalizedAttributePath, T attribute, Gson gson) { - - LinkedAttributePathToken pathToken = N5URL.getAttributePathTokens(normalizedAttributePath); - /* No path to traverse or build; just write the value */ - if (pathToken == null) - return gson.toJsonTree(attribute); - - JsonElement json = root; - while (pathToken != null) { - - JsonElement parent = pathToken.setAndCreateParentElement(json); - - /* We may need to create or override the existing root if it is non-existent or incompatible. */ - final boolean rootOverriden = json == root && parent != json; - if (root == null || rootOverriden) { - root = parent; - } - - json = pathToken.writeChild(gson, attribute); - - pathToken = pathToken.next(); - } - return root; - } - - static T getJsonAsArray(Gson gson, JsonArray array, Class cls) { - - return getJsonAsArray(gson, array, TypeToken.get(cls).getType()); - } - - static T getJsonAsArray(Gson gson, JsonArray array, Type type) { - - final Class clazz = (type instanceof Class) ? ((Class)type) : null; - - if (type == boolean[].class) { - final boolean[] retArray = new boolean[array.size()]; - for (int i = 0; i < array.size(); i++) { - final Boolean value = gson.fromJson(array.get(i), boolean.class); - retArray[i] = value; - } - return (T)retArray; - } else if (type == double[].class) { - final double[] retArray = new double[array.size()]; - for (int i = 0; i < array.size(); i++) { - final double value = gson.fromJson(array.get(i), double.class); - retArray[i] = value; - } - return (T)retArray; - } else if (type == float[].class) { - final float[] retArray = new float[array.size()]; - for (int i = 0; i < array.size(); i++) { - final float value = gson.fromJson(array.get(i), float.class); - retArray[i] = value; - } - return (T)retArray; - } else if (type == long[].class) { - final long[] retArray = new long[array.size()]; - for (int i = 0; i < array.size(); i++) { - final long value = gson.fromJson(array.get(i), long.class); - retArray[i] = value; - } - return (T)retArray; - } else if (type == short[].class) { - final short[] retArray = new short[array.size()]; - for (int i = 0; i < array.size(); i++) { - final short value = gson.fromJson(array.get(i), short.class); - retArray[i] = value; - } - return (T)retArray; - } else if (type == int[].class) { - final int[] retArray = new int[array.size()]; - for (int i = 0; i < array.size(); i++) { - final int value = gson.fromJson(array.get(i), int.class); - retArray[i] = value; - } - return (T)retArray; - } else if (type == byte[].class) { - final byte[] retArray = new byte[array.size()]; - for (int i = 0; i < array.size(); i++) { - final byte value = gson.fromJson(array.get(i), byte.class); - retArray[i] = value; - } - return (T)retArray; - } else if (type == char[].class) { - final char[] retArray = new char[array.size()]; - for (int i = 0; i < array.size(); i++) { - final char value = gson.fromJson(array.get(i), char.class); - retArray[i] = value; - } - return (T)retArray; - } else if (clazz != null && clazz.isArray()) { - final Class componentCls = clazz.getComponentType(); - final Object[] clsArray = (Object[])Array.newInstance(componentCls, array.size()); - for (int i = 0; i < array.size(); i++) { - clsArray[i] = gson.fromJson(array.get(i), componentCls); - } - //noinspection unchecked - return (T)clsArray; - } - return null; - } - /** * Return a reasonable class for a {@link JsonPrimitive}. Possible return * types are @@ -433,6 +171,7 @@ static T getJsonAsArray(Gson gson, JsonArray array, Type type) { * @param jsonPrimitive * @return */ + @Deprecated public static Class classForJsonPrimitive(final JsonPrimitive jsonPrimitive) { if (jsonPrimitive.isBoolean()) @@ -445,8 +184,7 @@ else if (jsonPrimitive.isNumber()) { return double.class; } else if (jsonPrimitive.isString()) return String.class; - else - return Object.class; + else return Object.class; } /** @@ -465,54 +203,50 @@ else if (jsonPrimitive.isNumber()) { *

    */ @Override + @Deprecated public default Map> listAttributes(final String pathName) throws IOException { - final JsonElement root = getAttributes(pathName); - if (!root.isJsonObject()) - return new HashMap<>(); - - final JsonObject rootObj = root.getAsJsonObject(); + final HashMap jsonElementMap = getAttributes(pathName); final HashMap> attributes = new HashMap<>(); - for (final Entry entry : rootObj.entrySet()) { - final String key = entry.getKey(); - final JsonElement jsonElement = entry.getValue(); - - final Class clazz; - if (jsonElement.isJsonNull()) - clazz = null; - else if (jsonElement.isJsonPrimitive()) - clazz = classForJsonPrimitive((JsonPrimitive)jsonElement); - else if (jsonElement.isJsonArray()) { - final JsonArray jsonArray = (JsonArray)jsonElement; - Class arrayElementClass = Object.class; - if (jsonArray.size() > 0) { - final JsonElement firstElement = jsonArray.get(0); - if (firstElement.isJsonPrimitive()) { - arrayElementClass = classForJsonPrimitive(firstElement.getAsJsonPrimitive()); - for (int i = 1; i < jsonArray.size() && arrayElementClass != Object.class; ++i) { - final JsonElement element = jsonArray.get(i); - if (element.isJsonPrimitive()) { - final Class nextArrayElementClass = classForJsonPrimitive(element.getAsJsonPrimitive()); - if (nextArrayElementClass != arrayElementClass) - if (nextArrayElementClass == double.class && arrayElementClass == long.class) - arrayElementClass = double.class; - else { + jsonElementMap.forEach( + (key, jsonElement) -> { + final Class clazz; + if (jsonElement.isJsonNull()) + clazz = null; + else if (jsonElement.isJsonPrimitive()) + clazz = classForJsonPrimitive((JsonPrimitive)jsonElement); + else if (jsonElement.isJsonArray()) { + final JsonArray jsonArray = (JsonArray)jsonElement; + Class arrayElementClass = Object.class; + if (jsonArray.size() > 0) { + final JsonElement firstElement = jsonArray.get(0); + if (firstElement.isJsonPrimitive()) { + arrayElementClass = classForJsonPrimitive(firstElement.getAsJsonPrimitive()); + for (int i = 1; i < jsonArray.size() && arrayElementClass != Object.class; ++i) { + final JsonElement element = jsonArray.get(i); + if (element.isJsonPrimitive()) { + final Class nextArrayElementClass = classForJsonPrimitive(element.getAsJsonPrimitive()); + if (nextArrayElementClass != arrayElementClass) + if (nextArrayElementClass == double.class && arrayElementClass == long.class) + arrayElementClass = double.class; + else { + arrayElementClass = Object.class; + break; + } + } else { arrayElementClass = Object.class; break; } - } else { - arrayElementClass = Object.class; - break; + } } - } + clazz = Array.newInstance(arrayElementClass, 0).getClass(); + } else + clazz = Object[].class; } - clazz = Array.newInstance(arrayElementClass, 0).getClass(); - } else - clazz = Object[].class; - } else - clazz = Object.class; - attributes.put(key, clazz); - } + else + clazz = Object.class; + attributes.put(key, clazz); + }); return attributes; } } From 5f509cf94deb72b59f42e11da39b9846883d6d5b Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Tue, 7 Mar 2023 13:49:20 -0500 Subject: [PATCH 103/243] build: fix javadoc errors --- .../n5/FileSystemKeyValueAccess.java | 2 +- .../saalfeldlab/n5/KeyValueAccess.java | 2 -- .../org/janelia/saalfeldlab/n5/N5URL.java | 20 ++++++++++--------- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java index 9c2e9a24..c3bc9954 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java @@ -329,7 +329,7 @@ protected static void tryDelete(final Path path) throws IOException { * Workaround for https://bugs.openjdk.java.net/browse/JDK-8130464 * * Creates a directory by creating all nonexistent parent directories first. - * Unlike the {@link #createDirectory createDirectory} method, an exception + * Unlike the {@link Files#createDirectories} method, an exception * is not thrown if the directory could not be created because it already * exists. * diff --git a/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccess.java index b8b2035e..117c1927 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccess.java @@ -166,7 +166,6 @@ public interface KeyValueAccess { * * @param normalPath is expected to be in normalized form, no further * efforts are made to normalize it. - * @return * @throws IOException */ public void createDirectories(final String normalPath) throws IOException; @@ -176,7 +175,6 @@ public interface KeyValueAccess { * * @param normalPath is expected to be in normalized form, no further * efforts are made to normalize it. - * @return */ public void delete(final String normalPath) throws IOException; } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java b/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java index 88ddf9af..a58d9042 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java @@ -375,8 +375,10 @@ public static String normalizePath(String path) { *
  • "." which represents the current element
    • *
  • ".." which represent the previous elemnt
    • *
  • "/" which is used to separate elements in the json tree
    • - *
  • [N] where N is an integer, refer to an index in the previous element in the tree; the previous element must be an array.
    • + *
  • [N] where N is an integer, refer to an index in the previous element in the tree; the previous element must be an array. + *

    * Note: [N] also separates the previous and following elements, regardless of whether it is preceded by "/" or not. + *

    • *
  • "\" which is an escape character, which indicates the subquent '/' or '[N]' should not be interpreted as a path delimeter, * but as part of the current path name.
    • * @@ -391,14 +393,14 @@ public static String normalizePath(String path) { *

    * Examples of valid attribute paths, and their normalizations *

      - *
    • /a/b/c -> /a/b/c
      • - *
    • /a/b/c/ -> /a/b/c
      • - *
    • ///a///b///c -> /a/b/c
      • - *
    • /a/././b/c -> /a/b/c
      • - *
    • /a/b[1]c -> /a/b/[1]/c
      • - *
    • /a/b/[1]c -> /a/b/[1]/c
      • - *
    • /a/b[1]/c -> /a/b/[1]/c
      • - *
    • /a/b[1]/c/.. -> /a/b/[1]
      • + *
    • /a/b/c becomes /a/b/c
      • + *
    • /a/b/c/ becomes /a/b/c
      • + *
    • ///a///b///c becomes /a/b/c
      • + *
    • /a/././b/c becomes /a/b/c
      • + *
    • /a/b[1]c becomes /a/b/[1]/c
      • + *
    • /a/b/[1]c becomes /a/b/[1]/c
      • + *
    • /a/b[1]/c becomes /a/b/[1]/c
      • + *
    • /a/b[1]/c/.. becomes /a/b/[1]
      • *
    * * @param attributePath to normalize From 0aaadc33f0186833ae0356e876b29d0c24b93bed Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Tue, 7 Mar 2023 15:28:18 -0500 Subject: [PATCH 104/243] feat!,doc,test: refactor caching to separate methods, clean some documentation --- .../saalfeldlab/n5/N5KeyValueReader.java | 415 +++++++++--------- .../saalfeldlab/n5/N5KeyValueWriter.java | 206 ++++----- .../saalfeldlab/n5/AbstractN5Test.java | 115 ++--- 3 files changed, 346 insertions(+), 390 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java index d3d55047..29785582 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java @@ -1,16 +1,16 @@ /** * Copyright (c) 2017--2021, Stephan Saalfeld * All rights reserved. - * + *

    * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: - * + *

    * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. + * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + *

    * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE @@ -28,6 +28,7 @@ import com.google.gson.Gson; import com.google.gson.GsonBuilder; import com.google.gson.JsonElement; +import com.google.gson.JsonObject; import java.io.IOException; import java.io.Reader; @@ -35,7 +36,6 @@ import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; -import java.util.Set; /** * {@link N5Reader} implementation through {@link KeyValueAccess} with JSON @@ -99,10 +99,7 @@ public N5KeyValueReader( this.keyValueAccess = keyValueAccess; this.basePath = keyValueAccess.normalize(basePath); - gsonBuilder.registerTypeAdapter(DataType.class, new DataType.JsonAdapter()); - gsonBuilder.registerTypeHierarchyAdapter(Compression.class, CompressionAdapter.getJsonAdapter()); - gsonBuilder.disableHtmlEscaping(); - this.gson = gsonBuilder.create(); + this.gson = GsonN5Reader.registerGson(gsonBuilder); this.cacheMeta = cacheMeta; if (exists("/")) { final Version version = getVersion(); @@ -125,55 +122,29 @@ public String getBasePath() { return this.basePath; } - /** - * Parses an attribute from the given attributes map. + * Reads the attributes at a given {@code normalAttributesPath}. * - * @param map - * @param key - * @param clazz - * @return - * @throws IOException + * @param normalAttributesPath normalized path of attributes location + * @return the attribute root JsonElement + * @throws IOException if unable to read the {@code normalAttributePath} */ - protected T parseAttribute( - final HashMap map, - final String key, - final Class clazz) throws IOException { + protected JsonElement readAttributes(final String normalAttributesPath) throws IOException { - final JsonElement attribute = map.get(key); - if (attribute != null) - return gson.fromJson(attribute, clazz); - else + if (!keyValueAccess.exists(normalAttributesPath)) return null; - } - /** - * Parses an attribute from the given attributes map. - * - * @param map - * @param key - * @param type - * @return - * @throws IOException - */ - protected T parseAttribute( - final HashMap map, - final String key, - final Type type) throws IOException { - - final JsonElement attribute = map.get(key); - if (attribute != null) - return gson.fromJson(attribute, type); - else - return null; + try (final LockedChannel lockedChannel = keyValueAccess.lockForReading(normalAttributesPath)) { + return readAttributes(lockedChannel.newReader()); + } } /** - * Reads the attributes map from a given {@link Reader}. + * Reads the attributes from a given {@link Reader}. * - * @param reader - * @return - * @throws IOException + * @param reader containing attriubtes + * @return the attributes root JsonElement + * @throws IOException if unable to read from the {@code reader} */ protected JsonElement readAttributes(final Reader reader) throws IOException { @@ -183,138 +154,85 @@ protected JsonElement readAttributes(final Reader reader) throws IOException { @Override public DatasetAttributes getDatasetAttributes(final String pathName) throws IOException { - final long[] dimensions; - final DataType dataType; - int[] blockSize; - Compression compression; - final String compressionVersion0Name; - final String normalPathName = N5URL.normalizePath(pathName); + final String normalPath = normalize(pathName); + N5GroupInfo info = null; if (cacheMeta) { - final N5GroupInfo info = getCachedN5GroupInfo(normalPathName); - if (info == emptyGroupInfo) - return null; - - final JsonElement cachedMap; - if (info.isDataset == null) { - - synchronized (info) { - - cachedMap = getCachedAttributes(info, normalPathName); - if (cachedMap == null) { - info.isDataset = false; - return null; - } - - dimensions = GsonN5Reader.readAttribute(cachedMap, DatasetAttributes.dimensionsKey, long[].class, gson); - if (dimensions == null) { - info.isDataset = false; - return null; - } - - dataType = GsonN5Reader.readAttribute(cachedMap, DatasetAttributes.dataTypeKey, DataType.class, gson); - if (dataType == null) { - info.isDataset = false; - return null; - } - - info.isDataset = true; - } - } else if (!info.isDataset) { + info = getCachedN5GroupInfo(normalPath); + if (info == emptyGroupInfo || (info != null && info.isDataset != null && !info.isDataset)) return null; - } else { - - cachedMap = getCachedAttributes(info, normalPathName); - dimensions = GsonN5Reader.readAttribute(cachedMap, DatasetAttributes.dimensionsKey, long[].class, gson); - dataType = GsonN5Reader.readAttribute(cachedMap, DatasetAttributes.dataTypeKey, DataType.class, gson); - } - - blockSize = GsonN5Reader.readAttribute(cachedMap, DatasetAttributes.blockSizeKey, int[].class, gson); - - compression = GsonN5Reader.readAttribute(cachedMap, DatasetAttributes.compressionKey, Compression.class, gson); + } - /* version 0 */ - compressionVersion0Name = compression - == null - ? GsonN5Reader.readAttribute(cachedMap, DatasetAttributes.compressionTypeKey, String.class, gson) - : null; + final JsonElement attributes = getAttributes(normalPath); + final long[] dimensions = GsonN5Reader.readAttribute(attributes, DatasetAttributes.dimensionsKey, long[].class, gson); + if (dimensions == null) { + setGroupInfoIsDataset(info, false); + return null; + } - } else { - final JsonElement attributes = getAttributes(normalPathName); + final DataType dataType = GsonN5Reader.readAttribute(attributes, DatasetAttributes.dataTypeKey, DataType.class, gson); + if (dataType == null) { + setGroupInfoIsDataset(info, false); + return null; + } - dimensions = GsonN5Reader.readAttribute(attributes, DatasetAttributes.dimensionsKey, long[].class, gson); - if (dimensions == null) - return null; + setGroupInfoIsDataset(info, true); - dataType = GsonN5Reader.readAttribute(attributes, DatasetAttributes.dataTypeKey, DataType.class, gson); - if (dataType == null) - return null; + final int[] blockSize = GsonN5Reader.readAttribute(attributes, DatasetAttributes.blockSizeKey, int[].class, gson); - blockSize = GsonN5Reader.readAttribute(attributes, DatasetAttributes.blockSizeKey, int[].class, gson); + final Compression compression = GsonN5Reader.readAttribute(attributes, DatasetAttributes.compressionKey, Compression.class, gson); - compression = GsonN5Reader.readAttribute(attributes, DatasetAttributes.compressionKey, Compression.class, gson); + /* version 0 */ + final String compressionVersion0Name = compression + == null + ? GsonN5Reader.readAttribute(attributes, DatasetAttributes.compressionTypeKey, String.class, gson) + : null; - /* version 0 */ - compressionVersion0Name = compression == null - ? GsonN5Reader.readAttribute(attributes, DatasetAttributes.compressionTypeKey, String.class, gson) - : null; - } + return createDatasetAttributes(dimensions, dataType, blockSize, compression, compressionVersion0Name); - if (blockSize == null) - blockSize = Arrays.stream(dimensions).mapToInt(a -> (int)a).toArray(); + } - /* version 0 */ - if (compression == null) { - switch (compressionVersion0Name) { - case "raw": - compression = new RawCompression(); - break; - case "gzip": - compression = new GzipCompression(); - break; - case "bzip2": - compression = new Bzip2Compression(); - break; - case "lz4": - compression = new Lz4Compression(); - break; - case "xz": - compression = new XzCompression(); - break; - } + private void setGroupInfoIsDataset(N5GroupInfo info, boolean normalPathName) { + if (info == null) + return; + synchronized (info) { + if (info != null) + info.isDataset = false; } - - return new DatasetAttributes(dimensions, blockSize, dataType, compression); } - protected JsonElement readAttributes(final String absoluteNormalPath) throws IOException { - - if (!keyValueAccess.exists(absoluteNormalPath)) - return null; + /** + * Get cached attributes for the group identified by a normalPath + * + * @param normalPath normalized group path without leading slash + * @return cached attributes + * empty map if the group exists but no attributes are set + * null if the group does not exist, or if attributes have not been cached + */ + protected JsonElement getCachedAttributes(final String normalPath) { - try (final LockedChannel lockedChannel = keyValueAccess.lockForReading(absoluteNormalPath)) { - return readAttributes(lockedChannel.newReader()); - } + return metaCache.get(normalPath).attributesCache; } /** - * Get and cache attributes for a group identified by an info object and a + * Cache and return attributes for a group identified by an info object and a * pathName. - * + *

    * This helper method does not intelligently handle the case that the group * does not exist (as indicated by info == emptyGroupInfo) which should be * done in calling code. * * @param info - * @param pathName normalized group path without leading slash + * @param normalPath normalized group path without leading slash * @return cached attributes - * empty map if the group exists but not attributes are set - * null if the group does not exist + * empty map if the group exists but not attributes are set + * null if the group does not exist * @throws IOException */ - protected JsonElement getCachedAttributes( + protected JsonElement cacheAttributes( final N5GroupInfo info, - final String normalPath) throws IOException { + final String normalPath + ) throws IOException { JsonElement metadataCache = info.attributesCache; if (metadataCache == null) { @@ -336,37 +254,42 @@ public T getAttribute( final String key, final Class clazz) throws IOException { - final String normalPathName = N5URL.normalizePath(pathName); + final String normalPathName = normalize(pathName); + final String normalizedAttributePath = N5URL.normalizeAttributePath(key); + if (cacheMeta) { - final N5GroupInfo info = getCachedN5GroupInfo(normalPathName); - if (info == emptyGroupInfo) - return null; - final JsonElement metadataCache = getCachedAttributes(info, normalPathName); - if (metadataCache == null) - return null; - return GsonN5Reader.readAttribute(metadataCache, N5URL.normalizeAttributePath(key), clazz, gson); + return getCachedAttribute(normalPathName, normalizedAttributePath, clazz); } else { - return GsonN5Reader.readAttribute(getAttributes(normalPathName), N5URL.normalizeAttributePath(key), clazz, gson); + return GsonN5Reader.readAttribute(getAttributes(normalPathName), normalizedAttributePath, clazz, gson); } } + private T getCachedAttribute(String normalPathName, String normalKey, Class clazz) throws IOException { + + return getCachedAttribute(normalPathName, normalKey, (Type)clazz); + + } + + private T getCachedAttribute(String normalPathName, String normalKey, Type type) throws IOException { + + final N5GroupInfo info = getCachedN5GroupInfo(normalPathName); + if (info == emptyGroupInfo) + return null; + return GsonN5Reader.readAttribute(getAttributes(normalPathName), N5URL.normalizeAttributePath(normalKey), type, gson); + } + @Override public T getAttribute( final String pathName, final String key, final Type type) throws IOException { - final String normalPathName = N5URL.normalizePath(pathName); + final String normalPathName = normalize(pathName); + final String normalizedAttributePath = N5URL.normalizeAttributePath(key); if (cacheMeta) { - final N5GroupInfo info = getCachedN5GroupInfo(normalPathName); - if (info == emptyGroupInfo) - return null; - final JsonElement metadataCache = getCachedAttributes(info, normalPathName); - if (metadataCache == null) - return null; - return GsonN5Reader.readAttribute(metadataCache, N5URL.normalizeAttributePath(key), type, gson); + return getCachedAttribute(normalPathName, key, type); } else { - return GsonN5Reader.readAttribute(getAttributes(normalPathName), N5URL.normalizeAttributePath(key), type, gson); + return GsonN5Reader.readAttribute(getAttributes(normalPathName), normalizedAttributePath, type, gson); } } @@ -406,7 +329,7 @@ protected N5GroupInfo getCachedN5GroupInfo(final String normalPathName) { @Override public boolean exists(final String pathName) { - final String normalPathName = N5URL.normalizePath(pathName); + final String normalPathName = normalize(pathName); if (cacheMeta) return getCachedN5GroupInfo(normalPathName) != emptyGroupInfo; else @@ -417,20 +340,14 @@ public boolean exists(final String pathName) { public boolean datasetExists(final String pathName) throws IOException { if (cacheMeta) { - final String normalPathName = N5URL.normalizePath(pathName); + final String normalPathName = normalize(pathName); final N5GroupInfo info = getCachedN5GroupInfo(normalPathName); if (info == emptyGroupInfo) return false; - if (info.isDataset == null) { - synchronized (info) { - if (info.isDataset == null ) { - - } - else - return info.isDataset; - } - } else - return info.isDataset; + synchronized (info) { + if (info.isDataset != null) + return info.isDataset; + } } return exists(pathName) && getDatasetAttributes(pathName) != null; } @@ -444,12 +361,24 @@ public boolean datasetExists(final String pathName) throws IOException { */ @Override public JsonElement getAttributes(final String pathName) throws IOException { - final String path = attributesPath(N5URL.normalizePath(pathName)); - if (exists(pathName) && !keyValueAccess.exists(path)) + final String groupPath = normalize(pathName); + final String attributesPath = attributesPath(groupPath); + + /* If cached, return the cache*/ + final N5GroupInfo groupInfo = getCachedN5GroupInfo(groupPath); + if (cacheMeta) { + if (groupInfo != null && groupInfo.attributesCache != null) + return groupInfo.attributesCache; + } + + if (exists(pathName) && !keyValueAccess.exists(attributesPath)) return null; - try (final LockedChannel lockedChannel = keyValueAccess.lockForReading(path)) { - return readAttributes(lockedChannel.newReader()); + try (final LockedChannel lockedChannel = keyValueAccess.lockForReading(attributesPath)) { + final JsonElement attributes = readAttributes(lockedChannel.newReader()); + /* If we are reading from the access, update the cache*/ + groupInfo.attributesCache = attributes; + return attributes; } } @@ -459,7 +388,7 @@ public DataBlock readBlock( final DatasetAttributes datasetAttributes, final long... gridPosition) throws IOException { - final String path = getDataBlockPath(N5URL.normalizePath(pathName), gridPosition); + final String path = getDataBlockPath(normalize(pathName), gridPosition); if (!keyValueAccess.exists(path)) return null; @@ -469,9 +398,8 @@ public DataBlock readBlock( } /** - * * @param normalPath normalized path name - * @return + * @return the normalized groups in {@code normalPath} * @throws IOException */ protected String[] normalList(final String normalPath) throws IOException { @@ -482,43 +410,43 @@ protected String[] normalList(final String normalPath) throws IOException { @Override public String[] list(final String pathName) throws IOException { + final String normalPath = normalize(pathName); if (cacheMeta) { - final N5GroupInfo info = getCachedN5GroupInfo(N5URL.normalizePath(pathName)); - if (info == emptyGroupInfo) - throw new IOException("Group '" + pathName +"' does not exist."); - else { - Set children = info.children; - final String[] list; - if (children == null) { - synchronized (info) { - children = info.children; - if (children == null) { - list = normalList(N5URL.normalizePath(pathName)); - info.children = new HashSet<>(Arrays.asList(list)); - } else - list = children.toArray(new String[children.size()]); - } - } else - list = children.toArray(new String[children.size()]); + return getCachedList(normalPath); + } else { + return normalList(normalPath); + } + } + + private String[] getCachedList(String normalPath) throws IOException { + + final N5GroupInfo info = getCachedN5GroupInfo(normalPath); + if (info == emptyGroupInfo) + throw new IOException("Group '" + normalPath + "' does not exist."); + else { + if (info.children != null) + return (info.children).toArray(new String[(info.children).size()]); + + synchronized (info) { + final String[] list = normalList(normalPath); + info.children = new HashSet<>(Arrays.asList(list)); return list; } - } else { - return normalList(N5URL.normalizePath(pathName)); } } /** * Constructs the path for a data block in a dataset at a given grid position. - * + *

    * The returned path is * 

     	 * $basePath/datasetPathName/$gridPosition[0]/$gridPosition[1]/.../$gridPosition[n]
 	 *
    - * + *

    * This is the file into which the data block will be stored. * - * @param normalDatasetPathName normalized dataset path without leading slash + * @param normalPath normalized dataset path * @param gridPosition * @return */ @@ -540,12 +468,12 @@ protected String getDataBlockPath( * Constructs the absolute path (in terms of this store) for the group or * dataset. * - * @param normalPath normalized group path without leading slash - * @return + * @param normalGroupPath normalized group path without leading slash + * @return the absolute path to the group */ - protected String groupPath(final String normalPath) { + protected String groupPath(final String normalGroupPath) { - return keyValueAccess.compose(basePath, normalPath); + return keyValueAccess.compose(basePath, normalGroupPath); } /** @@ -553,7 +481,7 @@ protected String groupPath(final String normalPath) { * file of a group or dataset. * * @param normalPath normalized group path without leading slash - * @return + * @return the absolute path to the attributes */ protected String attributesPath(final String normalPath) { @@ -568,7 +496,7 @@ protected String attributesPath(final String normalPath) { * {@code InvalidPathException}. * * @param path - * @return + * @return the normalized path, without leading slash */ protected String normalize(final String path) { @@ -580,4 +508,55 @@ public String toString() { return String.format("%s[access=%s, basePath=%s]", getClass().getSimpleName(), keyValueAccess, basePath); } + + private static DatasetAttributes createDatasetAttributes( + final long[] dimensions, + final DataType dataType, + int[] blockSize, + Compression compression, + final String compressionVersion0Name + ) { + + if (blockSize == null) + blockSize = Arrays.stream(dimensions).mapToInt(a -> (int)a).toArray(); + + /* version 0 */ + if (compression == null) { + switch (compressionVersion0Name) { + case "raw": + compression = new RawCompression(); + break; + case "gzip": + compression = new GzipCompression(); + break; + case "bzip2": + compression = new Bzip2Compression(); + break; + case "lz4": + compression = new Lz4Compression(); + break; + case "xz": + compression = new XzCompression(); + break; + } + } + + return new DatasetAttributes(dimensions, blockSize, dataType, compression); + } + + /** + * Check for attributes that are required for a group to be a dataset. + * + * @param attributes to check for dataset attributes + * @return if {@link DatasetAttributes#dimensionsKey} and {@link DatasetAttributes#dataTypeKey} are present + */ + protected static boolean hasDatasetAttributes(final JsonElement attributes) { + + if (attributes == null || !attributes.isJsonObject()) { + return false; + } + + final JsonObject metadataCache = attributes.getAsJsonObject(); + return metadataCache.has(DatasetAttributes.dimensionsKey) && metadataCache.has(DatasetAttributes.dataTypeKey); + } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriter.java b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriter.java index 9135ab5d..12cfb2b1 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriter.java @@ -6,10 +6,10 @@ * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. + * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE @@ -104,7 +104,7 @@ protected N5GroupInfo createCachedGroup(final String normalPath) throws IOExcept info = new N5GroupInfo(); metaCache.put(normalPath, info); } - for (String childPathName = normalPath; !(childPathName == null || childPathName.equals(""));) { + for (String childPathName = normalPath; !(childPathName == null || childPathName.equals("")); ) { final String parentPathName = keyValueAccess.parent(childPathName); if (parentPathName == null) break; @@ -133,7 +133,7 @@ protected N5GroupInfo createCachedGroup(final String normalPath) throws IOExcept @Override public void createGroup(final String path) throws IOException { - final String normalPath = N5URL.normalizePath(path); + final String normalPath = normalize(path); if (cacheMeta) { final N5GroupInfo info = createCachedGroup(normalPath); synchronized (info) { @@ -149,7 +149,7 @@ public void createDataset( final String path, final DatasetAttributes datasetAttributes) throws IOException { - final String normalPath = N5URL.normalizePath(path); + final String normalPath = normalize(path); if (cacheMeta) { final N5GroupInfo info = createCachedGroup(normalPath); synchronized (info) { @@ -163,54 +163,47 @@ public void createDataset( } /** - * Helper method that reads the existing map of attributes, JSON encodes, + * Helper method that reads an existing JsonElement representing the root attributes for {@code normalGroupPath}, * inserts and overrides the provided attributes, and writes them back into * the attributes store. * - * @param path - * @param attributes - * @throws IOException + * @param normalGroupPath to write the attributes to + * @param attributes to write + * @throws IOException if unable to read the attributes at {@code normalGroupPath} */ protected void writeAttributes( - final String path, + final String normalGroupPath, final JsonElement attributes) throws IOException { - try (final LockedChannel lock = keyValueAccess.lockForWriting(attributesPath(path))) { + try (final LockedChannel lock = keyValueAccess.lockForWriting(attributesPath(normalGroupPath))) { final JsonElement root = GsonN5Writer.insertAttribute(readAttributes(lock.newReader()), "/", attributes, gson); GsonN5Writer.writeAttributes(lock.newWriter(), root, gson); } } - protected void writeAttributes( - final String normalPath, - final Map attributes) throws IOException { - if (!attributes.isEmpty()) { - createGroup(normalPath); - final JsonElement existingAttributes = getAttributes(normalPath); - JsonElement newAttributesJson = existingAttributes != null && existingAttributes.isJsonObject() ? existingAttributes.getAsJsonObject() : new JsonObject(); - newAttributesJson = GsonN5Writer.insertAttributes(newAttributesJson, attributes, gson); - writeAttributes(normalPath, newAttributesJson); - } - } - /** - * Check for attributes that are required for a group to be a dataset. + * Helper method that reads the existing map of attributes, JSON encodes, + * inserts and overrides the provided attributes, and writes them back into + * the attributes store. * - * @param cachedAttributes - * @return + * @param normalGroupPath to write the attributes to + * @param attributes to write + * @throws IOException if unable to read the attributes at {@code normalGroupPath} */ - protected static boolean hasCachedDatasetAttributes(final JsonElement cachedAttributes) { - + protected void writeAttributes( + final String normalGroupPath, + final Map attributes) throws IOException { - if (cachedAttributes == null || !cachedAttributes.isJsonObject()) { - return false; + if (!attributes.isEmpty()) { + createGroup(normalGroupPath); + final JsonElement existingAttributes = getAttributes(normalGroupPath); + JsonElement newAttributes = + existingAttributes != null && existingAttributes.isJsonObject() ? existingAttributes.getAsJsonObject() : new JsonObject(); + newAttributes = GsonN5Writer.insertAttributes(newAttributes, attributes, gson); + writeAttributes(normalGroupPath, newAttributes); } - - final JsonObject metadataCache = cachedAttributes.getAsJsonObject(); - return metadataCache.has(DatasetAttributes.dimensionsKey) && metadataCache.has(DatasetAttributes.dataTypeKey); } - /** * Helper method to cache and write attributes. * @@ -219,9 +212,20 @@ protected static boolean hasCachedDatasetAttributes(final JsonElement cachedAttr * @return * @throws IOException */ - protected N5GroupInfo setCachedAttributes( - final String normalPath, - final Map attributes) throws IOException { + protected N5GroupInfo setCachedAttributes(final String normalPath, final Map attributes) throws IOException { + + N5GroupInfo info = getN5GroupInfo(normalPath); + final JsonElement metadata = getAttributes(normalPath); + synchronized (info) { + /* Necessary ensure `nulls` are treated consistently regardless of reading from the cache or not */ + info.attributesCache = gson.toJsonTree(GsonN5Writer.insertAttributes(metadata, attributes, gson)); + writeAttributes(normalPath, info.attributesCache); + info.isDataset = hasDatasetAttributes(info.attributesCache); + } + return info; + } + + private N5GroupInfo getN5GroupInfo(String normalPath) throws IOException { N5GroupInfo info = getCachedN5GroupInfo(normalPath); if (info == emptyGroupInfo) { @@ -232,13 +236,6 @@ protected N5GroupInfo setCachedAttributes( throw new IOException("N5 group '" + normalPath + "' does not exist. Cannot set attributes."); } } - final JsonElement metadataCache = getCachedAttributes(info, normalPath); - synchronized (info) { - /* Necessary ensure `nulls` are treated consistently regardless of reading from the cache or not */ - info.attributesCache = gson.toJsonTree(GsonN5Writer.insertAttributes(metadataCache, attributes, gson)); - writeAttributes(normalPath, info.attributesCache); - info.isDataset = hasCachedDatasetAttributes(info.attributesCache); - } return info; } @@ -247,7 +244,7 @@ public void setAttributes( final String path, final Map attributes) throws IOException { - final String normalPath = N5URL.normalizePath(path); + final String normalPath = normalize(path); if (cacheMeta) setCachedAttributes(normalPath, attributes); else @@ -256,64 +253,36 @@ public void setAttributes( @Override public boolean removeAttribute(String pathName, String key) throws IOException { - final String normalPath = N5URL.normalizePath(pathName); + final String normalPath = normalize(pathName); final String normalKey = N5URL.normalizeAttributePath(key); + if (!keyValueAccess.exists(normalPath)) + return false; - if (cacheMeta) { - N5GroupInfo info; - synchronized (metaCache) { - info = getCachedN5GroupInfo(normalPath); - if (info == emptyGroupInfo) { - throw new IOException("N5 group '" + normalPath + "' does not exist. Cannot set attributes."); - } - } - final JsonElement metadataCache = getCachedAttributes(info, normalPath); - if (GsonN5Writer.removeAttribute(metadataCache, normalKey) != null) { - writeAttributes(normalPath, metadataCache); - return true; - } - } else { - final JsonElement attributes = getAttributes(normalPath); - if (GsonN5Writer.removeAttribute(attributes, normalKey) != null) { - writeAttributes(normalPath, attributes); - return true; - } + final JsonElement attributes = getAttributes(normalPath); + if (GsonN5Writer.removeAttribute(attributes, normalKey) != null) { + writeAttributes(normalPath, attributes); + return true; } return false; } @Override public T removeAttribute(String pathName, String key, Class cls) throws IOException { - final String normalPath = N5URL.normalizePath(pathName); + + final String normalPath = normalize(pathName); final String normalKey = N5URL.normalizeAttributePath(key); - if (cacheMeta) { - N5GroupInfo info; - synchronized (metaCache) { - info = getCachedN5GroupInfo(normalPath); - if (info == emptyGroupInfo) { - throw new IOException("N5 group '" + normalPath + "' does not exist. Cannot set attributes."); - } - } - final JsonElement metadataCache = getCachedAttributes(info, normalPath); - final T obj = GsonN5Writer.removeAttribute(metadataCache, normalKey, cls, gson); - if (obj != null) { - writeAttributes(normalPath, metadataCache); - return obj; - } - } else { - final JsonElement attributes = getAttributes(normalPath); - final T obj = GsonN5Writer.removeAttribute(attributes, normalKey, cls, gson); - if (obj != null) { - writeAttributes(normalPath, attributes); - return obj; - } + final JsonElement attributes = getAttributes(normalPath); + final T obj = GsonN5Writer.removeAttribute(attributes, normalKey, cls, gson); + if (obj != null) { + writeAttributes(normalPath, attributes); } - return null; + return obj; } @Override public boolean removeAttributes(String pathName, List attributes) throws IOException { - final String normalPath = N5URL.normalizePath(pathName); + + final String normalPath = normalize(pathName); boolean removed = false; for (String attribute : attributes) { final String normalKey = N5URL.normalizeAttributePath(attribute); @@ -328,7 +297,7 @@ public void writeBlock( final DatasetAttributes datasetAttributes, final DataBlock dataBlock) throws IOException { - final String blockPath = getDataBlockPath(N5URL.normalizePath(path), dataBlock.getGridPosition()); + final String blockPath = getDataBlockPath(normalize(path), dataBlock.getGridPosition()); try (final LockedChannel lock = keyValueAccess.lockForWriting(blockPath)) { DefaultBlockWriter.writeBlock(lock.newOutputStream(), datasetAttributes, dataBlock); @@ -338,32 +307,10 @@ public void writeBlock( @Override public boolean remove(final String path) throws IOException { - final String normalPath = N5URL.normalizePath(path); + final String normalPath = normalize(path); final String groupPath = groupPath(normalPath); if (cacheMeta) { - synchronized (metaCache) { - if (keyValueAccess.exists(groupPath)) { - keyValueAccess.delete(groupPath); - - /* cache nonexistence for all prior children */ - for (final String key : metaCache.keySet()) { - if (key.startsWith(normalPath)) - metaCache.put(key, emptyGroupInfo); - } - - /* remove child from parent */ - final String parentPath = keyValueAccess.parent(normalPath); - final N5GroupInfo parent = metaCache.get(parentPath); - if (parent != null) { - final HashSet children = parent.children; - if (children != null) { - synchronized (children) { - children.remove(keyValueAccess.relativize(normalPath, parentPath)); - } - } - } - } - } + removeCachedGroup(normalPath, groupPath); } else { if (keyValueAccess.exists(groupPath)) keyValueAccess.delete(groupPath); @@ -373,12 +320,39 @@ public boolean remove(final String path) throws IOException { return true; } + private void removeCachedGroup(String normalPath, String groupPath) throws IOException { + + synchronized (metaCache) { + if (keyValueAccess.exists(groupPath)) { + keyValueAccess.delete(groupPath); + + /* cache nonexistence for all prior children */ + for (final String key : metaCache.keySet()) { + if (key.startsWith(normalPath)) + metaCache.put(key, emptyGroupInfo); + } + + /* remove child from parent */ + final String parentPath = keyValueAccess.parent(normalPath); + final N5GroupInfo parent = metaCache.get(parentPath); + if (parent != null) { + final HashSet children = parent.children; + if (children != null) { + synchronized (children) { + children.remove(keyValueAccess.relativize(normalPath, parentPath)); + } + } + } + } + } + } + @Override public boolean deleteBlock( final String path, final long... gridPosition) throws IOException { - final String blockPath = getDataBlockPath(N5URL.normalizePath(path), gridPosition); + final String blockPath = getDataBlockPath(normalize(path), gridPosition); if (keyValueAccess.exists(blockPath)) keyValueAccess.delete(blockPath); diff --git a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java index f57a652d..82a6fe9d 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java @@ -905,12 +905,12 @@ public void testDeepList() throws IOException, ExecutionException, InterruptedEx } @Test - public void testExists() { + public void testExists() throws IOException { final String groupName2 = groupName + "-2"; final String datasetName2 = datasetName + "-2"; final String notExists = groupName + "-notexists"; - try { + try (N5Writer n5 = createN5Writer()){ n5.createDataset(datasetName2, dimensions, blockSize, DataType.UINT64, new RawCompression()); Assert.assertTrue(n5.exists(datasetName2)); Assert.assertTrue(n5.datasetExists(datasetName2)); @@ -921,8 +921,7 @@ public void testExists() { assertFalse(n5.exists(notExists)); assertFalse(n5.datasetExists(notExists)); - } catch (final IOException e) { - fail(e.getMessage()); + } } @@ -1243,58 +1242,62 @@ public void testAttributePathEscaping() throws IOException { final String doubleBrackets = jsonKeyVal(doubleBracketsKey, dataString); final String doubleBackslash = jsonKeyVal(doubleBackslashKey, dataString); - // "/" as key - String grp = "a"; - n5.createGroup(grp); - n5.setAttribute(grp, "\\/", dataString); - assertEquals(dataString, n5.getAttribute(grp, "\\/", String.class)); - String jsonContents = n5.getAttribute(grp, "/", String.class); - assertEquals(rootSlash, jsonContents); - - // "abc/def" as key - grp = "b"; - n5.createGroup(grp); - n5.setAttribute(grp, "abc\\/def", dataString); - assertEquals(dataString, n5.getAttribute(grp, "abc\\/def", String.class)); - jsonContents = n5.getAttribute(grp, "/", String.class); - assertEquals(abcdef, jsonContents); - - // "[0]" as a key - grp = "c"; - n5.createGroup(grp); - n5.setAttribute(grp, "\\[0]", dataString); - assertEquals(dataString, n5.getAttribute(grp, "\\[0]", String.class)); - jsonContents = n5.getAttribute(grp, "/", String.class); - assertEquals(zero, jsonContents); - - // "]] [] [[" as a key - grp = "d"; - n5.createGroup(grp); - n5.setAttribute(grp, bracketsKey, dataString); - assertEquals(dataString, n5.getAttribute(grp, bracketsKey, String.class)); - jsonContents = n5.getAttribute(grp, "/", String.class); - assertEquals(brackets, jsonContents); - - // "[[2][33]]" - grp = "e"; - n5.createGroup(grp); - n5.setAttribute(grp, "[\\[2]\\[33]]", dataString); - assertEquals(dataString, n5.getAttribute(grp, "[\\[2]\\[33]]", String.class)); - jsonContents = n5.getAttribute(grp, "/", String.class); - assertEquals(doubleBrackets, jsonContents); - - // "\\" as key - grp = "f"; - n5.createGroup(grp); - n5.setAttribute(grp, "\\\\", dataString); - assertEquals(dataString, n5.getAttribute(grp, "\\\\", String.class)); - jsonContents = n5.getAttribute(grp, "/", String.class); - assertEquals(doubleBackslash, jsonContents); - - // clear - n5.setAttribute(grp, "/", emptyObj); - jsonContents = n5.getAttribute(grp, "/", String.class); - assertEquals(empty, jsonContents); + try (N5Writer n5 = createN5Writer()) { + + // "/" as key + String grp = "a"; + n5.createGroup(grp); + n5.setAttribute(grp, "\\/", dataString); + assertEquals(dataString, n5.getAttribute(grp, "\\/", String.class)); + String jsonContents = n5.getAttribute(grp, "/", String.class); + assertEquals(rootSlash, jsonContents); + + // "abc/def" as key + grp = "b"; + n5.createGroup(grp); + n5.setAttribute(grp, "abc\\/def", dataString); + assertEquals(dataString, n5.getAttribute(grp, "abc\\/def", String.class)); + jsonContents = n5.getAttribute(grp, "/", String.class); + assertEquals(abcdef, jsonContents); + + // "[0]" as a key + grp = "c"; + n5.createGroup(grp); + n5.setAttribute(grp, "\\[0]", dataString); + assertEquals(dataString, n5.getAttribute(grp, "\\[0]", String.class)); + jsonContents = n5.getAttribute(grp, "/", String.class); + assertEquals(zero, jsonContents); + + // "]] [] [[" as a key + grp = "d"; + n5.createGroup(grp); + n5.setAttribute(grp, bracketsKey, dataString); + assertEquals(dataString, n5.getAttribute(grp, bracketsKey, String.class)); + jsonContents = n5.getAttribute(grp, "/", String.class); + assertEquals(brackets, jsonContents); + + // "[[2][33]]" + grp = "e"; + n5.createGroup(grp); + n5.setAttribute(grp, "[\\[2]\\[33]]", dataString); + assertEquals(dataString, n5.getAttribute(grp, "[\\[2]\\[33]]", String.class)); + jsonContents = n5.getAttribute(grp, "/", String.class); + assertEquals(doubleBrackets, jsonContents); + + // "\\" as key + grp = "f"; + n5.createGroup(grp); + n5.setAttribute(grp, "\\\\", dataString); + assertEquals(dataString, n5.getAttribute(grp, "\\\\", String.class)); + jsonContents = n5.getAttribute(grp, "/", String.class); + assertEquals(doubleBackslash, jsonContents); + + // clear + n5.setAttribute(grp, "/", emptyObj); + jsonContents = n5.getAttribute(grp, "/", String.class); + assertEquals(empty, jsonContents); + + } } /* From d961716c148a95ef528e732bedcae646a3beda6c Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Tue, 7 Mar 2023 15:50:31 -0500 Subject: [PATCH 105/243] feat!: remove redundant methods --- .../saalfeldlab/n5/N5KeyValueReader.java | 51 ------------------- 1 file changed, 51 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java index 29785582..4e292d8c 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java @@ -122,23 +122,6 @@ public String getBasePath() { return this.basePath; } - /** - * Reads the attributes at a given {@code normalAttributesPath}. - * - * @param normalAttributesPath normalized path of attributes location - * @return the attribute root JsonElement - * @throws IOException if unable to read the {@code normalAttributePath} - */ - protected JsonElement readAttributes(final String normalAttributesPath) throws IOException { - - if (!keyValueAccess.exists(normalAttributesPath)) - return null; - - try (final LockedChannel lockedChannel = keyValueAccess.lockForReading(normalAttributesPath)) { - return readAttributes(lockedChannel.newReader()); - } - } - /** * Reads the attributes from a given {@link Reader}. * @@ -214,40 +197,6 @@ protected JsonElement getCachedAttributes(final String normalPath) { return metaCache.get(normalPath).attributesCache; } - /** - * Cache and return attributes for a group identified by an info object and a - * pathName. - *

    - * This helper method does not intelligently handle the case that the group - * does not exist (as indicated by info == emptyGroupInfo) which should be - * done in calling code. - * - * @param info - * @param normalPath normalized group path without leading slash - * @return cached attributes - * empty map if the group exists but not attributes are set - * null if the group does not exist - * @throws IOException - */ - protected JsonElement cacheAttributes( - final N5GroupInfo info, - final String normalPath - ) throws IOException { - - JsonElement metadataCache = info.attributesCache; - if (metadataCache == null) { - synchronized (info) { - metadataCache = info.attributesCache; - if (metadataCache == null) { - final String absoluteNormalPath = attributesPath(normalPath); - metadataCache = readAttributes(absoluteNormalPath); - info.attributesCache = metadataCache; - } - } - } - return metadataCache; - } - @Override public T getAttribute( final String pathName, From c6ccc8bdaedbb97b087c7f5cc3ec40aec6e3686b Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Tue, 7 Mar 2023 15:51:19 -0500 Subject: [PATCH 106/243] feat(test): add cache test --- .../saalfeldlab/n5/AbstractN5Test.java | 7 +++- .../saalfeldlab/n5/N5CachedFSTest.java | 40 ++++++++++++++++++- 2 files changed, 45 insertions(+), 2 deletions(-) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java index 82a6fe9d..e8512d75 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java @@ -1108,11 +1108,16 @@ protected static void addAndTest(N5Writer writer, ArrayList> existin throw new RuntimeException(e); } }); + runTests(writer, existingTests); + existingTests.add(testData); + } + + protected static void runTests(N5Writer writer, ArrayList> existingTests) throws IOException { + for (TestData test : existingTests) { Assert.assertEquals(test.attributeValue, writer.getAttribute(test.groupPath, test.attributePath, test.attributeClass)); Assert.assertEquals(test.attributeValue, writer.getAttribute(test.groupPath, test.attributePath, TypeToken.get(test.attributeClass).getType())); } - existingTests.add(testData); } @Test diff --git a/src/test/java/org/janelia/saalfeldlab/n5/N5CachedFSTest.java b/src/test/java/org/janelia/saalfeldlab/n5/N5CachedFSTest.java index 881bce7d..a345a125 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/N5CachedFSTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/N5CachedFSTest.java @@ -1,13 +1,17 @@ package org.janelia.saalfeldlab.n5; import com.google.gson.GsonBuilder; +import org.junit.Assert; +import org.junit.Test; import java.io.File; import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.ArrayList; public class N5CachedFSTest extends N5FSTest { - @Override protected N5Writer createN5Writer() throws IOException { return new N5FSWriter(createTempN5DirectoryPath(), true); @@ -25,4 +29,38 @@ public class N5CachedFSTest extends N5FSTest { return new N5FSReader(location, gson, true); } + + @Test + public void cacheTest() throws IOException { + /* Test the cache by setting many attributes, then manually deleting the underlying file. + * The only possible way for the test to succeed is if it never again attempts to read the file, and relies on the cache. */ + + try (N5KeyValueWriter n5 = (N5KeyValueWriter) createN5Writer()) { + final String cachedGroup = "cachedGroup"; + final String attributesPath = n5.attributesPath(cachedGroup); + + + + final ArrayList> tests = new ArrayList<>(); + addAndTest(n5, tests, new TestData<>(cachedGroup, "a/b/c", 100)); + addAndTest(n5, tests, new TestData<>(cachedGroup, "a/a[5]", "asdf")); + addAndTest(n5, tests, new TestData<>(cachedGroup, "a/a[2]", 0)); + + Files.delete(Paths.get(attributesPath)); + runTests(n5, tests); + } + + try (N5KeyValueWriter n5 = new N5FSWriter(createTempN5DirectoryPath(), false)) { + final String cachedGroup = "cachedGroup"; + final String attributesPath = n5.attributesPath(cachedGroup); + + final ArrayList> tests = new ArrayList<>(); + addAndTest(n5, tests, new TestData<>(cachedGroup, "a/b/c", 100)); + addAndTest(n5, tests, new TestData<>(cachedGroup, "a/a[5]", "asdf")); + addAndTest(n5, tests, new TestData<>(cachedGroup, "a/a[2]", 0)); + + Files.delete(Paths.get(attributesPath)); + Assert.assertThrows(AssertionError.class, () -> runTests(n5, tests)); + } + } } From 9448eb13480a2311359f53917417007ca87db0c1 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Fri, 10 Mar 2023 09:16:05 -0500 Subject: [PATCH 107/243] feat: expose N5Url Array Pattern --- src/main/java/org/janelia/saalfeldlab/n5/N5URL.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java b/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java index a58d9042..cdad3f09 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java @@ -20,7 +20,7 @@ public class N5URL { - static final Pattern ARRAY_INDEX = Pattern.compile("\\[([0-9]+)]"); + public static final Pattern ARRAY_INDEX = Pattern.compile("\\[([0-9]+)]"); final URI uri; private final String scheme; private final String container; From 34c01ef7a84231fcd2b1469e54a14701f20e0af9 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Fri, 10 Mar 2023 09:16:39 -0500 Subject: [PATCH 108/243] feat(test): add root non-JsonObject tests --- .../saalfeldlab/n5/AbstractN5Test.java | 41 +++++++++++++++---- 1 file changed, 33 insertions(+), 8 deletions(-) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java index e8512d75..1efec982 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java @@ -443,7 +443,7 @@ public void testOverwriteBlock() { } @Test - public void testAttributes() { + public void testAttributes() throws IOException { try (final N5Writer n5 = createN5Writer()) { n5.createGroup(groupName); @@ -501,9 +501,6 @@ public void testAttributes() { n5.setAttribute(groupName, "key2", null); n5.setAttribute(groupName, "key3", null); Assert.assertEquals(0, n5.listAttributes(groupName).size()); - - } catch (final IOException e) { - fail(e.getMessage()); } } @@ -1198,8 +1195,7 @@ public void testAttributePaths() throws IOException { /* We intentionally skipped index 3, it should be null */ Assert.assertNull(writer.getAttribute(testGroup, "/filled/double_array[3]", JsonNull.class)); - /* Lastly, we wish to ensure that escaping does NOT interpret the json path structure, but rather it adds the keys opaquely*/ - + /* Ensure that escaping does NOT interpret the json path structure, but rather it adds the keys opaquely*/ final HashMap testAttributes = new HashMap<>(); testAttributes.put("\\/z\\/y\\/x", 10); testAttributes.put("q\\/r\\/t", 11); @@ -1219,7 +1215,7 @@ public void testAttributePaths() throws IOException { * to try and grab the value as a json structure. I should grab the root, and match the empty string case */ assertEquals(writer.getAttribute(testGroup, "", JsonObject.class), writer.getAttribute(testGroup, "/", JsonObject.class)); - /* Lastly, ensure grabing nonsence results in an exception */ + /* Lastly, ensure grabing nonsense results in an exception */ assertNull(writer.getAttribute(testGroup, "/this/key/does/not/exist", Object.class)); writer.remove(testGroup); @@ -1324,7 +1320,36 @@ private String readAttributesAsString(final String group) { } @Test - public void testRootLeaves() throws IOException { + public void + testRootLeaves() throws IOException { + + /* Test retrieving non-JsonObject root leaves */ + try (final N5Writer n5 = createN5Writer()) { + n5.setAttribute(groupName, "/", "String"); + + final JsonElement stringPrimitive = n5.getAttribute(groupName, "/", JsonElement.class); + assertTrue(stringPrimitive.isJsonPrimitive()); + assertEquals("String", stringPrimitive.getAsString()); + n5.setAttribute(groupName, "/", 0); + final JsonElement intPrimitive = n5.getAttribute(groupName, "/", JsonElement.class); + assertTrue(intPrimitive.isJsonPrimitive()); + assertEquals(0, intPrimitive.getAsInt()); + n5.setAttribute(groupName, "/", true); + final JsonElement booleanPrimitive = n5.getAttribute(groupName, "/", JsonElement.class); + assertTrue(booleanPrimitive.isJsonPrimitive()); + assertEquals(true, booleanPrimitive.getAsBoolean()); + n5.setAttribute(groupName, "/", null); + final JsonElement jsonNull = n5.getAttribute(groupName, "/", JsonElement.class); + assertTrue(jsonNull.isJsonNull()); + assertEquals(JsonNull.INSTANCE, jsonNull); + n5.setAttribute(groupName, "[5]", "array"); + final JsonElement rootJsonArray = n5.getAttribute(groupName, "/", JsonElement.class); + assertTrue(rootJsonArray.isJsonArray()); + final JsonArray rootArray = rootJsonArray.getAsJsonArray(); + assertEquals("array", rootArray.get(5).getAsString()); + assertEquals(JsonNull.INSTANCE, rootArray.get(3)); + assertThrows(IndexOutOfBoundsException.class, () -> rootArray.get(10)); + } /* Test with new root's each time */ final ArrayList> tests = new ArrayList<>(); From a66ac1cb66e0efff01762962d4f6da613734560c Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Fri, 10 Mar 2023 10:53:47 -0500 Subject: [PATCH 109/243] feat(test): move/rename some useful methods for creating temporary N5 paths --- .../saalfeldlab/n5/AbstractN5Test.java | 65 ++++++++++--------- .../saalfeldlab/n5/N5CachedFSTest.java | 4 +- .../org/janelia/saalfeldlab/n5/N5FSTest.java | 19 +----- 3 files changed, 38 insertions(+), 50 deletions(-) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java index 1efec982..bd52d84a 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java @@ -46,9 +46,11 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Random; +import java.util.Set; import java.util.concurrent.ExecutionException; import java.util.concurrent.Executors; import java.util.function.Predicate; @@ -92,7 +94,14 @@ public abstract class AbstractN5Test { protected N5Writer createN5Writer(String location) throws IOException { - return createN5Writer(location, new GsonBuilder()); + final Path testN5Path = Paths.get(location); + final boolean existsBefore = testN5Path.toFile().exists(); + final N5Writer n5Writer = createN5Writer(location, new GsonBuilder()); + final boolean existsAfter = testN5Path.toFile().exists(); + if (!existsBefore && existsAfter) { + tmpFiles.add(location); + } + return n5Writer; } protected abstract N5Writer createN5Writer(String location, GsonBuilder gson) throws IOException; @@ -116,6 +125,19 @@ protected Compression[] getCompressions() { }; } + protected static Set tmpFiles = new HashSet<>(); + protected static String tempN5PathName() { + try { + final File tmpFile = Files.createTempDirectory("n5-test-").toFile(); + tmpFile.deleteOnExit(); + final String tmpPath = tmpFile.getCanonicalPath(); + tmpFiles.add(tmpPath); + return tmpPath; + } catch (Exception e) { + throw new RuntimeException(e); + } + } + /** * @throws IOException */ @@ -507,11 +529,8 @@ public void testAttributes() throws IOException { @Test public void testNullAttributes() throws IOException { - File tmpFile = Files.createTempDirectory("nulls-test-").toFile(); - tmpFile.delete(); - String canonicalPath = tmpFile.getCanonicalPath(); /* serializeNulls*/ - try (N5Writer writer = createN5Writer(canonicalPath, new GsonBuilder().serializeNulls())) { + try (N5Writer writer = createN5Writer(tempN5PathName(), new GsonBuilder().serializeNulls())) { writer.setAttribute(groupName, "nullValue", null); assertEquals(null, writer.getAttribute(groupName, "nullValue", Object.class)); @@ -549,11 +568,8 @@ public void testNullAttributes() throws IOException { assertEquals(JsonNull.INSTANCE, writer.getAttribute(groupName, "existingValue", JsonElement.class)); } - tmpFile = Files.createTempDirectory("nulls-test-").toFile(); - tmpFile.delete(); - canonicalPath = tmpFile.getCanonicalPath(); /* without serializeNulls*/ - try (N5Writer writer = createN5Writer(canonicalPath, new GsonBuilder())) { + try (N5Writer writer = createN5Writer()) { writer.setAttribute(groupName, "nullValue", null); assertEquals(null, writer.getAttribute(groupName, "nullValue", Object.class)); @@ -596,10 +612,7 @@ public void testNullAttributes() throws IOException { @Test public void testRemoveAttributes() throws IOException { - final File tmpFile = Files.createTempDirectory("nulls-test-").toFile(); - tmpFile.delete(); - final String canonicalPath = tmpFile.getCanonicalPath(); - try (N5Writer writer = createN5Writer(canonicalPath, new GsonBuilder().serializeNulls())) { + try (N5Writer writer = createN5Writer(tempN5PathName(), new GsonBuilder().serializeNulls())) { writer.setAttribute("", "a/b/c", 100); assertEquals((Integer)100, writer.getAttribute("", "a/b/c", Integer.class)); @@ -721,10 +734,7 @@ public void testRemove() throws IOException { @Test public void testList() throws IOException { - final File tmpFile = Files.createTempDirectory("list-test-").toFile(); - tmpFile.delete(); - final String canonicalPath = tmpFile.getCanonicalPath(); - try (final N5Writer listN5 = createN5Writer(canonicalPath)) { + try (final N5Writer listN5 = createN5Writer()) { listN5.createGroup(groupName); for (final String subGroup : subGroupNames) listN5.createGroup(groupName + "/" + subGroup); @@ -985,6 +995,8 @@ public void testVersion() throws NumberFormatException, IOException { writer.setAttribute("/", N5Reader.VERSION_KEY, incompatibleVersion.toString()); final Version version = writer.getVersion(); assertFalse(N5Reader.VERSION.isCompatible(version)); + final Version compatibleVersion = new Version(N5Reader.VERSION.getMajor(), N5Reader.VERSION.getMinor(), N5Reader.VERSION.getPatch()); + writer.setAttribute("/", N5Reader.VERSION_KEY, compatibleVersion.toString()); } } @@ -992,9 +1004,7 @@ public void testVersion() throws NumberFormatException, IOException { @Test public void testReaderCreation() throws IOException { - final File tmpFile = Files.createTempDirectory("reader-create-test-").toFile(); - tmpFile.delete(); - final String canonicalPath = tmpFile.getCanonicalPath(); + final String canonicalPath = tempN5PathName(); try (N5Writer writer = createN5Writer(canonicalPath)) { final N5Reader n5r = createN5Reader(canonicalPath); @@ -1358,10 +1368,7 @@ private String readAttributesAsString(final String group) { tests.add(new TestData<>(groupName, "[0]", "array_root")); for (TestData testData : tests) { - final File tmpFile = Files.createTempDirectory("root-leaf-test-").toFile(); - tmpFile.delete(); - final String canonicalPath = tmpFile.getCanonicalPath(); - try (final N5Writer writer = createN5Writer(canonicalPath)) { + try (final N5Writer writer = createN5Writer()) { writer.setAttribute(testData.groupPath, testData.attributePath, testData.attributeValue); Assert.assertEquals(testData.attributeValue, writer.getAttribute(testData.groupPath, testData.attributePath, testData.attributeClass)); Assert.assertEquals(testData.attributeValue, @@ -1375,10 +1382,7 @@ private String readAttributesAsString(final String group) { tests.add(new TestData<>(groupName, "/", "replace_empty_root")); tests.add(new TestData<>(groupName, "[0]", "array_root")); - final File tmpFile = Files.createTempDirectory("reuse-root-leaf-test-").toFile(); - tmpFile.delete(); - final String canonicalPath = tmpFile.getCanonicalPath(); - try (final N5Writer writer = createN5Writer(canonicalPath)) { + try (final N5Writer writer = createN5Writer()) { for (TestData testData : tests) { writer.setAttribute(testData.groupPath, testData.attributePath, testData.attributeValue); Assert.assertEquals(testData.attributeValue, writer.getAttribute(testData.groupPath, testData.attributePath, testData.attributeClass)); @@ -1394,10 +1398,7 @@ private String readAttributesAsString(final String group) { final TestData rootAsArray = new TestData<>(groupName, "/", 300); tests.add(rootAsPrimitive); tests.add(rootAsArray); - final File tmpFile2 = Files.createTempDirectory("reuse-root-leaf-test-").toFile(); - tmpFile2.delete(); - final String canonicalPath2 = tmpFile2.getCanonicalPath(); - try (final N5Writer writer = createN5Writer(canonicalPath2)) { + try (final N5Writer writer = createN5Writer()) { for (TestData test : tests) { /* Set the root as Object*/ writer.setAttribute(rootAsObject.groupPath, rootAsObject.attributePath, rootAsObject.attributeValue); diff --git a/src/test/java/org/janelia/saalfeldlab/n5/N5CachedFSTest.java b/src/test/java/org/janelia/saalfeldlab/n5/N5CachedFSTest.java index a345a125..2f6cbd29 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/N5CachedFSTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/N5CachedFSTest.java @@ -14,7 +14,7 @@ public class N5CachedFSTest extends N5FSTest { @Override protected N5Writer createN5Writer() throws IOException { - return new N5FSWriter(createTempN5DirectoryPath(), true); + return new N5FSWriter(tempN5PathName(), true); } @Override protected N5Writer createN5Writer(String location, GsonBuilder gson) throws IOException { @@ -50,7 +50,7 @@ public void cacheTest() throws IOException { runTests(n5, tests); } - try (N5KeyValueWriter n5 = new N5FSWriter(createTempN5DirectoryPath(), false)) { + try (N5KeyValueWriter n5 = new N5FSWriter(tempN5PathName(), false)) { final String cachedGroup = "cachedGroup"; final String attributesPath = n5.attributesPath(cachedGroup); diff --git a/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java b/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java index b0a289c4..138f5838 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java @@ -33,8 +33,6 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import java.util.HashSet; -import java.util.Set; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -58,9 +56,8 @@ */ public class N5FSTest extends AbstractN5Test { - protected static Set tmpFiles = new HashSet<>(); private static FileSystemKeyValueAccess access = new FileSystemKeyValueAccess(FileSystems.getDefault()); - private static String testDirPath = createTempN5DirectoryPath(); + private static String testDirPath = tempN5PathName(); @AfterClass public static void cleanup() { @@ -77,20 +74,10 @@ public static void cleanup() { */ @Override protected N5Writer createN5Writer() throws IOException { - return new N5FSWriter(createTempN5DirectoryPath(), false); + return new N5FSWriter(tempN5PathName(), false); } - protected static String createTempN5DirectoryPath() { - try { - final File tmpFile = Files.createTempDirectory("n5-test-").toFile(); - tmpFile.deleteOnExit(); - final String tmpPath = tmpFile.getCanonicalPath(); - tmpFiles.add(tmpPath); - return tmpPath; - } catch (Exception e) { - throw new RuntimeException(e); - } - } + @Override From 4f0bd4fff7f44059b3e975fc519311d958911e45 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Fri, 10 Mar 2023 14:49:40 -0500 Subject: [PATCH 110/243] fix: return null if not attributes --- src/main/java/org/janelia/saalfeldlab/n5/GsonN5Reader.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonN5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonN5Reader.java index 855c8274..35418bbe 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonN5Reader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonN5Reader.java @@ -270,7 +270,7 @@ static JsonElement getAttribute(JsonElement root, String normalizedAttributePath */ static Map> listAttributes(JsonElement root) throws IOException { - if (root != null && !root.isJsonObject()) { + if (root == null || !root.isJsonObject()) { return null; } From 43373df61136ea7bfca07a38cf7865e90e45e52b Mon Sep 17 00:00:00 2001 From: Stephan Saalfeld Date: Tue, 21 Mar 2023 15:42:40 -0400 Subject: [PATCH 111/243] improve comments --- .../java/org/janelia/saalfeldlab/n5/KeyValueAccess.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccess.java index 117c1927..4e904855 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccess.java @@ -74,7 +74,7 @@ public interface KeyValueAccess { /** * Normalize a path to canonical form. All paths pointing to the same * location return the same output. This is most important for cached - * data pointing getting the same key. + * data pointing at the same location getting the same key. * * @param path * @return @@ -162,7 +162,10 @@ public interface KeyValueAccess { public String[] list(final String normalPath) throws IOException; /** - * Create a director and all parent paths along the way. + * Create a directory and all parent paths along the way. The directory + * and parent paths are discoverable. On a filesystem, this usually means + * that the directories exist, on a key value store that is unaware of + * directories, this may be implemented as creating an object for each path. * * @param normalPath is expected to be in normalized form, no further * efforts are made to normalize it. From 548d489f1ec7742677b8e5662153decc22b24bc0 Mon Sep 17 00:00:00 2001 From: Stephan Saalfeld Date: Wed, 22 Mar 2023 17:46:53 -0400 Subject: [PATCH 112/243] build: can compile with Java >=11, replace sun.nio.cs.ThreadLocalCoders --- .../org/janelia/saalfeldlab/n5/N5URL.java | 43 +++++++++---------- 1 file changed, 21 insertions(+), 22 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java b/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java index cdad3f09..fb40706f 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java @@ -1,11 +1,10 @@ package org.janelia.saalfeldlab.n5; -import sun.nio.cs.ThreadLocalCoders; - import java.net.URI; import java.net.URISyntaxException; import java.nio.ByteBuffer; import java.nio.CharBuffer; +import java.nio.charset.Charset; import java.nio.charset.CharsetDecoder; import java.nio.charset.CoderResult; import java.nio.charset.CodingErrorAction; @@ -20,6 +19,7 @@ public class N5URL { + private static final Charset UTF8 = Charset.forName("UTF-8"); public static final Pattern ARRAY_INDEX = Pattern.compile("\\[([0-9]+)]"); final URI uri; private final String scheme; @@ -27,12 +27,12 @@ public class N5URL { private final String group; private final String attribute; - public N5URL(String uri) throws URISyntaxException { + public N5URL(final String uri) throws URISyntaxException { this(encodeAsUri(uri)); } - public N5URL(URI uri) { + public N5URL(final URI uri) { this.uri = uri; scheme = uri.getScheme() == null ? null : uri.getScheme(); @@ -112,7 +112,7 @@ public LinkedAttributePathToken getAttributePathTokens() { * @param normalizedAttributePath to parse into {@link LinkedAttributePathToken}s * @return the head of the {@link LinkedAttributePathToken}s */ - public static LinkedAttributePathToken getAttributePathTokens(String normalizedAttributePath) { + public static LinkedAttributePathToken getAttributePathTokens(final String normalizedAttributePath) { final String[] attributePathParts = normalizedAttributePath.replaceAll("^/", "").split("(? tokens = new ArrayList<>(); - StringBuilder curToken = new StringBuilder(); + final StringBuilder curToken = new StringBuilder(); boolean escape = false; for (final char character : pathChars) { /* Skip if we last saw escape*/ @@ -406,7 +406,7 @@ public static String normalizePath(String path) { * @param attributePath to normalize * @return the normalized attribute path */ - public static String normalizeAttributePath(String attributePath) { + public static String normalizeAttributePath(final String attributePath) { /* Short circuit if there are no non-escaped `/` or array indices (e.g. [N] where N is a non-negative integer) */ if (!attributePath.matches(".*((?= '0') && (c <= '9')) return c - '0'; @@ -526,7 +526,7 @@ private static int decode(char c) { * * @see URI#decode(char, char) */ - private static byte decode(char c1, char c2) { + private static byte decode(final char c1, final char c2) { return (byte)(((decode(c1) & 0xf) << 4) | ((decode(c2) & 0xf) << 0)); @@ -544,26 +544,26 @@ private static byte decode(char c1, char c2) { * * @see URI#decode(char, char) */ - private static String decodeFragment(String rawFragment) { + private static String decodeFragment(final String rawFragment) { if (rawFragment == null) return rawFragment; - int n = rawFragment.length(); + final int n = rawFragment.length(); if (n == 0) return rawFragment; if (rawFragment.indexOf('%') < 0) return rawFragment; - StringBuffer sb = new StringBuffer(n); - ByteBuffer bb = ByteBuffer.allocate(n); - CharBuffer cb = CharBuffer.allocate(n); - CharsetDecoder dec = ThreadLocalCoders.decoderFor("UTF-8") + final StringBuffer sb = new StringBuffer(n); + final ByteBuffer bb = ByteBuffer.allocate(n); + final CharBuffer cb = CharBuffer.allocate(n); + + final CharsetDecoder dec = UTF8.newDecoder() .onMalformedInput(CodingErrorAction.REPLACE) .onUnmappableCharacter(CodingErrorAction.REPLACE); // This is not horribly efficient, but it will do for now char c = rawFragment.charAt(0); - boolean betweenBrackets = false; for (int i = 0; i < n; ) { assert c == rawFragment.charAt(i); // Loop invariant @@ -575,7 +575,6 @@ private static String decodeFragment(String rawFragment) { continue; } bb.clear(); - int ui = i; for (; ; ) { assert (n - i >= 2); bb.put(decode(rawFragment.charAt(++i), rawFragment.charAt(++i))); From 36ba7ae075278784816b4cfa0c04747667c9e75b Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Thu, 23 Mar 2023 16:50:54 -0400 Subject: [PATCH 113/243] fix remove attribute for non-root paths * add a test --- .../java/org/janelia/saalfeldlab/n5/N5KeyValueWriter.java | 3 ++- src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java | 4 ++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriter.java b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriter.java index 12cfb2b1..d940509f 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriter.java @@ -254,9 +254,10 @@ public void setAttributes( @Override public boolean removeAttribute(String pathName, String key) throws IOException { final String normalPath = normalize(pathName); + final String absoluteNormalPath = keyValueAccess.compose(basePath, normalPath); final String normalKey = N5URL.normalizeAttributePath(key); - if (!keyValueAccess.exists(normalPath)) + if (!keyValueAccess.exists(absoluteNormalPath)) return false; final JsonElement attributes = getAttributes(normalPath); diff --git a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java index bd52d84a..ee7a2e32 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java @@ -715,6 +715,10 @@ public void testRemoveAttributes() throws IOException { assertEquals((Integer)100, writer.removeAttribute("", "a////b/x/../c", Integer.class)); assertNull(writer.getAttribute("", "a/b/c", Integer.class)); + writer.createGroup("foo"); + writer.setAttribute("foo", "a", 100); + writer.removeAttribute("foo", "a"); + assertNull(writer.getAttribute("foo", "a", Integer.class)); } } From 869bbac1e3ca3b37cb241e92aa1bb45dfa058509 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Mon, 27 Mar 2023 13:39:48 -0400 Subject: [PATCH 114/243] test: make testAttributePathEscaping more permissive * check that the key appears correctly in the json that is written * allow other values to potentially be in the json as well * this is helpful for the zarr test --- .../janelia/saalfeldlab/n5/AbstractN5Test.java | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java index ee7a2e32..1346af25 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java @@ -1240,7 +1240,6 @@ public void testAttributePaths() throws IOException { public void testAttributePathEscaping() throws IOException { final JsonObject emptyObj = new JsonObject(); - final String empty = "{}"; final String slashKey = "/"; final String abcdefKey = "abc/def"; @@ -1265,7 +1264,7 @@ public void testAttributePathEscaping() throws IOException { n5.setAttribute(grp, "\\/", dataString); assertEquals(dataString, n5.getAttribute(grp, "\\/", String.class)); String jsonContents = n5.getAttribute(grp, "/", String.class); - assertEquals(rootSlash, jsonContents); + assertTrue(jsonContents.contains(rootSlash)); // "abc/def" as key grp = "b"; @@ -1273,7 +1272,7 @@ public void testAttributePathEscaping() throws IOException { n5.setAttribute(grp, "abc\\/def", dataString); assertEquals(dataString, n5.getAttribute(grp, "abc\\/def", String.class)); jsonContents = n5.getAttribute(grp, "/", String.class); - assertEquals(abcdef, jsonContents); + assertTrue(jsonContents.contains(abcdef)); // "[0]" as a key grp = "c"; @@ -1281,7 +1280,7 @@ public void testAttributePathEscaping() throws IOException { n5.setAttribute(grp, "\\[0]", dataString); assertEquals(dataString, n5.getAttribute(grp, "\\[0]", String.class)); jsonContents = n5.getAttribute(grp, "/", String.class); - assertEquals(zero, jsonContents); + assertTrue(jsonContents.contains(zero)); // "]] [] [[" as a key grp = "d"; @@ -1289,7 +1288,7 @@ public void testAttributePathEscaping() throws IOException { n5.setAttribute(grp, bracketsKey, dataString); assertEquals(dataString, n5.getAttribute(grp, bracketsKey, String.class)); jsonContents = n5.getAttribute(grp, "/", String.class); - assertEquals(brackets, jsonContents); + assertTrue(jsonContents.contains(brackets)); // "[[2][33]]" grp = "e"; @@ -1297,7 +1296,7 @@ public void testAttributePathEscaping() throws IOException { n5.setAttribute(grp, "[\\[2]\\[33]]", dataString); assertEquals(dataString, n5.getAttribute(grp, "[\\[2]\\[33]]", String.class)); jsonContents = n5.getAttribute(grp, "/", String.class); - assertEquals(doubleBrackets, jsonContents); + assertTrue(jsonContents.contains(doubleBrackets)); // "\\" as key grp = "f"; @@ -1305,12 +1304,12 @@ public void testAttributePathEscaping() throws IOException { n5.setAttribute(grp, "\\\\", dataString); assertEquals(dataString, n5.getAttribute(grp, "\\\\", String.class)); jsonContents = n5.getAttribute(grp, "/", String.class); - assertEquals(doubleBackslash, jsonContents); + assertTrue(jsonContents.contains(doubleBackslash)); // clear n5.setAttribute(grp, "/", emptyObj); jsonContents = n5.getAttribute(grp, "/", String.class); - assertEquals(empty, jsonContents); + assertFalse(jsonContents.contains(doubleBackslash)); } } @@ -1320,7 +1319,7 @@ public void testAttributePathEscaping() throws IOException { */ private String jsonKeyVal(String key, String val) { - return String.format("{\"%s\":\"%s\"}", key, val); + return String.format("\"%s\":\"%s\"", key, val); } private String readAttributesAsString(final String group) { From 696ea39aae1e821e84ab8399d624e5cb20473b7c Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Mon, 27 Mar 2023 17:03:50 -0400 Subject: [PATCH 115/243] style: cleanup documentation --- src/main/java/org/janelia/saalfeldlab/n5/N5URL.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java b/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java index fb40706f..c9c72267 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java @@ -533,9 +533,9 @@ private static byte decode(final char c1, final char c2) { } /** - * Modifier from {@link URI#decode(String)} to ignore the listed Exception, where it doesn't decode escape values inside square braces. + * Modified from {@link URI#decode(String)} to ignore the listed exception, where it doesn't decode escape values inside square braces. *

    - * As an example of the origin implement, a backslash inside a square brace would be encoded to "[%5C]", and + * As an example of the original implementation, a backslash inside a square brace would be encoded to "[%5C]", and * when calling {@code decode("[%5C]")} it would not decode to "[\]" since the encode escape sequence is inside square braces. *

    * We keep all the decoding logic in this modified version, EXCEPT, that we don't check for and ignore encoded sequences inside square braces. @@ -591,7 +591,7 @@ private static String decodeFragment(final String rawFragment) { assert cr.isUnderflow(); cr = dec.flush(cb); assert cr.isUnderflow(); - sb.append(cb.flip().toString()); + sb.append(cb.flip()); } return sb.toString(); From 3a81347a86644eb724b1eed4ca4fe58fad34c9d2 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Mon, 27 Mar 2023 17:04:40 -0400 Subject: [PATCH 116/243] fix: replace root with null when removing "/" attribute --- .../java/org/janelia/saalfeldlab/n5/N5KeyValueWriter.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriter.java b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriter.java index d940509f..eeb0300b 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriter.java @@ -27,6 +27,7 @@ import com.google.gson.GsonBuilder; import com.google.gson.JsonElement; +import com.google.gson.JsonNull; import com.google.gson.JsonObject; import java.io.IOException; @@ -260,6 +261,11 @@ public void setAttributes( if (!keyValueAccess.exists(absoluteNormalPath)) return false; + if (key.equals("/")) { + writeAttributes(normalPath, JsonNull.INSTANCE); + return true; + } + final JsonElement attributes = getAttributes(normalPath); if (GsonN5Writer.removeAttribute(attributes, normalKey) != null) { writeAttributes(normalPath, attributes); From 9e77e19afedcb2d27207d03e93053ac8bd91321c Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Mon, 27 Mar 2023 17:06:47 -0400 Subject: [PATCH 117/243] feat: throw IOException if container doesn't exist when opening reader. Create (before construction) if doesn't exist for writer --- .../org/janelia/saalfeldlab/n5/N5KeyValueReader.java | 2 ++ .../org/janelia/saalfeldlab/n5/N5KeyValueWriter.java | 9 ++++++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java index 4e292d8c..9ed9c715 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java @@ -105,6 +105,8 @@ public N5KeyValueReader( final Version version = getVersion(); if (!VERSION.isCompatible(version)) throw new IOException("Incompatible version " + version + " (this is " + VERSION + ")."); + } else { + throw new IOException("basePath " + basePath + " does not exist. "); } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriter.java b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriter.java index eeb0300b..ab385f71 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriter.java @@ -72,12 +72,19 @@ public N5KeyValueWriter( final GsonBuilder gsonBuilder, final boolean cacheAttributes) throws IOException { - super(keyValueAccess, basePath, gsonBuilder, cacheAttributes); + super(keyValueAccess, createDirectories(keyValueAccess, basePath), gsonBuilder, cacheAttributes); createGroup("/"); if (!VERSION.equals(getVersion())) setAttribute("/", VERSION_KEY, VERSION.toString()); } + private static String createDirectories(KeyValueAccess keyValueAccess, String basePath) throws IOException { + + final String normBasePath = keyValueAccess.normalize(basePath); + keyValueAccess.createDirectories(normBasePath); + return normBasePath; + } + /** * Helper method to create and cache a group. * From ef1741fe831095c1d7b762ac0725cd63b4443671 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Mon, 27 Mar 2023 17:07:32 -0400 Subject: [PATCH 118/243] feat: don't create groups when writing attributes if group doesn't exist. --- .../saalfeldlab/n5/N5KeyValueWriter.java | 1 - .../janelia/saalfeldlab/n5/AbstractN5Test.java | 17 +++++++++++++++++ .../org/janelia/saalfeldlab/n5/N5FSTest.java | 1 + 3 files changed, 18 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriter.java b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriter.java index ab385f71..d296f3b2 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriter.java @@ -203,7 +203,6 @@ protected void writeAttributes( final Map attributes) throws IOException { if (!attributes.isEmpty()) { - createGroup(normalGroupPath); final JsonElement existingAttributes = getAttributes(normalGroupPath); JsonElement newAttributes = existingAttributes != null && existingAttributes.isJsonObject() ? existingAttributes.getAsJsonObject() : new JsonObject(); diff --git a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java index 1346af25..97df75ae 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java @@ -193,6 +193,17 @@ public void testCreateGroup() { fail("Group does not exist"); } + @Test + public void testSetAttributeDoesntCreateGroup() throws IOException { + + try (final N5Writer writer = createN5Writer()) { + final String testGroup = "/group/should/not/exit"; + assertFalse(writer.exists(testGroup)); + assertThrows(IOException.class, () -> writer.setAttribute(testGroup, "test", "test")); + assertFalse(writer.exists(testGroup)); + } + } + @Test public void testCreateDataset() throws IOException { @@ -532,6 +543,7 @@ public void testNullAttributes() throws IOException { /* serializeNulls*/ try (N5Writer writer = createN5Writer(tempN5PathName(), new GsonBuilder().serializeNulls())) { + writer.createGroup(groupName); writer.setAttribute(groupName, "nullValue", null); assertEquals(null, writer.getAttribute(groupName, "nullValue", Object.class)); assertEquals(JsonNull.INSTANCE, writer.getAttribute(groupName, "nullValue", JsonElement.class)); @@ -571,6 +583,7 @@ public void testNullAttributes() throws IOException { /* without serializeNulls*/ try (N5Writer writer = createN5Writer()) { + writer.createGroup(groupName); writer.setAttribute(groupName, "nullValue", null); assertEquals(null, writer.getAttribute(groupName, "nullValue", Object.class)); assertEquals(null, writer.getAttribute(groupName, "nullValue", JsonElement.class)); @@ -1338,6 +1351,7 @@ private String readAttributesAsString(final String group) { /* Test retrieving non-JsonObject root leaves */ try (final N5Writer n5 = createN5Writer()) { + n5.createGroup(groupName); n5.setAttribute(groupName, "/", "String"); final JsonElement stringPrimitive = n5.getAttribute(groupName, "/", JsonElement.class); @@ -1372,6 +1386,7 @@ private String readAttributesAsString(final String group) { for (TestData testData : tests) { try (final N5Writer writer = createN5Writer()) { + writer.createGroup(testData.groupPath); writer.setAttribute(testData.groupPath, testData.attributePath, testData.attributeValue); Assert.assertEquals(testData.attributeValue, writer.getAttribute(testData.groupPath, testData.attributePath, testData.attributeClass)); Assert.assertEquals(testData.attributeValue, @@ -1386,6 +1401,7 @@ private String readAttributesAsString(final String group) { tests.add(new TestData<>(groupName, "[0]", "array_root")); try (final N5Writer writer = createN5Writer()) { + writer.createGroup(groupName); for (TestData testData : tests) { writer.setAttribute(testData.groupPath, testData.attributePath, testData.attributeValue); Assert.assertEquals(testData.attributeValue, writer.getAttribute(testData.groupPath, testData.attributePath, testData.attributeClass)); @@ -1402,6 +1418,7 @@ private String readAttributesAsString(final String group) { tests.add(rootAsPrimitive); tests.add(rootAsArray); try (final N5Writer writer = createN5Writer()) { + writer.createGroup(groupName); for (TestData test : tests) { /* Set the root as Object*/ writer.setAttribute(rootAsObject.groupPath, rootAsObject.attributePath, rootAsObject.attributeValue); diff --git a/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java b/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java index 138f5838..fc87a229 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java @@ -103,6 +103,7 @@ public void customObjectTest() throws IOException { final UrlAttributeTest.TestDoubles doubles2 = new UrlAttributeTest.TestDoubles("doubles", "doubles2", new double[]{5.8, 4.6, 3.5}); final UrlAttributeTest.TestDoubles doubles3 = new UrlAttributeTest.TestDoubles("doubles", "doubles3", new double[]{5.9, 4.7, 3.6}); final UrlAttributeTest.TestDoubles doubles4 = new UrlAttributeTest.TestDoubles("doubles", "doubles4", new double[]{5.10, 4.8, 3.7}); + n5.createGroup(testGroup); addAndTest(n5, existingTests, new TestData<>(testGroup, "/doubles[1]", doubles1)); addAndTest(n5, existingTests, new TestData<>(testGroup, "/doubles[2]", doubles2)); addAndTest(n5, existingTests, new TestData<>(testGroup, "/doubles[3]", doubles3)); From 2846d267ee38d2584f03a2a4629d3d44140e7490 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Mon, 27 Mar 2023 17:10:29 -0400 Subject: [PATCH 119/243] test: add remove container test --- .../org/janelia/saalfeldlab/n5/AbstractN5Test.java | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java index 97df75ae..829f473f 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java @@ -736,7 +736,19 @@ public void testRemoveAttributes() throws IOException { } @Test - public void testRemove() throws IOException { + public void testRemoveContainer() throws IOException { + + final String location = tempN5PathName(); + try (final N5Writer n5 = createN5Writer(location)) { + assertNotNull(createN5Reader(location)); + n5.remove(); + assertThrows(Exception.class, () -> createN5Reader(location)); + } + assertThrows(Exception.class, () -> createN5Reader(location)); + } + + @Test + public void testRemoveGroup() throws IOException { try (final N5Writer n5 = createN5Writer()) { n5.createDataset(datasetName, dimensions, blockSize, DataType.UINT64, new RawCompression()); From 53f72e298a3c6cacd5e63f0b9a456790697debfc Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Mon, 27 Mar 2023 17:11:27 -0400 Subject: [PATCH 120/243] test: improve reader creation test to more generically support reader implementations --- .../saalfeldlab/n5/AbstractN5Test.java | 24 ++++++++----------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java index 829f473f..99bc1df2 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java @@ -1041,36 +1041,32 @@ public void testReaderCreation() throws IOException { // existing directory without attributes is okay; // Remove and create to remove attributes store - writer.remove("/"); - writer.createGroup("/"); + writer.removeAttribute("/", "/"); final N5Reader na = createN5Reader(canonicalPath); assertNotNull(na); // existing location with attributes, but no version - writer.remove("/"); - writer.createGroup("/"); + writer.removeAttribute("/", "/"); writer.setAttribute("/", "mystring", "ms"); final N5Reader wa = createN5Reader(canonicalPath); assertNotNull(wa); // existing directory with incompatible version should fail - writer.remove("/"); - writer.createGroup("/"); + writer.removeAttribute("/", "/"); writer.setAttribute("/", N5Reader.VERSION_KEY, new Version(N5Reader.VERSION.getMajor() + 1, N5Reader.VERSION.getMinor(), N5Reader.VERSION.getPatch()).toString()); assertThrows("Incompatible version throws error", IOException.class, () -> { createN5Reader(canonicalPath); }); - - // non-existent directory should fail - writer.remove("/"); - assertThrows("Non-existant location throws error", IOException.class, - () -> { - final N5Reader test = createN5Reader(canonicalPath); - test.list("/"); - }); + writer.remove(); } + // non-existent group should fail + assertThrows("Non-existant location throws error", IOException.class, + () -> { + final N5Reader test = createN5Reader(canonicalPath); + test.list("/"); + }); } @Test From c82a87f8e21e798d580edad10b77bde3c6d122d5 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Mon, 27 Mar 2023 17:15:50 -0400 Subject: [PATCH 121/243] fix: groups aren't created when setting attributes, even when caching --- src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriter.java | 1 - src/test/java/org/janelia/saalfeldlab/n5/N5CachedFSTest.java | 2 ++ 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriter.java b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriter.java index d296f3b2..3789a2d9 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriter.java @@ -236,7 +236,6 @@ private N5GroupInfo getN5GroupInfo(String normalPath) throws IOException { N5GroupInfo info = getCachedN5GroupInfo(normalPath); if (info == emptyGroupInfo) { - createGroup(normalPath); synchronized (metaCache) { info = getCachedN5GroupInfo(normalPath); if (info == emptyGroupInfo) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/N5CachedFSTest.java b/src/test/java/org/janelia/saalfeldlab/n5/N5CachedFSTest.java index 2f6cbd29..98e36531 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/N5CachedFSTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/N5CachedFSTest.java @@ -42,6 +42,7 @@ public void cacheTest() throws IOException { final ArrayList> tests = new ArrayList<>(); + n5.createGroup(cachedGroup); addAndTest(n5, tests, new TestData<>(cachedGroup, "a/b/c", 100)); addAndTest(n5, tests, new TestData<>(cachedGroup, "a/a[5]", "asdf")); addAndTest(n5, tests, new TestData<>(cachedGroup, "a/a[2]", 0)); @@ -55,6 +56,7 @@ public void cacheTest() throws IOException { final String attributesPath = n5.attributesPath(cachedGroup); final ArrayList> tests = new ArrayList<>(); + n5.createGroup(cachedGroup); addAndTest(n5, tests, new TestData<>(cachedGroup, "a/b/c", 100)); addAndTest(n5, tests, new TestData<>(cachedGroup, "a/a[5]", "asdf")); addAndTest(n5, tests, new TestData<>(cachedGroup, "a/a[2]", 0)); From 607233c38b8b263042908d9a0fd8679c495e830f Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Wed, 29 Mar 2023 09:48:32 -0400 Subject: [PATCH 122/243] feat!: add getBasePath to N5Reader API --- .../java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java | 5 +---- src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java | 7 +++++++ 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java index 4e292d8c..31ef9ac9 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java @@ -113,10 +113,7 @@ public N5KeyValueReader( return gson; } - /** - * - * @return N5 base path - */ + @Override public String getBasePath() { return this.basePath; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java index f1a18b73..32c63693 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java @@ -209,6 +209,13 @@ public default Version getVersion() throws IOException { return new Version(getAttribute("/", VERSION_KEY, String.class)); } + /** + * Returns a String representation of the path to the root of the container. + * + * @return the base path + */ + public String getBasePath(); + /** * Reads an attribute. * From 38faccad0b3a2547ec5e4daa2aed1437ee98ebb7 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Wed, 29 Mar 2023 09:52:52 -0400 Subject: [PATCH 123/243] feat!: move normalize to GsonN5Reader * expose KeyValueAccess to avoid code duplication --- .../janelia/saalfeldlab/n5/GsonN5Reader.java | 16 ++++++++++++++++ .../saalfeldlab/n5/N5KeyValueReader.java | 19 +++++-------------- 2 files changed, 21 insertions(+), 14 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonN5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonN5Reader.java index 35418bbe..6a0b0f37 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonN5Reader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonN5Reader.java @@ -145,6 +145,22 @@ default DatasetAttributes getDatasetAttributes(final String pathName) throws IOE return new DatasetAttributes(dimensions, blockSize, dataType, compression); } + public abstract KeyValueAccess getKeyValueAccess(); + + /** + * Removes the leading slash from a given path and returns the normalized + * path. It ensures correctness on both Unix and Windows, otherwise + * {@code pathName} is treated as UNC path on Windows, and + * {@code Paths.get(pathName, ...)} fails with + * {@code InvalidPathException}. + * + * @param path + * @return the normalized path, without leading slash + */ + public default String normalize(final String path) { + + return getKeyValueAccess().normalize(path.startsWith("/") || path.startsWith("\\") ? path.substring(1) : path); + } static Gson registerGson(final GsonBuilder gsonBuilder) { gsonBuilder.registerTypeAdapter(DataType.class, new DataType.JsonAdapter()); diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java index 31ef9ac9..93a6f5b4 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java @@ -113,6 +113,11 @@ public N5KeyValueReader( return gson; } + @Override + public KeyValueAccess getKeyValueAccess() { + return keyValueAccess; + } + @Override public String getBasePath() { @@ -434,20 +439,6 @@ protected String attributesPath(final String normalPath) { return keyValueAccess.compose(basePath, normalPath, jsonFile); } - /** - * Removes the leading slash from a given path and returns the normalized - * path. It ensures correctness on both Unix and Windows, otherwise - * {@code pathName} is treated as UNC path on Windows, and - * {@code Paths.get(pathName, ...)} fails with - * {@code InvalidPathException}. - * - * @param path - * @return the normalized path, without leading slash - */ - protected String normalize(final String path) { - - return keyValueAccess.normalize(path.startsWith("/") || path.startsWith("\\") ? path.substring(1) : path); - } @Override public String toString() { From db74311d538ca1163e6ce875f46c1b5353dc5fa3 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Wed, 29 Mar 2023 10:05:26 -0400 Subject: [PATCH 124/243] feat: add setVersion method * enables subclasses (zarr) to change how version is set --- .../java/org/janelia/saalfeldlab/n5/N5KeyValueWriter.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriter.java b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriter.java index d940509f..e0c37ba6 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriter.java @@ -73,6 +73,11 @@ public N5KeyValueWriter( super(keyValueAccess, basePath, gsonBuilder, cacheAttributes); createGroup("/"); + setVersion("/"); + } + + protected void setVersion( final String path ) throws IOException + { if (!VERSION.equals(getVersion())) setAttribute("/", VERSION_KEY, VERSION.toString()); } From 04ac63ad6e01794b0121be91013b4951c2bdc59d Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Wed, 29 Mar 2023 10:06:17 -0400 Subject: [PATCH 125/243] feat: add initializeGroup method * enables subclasses (zarr) to change how groups are created --- .../saalfeldlab/n5/N5KeyValueWriter.java | 31 +++++++++++++++---- 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriter.java b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriter.java index e0c37ba6..af792124 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriter.java @@ -94,13 +94,12 @@ protected N5GroupInfo createCachedGroup(final String normalPath) throws IOExcept N5GroupInfo info = getCachedN5GroupInfo(normalPath); if (info == emptyGroupInfo) { - /* The directories may be created multiple times concurrently, - * but a new cache entry is inserted only if none has been - * inserted in the meantime (because that may already include - * more cached data). + /* + * The directories may be created multiple times concurrently, but a new cache + * entry is inserted only if none has been inserted in the meantime (because + * that may already include more cached data). * - * This avoids synchronizing on the cache for independent - * group creation. + * This avoids synchronizing on the cache for independent group creation. */ keyValueAccess.createDirectories(groupPath(normalPath)); synchronized (metaCache) { @@ -131,10 +130,27 @@ protected N5GroupInfo createCachedGroup(final String normalPath) throws IOExcept childPathName = parentPathName; } } + + /* + * initialize after updating the cache so that the correct N5GroupInfo instance + * can be updated if necessary. + */ + initializeGroup(normalPath); } return info; } + /** + * Performs any necessary initialization to ensure the key given by the argument {@code normalPath} + * is a valid group. Called by {@link createGroup}. + * + * @param normalPath the group path. + */ + protected void initializeGroup( final String normalPath ) + { + // Nothing to do here, but other implementations (e.g. zarr) use this. + } + @Override public void createGroup(final String path) throws IOException { @@ -146,7 +162,10 @@ public void createGroup(final String path) throws IOException { info.isDataset = false; } } else + { keyValueAccess.createDirectories(groupPath(normalPath)); + initializeGroup(normalPath); + } } @Override From 9aee65cdcde38bf9e6aebc02303fd58b3c869a11 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Wed, 29 Mar 2023 10:07:46 -0400 Subject: [PATCH 126/243] chore: formatting --- .../java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java index 93a6f5b4..678d2f9b 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java @@ -108,7 +108,8 @@ public N5KeyValueReader( } } - @Override public Gson getGson() { + @Override + public Gson getGson() { return gson; } @@ -217,8 +218,7 @@ public T getAttribute( private T getCachedAttribute(String normalPathName, String normalKey, Class clazz) throws IOException { - return getCachedAttribute(normalPathName, normalKey, (Type)clazz); - + return getCachedAttribute(normalPathName, normalKey, (Type) clazz); } private T getCachedAttribute(String normalPathName, String normalKey, Type type) throws IOException { @@ -439,7 +439,6 @@ protected String attributesPath(final String normalPath) { return keyValueAccess.compose(basePath, normalPath, jsonFile); } - @Override public String toString() { From a6d105310c87d80876fece4d7f1a2fb713b19a19 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Fri, 31 Mar 2023 09:34:51 -0400 Subject: [PATCH 127/243] feat: changes to make zarr impl easier * rename createDirectories to initializeContainer * make initializeContainer protected --- .../org/janelia/saalfeldlab/n5/N5KeyValueReader.java | 9 +++++++-- .../org/janelia/saalfeldlab/n5/N5KeyValueWriter.java | 7 +++++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java index 8e8fab7e..907c0380 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java @@ -286,7 +286,11 @@ public boolean exists(final String pathName) { if (cacheMeta) return getCachedN5GroupInfo(normalPathName) != emptyGroupInfo; else - return groupExists(groupPath(normalPathName)); + try { + return groupExists(groupPath(normalPathName)) || datasetExists(normalPathName); + } catch (IOException e) { + return false; + } } @Override @@ -302,7 +306,8 @@ public boolean datasetExists(final String pathName) throws IOException { return info.isDataset; } } - return exists(pathName) && getDatasetAttributes(pathName) != null; + // for n5, every dataset must be a group + return groupExists(groupPath(pathName)) && getDatasetAttributes(pathName) != null; } /** diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriter.java b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriter.java index 75570396..ac2c18cd 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriter.java @@ -72,7 +72,7 @@ public N5KeyValueWriter( final GsonBuilder gsonBuilder, final boolean cacheAttributes) throws IOException { - super(keyValueAccess, createDirectories(keyValueAccess, basePath), gsonBuilder, cacheAttributes); + super(keyValueAccess, initializeContainer(keyValueAccess, basePath), gsonBuilder, cacheAttributes); createGroup("/"); setVersion("/"); } @@ -83,7 +83,7 @@ protected void setVersion( final String path ) throws IOException setAttribute("/", VERSION_KEY, VERSION.toString()); } - private static String createDirectories(KeyValueAccess keyValueAccess, String basePath) throws IOException { + protected static String initializeContainer(KeyValueAccess keyValueAccess, String basePath) throws IOException { final String normBasePath = keyValueAccess.normalize(basePath); keyValueAccess.createDirectories(normBasePath); @@ -275,6 +275,9 @@ public void setAttributes( final Map attributes) throws IOException { final String normalPath = normalize(path); + if( !exists(normalPath)) + throw new IOException("" + normalPath + " is not a group or dataset."); + if (cacheMeta) setCachedAttributes(normalPath, attributes); else From d55a3e1c10fa7f11cc04e8578455f94d3bb9b6dc Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Fri, 31 Mar 2023 15:01:24 -0400 Subject: [PATCH 128/243] fix: move existence check to N5FSReader * helpful for downstream (zarr) impl --- .../org/janelia/saalfeldlab/n5/N5FSReader.java | 3 +++ .../janelia/saalfeldlab/n5/N5KeyValueReader.java | 15 ++++++++------- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5FSReader.java b/src/main/java/org/janelia/saalfeldlab/n5/N5FSReader.java index 18314db7..f1cde693 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5FSReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5FSReader.java @@ -66,6 +66,9 @@ public N5FSReader(final String basePath, final GsonBuilder gsonBuilder, final bo basePath, gsonBuilder, cacheMeta); + + if( !exists("/")) + throw new IOException("No container exists at " + basePath ); } /** diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java index 907c0380..79a98ee9 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java @@ -37,6 +37,8 @@ import java.util.HashMap; import java.util.HashSet; +import org.janelia.saalfeldlab.n5.N5Reader.Version; + /** * {@link N5Reader} implementation through {@link KeyValueAccess} with JSON * attributes parsed with {@link Gson}. @@ -101,13 +103,12 @@ public N5KeyValueReader( this.basePath = keyValueAccess.normalize(basePath); this.gson = GsonN5Reader.registerGson(gsonBuilder); this.cacheMeta = cacheMeta; - if (exists("/")) { - final Version version = getVersion(); - if (!VERSION.isCompatible(version)) - throw new IOException("Incompatible version " + version + " (this is " + VERSION + ")."); - } else { - throw new IOException("basePath " + basePath + " does not exist. "); - } + + /* Existence checks, if any, go in subclasses */ + /* Check that version (if there is one) is compatible. */ + final Version version = getVersion(); + if (!VERSION.isCompatible(version)) + throw new IOException("Incompatible version " + version + " (this is " + VERSION + ")."); } @Override From 399199402709b78ec1cb7f22beba47a2aaaac34f Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Fri, 31 Mar 2023 15:01:49 -0400 Subject: [PATCH 129/243] test: that opening a writing on an incompatible container fails --- src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java index 99bc1df2..a3bb4a7b 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java @@ -1024,6 +1024,9 @@ public void testVersion() throws NumberFormatException, IOException { writer.setAttribute("/", N5Reader.VERSION_KEY, incompatibleVersion.toString()); final Version version = writer.getVersion(); assertFalse(N5Reader.VERSION.isCompatible(version)); + + assertThrows(IOException.class, () -> createN5Writer( writer.getBasePath() )); + final Version compatibleVersion = new Version(N5Reader.VERSION.getMajor(), N5Reader.VERSION.getMinor(), N5Reader.VERSION.getPatch()); writer.setAttribute("/", N5Reader.VERSION_KEY, compatibleVersion.toString()); } From f74b8821f51c1e768a92afd3313f21270fe4ea6f Mon Sep 17 00:00:00 2001 From: Stephan Saalfeld Date: Mon, 3 Apr 2023 16:57:15 -0400 Subject: [PATCH 130/243] fix: remove leading slashes in FileSystemKeyValueAccess.normalize(path) --- .../saalfeldlab/n5/FileSystemKeyValueAccess.java | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java index c3bc9954..4ed7d6c4 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java @@ -256,10 +256,20 @@ public String relativize(final String path, final String base) { return basePath.relativize(fileSystem.getPath(path)).toString(); } + /** + * Removes the leading slash from a given path and returns the normalized + * path. It ensures correctness on both Unix and Windows, otherwise + * {@code pathName} is treated as UNC path on Windows, and + * {@code Paths.get(pathName, ...)} fails with + * {@code InvalidPathException}. + * + * @param path + * @return the normalized path, without leading slash + */ @Override public String normalize(final String path) { - return fileSystem.getPath(path).normalize().toString(); + return fileSystem.getPath(path.startsWith("/") || path.startsWith("\\") ? path.substring(1) : path).normalize().toString(); } @Override From 550fe323e3db48efeb2f78331051ec247180f2ad Mon Sep 17 00:00:00 2001 From: Stephan Saalfeld Date: Mon, 3 Apr 2023 17:02:09 -0400 Subject: [PATCH 131/243] BREAKING: remove GsonN5Reader/ Writer interfaces * combine static methods in GsonN5Reader and Writer into GsonN5Utils * remove default implementations and do not implement N5Reader/ Writer * move implementations into N5KeyValueReader/ Writer --- .../janelia/saalfeldlab/n5/GsonN5Writer.java | 238 ------------ .../n5/{GsonN5Reader.java => GsonUtils.java} | 343 +++++++++++------- .../janelia/saalfeldlab/n5/N5FSWriter.java | 6 +- .../saalfeldlab/n5/N5KeyValueReader.java | 81 ++--- .../saalfeldlab/n5/N5KeyValueWriter.java | 60 +-- 5 files changed, 281 insertions(+), 447 deletions(-) delete mode 100644 src/main/java/org/janelia/saalfeldlab/n5/GsonN5Writer.java rename src/main/java/org/janelia/saalfeldlab/n5/{GsonN5Reader.java => GsonUtils.java} (58%) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonN5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonN5Writer.java deleted file mode 100644 index d697911e..00000000 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonN5Writer.java +++ /dev/null @@ -1,238 +0,0 @@ -/** - * Copyright (c) 2017, Stephan Saalfeld - * All rights reserved. - *

    - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - *

    - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - *

    - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ -package org.janelia.saalfeldlab.n5; - -import com.google.gson.Gson; -import com.google.gson.JsonArray; -import com.google.gson.JsonElement; -import com.google.gson.JsonObject; - -import java.io.IOException; -import java.io.Writer; -import java.util.Map; -import java.util.regex.Matcher; - -/** - * {@link N5Reader} for JSON attributes parsed by {@link Gson}. - * - * @author Stephan Saalfeld - */ -public interface GsonN5Writer extends GsonN5Reader, N5Writer { - - - /** - * If there is an attribute in {@code root} such that it can be parsed and desrialized as {@link T}, - * then remove it from {@code root}, write {@code root} to the {@code writer}, and return the removed attribute. - *

    - * If there is an attribute at the location specified by {@code normalizedAttributePath} but it cannot be deserialized to {@link T}, then it is not removed. - *

    - * If nothing is removed, then {@code root} is not writen to the {@code writer}. - * - * @param writer to write the modified {@code root} to after removal of the attribute - * @param root to remove the attribute from - * @param normalizedAttributePath to the attribute location - * @param cls of the attribute to remove - * @param gson to deserialize the attribute with - * @param of the removed attribute - * @return the removed attribute, or null if nothing removed - * @throws IOException - */ - static T removeAttribute( - final Writer writer, - JsonElement root, - final String normalizedAttributePath, - final Class cls, - final Gson gson) throws IOException { - - final T removed = removeAttribute(root, normalizedAttributePath, cls, gson); - if (removed != null) { - writeAttributes(writer, root, gson); - } - return removed; - } - - /** - * If there is an attribute in {@code root} at location {@code normalizedAttributePath} then remove it from {@code root}.. - * - * @param writer to write the modified {@code root} to after removal of the attribute - * @param root to remove the attribute from - * @param normalizedAttributePath to the attribute location - * @param gson to deserialize the attribute with - * @return if the attribute was removed or not - */ - static boolean removeAttribute( - final Writer writer, - final JsonElement root, - final String normalizedAttributePath, - final Gson gson) throws IOException { - - final JsonElement removed = removeAttribute(root, normalizedAttributePath, JsonElement.class, gson); - if (removed != null) { - writeAttributes(writer, root, gson); - return true; - } - return false; - } - - /** - * If there is an attribute in {@code root} such that it can be parsed and desrialized as {@link T}, - * then remove it from {@code root} and return the removed attribute. - *

    - * If there is an attribute at the location specified by {@code normalizedAttributePath} but it cannot be deserialized to {@link T}, then it is not removed. - * - * @param root to remove the attribute from - * @param normalizedAttributePath to the attribute location - * @param cls of the attribute to remove - * @param gson to deserialize the attribute with - * @param of the removed attribute - * @return the removed attribute, or null if nothing removed - */ - static T removeAttribute(final JsonElement root, final String normalizedAttributePath, final Class cls, final Gson gson) { - - final T attribute = GsonN5Reader.readAttribute(root, normalizedAttributePath, cls, gson); - if (attribute != null) { - removeAttribute(root, normalizedAttributePath); - } - return attribute; - } - - /** - * Remove and return the attribute at {@code normalizedAttributePath} as a {@link JsonElement}. - * Does not attempt to parse the attribute. - * - * @param root to search for the {@link JsonElement} at location {@code normalizedAttributePath} - * @param normalizedAttributePath to the attribute - * @return the attribute as a {@link JsonElement}. - */ - static JsonElement removeAttribute(JsonElement root, String normalizedAttributePath) { - - final String[] pathParts = normalizedAttributePath.split("(?= jsonArray.size()) { - return null; - } - root = jsonArray.get(index); - if (i == pathParts.length - 1) { - jsonArray.remove(index); - } - } else { - return null; - } - } - } - return root; - } - - /** - * Inserts {@code attribute} into {@code root} at location {@code normalizedAttributePath} and write the resulting {@code root}. - *

    - * If {@code root} is not a {@link JsonObject}, then it is overwritten with an object containing {@code "normalizedAttributePath": attribute } - * - * @param writer - * @param root - * @param normalizedAttributePath - * @param attribute - * @param gson - * @throws IOException - */ - static void writeAttribute( - final Writer writer, - JsonElement root, - final String normalizedAttributePath, - final T attribute, - final Gson gson) throws IOException { - - root = insertAttribute(root, normalizedAttributePath, attribute, gson); - writeAttributes(writer, root, gson); - } - - /** - * Writes the attributes JsonElemnt to a given {@link Writer}. - * This will overwrite any existing attributes. - * - * @param writer - * @param root - * @throws IOException - */ - static void writeAttributes( - final Writer writer, - JsonElement root, - final Gson gson) throws IOException { - - gson.toJson(root, writer); - writer.flush(); - } - - static JsonElement insertAttributes(JsonElement root, Map attributes, Gson gson) { - - for (Map.Entry attribute : attributes.entrySet()) { - root = insertAttribute(root, N5URL.normalizeAttributePath(attribute.getKey()), attribute.getValue(), gson); - } - return root; - } - - static JsonElement insertAttribute(JsonElement root, String normalizedAttributePath, T attribute, Gson gson) { - - LinkedAttributePathToken pathToken = N5URL.getAttributePathTokens(normalizedAttributePath); - /* No path to traverse or build; just write the value */ - if (pathToken == null) - return gson.toJsonTree(attribute); - - JsonElement json = root; - while (pathToken != null) { - - JsonElement parent = pathToken.setAndCreateParentElement(json); - - /* We may need to create or override the existing root if it is non-existent or incompatible. */ - final boolean rootOverriden = json == root && parent != json; - if (root == null || rootOverriden) { - root = parent; - } - - json = pathToken.writeChild(gson, attribute); - - pathToken = pathToken.next(); - } - return root; - } -} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonN5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonUtils.java similarity index 58% rename from src/main/java/org/janelia/saalfeldlab/n5/GsonN5Reader.java rename to src/main/java/org/janelia/saalfeldlab/n5/GsonUtils.java index 6a0b0f37..76d19e97 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonN5Reader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonUtils.java @@ -25,6 +25,15 @@ */ package org.janelia.saalfeldlab.n5; +import java.io.IOException; +import java.io.Reader; +import java.io.Writer; +import java.lang.reflect.Array; +import java.lang.reflect.Type; +import java.util.HashMap; +import java.util.Map; +import java.util.regex.Matcher; + import com.google.gson.Gson; import com.google.gson.GsonBuilder; import com.google.gson.JsonArray; @@ -34,133 +43,12 @@ import com.google.gson.JsonSyntaxException; import com.google.gson.reflect.TypeToken; -import java.io.IOException; -import java.io.Reader; -import java.lang.reflect.Array; -import java.lang.reflect.Type; -import java.net.URISyntaxException; -import java.util.Arrays; -import java.util.HashMap; -import java.util.Map; -import java.util.regex.Matcher; - /** * {@link N5Reader} for JSON attributes parsed by {@link Gson}. * * @author Stephan Saalfeld */ -public interface GsonN5Reader extends N5Reader { - - Gson getGson(); - - /** - * Reads the attributes a group or dataset. - * - * @param pathName group path - * @return the root {@link JsonElement} of the attributes - * @throws IOException - */ - JsonElement getAttributes(final String pathName) throws IOException; - - @Override - default T getAttribute( - final String pathName, - final String key, - final Class clazz) throws IOException { - - return getAttribute(pathName, key, TypeToken.get(clazz).getType()); - } - - @Override - default T getAttribute( - final String pathName, - final String key, - final Type type) throws IOException { - - final String normalizedGroupPath; - final String normalizedAttributePath; - try { - final N5URL n5url = N5URL.from(null, pathName, key ); - normalizedGroupPath = n5url.normalizeGroupPath(); - normalizedAttributePath = n5url.normalizeAttributePath(); - } catch (URISyntaxException e) { - throw new IOException(e); - } - return GsonN5Reader.readAttribute( getAttributes( normalizedGroupPath ), normalizedAttributePath, type, getGson()); - } - - @Override - default Map> listAttributes(String pathName) throws IOException { - - return listAttributes(getAttributes(pathName)); - } - - - @Override - default DatasetAttributes getDatasetAttributes(final String pathName) throws IOException { - - final JsonElement root = getAttributes(pathName); - final Gson gson = getGson(); - - if (root == null || !root.isJsonObject()) - return null; - - final JsonObject rootObject = root.getAsJsonObject(); - - final long[] dimensions = GsonN5Reader.readAttribute(rootObject, DatasetAttributes.dimensionsKey, long[].class, gson); - if (dimensions == null) - return null; - - final DataType dataType = GsonN5Reader.readAttribute(rootObject, DatasetAttributes.dataTypeKey, DataType.class, gson); - if (dataType == null) - return null; - - int[] blockSize = GsonN5Reader.readAttribute(rootObject, DatasetAttributes.blockSizeKey, int[].class, gson); - if (blockSize == null) - blockSize = Arrays.stream(dimensions).mapToInt(a -> (int)a).toArray(); - - Compression compression = GsonN5Reader.readAttribute(rootObject, DatasetAttributes.compressionKey, Compression.class, gson); - - /* version 0 */ - if (compression == null) { - switch (GsonN5Reader.readAttribute(rootObject, DatasetAttributes.compressionTypeKey, String.class, gson)) { - case "raw": - compression = new RawCompression(); - break; - case "gzip": - compression = new GzipCompression(); - break; - case "bzip2": - compression = new Bzip2Compression(); - break; - case "lz4": - compression = new Lz4Compression(); - break; - case "xz": - compression = new XzCompression(); - break; - } - } - - return new DatasetAttributes(dimensions, blockSize, dataType, compression); - } - - public abstract KeyValueAccess getKeyValueAccess(); - - /** - * Removes the leading slash from a given path and returns the normalized - * path. It ensures correctness on both Unix and Windows, otherwise - * {@code pathName} is treated as UNC path on Windows, and - * {@code Paths.get(pathName, ...)} fails with - * {@code InvalidPathException}. - * - * @param path - * @return the normalized path, without leading slash - */ - public default String normalize(final String path) { - - return getKeyValueAccess().normalize(path.startsWith("/") || path.startsWith("\\") ? path.substring(1) : path); - } +public interface GsonUtils { static Gson registerGson(final GsonBuilder gsonBuilder) { gsonBuilder.registerTypeAdapter(DataType.class, new DataType.JsonAdapter()); @@ -202,29 +90,29 @@ static T readAttribute(final JsonElement root, final String normalizedAttrib * @param return type represented by {@link Type type} * @return the deserialized attribute object, or {@code null} if {@code attribute} cannot deserialize to {@link T} */ - static T parseAttributeElement(JsonElement attribute, Gson gson, Type type) { + static T parseAttributeElement(final JsonElement attribute, final Gson gson, final Type type) { if (attribute == null) return null; final Class clazz = (type instanceof Class) ? ((Class)type) : null; if (clazz != null && clazz.isAssignableFrom(HashMap.class)) { - Type mapType = new TypeToken>() { + final Type mapType = new TypeToken>() { }.getType(); - Map retMap = gson.fromJson(attribute, mapType); + final Map retMap = gson.fromJson(attribute, mapType); //noinspection unchecked return (T)retMap; } if (attribute instanceof JsonArray) { final JsonArray array = attribute.getAsJsonArray(); - T retArray = GsonN5Reader.getJsonAsArray(gson, array, type); + final T retArray = GsonUtils.getJsonAsArray(gson, array, type); if (retArray != null) return retArray; } try { return gson.fromJson(attribute, type); - } catch (JsonSyntaxException e) { + } catch (final JsonSyntaxException e) { if (type == String.class) return (T)gson.toJson(attribute); return null; @@ -239,7 +127,7 @@ static T parseAttributeElement(JsonElement attribute, Gson gson, Type type) * @param normalizedAttributePath to the attribute * @return the attribute as a {@link JsonElement}. */ - static JsonElement getAttribute(JsonElement root, String normalizedAttributePath) { + static JsonElement getAttribute(JsonElement root, final String normalizedAttributePath) { final String[] pathParts = normalizedAttributePath.split("(?Object[] * */ - static Map> listAttributes(JsonElement root) throws IOException { + static Map> listAttributes(final JsonElement root) throws IOException { if (root == null || !root.isJsonObject()) { return null; @@ -333,12 +221,12 @@ else if (jsonElement.isJsonArray()) { return attributes; } - static T getJsonAsArray(Gson gson, JsonArray array, Class cls) { + static T getJsonAsArray(final Gson gson, final JsonArray array, final Class cls) { return getJsonAsArray(gson, array, TypeToken.get(cls).getType()); } - static T getJsonAsArray(Gson gson, JsonArray array, Type type) { + static T getJsonAsArray(final Gson gson, final JsonArray array, final Type type) { final Class clazz = (type instanceof Class) ? ((Class)type) : null; @@ -438,4 +326,197 @@ else if (jsonPrimitive.isNumber()) { else return Object.class; } + + /** + * If there is an attribute in {@code root} such that it can be parsed and desrialized as {@link T}, + * then remove it from {@code root}, write {@code root} to the {@code writer}, and return the removed attribute. + *

    + * If there is an attribute at the location specified by {@code normalizedAttributePath} but it cannot be deserialized to {@link T}, then it is not removed. + *

    + * If nothing is removed, then {@code root} is not writen to the {@code writer}. + * + * @param writer to write the modified {@code root} to after removal of the attribute + * @param root to remove the attribute from + * @param normalizedAttributePath to the attribute location + * @param cls of the attribute to remove + * @param gson to deserialize the attribute with + * @param of the removed attribute + * @return the removed attribute, or null if nothing removed + * @throws IOException + */ + static T removeAttribute( + final Writer writer, + final JsonElement root, + final String normalizedAttributePath, + final Class cls, + final Gson gson) throws IOException { + + final T removed = removeAttribute(root, normalizedAttributePath, cls, gson); + if (removed != null) { + writeAttributes(writer, root, gson); + } + return removed; + } + + /** + * If there is an attribute in {@code root} at location {@code normalizedAttributePath} then remove it from {@code root}.. + * + * @param writer to write the modified {@code root} to after removal of the attribute + * @param root to remove the attribute from + * @param normalizedAttributePath to the attribute location + * @param gson to deserialize the attribute with + * @return if the attribute was removed or not + */ + static boolean removeAttribute( + final Writer writer, + final JsonElement root, + final String normalizedAttributePath, + final Gson gson) throws IOException { + + final JsonElement removed = removeAttribute(root, normalizedAttributePath, JsonElement.class, gson); + if (removed != null) { + writeAttributes(writer, root, gson); + return true; + } + return false; + } + + /** + * If there is an attribute in {@code root} such that it can be parsed and desrialized as {@link T}, + * then remove it from {@code root} and return the removed attribute. + *

    + * If there is an attribute at the location specified by {@code normalizedAttributePath} but it cannot be deserialized to {@link T}, then it is not removed. + * + * @param root to remove the attribute from + * @param normalizedAttributePath to the attribute location + * @param cls of the attribute to remove + * @param gson to deserialize the attribute with + * @param of the removed attribute + * @return the removed attribute, or null if nothing removed + */ + static T removeAttribute(final JsonElement root, final String normalizedAttributePath, final Class cls, final Gson gson) { + + final T attribute = GsonUtils.readAttribute(root, normalizedAttributePath, cls, gson); + if (attribute != null) { + removeAttribute(root, normalizedAttributePath); + } + return attribute; + } + + /** + * Remove and return the attribute at {@code normalizedAttributePath} as a {@link JsonElement}. + * Does not attempt to parse the attribute. + * + * @param root to search for the {@link JsonElement} at location {@code normalizedAttributePath} + * @param normalizedAttributePath to the attribute + * @return the attribute as a {@link JsonElement}. + */ + static JsonElement removeAttribute(JsonElement root, final String normalizedAttributePath) { + + final String[] pathParts = normalizedAttributePath.split("(?= jsonArray.size()) { + return null; + } + root = jsonArray.get(index); + if (i == pathParts.length - 1) { + jsonArray.remove(index); + } + } else { + return null; + } + } + } + return root; + } + + /** + * Inserts {@code attribute} into {@code root} at location {@code normalizedAttributePath} and write the resulting {@code root}. + *

    + * If {@code root} is not a {@link JsonObject}, then it is overwritten with an object containing {@code "normalizedAttributePath": attribute } + * + * @param writer + * @param root + * @param normalizedAttributePath + * @param attribute + * @param gson + * @throws IOException + */ + static void writeAttribute( + final Writer writer, + JsonElement root, + final String normalizedAttributePath, + final T attribute, + final Gson gson) throws IOException { + + root = insertAttribute(root, normalizedAttributePath, attribute, gson); + writeAttributes(writer, root, gson); + } + + /** + * Writes the attributes JsonElemnt to a given {@link Writer}. + * This will overwrite any existing attributes. + * + * @param writer + * @param root + * @throws IOException + */ + static void writeAttributes( + final Writer writer, + final JsonElement root, + final Gson gson) throws IOException { + + gson.toJson(root, writer); + writer.flush(); + } + + static JsonElement insertAttributes(JsonElement root, final Map attributes, final Gson gson) { + + for (final Map.Entry attribute : attributes.entrySet()) { + root = insertAttribute(root, N5URL.normalizeAttributePath(attribute.getKey()), attribute.getValue(), gson); + } + return root; + } + + static JsonElement insertAttribute(JsonElement root, final String normalizedAttributePath, final T attribute, final Gson gson) { + + LinkedAttributePathToken pathToken = N5URL.getAttributePathTokens(normalizedAttributePath); + /* No path to traverse or build; just write the value */ + if (pathToken == null) + return gson.toJsonTree(attribute); + + JsonElement json = root; + while (pathToken != null) { + + final JsonElement parent = pathToken.setAndCreateParentElement(json); + + /* We may need to create or override the existing root if it is non-existent or incompatible. */ + final boolean rootOverriden = json == root && parent != json; + if (root == null || rootOverriden) { + root = parent; + } + + json = pathToken.writeChild(gson, attribute); + + pathToken = pathToken.next(); + } + return root; + } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java b/src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java index dea1ee1c..ea7f1cc5 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java @@ -25,17 +25,17 @@ */ package org.janelia.saalfeldlab.n5; -import com.google.gson.GsonBuilder; - import java.io.IOException; import java.nio.file.FileSystems; +import com.google.gson.GsonBuilder; + /** * Filesystem {@link N5Writer} implementation with version compatibility check. * * @author Stephan Saalfeld */ -public class N5FSWriter extends N5KeyValueWriter implements GsonN5Writer { +public class N5FSWriter extends N5KeyValueWriter implements N5Writer { /** * Opens an {@link N5FSWriter} at a given base path with a custom diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java index 79a98ee9..92b57590 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java @@ -25,19 +25,17 @@ */ package org.janelia.saalfeldlab.n5; -import com.google.gson.Gson; -import com.google.gson.GsonBuilder; -import com.google.gson.JsonElement; -import com.google.gson.JsonObject; - import java.io.IOException; -import java.io.Reader; import java.lang.reflect.Type; import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; +import java.util.Map; -import org.janelia.saalfeldlab.n5.N5Reader.Version; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; /** * {@link N5Reader} implementation through {@link KeyValueAccess} with JSON @@ -47,7 +45,7 @@ * @author Igor Pisarev * @author Philipp Hanslovsky */ -public class N5KeyValueReader implements GsonN5Reader { +public class N5KeyValueReader implements N5Reader { /** * Data object for caching meta data. Elements that are null are not yet @@ -101,7 +99,7 @@ public N5KeyValueReader( this.keyValueAccess = keyValueAccess; this.basePath = keyValueAccess.normalize(basePath); - this.gson = GsonN5Reader.registerGson(gsonBuilder); + this.gson = GsonUtils.registerGson(gsonBuilder); this.cacheMeta = cacheMeta; /* Existence checks, if any, go in subclasses */ @@ -111,13 +109,17 @@ public N5KeyValueReader( throw new IOException("Incompatible version " + version + " (this is " + VERSION + ")."); } - @Override public Gson getGson() { return gson; } @Override + public Map> listAttributes(final String pathName) throws IOException { + + return GsonUtils.listAttributes(getAttributes(pathName)); + } + public KeyValueAccess getKeyValueAccess() { return keyValueAccess; } @@ -128,22 +130,10 @@ public String getBasePath() { return this.basePath; } - /** - * Reads the attributes from a given {@link Reader}. - * - * @param reader containing attriubtes - * @return the attributes root JsonElement - * @throws IOException if unable to read from the {@code reader} - */ - protected JsonElement readAttributes(final Reader reader) throws IOException { - - return GsonN5Reader.readAttributes(reader, gson); - } - @Override public DatasetAttributes getDatasetAttributes(final String pathName) throws IOException { - final String normalPath = normalize(pathName); + final String normalPath = keyValueAccess.normalize(pathName); N5GroupInfo info = null; if (cacheMeta) { info = getCachedN5GroupInfo(normalPath); @@ -153,13 +143,13 @@ public DatasetAttributes getDatasetAttributes(final String pathName) throws IOEx final JsonElement attributes = getAttributes(normalPath); - final long[] dimensions = GsonN5Reader.readAttribute(attributes, DatasetAttributes.dimensionsKey, long[].class, gson); + final long[] dimensions = GsonUtils.readAttribute(attributes, DatasetAttributes.dimensionsKey, long[].class, gson); if (dimensions == null) { setGroupInfoIsDataset(info, false); return null; } - final DataType dataType = GsonN5Reader.readAttribute(attributes, DatasetAttributes.dataTypeKey, DataType.class, gson); + final DataType dataType = GsonUtils.readAttribute(attributes, DatasetAttributes.dataTypeKey, DataType.class, gson); if (dataType == null) { setGroupInfoIsDataset(info, false); return null; @@ -167,21 +157,20 @@ public DatasetAttributes getDatasetAttributes(final String pathName) throws IOEx setGroupInfoIsDataset(info, true); - final int[] blockSize = GsonN5Reader.readAttribute(attributes, DatasetAttributes.blockSizeKey, int[].class, gson); + final int[] blockSize = GsonUtils.readAttribute(attributes, DatasetAttributes.blockSizeKey, int[].class, gson); - final Compression compression = GsonN5Reader.readAttribute(attributes, DatasetAttributes.compressionKey, Compression.class, gson); + final Compression compression = GsonUtils.readAttribute(attributes, DatasetAttributes.compressionKey, Compression.class, gson); /* version 0 */ final String compressionVersion0Name = compression == null - ? GsonN5Reader.readAttribute(attributes, DatasetAttributes.compressionTypeKey, String.class, gson) + ? GsonUtils.readAttribute(attributes, DatasetAttributes.compressionTypeKey, String.class, gson) : null; return createDatasetAttributes(dimensions, dataType, blockSize, compression, compressionVersion0Name); - } - private void setGroupInfoIsDataset(N5GroupInfo info, boolean normalPathName) { + private void setGroupInfoIsDataset(final N5GroupInfo info, final boolean normalPathName) { if (info == null) return; synchronized (info) { @@ -209,27 +198,27 @@ public T getAttribute( final String key, final Class clazz) throws IOException { - final String normalPathName = normalize(pathName); + final String normalPathName = keyValueAccess.normalize(pathName); final String normalizedAttributePath = N5URL.normalizeAttributePath(key); if (cacheMeta) { return getCachedAttribute(normalPathName, normalizedAttributePath, clazz); } else { - return GsonN5Reader.readAttribute(getAttributes(normalPathName), normalizedAttributePath, clazz, gson); + return GsonUtils.readAttribute(getAttributes(normalPathName), normalizedAttributePath, clazz, gson); } } - private T getCachedAttribute(String normalPathName, String normalKey, Class clazz) throws IOException { + private T getCachedAttribute(final String normalPathName, final String normalKey, final Class clazz) throws IOException { return getCachedAttribute(normalPathName, normalKey, (Type) clazz); } - private T getCachedAttribute(String normalPathName, String normalKey, Type type) throws IOException { + private T getCachedAttribute(final String normalPathName, final String normalKey, final Type type) throws IOException { final N5GroupInfo info = getCachedN5GroupInfo(normalPathName); if (info == emptyGroupInfo) return null; - return GsonN5Reader.readAttribute(getAttributes(normalPathName), N5URL.normalizeAttributePath(normalKey), type, gson); + return GsonUtils.readAttribute(getAttributes(normalPathName), N5URL.normalizeAttributePath(normalKey), type, gson); } @Override @@ -238,12 +227,12 @@ public T getAttribute( final String key, final Type type) throws IOException { - final String normalPathName = normalize(pathName); + final String normalPathName = keyValueAccess.normalize(pathName); final String normalizedAttributePath = N5URL.normalizeAttributePath(key); if (cacheMeta) { return getCachedAttribute(normalPathName, key, type); } else { - return GsonN5Reader.readAttribute(getAttributes(normalPathName), normalizedAttributePath, type, gson); + return GsonUtils.readAttribute(getAttributes(normalPathName), normalizedAttributePath, type, gson); } } @@ -283,13 +272,13 @@ protected N5GroupInfo getCachedN5GroupInfo(final String normalPathName) { @Override public boolean exists(final String pathName) { - final String normalPathName = normalize(pathName); + final String normalPathName = keyValueAccess.normalize(pathName); if (cacheMeta) return getCachedN5GroupInfo(normalPathName) != emptyGroupInfo; else try { return groupExists(groupPath(normalPathName)) || datasetExists(normalPathName); - } catch (IOException e) { + } catch (final IOException e) { return false; } } @@ -298,7 +287,7 @@ public boolean exists(final String pathName) { public boolean datasetExists(final String pathName) throws IOException { if (cacheMeta) { - final String normalPathName = normalize(pathName); + final String normalPathName = keyValueAccess.normalize(pathName); final N5GroupInfo info = getCachedN5GroupInfo(normalPathName); if (info == emptyGroupInfo) return false; @@ -318,9 +307,9 @@ public boolean datasetExists(final String pathName) throws IOException { * @return * @throws IOException */ - @Override public JsonElement getAttributes(final String pathName) throws IOException { + public JsonElement getAttributes(final String pathName) throws IOException { - final String groupPath = normalize(pathName); + final String groupPath = keyValueAccess.normalize(pathName); final String attributesPath = attributesPath(groupPath); /* If cached, return the cache*/ @@ -334,7 +323,7 @@ public boolean datasetExists(final String pathName) throws IOException { return null; try (final LockedChannel lockedChannel = keyValueAccess.lockForReading(attributesPath)) { - final JsonElement attributes = readAttributes(lockedChannel.newReader()); + final JsonElement attributes = GsonUtils.readAttributes(lockedChannel.newReader(), gson); /* If we are reading from the access, update the cache*/ groupInfo.attributesCache = attributes; return attributes; @@ -347,7 +336,7 @@ public DataBlock readBlock( final DatasetAttributes datasetAttributes, final long... gridPosition) throws IOException { - final String path = getDataBlockPath(normalize(pathName), gridPosition); + final String path = getDataBlockPath(keyValueAccess.normalize(pathName), gridPosition); if (!keyValueAccess.exists(path)) return null; @@ -369,7 +358,7 @@ protected String[] normalList(final String normalPath) throws IOException { @Override public String[] list(final String pathName) throws IOException { - final String normalPath = normalize(pathName); + final String normalPath = keyValueAccess.normalize(pathName); if (cacheMeta) { return getCachedList(normalPath); } else { @@ -377,7 +366,7 @@ public String[] list(final String pathName) throws IOException { } } - private String[] getCachedList(String normalPath) throws IOException { + private String[] getCachedList(final String normalPath) throws IOException { final N5GroupInfo info = getCachedN5GroupInfo(normalPath); if (info == emptyGroupInfo) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriter.java b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriter.java index ac2c18cd..cdf9d248 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriter.java @@ -25,22 +25,22 @@ */ package org.janelia.saalfeldlab.n5; -import com.google.gson.GsonBuilder; -import com.google.gson.JsonElement; -import com.google.gson.JsonNull; -import com.google.gson.JsonObject; - import java.io.IOException; import java.util.HashSet; import java.util.List; import java.util.Map; +import com.google.gson.GsonBuilder; +import com.google.gson.JsonElement; +import com.google.gson.JsonNull; +import com.google.gson.JsonObject; + /** * Filesystem {@link N5Writer} implementation with version compatibility check. * * @author Stephan Saalfeld */ -public class N5KeyValueWriter extends N5KeyValueReader implements GsonN5Writer { +public class N5KeyValueWriter extends N5KeyValueReader implements N5Writer { /** * Opens an {@link N5KeyValueWriter} at a given base path with a custom @@ -83,7 +83,7 @@ protected void setVersion( final String path ) throws IOException setAttribute("/", VERSION_KEY, VERSION.toString()); } - protected static String initializeContainer(KeyValueAccess keyValueAccess, String basePath) throws IOException { + protected static String initializeContainer(final KeyValueAccess keyValueAccess, final String basePath) throws IOException { final String normBasePath = keyValueAccess.normalize(basePath); keyValueAccess.createDirectories(normBasePath); @@ -162,7 +162,7 @@ protected void initializeGroup( final String normalPath ) @Override public void createGroup(final String path) throws IOException { - final String normalPath = normalize(path); + final String normalPath = keyValueAccess.normalize(path); if (cacheMeta) { final N5GroupInfo info = createCachedGroup(normalPath); synchronized (info) { @@ -181,7 +181,7 @@ public void createDataset( final String path, final DatasetAttributes datasetAttributes) throws IOException { - final String normalPath = normalize(path); + final String normalPath = keyValueAccess.normalize(path); if (cacheMeta) { final N5GroupInfo info = createCachedGroup(normalPath); synchronized (info) { @@ -202,14 +202,16 @@ public void createDataset( * @param normalGroupPath to write the attributes to * @param attributes to write * @throws IOException if unable to read the attributes at {@code normalGroupPath} + * + * @TODO consider cache (or you read the attributes twice?) */ protected void writeAttributes( final String normalGroupPath, final JsonElement attributes) throws IOException { try (final LockedChannel lock = keyValueAccess.lockForWriting(attributesPath(normalGroupPath))) { - final JsonElement root = GsonN5Writer.insertAttribute(readAttributes(lock.newReader()), "/", attributes, gson); - GsonN5Writer.writeAttributes(lock.newWriter(), root, gson); + final JsonElement root = GsonUtils.insertAttribute(GsonUtils.readAttributes(lock.newReader(), gson), "/", attributes, gson); + GsonUtils.writeAttributes(lock.newWriter(), root, gson); } } @@ -230,7 +232,7 @@ protected void writeAttributes( final JsonElement existingAttributes = getAttributes(normalGroupPath); JsonElement newAttributes = existingAttributes != null && existingAttributes.isJsonObject() ? existingAttributes.getAsJsonObject() : new JsonObject(); - newAttributes = GsonN5Writer.insertAttributes(newAttributes, attributes, gson); + newAttributes = GsonUtils.insertAttributes(newAttributes, attributes, gson); writeAttributes(normalGroupPath, newAttributes); } } @@ -245,18 +247,18 @@ protected void writeAttributes( */ protected N5GroupInfo setCachedAttributes(final String normalPath, final Map attributes) throws IOException { - N5GroupInfo info = getN5GroupInfo(normalPath); + final N5GroupInfo info = getN5GroupInfo(normalPath); final JsonElement metadata = getAttributes(normalPath); synchronized (info) { /* Necessary ensure `nulls` are treated consistently regardless of reading from the cache or not */ - info.attributesCache = gson.toJsonTree(GsonN5Writer.insertAttributes(metadata, attributes, gson)); + info.attributesCache = gson.toJsonTree(GsonUtils.insertAttributes(metadata, attributes, gson)); writeAttributes(normalPath, info.attributesCache); info.isDataset = hasDatasetAttributes(info.attributesCache); } return info; } - private N5GroupInfo getN5GroupInfo(String normalPath) throws IOException { + private N5GroupInfo getN5GroupInfo(final String normalPath) throws IOException { N5GroupInfo info = getCachedN5GroupInfo(normalPath); if (info == emptyGroupInfo) { @@ -274,7 +276,7 @@ public void setAttributes( final String path, final Map attributes) throws IOException { - final String normalPath = normalize(path); + final String normalPath = keyValueAccess.normalize(path); if( !exists(normalPath)) throw new IOException("" + normalPath + " is not a group or dataset."); @@ -284,9 +286,9 @@ public void setAttributes( writeAttributes(normalPath, attributes); } - @Override public boolean removeAttribute(String pathName, String key) throws IOException { + @Override public boolean removeAttribute(final String pathName, final String key) throws IOException { - final String normalPath = normalize(pathName); + final String normalPath = keyValueAccess.normalize(pathName); final String absoluteNormalPath = keyValueAccess.compose(basePath, normalPath); final String normalKey = N5URL.normalizeAttributePath(key); @@ -299,31 +301,31 @@ public void setAttributes( } final JsonElement attributes = getAttributes(normalPath); - if (GsonN5Writer.removeAttribute(attributes, normalKey) != null) { + if (GsonUtils.removeAttribute(attributes, normalKey) != null) { writeAttributes(normalPath, attributes); return true; } return false; } - @Override public T removeAttribute(String pathName, String key, Class cls) throws IOException { + @Override public T removeAttribute(final String pathName, final String key, final Class cls) throws IOException { - final String normalPath = normalize(pathName); + final String normalPath = keyValueAccess.normalize(pathName); final String normalKey = N5URL.normalizeAttributePath(key); final JsonElement attributes = getAttributes(normalPath); - final T obj = GsonN5Writer.removeAttribute(attributes, normalKey, cls, gson); + final T obj = GsonUtils.removeAttribute(attributes, normalKey, cls, gson); if (obj != null) { writeAttributes(normalPath, attributes); } return obj; } - @Override public boolean removeAttributes(String pathName, List attributes) throws IOException { + @Override public boolean removeAttributes(final String pathName, final List attributes) throws IOException { - final String normalPath = normalize(pathName); + final String normalPath = keyValueAccess.normalize(pathName); boolean removed = false; - for (String attribute : attributes) { + for (final String attribute : attributes) { final String normalKey = N5URL.normalizeAttributePath(attribute); removed |= removeAttribute(normalPath, attribute); } @@ -336,7 +338,7 @@ public void writeBlock( final DatasetAttributes datasetAttributes, final DataBlock dataBlock) throws IOException { - final String blockPath = getDataBlockPath(normalize(path), dataBlock.getGridPosition()); + final String blockPath = getDataBlockPath(keyValueAccess.normalize(path), dataBlock.getGridPosition()); try (final LockedChannel lock = keyValueAccess.lockForWriting(blockPath)) { DefaultBlockWriter.writeBlock(lock.newOutputStream(), datasetAttributes, dataBlock); @@ -346,7 +348,7 @@ public void writeBlock( @Override public boolean remove(final String path) throws IOException { - final String normalPath = normalize(path); + final String normalPath = keyValueAccess.normalize(path); final String groupPath = groupPath(normalPath); if (cacheMeta) { removeCachedGroup(normalPath, groupPath); @@ -359,7 +361,7 @@ public boolean remove(final String path) throws IOException { return true; } - private void removeCachedGroup(String normalPath, String groupPath) throws IOException { + private void removeCachedGroup(final String normalPath, final String groupPath) throws IOException { synchronized (metaCache) { if (keyValueAccess.exists(groupPath)) { @@ -391,7 +393,7 @@ public boolean deleteBlock( final String path, final long... gridPosition) throws IOException { - final String blockPath = getDataBlockPath(normalize(path), gridPosition); + final String blockPath = getDataBlockPath(keyValueAccess.normalize(path), gridPosition); if (keyValueAccess.exists(blockPath)) keyValueAccess.delete(blockPath); From 2ccdf4e195d21797809c41d2b52ceab5e4af2f3a Mon Sep 17 00:00:00 2001 From: Stephan Saalfeld Date: Mon, 3 Apr 2023 17:02:53 -0400 Subject: [PATCH 132/243] style: remove public from interface method declarations --- .../org/janelia/saalfeldlab/n5/N5Reader.java | 46 +++++++++---------- .../org/janelia/saalfeldlab/n5/N5Writer.java | 24 +++++----- 2 files changed, 35 insertions(+), 35 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java index 32c63693..a31d8078 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java @@ -50,7 +50,7 @@ */ public interface N5Reader extends AutoCloseable { - static public class Version { + public static class Version { private final int major; private final int minor; @@ -204,7 +204,7 @@ public boolean isCompatible(final Version version) { * @return * @throws IOException */ - public default Version getVersion() throws IOException { + default Version getVersion() throws IOException { return new Version(getAttribute("/", VERSION_KEY, String.class)); } @@ -214,7 +214,7 @@ public default Version getVersion() throws IOException { * * @return the base path */ - public String getBasePath(); + String getBasePath(); /** * Reads an attribute. @@ -227,7 +227,7 @@ public default Version getVersion() throws IOException { * @return * @throws IOException */ - public T getAttribute( + T getAttribute( final String pathName, final String key, final Class clazz) throws IOException; @@ -243,7 +243,7 @@ public T getAttribute( * @return * @throws IOException */ - public T getAttribute( + T getAttribute( final String pathName, final String key, final Type type) throws IOException; @@ -257,7 +257,7 @@ public T getAttribute( * not set * @throws IOException */ - public DatasetAttributes getDatasetAttributes(final String pathName) throws IOException; + DatasetAttributes getDatasetAttributes(final String pathName) throws IOException; /** * Reads a {@link DataBlock}. @@ -269,7 +269,7 @@ public T getAttribute( * @return * @throws IOException */ - public DataBlock readBlock( + DataBlock readBlock( final String pathName, final DatasetAttributes datasetAttributes, final long... gridPosition) throws IOException; @@ -285,7 +285,7 @@ public DataBlock readBlock( * @throws ClassNotFoundException */ @SuppressWarnings("unchecked") - public default T readSerializedBlock( + default T readSerializedBlock( final String dataset, final DatasetAttributes attributes, final long... gridPosition) throws IOException, ClassNotFoundException { @@ -307,7 +307,7 @@ public default T readSerializedBlock( * group path * @return */ - public boolean exists(final String pathName); + boolean exists(final String pathName); /** * Test whether a dataset exists. @@ -317,7 +317,7 @@ public default T readSerializedBlock( * @return * @throws IOException */ - public default boolean datasetExists(final String pathName) throws IOException { + default boolean datasetExists(final String pathName) throws IOException { return exists(pathName) && getDatasetAttributes(pathName) != null; } @@ -330,7 +330,7 @@ public default boolean datasetExists(final String pathName) throws IOException { * @return * @throws IOException */ - public String[] list(final String pathName) throws IOException; + String[] list(final String pathName) throws IOException; /** @@ -346,7 +346,7 @@ public default boolean datasetExists(final String pathName) throws IOException { * @return list of groups * @throws IOException */ - public default String[] deepList( + default String[] deepList( final String pathName, final Predicate filter) throws IOException { @@ -370,7 +370,7 @@ public default String[] deepList( * @return list of groups * @throws IOException */ - public default String[] deepList(final String pathName) throws IOException { + default String[] deepList(final String pathName) throws IOException { return deepList(pathName, a -> true); } @@ -401,7 +401,7 @@ public default String[] deepList(final String pathName) throws IOException { * @return list of groups * @throws IOException */ - public default String[] deepListDatasets( + default String[] deepListDatasets( final String pathName, final Predicate filter) throws IOException { @@ -438,7 +438,7 @@ public default String[] deepListDatasets( * @return list of groups * @throws IOException */ - public default String[] deepListDatasets(final String pathName) throws IOException { + default String[] deepListDatasets(final String pathName) throws IOException { return deepListDatasets(pathName, a -> true); } @@ -490,7 +490,7 @@ static ArrayList deepList( * @throws ExecutionException * @throws InterruptedException */ - public default String[] deepList( + default String[] deepList( final String pathName, final Predicate filter, final ExecutorService executor) throws IOException, InterruptedException, ExecutionException { @@ -524,7 +524,7 @@ public default String[] deepList( * @throws ExecutionException * @throws InterruptedException */ - public default String[] deepList( + default String[] deepList( final String pathName, final ExecutorService executor) throws IOException, InterruptedException, ExecutionException { @@ -563,7 +563,7 @@ public default String[] deepList( * @throws ExecutionException * @throws InterruptedException */ - public default String[] deepListDatasets( + default String[] deepListDatasets( final String pathName, final Predicate filter, final ExecutorService executor) throws IOException, InterruptedException, ExecutionException { @@ -611,7 +611,7 @@ public default String[] deepListDatasets( * @throws ExecutionException * @throws InterruptedException */ - public default String[] deepListDatasets( + default String[] deepListDatasets( final String pathName, final ExecutorService executor) throws IOException, InterruptedException, ExecutionException { @@ -671,14 +671,14 @@ static void deepListHelper( * @return * @throws IOException */ - public Map> listAttributes(final String pathName) throws IOException; + Map> listAttributes(final String pathName) throws IOException; /** * Returns the symbol that is used to separate nodes in a group path. * * @return */ - public default String getGroupSeparator() { + default String getGroupSeparator() { return "/"; } @@ -691,7 +691,7 @@ public default String getGroupSeparator() { * @param nodes * @return */ - public default String groupPath(final String... nodes) { + default String groupPath(final String... nodes) { if (nodes == null || nodes.length == 0) return ""; @@ -713,5 +713,5 @@ public default String groupPath(final String... nodes) { * implementations that do not hold any closable resources. */ @Override - public default void close() {} + default void close() {} } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java index 53cc3d46..ae9c8bb3 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java @@ -50,7 +50,7 @@ public interface N5Writer extends N5Reader { * @param attribute * @throws IOException */ - public default void setAttribute( + default void setAttribute( final String pathName, final String key, final T attribute) throws IOException { @@ -65,7 +65,7 @@ public default void setAttribute( * @param attributes * @throws IOException */ - public void setAttributes( + void setAttributes( final String pathName, final Map attributes) throws IOException; @@ -86,12 +86,12 @@ public void setAttributes( * * @param pathName group path * @param key of attribute to remove - * @param cls of the attribute to remove + * @param clazz of the attribute to remove * @param of the attribute * @return the removed attribute, as {@link T}, or {@code null} if no matching attribute * @throws IOException */ - T removeAttribute(String pathName, String key, Class cls) throws IOException; + T removeAttribute(String pathName, String key, Class clazz) throws IOException; /** * Remove attributes as provided by {@code attributes}. @@ -114,7 +114,7 @@ public void setAttributes( * @param datasetAttributes * @throws IOException */ - public default void setDatasetAttributes( + default void setDatasetAttributes( final String pathName, final DatasetAttributes datasetAttributes) throws IOException { @@ -127,7 +127,7 @@ public default void setDatasetAttributes( * @param pathName * @throws IOException */ - public void createGroup(final String pathName) throws IOException; + void createGroup(final String pathName) throws IOException; /** * Removes a group or dataset (directory and all contained files). @@ -144,7 +144,7 @@ public default void setDatasetAttributes( * @return true if removal was successful, false otherwise * @throws IOException */ - public boolean remove(final String pathName) throws IOException; + boolean remove(final String pathName) throws IOException; /** * Removes the N5 container. @@ -152,7 +152,7 @@ public default void setDatasetAttributes( * @return true if removal was successful, false otherwise * @throws IOException */ - public default boolean remove() throws IOException { + default boolean remove() throws IOException { return remove("/"); } @@ -165,7 +165,7 @@ public default boolean remove() throws IOException { * @param datasetAttributes * @throws IOException */ - public default void createDataset( + default void createDataset( final String pathName, final DatasetAttributes datasetAttributes) throws IOException { @@ -183,7 +183,7 @@ public default void createDataset( * @param dataType * @throws IOException */ - public default void createDataset( + default void createDataset( final String pathName, final long[] dimensions, final int[] blockSize, @@ -201,7 +201,7 @@ public default void createDataset( * @param dataBlock * @throws IOException */ - public void writeBlock( + void writeBlock( final String pathName, final DatasetAttributes datasetAttributes, final DataBlock dataBlock) throws IOException; @@ -233,7 +233,7 @@ boolean deleteBlock( * @param gridPosition * @throws IOException */ - public default void writeSerializedBlock( + default void writeSerializedBlock( final Serializable object, final String dataset, final DatasetAttributes datasetAttributes, From f99c34ef583bed0fcdd712d33a384a8280e7340e Mon Sep 17 00:00:00 2001 From: Stephan Saalfeld Date: Mon, 3 Apr 2023 17:10:58 -0400 Subject: [PATCH 133/243] style: --- .../saalfeldlab/n5/N5KeyValueWriter.java | 123 +++++++++++------- 1 file changed, 74 insertions(+), 49 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriter.java b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriter.java index cdf9d248..b2b428b3 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriter.java @@ -53,37 +53,41 @@ public class N5KeyValueWriter extends N5KeyValueReader implements N5Writer { * will be set to the current N5 version of this implementation. * * @param keyValueAccess - * @param basePath n5 base path + * @param basePath + * n5 base path * @param gsonBuilder - * @param cacheAttributes cache attributes - * Setting this to true avoids frequent reading and parsing of JSON - * encoded attributes, this is most interesting for high latency file - * systems. Changes of attributes by an independent writer will not be - * tracked. + * @param cacheAttributes + * Setting this to true avoids frequent reading and parsing of + * JSON encoded attributes, this is most interesting for high + * latency file systems. Changes of attributes by an independent + * writer will not be tracked. * * @throws IOException - * if the base path cannot be written to or cannot be created, - * if the N5 version of the container is not compatible with this - * implementation. + * if the base path cannot be written to or cannot be created, + * if the N5 version of the container is not compatible with + * this implementation. */ public N5KeyValueWriter( final KeyValueAccess keyValueAccess, final String basePath, final GsonBuilder gsonBuilder, - final boolean cacheAttributes) throws IOException { + final boolean cacheAttributes) + throws IOException { super(keyValueAccess, initializeContainer(keyValueAccess, basePath), gsonBuilder, cacheAttributes); createGroup("/"); setVersion("/"); } - protected void setVersion( final String path ) throws IOException - { + protected void setVersion(final String path) throws IOException { + if (!VERSION.equals(getVersion())) setAttribute("/", VERSION_KEY, VERSION.toString()); } - protected static String initializeContainer(final KeyValueAccess keyValueAccess, final String basePath) throws IOException { + protected static String initializeContainer( + final KeyValueAccess keyValueAccess, + final String basePath) throws IOException { final String normBasePath = keyValueAccess.normalize(basePath); keyValueAccess.createDirectories(normBasePath); @@ -93,7 +97,8 @@ protected static String initializeContainer(final KeyValueAccess keyValueAccess, /** * Helper method to create and cache a group. * - * @param normalPath normalized group path without leading slash + * @param normalPath + * normalized group path without leading slash * @return * @throws IOException */ @@ -103,11 +108,12 @@ protected N5GroupInfo createCachedGroup(final String normalPath) throws IOExcept if (info == emptyGroupInfo) { /* - * The directories may be created multiple times concurrently, but a new cache - * entry is inserted only if none has been inserted in the meantime (because - * that may already include more cached data). + * The directories may be created multiple times concurrently, but a + * new cache entry is inserted only if none has been inserted in the + * meantime (because that may already include more cached data). * - * This avoids synchronizing on the cache for independent group creation. + * This avoids synchronizing on the cache for independent group + * creation. */ keyValueAccess.createDirectories(groupPath(normalPath)); synchronized (metaCache) { @@ -116,7 +122,7 @@ protected N5GroupInfo createCachedGroup(final String normalPath) throws IOExcept info = new N5GroupInfo(); metaCache.put(normalPath, info); } - for (String childPathName = normalPath; !(childPathName == null || childPathName.equals("")); ) { + for (String childPathName = normalPath; !(childPathName == null || childPathName.equals(""));) { final String parentPathName = keyValueAccess.parent(childPathName); if (parentPathName == null) break; @@ -131,8 +137,9 @@ protected N5GroupInfo createCachedGroup(final String normalPath) throws IOExcept children = new HashSet<>(); } synchronized (children) { - children.add( - keyValueAccess.relativize(childPathName, parentPathName)); + children + .add( + keyValueAccess.relativize(childPathName, parentPathName)); parentInfo.children = children; } childPathName = parentPathName; @@ -140,8 +147,8 @@ protected N5GroupInfo createCachedGroup(final String normalPath) throws IOExcept } /* - * initialize after updating the cache so that the correct N5GroupInfo instance - * can be updated if necessary. + * initialize after updating the cache so that the correct + * N5GroupInfo instance can be updated if necessary. */ initializeGroup(normalPath); } @@ -149,13 +156,15 @@ protected N5GroupInfo createCachedGroup(final String normalPath) throws IOExcept } /** - * Performs any necessary initialization to ensure the key given by the argument {@code normalPath} - * is a valid group. Called by {@link createGroup}. + * Performs any necessary initialization to ensure the key given by the + * argument {@code normalPath} is a valid group. Called by + * {@link createGroup}. * - * @param normalPath the group path. + * @param normalPath + * the group path. */ - protected void initializeGroup( final String normalPath ) - { + protected void initializeGroup(final String normalPath) { + // Nothing to do here, but other implementations (e.g. zarr) use this. } @@ -169,8 +178,7 @@ public void createGroup(final String path) throws IOException { if (info.isDataset == null) info.isDataset = false; } - } else - { + } else { keyValueAccess.createDirectories(groupPath(normalPath)); initializeGroup(normalPath); } @@ -195,13 +203,16 @@ public void createDataset( } /** - * Helper method that reads an existing JsonElement representing the root attributes for {@code normalGroupPath}, - * inserts and overrides the provided attributes, and writes them back into - * the attributes store. + * Helper method that reads an existing JsonElement representing the root + * attributes for {@code normalGroupPath}, inserts and overrides the + * provided attributes, and writes them back into the attributes store. * - * @param normalGroupPath to write the attributes to - * @param attributes to write - * @throws IOException if unable to read the attributes at {@code normalGroupPath} + * @param normalGroupPath + * to write the attributes to + * @param attributes + * to write + * @throws IOException + * if unable to read the attributes at {@code normalGroupPath} * * @TODO consider cache (or you read the attributes twice?) */ @@ -210,7 +221,8 @@ protected void writeAttributes( final JsonElement attributes) throws IOException { try (final LockedChannel lock = keyValueAccess.lockForWriting(attributesPath(normalGroupPath))) { - final JsonElement root = GsonUtils.insertAttribute(GsonUtils.readAttributes(lock.newReader(), gson), "/", attributes, gson); + final JsonElement root = GsonUtils + .insertAttribute(GsonUtils.readAttributes(lock.newReader(), gson), "/", attributes, gson); GsonUtils.writeAttributes(lock.newWriter(), root, gson); } } @@ -220,9 +232,12 @@ protected void writeAttributes( * inserts and overrides the provided attributes, and writes them back into * the attributes store. * - * @param normalGroupPath to write the attributes to - * @param attributes to write - * @throws IOException if unable to read the attributes at {@code normalGroupPath} + * @param normalGroupPath + * to write the attributes to + * @param attributes + * to write + * @throws IOException + * if unable to read the attributes at {@code normalGroupPath} */ protected void writeAttributes( final String normalGroupPath, @@ -230,8 +245,9 @@ protected void writeAttributes( if (!attributes.isEmpty()) { final JsonElement existingAttributes = getAttributes(normalGroupPath); - JsonElement newAttributes = - existingAttributes != null && existingAttributes.isJsonObject() ? existingAttributes.getAsJsonObject() : new JsonObject(); + JsonElement newAttributes = existingAttributes != null && existingAttributes.isJsonObject() + ? existingAttributes.getAsJsonObject() + : new JsonObject(); newAttributes = GsonUtils.insertAttributes(newAttributes, attributes, gson); writeAttributes(normalGroupPath, newAttributes); } @@ -240,17 +256,23 @@ protected void writeAttributes( /** * Helper method to cache and write attributes. * - * @param normalPath normalized group path without leading slash + * @param normalPath + * normalized group path without leading slash * @param attributes * @return * @throws IOException */ - protected N5GroupInfo setCachedAttributes(final String normalPath, final Map attributes) throws IOException { + protected N5GroupInfo setCachedAttributes( + final String normalPath, + final Map attributes) throws IOException { final N5GroupInfo info = getN5GroupInfo(normalPath); final JsonElement metadata = getAttributes(normalPath); synchronized (info) { - /* Necessary ensure `nulls` are treated consistently regardless of reading from the cache or not */ + /* + * Necessary ensure `nulls` are treated consistently regardless of + * reading from the cache or not + */ info.attributesCache = gson.toJsonTree(GsonUtils.insertAttributes(metadata, attributes, gson)); writeAttributes(normalPath, info.attributesCache); info.isDataset = hasDatasetAttributes(info.attributesCache); @@ -277,7 +299,7 @@ public void setAttributes( final Map attributes) throws IOException { final String normalPath = keyValueAccess.normalize(path); - if( !exists(normalPath)) + if (!exists(normalPath)) throw new IOException("" + normalPath + " is not a group or dataset."); if (cacheMeta) @@ -286,7 +308,8 @@ public void setAttributes( writeAttributes(normalPath, attributes); } - @Override public boolean removeAttribute(final String pathName, final String key) throws IOException { + @Override + public boolean removeAttribute(final String pathName, final String key) throws IOException { final String normalPath = keyValueAccess.normalize(pathName); final String absoluteNormalPath = keyValueAccess.compose(basePath, normalPath); @@ -308,7 +331,8 @@ public void setAttributes( return false; } - @Override public T removeAttribute(final String pathName, final String key, final Class cls) throws IOException { + @Override + public T removeAttribute(final String pathName, final String key, final Class cls) throws IOException { final String normalPath = keyValueAccess.normalize(pathName); final String normalKey = N5URL.normalizeAttributePath(key); @@ -321,7 +345,8 @@ public void setAttributes( return obj; } - @Override public boolean removeAttributes(final String pathName, final List attributes) throws IOException { + @Override + public boolean removeAttributes(final String pathName, final List attributes) throws IOException { final String normalPath = keyValueAccess.normalize(pathName); boolean removed = false; From 57878343f5ac77f91740b1c0bdd13bfa67af8cc9 Mon Sep 17 00:00:00 2001 From: Stephan Saalfeld Date: Mon, 3 Apr 2023 17:38:32 -0400 Subject: [PATCH 134/243] build: fix Javadoc errors --- .../org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java | 4 ++-- .../java/org/janelia/saalfeldlab/n5/N5KeyValueWriter.java | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java index 4ed7d6c4..42166c96 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java @@ -431,8 +431,8 @@ protected static Path createDirectories(Path dir, final FileAttribute... attr } /** - * This is a copy of - * {@link Files#createAndCheckIsDirectory(Path, FileAttribute...)} that + * This is a copy of a previous + * Files#createAndCheckIsDirectory(Path, FileAttribute...) method that * follows symlinks. * * Workaround for https://bugs.openjdk.java.net/browse/JDK-8130464 diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriter.java b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriter.java index b2b428b3..179be5e5 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriter.java @@ -214,7 +214,7 @@ public void createDataset( * @throws IOException * if unable to read the attributes at {@code normalGroupPath} * - * @TODO consider cache (or you read the attributes twice?) + * TODO consider cache (or you read the attributes twice?) */ protected void writeAttributes( final String normalGroupPath, From 6d470b3a1c72b9285fa45009d3d1e3f75cef0d88 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Wed, 5 Apr 2023 15:28:23 -0400 Subject: [PATCH 135/243] fix: catch NumberFormatException when parsing string[] as double[] * instead return null * test primitive parsing --- .../org/janelia/saalfeldlab/n5/GsonUtils.java | 2 + .../saalfeldlab/n5/AbstractN5Test.java | 76 +++++++++++++++++++ 2 files changed, 78 insertions(+) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonUtils.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonUtils.java index 76d19e97..f6773dff 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonUtils.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonUtils.java @@ -116,6 +116,8 @@ static T parseAttributeElement(final JsonElement attribute, final Gson gson, if (type == String.class) return (T)gson.toJson(attribute); return null; + } catch ( final NumberFormatException nfe ) { + return null; } } diff --git a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java index a3bb4a7b..09ffdbe9 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java @@ -475,6 +475,82 @@ public void testOverwriteBlock() { } } + @Test + public void testAttributeParsingPrimitive() throws IOException { + + try (final N5Writer n5 = createN5Writer()) { + + n5.createGroup(groupName); + + /* Test parsing of int, int[], double, double[], String, and String[] types + * + * All types should be parseable as JsonElements + * + * ints should be parseable as doubles and Strings + * doubles should be parseable as doubles and Strings + * Strings sould be parsable as Strings + * + * int[]s should be parseable as double[]s and String[]s + * double[]s should be parseable as double[]s and String[]s + * String[]s should be parsable as String[]s + */ + + n5.setAttribute(groupName, "key", "value"); + Assert.assertNull(n5.getAttribute(groupName, "key", Integer.class )); + Assert.assertNull(n5.getAttribute(groupName, "key", int[].class )); + Assert.assertNull(n5.getAttribute(groupName, "key", Double.class )); + Assert.assertNull(n5.getAttribute(groupName, "key", double[].class )); + Assert.assertNotNull(n5.getAttribute(groupName, "key", String.class )); + Assert.assertNull(n5.getAttribute(groupName, "key", String[].class )); + Assert.assertNotNull(n5.getAttribute(groupName, "key", JsonElement.class )); + + n5.setAttribute(groupName, "key", new String[]{"value" }); + Assert.assertNull(n5.getAttribute(groupName, "key", Integer.class )); + Assert.assertNull(n5.getAttribute(groupName, "key", int[].class )); + Assert.assertNull(n5.getAttribute(groupName, "key", Double.class )); + Assert.assertNull(n5.getAttribute(groupName, "key", double[].class )); + Assert.assertNotNull(n5.getAttribute(groupName, "key", String.class )); + Assert.assertNotNull(n5.getAttribute(groupName, "key", String[].class )); + Assert.assertNotNull(n5.getAttribute(groupName, "key", JsonElement.class )); + + n5.setAttribute(groupName, "key", 1); + Assert.assertNotNull(n5.getAttribute(groupName, "key", Integer.class )); + Assert.assertNull(n5.getAttribute(groupName, "key", int[].class )); + Assert.assertNotNull(n5.getAttribute(groupName, "key", Double.class )); + Assert.assertNull(n5.getAttribute(groupName, "key", double[].class )); + Assert.assertNotNull(n5.getAttribute(groupName, "key", String.class )); + Assert.assertNull(n5.getAttribute(groupName, "key", String[].class )); + Assert.assertNotNull(n5.getAttribute(groupName, "key", JsonElement.class )); + + n5.setAttribute(groupName, "key", new int[]{2, 3}); + Assert.assertNull(n5.getAttribute(groupName, "key", Integer.class )); + Assert.assertNotNull(n5.getAttribute(groupName, "key", int[].class )); + Assert.assertNull(n5.getAttribute(groupName, "key", Double.class )); + Assert.assertNotNull(n5.getAttribute(groupName, "key", double[].class )); + Assert.assertNotNull(n5.getAttribute(groupName, "key", String.class )); + Assert.assertNotNull(n5.getAttribute(groupName, "key", String[].class )); + Assert.assertNotNull(n5.getAttribute(groupName, "key", JsonElement.class )); + + n5.setAttribute(groupName, "key", 0.1); +// Assert.assertNull(n5.getAttribute(groupName, "key", Integer.class )); // TODO returns 0, is this right + Assert.assertNull(n5.getAttribute(groupName, "key", int[].class )); + Assert.assertNotNull(n5.getAttribute(groupName, "key", Double.class )); + Assert.assertNull(n5.getAttribute(groupName, "key", double[].class )); + Assert.assertNotNull(n5.getAttribute(groupName, "key", String.class )); + Assert.assertNull(n5.getAttribute(groupName, "key", String[].class )); + Assert.assertNotNull(n5.getAttribute(groupName, "key", JsonElement.class )); + + n5.setAttribute(groupName, "key", new double[]{0.2, 0.3}); + Assert.assertNull(n5.getAttribute(groupName, "key", Integer.class )); +// Assert.assertNull(n5.getAttribute(groupName, "key", int[].class )); // TODO returns not null, is this right? + Assert.assertNull(n5.getAttribute(groupName, "key", Double.class )); + Assert.assertNotNull(n5.getAttribute(groupName, "key", double[].class )); + Assert.assertNotNull(n5.getAttribute(groupName, "key", String.class )); + Assert.assertNotNull(n5.getAttribute(groupName, "key", String[].class )); + Assert.assertNotNull(n5.getAttribute(groupName, "key", JsonElement.class )); + } + } + @Test public void testAttributes() throws IOException { From 2bf619709920cb0c9616d6492f2dceb8b22109bd Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Wed, 5 Apr 2023 15:56:07 -0400 Subject: [PATCH 136/243] fix: catch Exceptionhs in JsonArray parsing --- .../java/org/janelia/saalfeldlab/n5/GsonUtils.java | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonUtils.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonUtils.java index f6773dff..f45de6de 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonUtils.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonUtils.java @@ -106,9 +106,17 @@ static T parseAttributeElement(final JsonElement attribute, final Gson gson, } if (attribute instanceof JsonArray) { final JsonArray array = attribute.getAsJsonArray(); - final T retArray = GsonUtils.getJsonAsArray(gson, array, type); - if (retArray != null) - return retArray; + try { + final T retArray = GsonUtils.getJsonAsArray(gson, array, type); + if (retArray != null) + return retArray; + } catch (final JsonSyntaxException e) { + if (type == String.class) + return (T)gson.toJson(attribute); + return null; + } catch ( final NumberFormatException nfe ) { + return null; + } } try { return gson.fromJson(attribute, type); From 159ad03a176b62a91f840c77a9ff7647d5f5b0cd Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Thu, 6 Apr 2023 15:37:08 -0400 Subject: [PATCH 137/243] feat: N5Exceptions --- .../org/janelia/saalfeldlab/n5/GsonUtils.java | 2 +- .../janelia/saalfeldlab/n5/N5Exception.java | 57 +++++++++++++++++++ .../janelia/saalfeldlab/n5/N5FSReader.java | 2 +- .../saalfeldlab/n5/N5KeyValueReader.java | 11 ++-- .../saalfeldlab/n5/N5KeyValueWriter.java | 2 +- .../saalfeldlab/n5/AbstractN5Test.java | 8 +-- 6 files changed, 69 insertions(+), 13 deletions(-) create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/N5Exception.java diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonUtils.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonUtils.java index f45de6de..1a52f35a 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonUtils.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonUtils.java @@ -182,7 +182,7 @@ static JsonElement getAttribute(JsonElement root, final String normalizedAttribu *

  • Object[]
    • * */ - static Map> listAttributes(final JsonElement root) throws IOException { + static Map> listAttributes(final JsonElement root) throws N5Exception.N5IOException { if (root == null || !root.isJsonObject()) { return null; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5Exception.java b/src/main/java/org/janelia/saalfeldlab/n5/N5Exception.java new file mode 100644 index 00000000..93e87fc3 --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5Exception.java @@ -0,0 +1,57 @@ +package org.janelia.saalfeldlab.n5; + +import java.io.IOException; + +public class N5Exception extends RuntimeException{ + + public N5Exception() { + + super(); + } + + public N5Exception(String message) { + + super(message); + } + + public N5Exception(String message, Throwable cause) { + + super(message, cause); + } + + public N5Exception(Throwable cause) { + + super(cause); + } + + protected N5Exception(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) { + + super(message, cause, enableSuppression, writableStackTrace); + } + + public static class N5IOException extends N5Exception { + + public N5IOException(String message) { + + super(message); + } + + public N5IOException(String message, IOException cause) { + + super(message, cause); + } + + public N5IOException(IOException cause) { + + super(cause); + } + + protected N5IOException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) { + + super(message, cause, enableSuppression, writableStackTrace); + } + + + } + +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5FSReader.java b/src/main/java/org/janelia/saalfeldlab/n5/N5FSReader.java index f1cde693..083c92b7 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5FSReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5FSReader.java @@ -68,7 +68,7 @@ public N5FSReader(final String basePath, final GsonBuilder gsonBuilder, final bo cacheMeta); if( !exists("/")) - throw new IOException("No container exists at " + basePath ); + throw new N5Exception.N5IOException("No container exists at " + basePath ); } /** diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java index 92b57590..aea70833 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java @@ -106,7 +106,7 @@ public N5KeyValueReader( /* Check that version (if there is one) is compatible. */ final Version version = getVersion(); if (!VERSION.isCompatible(version)) - throw new IOException("Incompatible version " + version + " (this is " + VERSION + ")."); + throw new N5Exception.N5IOException("Incompatible version " + version + " (this is " + VERSION + ")."); } public Gson getGson() { @@ -284,7 +284,7 @@ public boolean exists(final String pathName) { } @Override - public boolean datasetExists(final String pathName) throws IOException { + public boolean datasetExists(final String pathName) throws N5Exception.N5IOException { if (cacheMeta) { final String normalPathName = keyValueAccess.normalize(pathName); @@ -323,10 +323,9 @@ public JsonElement getAttributes(final String pathName) throws IOException { return null; try (final LockedChannel lockedChannel = keyValueAccess.lockForReading(attributesPath)) { - final JsonElement attributes = GsonUtils.readAttributes(lockedChannel.newReader(), gson); - /* If we are reading from the access, update the cache*/ - groupInfo.attributesCache = attributes; - return attributes; + return GsonUtils.readAttributes(lockedChannel.newReader(), gson); + } catch (IOException e) { + throw new N5Exception.N5IOException("Cannot open lock for Reading", e); } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriter.java b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriter.java index 179be5e5..db398ce7 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriter.java @@ -300,7 +300,7 @@ public void setAttributes( final String normalPath = keyValueAccess.normalize(path); if (!exists(normalPath)) - throw new IOException("" + normalPath + " is not a group or dataset."); + throw new N5Exception.N5IOException("" + normalPath + " is not a group or dataset."); if (cacheMeta) setCachedAttributes(normalPath, attributes); diff --git a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java index 09ffdbe9..e0a29f89 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java @@ -199,7 +199,7 @@ public void testSetAttributeDoesntCreateGroup() throws IOException { try (final N5Writer writer = createN5Writer()) { final String testGroup = "/group/should/not/exit"; assertFalse(writer.exists(testGroup)); - assertThrows(IOException.class, () -> writer.setAttribute(testGroup, "test", "test")); + assertThrows(N5Exception.N5IOException.class, () -> writer.setAttribute(testGroup, "test", "test")); assertFalse(writer.exists(testGroup)); } } @@ -1101,7 +1101,7 @@ public void testVersion() throws NumberFormatException, IOException { final Version version = writer.getVersion(); assertFalse(N5Reader.VERSION.isCompatible(version)); - assertThrows(IOException.class, () -> createN5Writer( writer.getBasePath() )); + assertThrows(N5Exception.N5IOException.class, () -> createN5Writer( writer.getBasePath() )); final Version compatibleVersion = new Version(N5Reader.VERSION.getMajor(), N5Reader.VERSION.getMinor(), N5Reader.VERSION.getPatch()); writer.setAttribute("/", N5Reader.VERSION_KEY, compatibleVersion.toString()); @@ -1134,14 +1134,14 @@ public void testReaderCreation() throws IOException { writer.removeAttribute("/", "/"); writer.setAttribute("/", N5Reader.VERSION_KEY, new Version(N5Reader.VERSION.getMajor() + 1, N5Reader.VERSION.getMinor(), N5Reader.VERSION.getPatch()).toString()); - assertThrows("Incompatible version throws error", IOException.class, + assertThrows("Incompatible version throws error", N5Exception.N5IOException.class, () -> { createN5Reader(canonicalPath); }); writer.remove(); } // non-existent group should fail - assertThrows("Non-existant location throws error", IOException.class, + assertThrows("Non-existant location throws error", N5Exception.N5IOException.class, () -> { final N5Reader test = createN5Reader(canonicalPath); test.list("/"); From 598d70dbc6b57166cae3f81e955bd83bb5e19e4f Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Thu, 6 Apr 2023 15:37:15 -0400 Subject: [PATCH 138/243] feat: remove leading `/` or `\\` from group (credit: bogovicj) --- .../n5/FileSystemKeyValueAccess.java | 10 ++++----- .../saalfeldlab/n5/N5KeyValueReader.java | 10 ++++----- .../org/janelia/saalfeldlab/n5/N5URL.java | 21 +++++++++++++++++-- 3 files changed, 28 insertions(+), 13 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java index 42166c96..3dc753eb 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java @@ -257,11 +257,9 @@ public String relativize(final String path, final String base) { } /** - * Removes the leading slash from a given path and returns the normalized - * path. It ensures correctness on both Unix and Windows, otherwise - * {@code pathName} is treated as UNC path on Windows, and - * {@code Paths.get(pathName, ...)} fails with - * {@code InvalidPathException}. + * Returns a normalized path. It ensures correctness on both Unix and Windows, + * otherwise {@code pathName} is treated as UNC path on Windows, and + * {@code Paths.get(pathName, ...)} fails with {@code InvalidPathException}. * * @param path * @return the normalized path, without leading slash @@ -269,7 +267,7 @@ public String relativize(final String path, final String base) { @Override public String normalize(final String path) { - return fileSystem.getPath(path.startsWith("/") || path.startsWith("\\") ? path.substring(1) : path).normalize().toString(); + return fileSystem.getPath(path).normalize().toString(); } @Override diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java index aea70833..c0228fbd 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java @@ -198,7 +198,7 @@ public T getAttribute( final String key, final Class clazz) throws IOException { - final String normalPathName = keyValueAccess.normalize(pathName); + final String normalPathName = N5URL.normalizeGroupPath(pathName); final String normalizedAttributePath = N5URL.normalizeAttributePath(key); if (cacheMeta) { @@ -227,7 +227,7 @@ public T getAttribute( final String key, final Type type) throws IOException { - final String normalPathName = keyValueAccess.normalize(pathName); + final String normalPathName = N5URL.normalizeGroupPath(pathName); final String normalizedAttributePath = N5URL.normalizeAttributePath(key); if (cacheMeta) { return getCachedAttribute(normalPathName, key, type); @@ -272,7 +272,7 @@ protected N5GroupInfo getCachedN5GroupInfo(final String normalPathName) { @Override public boolean exists(final String pathName) { - final String normalPathName = keyValueAccess.normalize(pathName); + final String normalPathName = N5URL.normalizeGroupPath(pathName); if (cacheMeta) return getCachedN5GroupInfo(normalPathName) != emptyGroupInfo; else @@ -335,7 +335,7 @@ public DataBlock readBlock( final DatasetAttributes datasetAttributes, final long... gridPosition) throws IOException { - final String path = getDataBlockPath(keyValueAccess.normalize(pathName), gridPosition); + final String path = getDataBlockPath(N5URL.normalizeGroupPath(pathName), gridPosition); if (!keyValueAccess.exists(path)) return null; @@ -357,7 +357,7 @@ protected String[] normalList(final String normalPath) throws IOException { @Override public String[] list(final String pathName) throws IOException { - final String normalPath = keyValueAccess.normalize(pathName); + final String normalPath = N5URL.normalizeGroupPath(pathName); if (cacheMeta) { return getCachedList(normalPath); } else { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java b/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java index c9c72267..36fbecf8 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java @@ -75,7 +75,7 @@ public String normalizeContainerPath() { */ public String normalizeGroupPath() { - return normalizePath(getGroupPath()); + return normalizeGroupPath(getGroupPath()); } /** @@ -367,6 +367,24 @@ public static String normalizePath(String path) { .reduce((l, r) -> l + "/" + r).orElse(""); } + /** + * Normalize a group path relative to a container's root, resulting in removal of redundant "/", "./", resolution of relative "../", + * and removal of leading slashes. + * + * @param path to normalize + * @return the normalized path + */ + public static String normalizeGroupPath( final String path ) { + + /* Alternatively, could do something like the below in every KeyValueReader implementation + * + * return keyValueAccess.relativize( N5URL.normalizeGroupPath(path), basePath); + * + * has to be in the implementations, since KeyValueAccess doesn't have a basePath. + */ + return normalizePath(path.startsWith("/") || path.startsWith("\\") ? path.substring(1) : path); + } + /** * Normalize the {@link String attributePath}. *

    @@ -433,7 +451,6 @@ public static String normalizeAttributePath(final String attributePath) { final Pattern relativePathPattern = Pattern.compile( "((?<=/)/+|(?<=(/|^))(\\./)+|((/|(?<=/))\\.)$|(? Date: Thu, 6 Apr 2023 15:37:20 -0400 Subject: [PATCH 139/243] refactor: expose dataset keys --- .../saalfeldlab/n5/AbstractGsonReader.java | 8 +++---- .../saalfeldlab/n5/DatasetAttributes.java | 16 ++++++------- .../saalfeldlab/n5/N5KeyValueReader.java | 23 +++++++++++-------- 3 files changed, 25 insertions(+), 22 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/AbstractGsonReader.java b/src/main/java/org/janelia/saalfeldlab/n5/AbstractGsonReader.java index 85b9a083..f4e660b7 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/AbstractGsonReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/AbstractGsonReader.java @@ -86,19 +86,19 @@ public DatasetAttributes getDatasetAttributes(final String pathName) throws IOEx final HashMap map = getAttributes(pathName); final Gson gson = getGson(); - final long[] dimensions = GsonAttributesParser.parseAttribute(map, DatasetAttributes.dimensionsKey, long[].class, gson); + final long[] dimensions = GsonAttributesParser.parseAttribute(map, DatasetAttributes.DIMENSIONS_KEY, long[].class, gson); if (dimensions == null) return null; - final DataType dataType = GsonAttributesParser.parseAttribute(map, DatasetAttributes.dataTypeKey, DataType.class, gson); + final DataType dataType = GsonAttributesParser.parseAttribute(map, DatasetAttributes.DATA_TYPE_KEY, DataType.class, gson); if (dataType == null) return null; - int[] blockSize = GsonAttributesParser.parseAttribute(map, DatasetAttributes.blockSizeKey, int[].class, gson); + int[] blockSize = GsonAttributesParser.parseAttribute(map, DatasetAttributes.BLOCK_SIZE_KEY, int[].class, gson); if (blockSize == null) blockSize = Arrays.stream(dimensions).mapToInt(a -> (int)a).toArray(); - Compression compression = GsonAttributesParser.parseAttribute(map, DatasetAttributes.compressionKey, Compression.class, gson); + Compression compression = GsonAttributesParser.parseAttribute(map, DatasetAttributes.COMPRESSION_KEY, Compression.class, gson); /* version 0 */ if (compression == null) { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java b/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java index 4f771271..17f098b8 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java @@ -45,10 +45,10 @@ public class DatasetAttributes implements Serializable { private static final long serialVersionUID = -4521467080388947553L; - protected static final String dimensionsKey = "dimensions"; - protected static final String blockSizeKey = "blockSize"; - protected static final String dataTypeKey = "dataType"; - protected static final String compressionKey = "compression"; + public static final String DIMENSIONS_KEY = "dimensions"; + public static final String BLOCK_SIZE_KEY = "blockSize"; + public static final String DATA_TYPE_KEY = "dataType"; + public static final String COMPRESSION_KEY = "compression"; /* version 0 */ protected static final String compressionTypeKey = "compressionType"; @@ -98,10 +98,10 @@ public DataType getDataType() { public HashMap asMap() { final HashMap map = new HashMap<>(); - map.put(dimensionsKey, dimensions); - map.put(blockSizeKey, blockSize); - map.put(dataTypeKey, dataType); - map.put(compressionKey, compression); + map.put(DIMENSIONS_KEY, dimensions); + map.put(BLOCK_SIZE_KEY, blockSize); + map.put(DATA_TYPE_KEY, dataType); + map.put(COMPRESSION_KEY, compression); return map; } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java index c0228fbd..1fc30ca8 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java @@ -143,23 +143,26 @@ public DatasetAttributes getDatasetAttributes(final String pathName) throws IOEx final JsonElement attributes = getAttributes(normalPath); - final long[] dimensions = GsonUtils.readAttribute(attributes, DatasetAttributes.dimensionsKey, long[].class, gson); + final String normalPath = N5URL.normalizeGroupPath(pathName); + final JsonElement attributes = normalGetAttributes(normalPath); + return createDatasetAttributes(attributes); + } + + private DatasetAttributes createDatasetAttributes(JsonElement attributes) { + + final long[] dimensions = GsonUtils.readAttribute(attributes, DatasetAttributes.DIMENSIONS_KEY, long[].class, gson); if (dimensions == null) { - setGroupInfoIsDataset(info, false); return null; } - final DataType dataType = GsonUtils.readAttribute(attributes, DatasetAttributes.dataTypeKey, DataType.class, gson); + final DataType dataType = GsonUtils.readAttribute(attributes, DatasetAttributes.DATA_TYPE_KEY, DataType.class, gson); if (dataType == null) { - setGroupInfoIsDataset(info, false); return null; } - setGroupInfoIsDataset(info, true); - - final int[] blockSize = GsonUtils.readAttribute(attributes, DatasetAttributes.blockSizeKey, int[].class, gson); + final int[] blockSize = GsonUtils.readAttribute(attributes, DatasetAttributes.BLOCK_SIZE_KEY, int[].class, gson); - final Compression compression = GsonUtils.readAttribute(attributes, DatasetAttributes.compressionKey, Compression.class, gson); + final Compression compression = GsonUtils.readAttribute(attributes, DatasetAttributes.COMPRESSION_KEY, Compression.class, gson); /* version 0 */ final String compressionVersion0Name = compression @@ -480,7 +483,7 @@ private static DatasetAttributes createDatasetAttributes( * Check for attributes that are required for a group to be a dataset. * * @param attributes to check for dataset attributes - * @return if {@link DatasetAttributes#dimensionsKey} and {@link DatasetAttributes#dataTypeKey} are present + * @return if {@link DatasetAttributes#DIMENSIONS_KEY} and {@link DatasetAttributes#DATA_TYPE_KEY} are present */ protected static boolean hasDatasetAttributes(final JsonElement attributes) { @@ -489,6 +492,6 @@ protected static boolean hasDatasetAttributes(final JsonElement attributes) { } final JsonObject metadataCache = attributes.getAsJsonObject(); - return metadataCache.has(DatasetAttributes.dimensionsKey) && metadataCache.has(DatasetAttributes.dataTypeKey); + return metadataCache.has(DatasetAttributes.DIMENSIONS_KEY) && metadataCache.has(DatasetAttributes.DATA_TYPE_KEY); } } From e80123f87a6e22249c7a5427f1561152f5eb326d Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Thu, 6 Apr 2023 15:37:28 -0400 Subject: [PATCH 140/243] style: formatting --- .../java/org/janelia/saalfeldlab/n5/GsonUtils.java | 1 + .../janelia/saalfeldlab/n5/N5KeyValueWriter.java | 14 +++++++------- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonUtils.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonUtils.java index 1a52f35a..e84a09bc 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonUtils.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonUtils.java @@ -51,6 +51,7 @@ public interface GsonUtils { static Gson registerGson(final GsonBuilder gsonBuilder) { + gsonBuilder.registerTypeAdapter(DataType.class, new DataType.JsonAdapter()); gsonBuilder.registerTypeHierarchyAdapter(Compression.class, CompressionAdapter.getJsonAdapter()); gsonBuilder.disableHtmlEscaping(); diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriter.java b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriter.java index db398ce7..1004a3b0 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriter.java @@ -1,16 +1,16 @@ /** * Copyright (c) 2017--2021, Stephan Saalfeld * All rights reserved. - * + *

    * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: - * + *

    * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright notice, * this list of conditions and the following disclaimer in the documentation * and/or other materials provided with the distribution. - * + *

    * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE @@ -45,9 +45,9 @@ public class N5KeyValueWriter extends N5KeyValueReader implements N5Writer { /** * Opens an {@link N5KeyValueWriter} at a given base path with a custom * {@link GsonBuilder} to support custom attributes. - * + *

    * If the base path does not exist, it will be created. - * + *

    * If the base path exists and if the N5 version of the container is * compatible with this implementation, the N5 version of this container * will be set to the current N5 version of this implementation. @@ -157,8 +157,8 @@ protected N5GroupInfo createCachedGroup(final String normalPath) throws IOExcept /** * Performs any necessary initialization to ensure the key given by the - * argument {@code normalPath} is a valid group. Called by - * {@link createGroup}. + * argument {@code normalPath} is a valid group after creation. Called by + * {@link #createGroup(String)}. * * @param normalPath * the group path. From f9e7a370a9870aada5380f6e946b7e3d3f497faa Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Thu, 6 Apr 2023 15:37:49 -0400 Subject: [PATCH 141/243] refactor: extract caching to separate class --- .../saalfeldlab/n5/N5KeyValueReader.java | 199 ++++--------- .../saalfeldlab/n5/N5KeyValueWriter.java | 207 +++---------- .../saalfeldlab/n5/cache/N5JsonCache.java | 278 ++++++++++++++++++ 3 files changed, 391 insertions(+), 293 deletions(-) create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/cache/N5JsonCache.java diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java index 1fc30ca8..b294cd36 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java @@ -28,14 +28,13 @@ import java.io.IOException; import java.lang.reflect.Type; import java.util.Arrays; -import java.util.HashMap; -import java.util.HashSet; import java.util.Map; import com.google.gson.Gson; import com.google.gson.GsonBuilder; import com.google.gson.JsonElement; import com.google.gson.JsonObject; +import org.janelia.saalfeldlab.n5.cache.N5JsonCache; /** * {@link N5Reader} implementation through {@link KeyValueAccess} with JSON @@ -47,29 +46,20 @@ */ public class N5KeyValueReader implements N5Reader { - /** - * Data object for caching meta data. Elements that are null are not yet - * cached. - */ - protected static class N5GroupInfo { - - public HashSet children = null; - public JsonElement attributesCache = null; - public Boolean isDataset = null; - } - - protected static final N5GroupInfo emptyGroupInfo = new N5GroupInfo(); - protected final KeyValueAccess keyValueAccess; protected final Gson gson; - protected final HashMap metaCache = new HashMap<>(); + protected final N5JsonCache cache = new N5JsonCache( + (groupPath,cacheKey) -> normalGetAttributes(groupPath), + this::normalExists, + this::normalExists, + this::normalDatasetExists, + this::normalList + ); protected final boolean cacheMeta; - protected static final String jsonFile = "attributes.json"; - protected final String basePath; /** @@ -133,15 +123,18 @@ public String getBasePath() { @Override public DatasetAttributes getDatasetAttributes(final String pathName) throws IOException { - final String normalPath = keyValueAccess.normalize(pathName); - N5GroupInfo info = null; - if (cacheMeta) { - info = getCachedN5GroupInfo(normalPath); - if (info == emptyGroupInfo || (info != null && info.isDataset != null && !info.isDataset)) - return null; + final String normalPath = N5URL.normalizeGroupPath(pathName); + final JsonElement attributes; + if (cacheMeta && cache.isDataset(normalPath)) { + attributes = cache.getAttributes(normalPath, N5JsonCache.jsonFile); + } else { + attributes = getAttributes(normalPath); } - final JsonElement attributes = getAttributes(normalPath); + return createDatasetAttributes(attributes); + } + + private DatasetAttributes normalGetDatasetAttributes(final String pathName) throws N5Exception.N5IOException { final String normalPath = N5URL.normalizeGroupPath(pathName); final JsonElement attributes = normalGetAttributes(normalPath); @@ -173,28 +166,6 @@ private DatasetAttributes createDatasetAttributes(JsonElement attributes) { return createDatasetAttributes(dimensions, dataType, blockSize, compression, compressionVersion0Name); } - private void setGroupInfoIsDataset(final N5GroupInfo info, final boolean normalPathName) { - if (info == null) - return; - synchronized (info) { - if (info != null) - info.isDataset = false; - } - } - - /** - * Get cached attributes for the group identified by a normalPath - * - * @param normalPath normalized group path without leading slash - * @return cached attributes - * empty map if the group exists but no attributes are set - * null if the group does not exist, or if attributes have not been cached - */ - protected JsonElement getCachedAttributes(final String normalPath) { - - return metaCache.get(normalPath).attributesCache; - } - @Override public T getAttribute( final String pathName, @@ -204,24 +175,13 @@ public T getAttribute( final String normalPathName = N5URL.normalizeGroupPath(pathName); final String normalizedAttributePath = N5URL.normalizeAttributePath(key); + final JsonElement attributes; if (cacheMeta) { - return getCachedAttribute(normalPathName, normalizedAttributePath, clazz); + attributes = cache.getAttributes(normalPathName, N5JsonCache.jsonFile); } else { - return GsonUtils.readAttribute(getAttributes(normalPathName), normalizedAttributePath, clazz, gson); + attributes = getAttributes(normalPathName); } - } - - private T getCachedAttribute(final String normalPathName, final String normalKey, final Class clazz) throws IOException { - - return getCachedAttribute(normalPathName, normalKey, (Type) clazz); - } - - private T getCachedAttribute(final String normalPathName, final String normalKey, final Type type) throws IOException { - - final N5GroupInfo info = getCachedN5GroupInfo(normalPathName); - if (info == emptyGroupInfo) - return null; - return GsonUtils.readAttribute(getAttributes(normalPathName), N5URL.normalizeAttributePath(normalKey), type, gson); + return GsonUtils.readAttribute(attributes, normalizedAttributePath, clazz, gson); } @Override @@ -232,11 +192,13 @@ public T getAttribute( final String normalPathName = N5URL.normalizeGroupPath(pathName); final String normalizedAttributePath = N5URL.normalizeAttributePath(key); + JsonElement attributes; if (cacheMeta) { - return getCachedAttribute(normalPathName, key, type); + attributes = cache.getAttributes(normalPathName, N5JsonCache.jsonFile); } else { - return GsonUtils.readAttribute(getAttributes(normalPathName), normalizedAttributePath, type, gson); + attributes = getAttributes(normalPathName); } + return GsonUtils.readAttribute(attributes, normalizedAttributePath, type, gson); } protected boolean groupExists(final String absoluteNormalPath) { @@ -244,63 +206,37 @@ protected boolean groupExists(final String absoluteNormalPath) { return keyValueAccess.exists(absoluteNormalPath) && keyValueAccess.isDirectory(absoluteNormalPath); } - /** - * Get an existing cached N5 group info object or create it. - * - * @param normalPathName normalized group path without leading slash - * @return - */ - protected N5GroupInfo getCachedN5GroupInfo(final String normalPathName) { - - N5GroupInfo info = metaCache.get(normalPathName); - if (info == null) { - - /* I do not have a better solution yet to allow parallel - * exists checks for independent paths than to accept the - * same exists check to potentially run multiple times. - */ - final boolean exists = groupExists(groupPath(normalPathName)); - - synchronized (metaCache) { - info = metaCache.get(normalPathName); - if (info == null) { - info = exists ? new N5GroupInfo() : emptyGroupInfo; - metaCache.put(normalPathName, info); - } - } - } - return info; - } - @Override public boolean exists(final String pathName) { final String normalPathName = N5URL.normalizeGroupPath(pathName); if (cacheMeta) - return getCachedN5GroupInfo(normalPathName) != emptyGroupInfo; - else - try { - return groupExists(groupPath(normalPathName)) || datasetExists(normalPathName); - } catch (final IOException e) { - return false; - } + return cache.exists(normalPathName); + else { + return normalExists(normalPathName); + } + } + + private boolean normalExists(String normalPathName) { + + return keyValueAccess.exists(groupPath(normalPathName)) || normalDatasetExists(normalPathName); } @Override public boolean datasetExists(final String pathName) throws N5Exception.N5IOException { if (cacheMeta) { - final String normalPathName = keyValueAccess.normalize(pathName); - final N5GroupInfo info = getCachedN5GroupInfo(normalPathName); - if (info == emptyGroupInfo) - return false; - synchronized (info) { - if (info.isDataset != null) - return info.isDataset; - } + final String normalPathName = N5URL.normalizeGroupPath(pathName); + return cache.isDataset(normalPathName); } + return normalDatasetExists(pathName); + } + + private boolean normalDatasetExists(String pathName) throws N5Exception.N5IOException { + // for n5, every dataset must be a group - return groupExists(groupPath(pathName)) && getDatasetAttributes(pathName) != null; + return groupExists(groupPath(pathName)) && normalGetDatasetAttributes(pathName) != null; + } /** @@ -312,17 +248,23 @@ public boolean datasetExists(final String pathName) throws N5Exception.N5IOExcep */ public JsonElement getAttributes(final String pathName) throws IOException { - final String groupPath = keyValueAccess.normalize(pathName); - final String attributesPath = attributesPath(groupPath); + final String groupPath = N5URL.normalizeGroupPath(pathName); /* If cached, return the cache*/ - final N5GroupInfo groupInfo = getCachedN5GroupInfo(groupPath); if (cacheMeta) { - if (groupInfo != null && groupInfo.attributesCache != null) - return groupInfo.attributesCache; + return cache.getAttributes(groupPath, N5JsonCache.jsonFile); + } else { + return normalGetAttributes(groupPath); } - if (exists(pathName) && !keyValueAccess.exists(attributesPath)) + } + + private JsonElement normalGetAttributes(String groupPath) throws N5Exception.N5IOException { + + final String attributesPath = attributesPath(groupPath); + if (!normalExists(groupPath)) + throw new N5Exception.N5IOException("Group " + groupPath + " does not exist"); + if (!keyValueAccess.exists(attributesPath)) return null; try (final LockedChannel lockedChannel = keyValueAccess.lockForReading(attributesPath)) { @@ -352,9 +294,13 @@ public DataBlock readBlock( * @return the normalized groups in {@code normalPath} * @throws IOException */ - protected String[] normalList(final String normalPath) throws IOException { + protected String[] normalList(final String normalPath) throws N5Exception.N5IOException { - return keyValueAccess.listDirectories(groupPath(normalPath)); + try { + return keyValueAccess.listDirectories(groupPath(normalPath)); + } catch (IOException e) { + throw new N5Exception.N5IOException("Cannot list directories for group " + normalPath, e); + } } @Override @@ -362,29 +308,12 @@ public String[] list(final String pathName) throws IOException { final String normalPath = N5URL.normalizeGroupPath(pathName); if (cacheMeta) { - return getCachedList(normalPath); + return cache.list(normalPath); } else { return normalList(normalPath); } } - private String[] getCachedList(final String normalPath) throws IOException { - - final N5GroupInfo info = getCachedN5GroupInfo(normalPath); - if (info == emptyGroupInfo) - throw new IOException("Group '" + normalPath + "' does not exist."); - else { - - if (info.children != null) - return (info.children).toArray(new String[(info.children).size()]); - - synchronized (info) { - final String[] list = normalList(normalPath); - info.children = new HashSet<>(Arrays.asList(list)); - return list; - } - } - } /** * Constructs the path for a data block in a dataset at a given grid position. @@ -435,7 +364,7 @@ protected String groupPath(final String normalGroupPath) { */ protected String attributesPath(final String normalPath) { - return keyValueAccess.compose(basePath, normalPath, jsonFile); + return keyValueAccess.compose(basePath, normalPath, N5JsonCache.jsonFile); } @Override diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriter.java b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriter.java index 1004a3b0..a6cc1164 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriter.java @@ -26,7 +26,6 @@ package org.janelia.saalfeldlab.n5; import java.io.IOException; -import java.util.HashSet; import java.util.List; import java.util.Map; @@ -34,6 +33,8 @@ import com.google.gson.JsonElement; import com.google.gson.JsonNull; import com.google.gson.JsonObject; +import org.janelia.saalfeldlab.n5.cache.N5JsonCache; +import java.util.Arrays; /** * Filesystem {@link N5Writer} implementation with version compatibility check. @@ -94,67 +95,6 @@ protected static String initializeContainer( return normBasePath; } - /** - * Helper method to create and cache a group. - * - * @param normalPath - * normalized group path without leading slash - * @return - * @throws IOException - */ - protected N5GroupInfo createCachedGroup(final String normalPath) throws IOException { - - N5GroupInfo info = getCachedN5GroupInfo(normalPath); - if (info == emptyGroupInfo) { - - /* - * The directories may be created multiple times concurrently, but a - * new cache entry is inserted only if none has been inserted in the - * meantime (because that may already include more cached data). - * - * This avoids synchronizing on the cache for independent group - * creation. - */ - keyValueAccess.createDirectories(groupPath(normalPath)); - synchronized (metaCache) { - info = getCachedN5GroupInfo(normalPath); - if (info == emptyGroupInfo) { - info = new N5GroupInfo(); - metaCache.put(normalPath, info); - } - for (String childPathName = normalPath; !(childPathName == null || childPathName.equals(""));) { - final String parentPathName = keyValueAccess.parent(childPathName); - if (parentPathName == null) - break; - N5GroupInfo parentInfo = getCachedN5GroupInfo(parentPathName); - if (parentInfo == emptyGroupInfo) { - parentInfo = new N5GroupInfo(); - parentInfo.isDataset = false; - metaCache.put(parentPathName, parentInfo); - } - HashSet children = parentInfo.children; - if (children == null) { - children = new HashSet<>(); - } - synchronized (children) { - children - .add( - keyValueAccess.relativize(childPathName, parentPathName)); - parentInfo.children = children; - } - childPathName = parentPathName; - } - } - - /* - * initialize after updating the cache so that the correct - * N5GroupInfo instance can be updated if necessary. - */ - initializeGroup(normalPath); - } - return info; - } - /** * Performs any necessary initialization to ensure the key given by the * argument {@code normalPath} is a valid group after creation. Called by @@ -172,16 +112,19 @@ protected void initializeGroup(final String normalPath) { public void createGroup(final String path) throws IOException { final String normalPath = keyValueAccess.normalize(path); + keyValueAccess.createDirectories(groupPath(normalPath)); if (cacheMeta) { - final N5GroupInfo info = createCachedGroup(normalPath); - synchronized (info) { - if (info.isDataset == null) - info.isDataset = false; + final String[] pathParts = keyValueAccess.components(normalPath); + if (pathParts.length > 1) { + String parent = keyValueAccess.normalize("/"); + for (String child : pathParts) { + final String childPath = keyValueAccess.compose(parent, child); + cache.addChild(parent, child); + parent = childPath; + } } - } else { - keyValueAccess.createDirectories(groupPath(normalPath)); - initializeGroup(normalPath); } + initializeGroup(normalPath); } @Override @@ -190,16 +133,8 @@ public void createDataset( final DatasetAttributes datasetAttributes) throws IOException { final String normalPath = keyValueAccess.normalize(path); - if (cacheMeta) { - final N5GroupInfo info = createCachedGroup(normalPath); - synchronized (info) { - setDatasetAttributes(normalPath, datasetAttributes); - info.isDataset = true; - } - } else { - createGroup(path); - setDatasetAttributes(normalPath, datasetAttributes); - } + createGroup(path); + setDatasetAttributes(normalPath, datasetAttributes); } /** @@ -220,11 +155,9 @@ protected void writeAttributes( final String normalGroupPath, final JsonElement attributes) throws IOException { - try (final LockedChannel lock = keyValueAccess.lockForWriting(attributesPath(normalGroupPath))) { - final JsonElement root = GsonUtils - .insertAttribute(GsonUtils.readAttributes(lock.newReader(), gson), "/", attributes, gson); - GsonUtils.writeAttributes(lock.newWriter(), root, gson); - } + JsonElement root = getAttributes(normalGroupPath); + root = GsonUtils.insertAttribute(root, "/", attributes, gson); + writeAndCacheAttributes(normalGroupPath, root); } /** @@ -243,54 +176,35 @@ protected void writeAttributes( final String normalGroupPath, final Map attributes) throws IOException { - if (!attributes.isEmpty()) { - final JsonElement existingAttributes = getAttributes(normalGroupPath); - JsonElement newAttributes = existingAttributes != null && existingAttributes.isJsonObject() + if (attributes != null && !attributes.isEmpty()) { + JsonElement existingAttributes = getAttributes(normalGroupPath); + existingAttributes = existingAttributes != null && existingAttributes.isJsonObject() ? existingAttributes.getAsJsonObject() : new JsonObject(); - newAttributes = GsonUtils.insertAttributes(newAttributes, attributes, gson); - writeAttributes(normalGroupPath, newAttributes); + JsonElement newAttributes = GsonUtils.insertAttributes(existingAttributes, attributes, gson); + writeAndCacheAttributes(normalGroupPath, newAttributes); } } - /** - * Helper method to cache and write attributes. - * - * @param normalPath - * normalized group path without leading slash - * @param attributes - * @return - * @throws IOException - */ - protected N5GroupInfo setCachedAttributes( - final String normalPath, - final Map attributes) throws IOException { + private void writeAndCacheAttributes(final String normalGroupPath, final JsonElement attributes) throws IOException { - final N5GroupInfo info = getN5GroupInfo(normalPath); - final JsonElement metadata = getAttributes(normalPath); - synchronized (info) { - /* - * Necessary ensure `nulls` are treated consistently regardless of - * reading from the cache or not - */ - info.attributesCache = gson.toJsonTree(GsonUtils.insertAttributes(metadata, attributes, gson)); - writeAttributes(normalPath, info.attributesCache); - info.isDataset = hasDatasetAttributes(info.attributesCache); + try (final LockedChannel lock = keyValueAccess.lockForWriting(attributesPath(normalGroupPath))) { + GsonUtils.writeAttributes(lock.newWriter(), attributes, gson); } - return info; - } - - private N5GroupInfo getN5GroupInfo(final String normalPath) throws IOException { - - N5GroupInfo info = getCachedN5GroupInfo(normalPath); - if (info == emptyGroupInfo) { - synchronized (metaCache) { - info = getCachedN5GroupInfo(normalPath); - if (info == emptyGroupInfo) - throw new IOException("N5 group '" + normalPath + "' does not exist. Cannot set attributes."); + if (cacheMeta) { + JsonElement nullRespectingAttributes = attributes; + /* Gson only filters out nulls when you write the JsonElement. This means it doesn't filter them out when caching. + * To handle this, we explicitly writer the existing JsonElement to a new JsonElement. + * The output is identical to the input if: + * - serializeNulls is true + * - no null values are present + * - cacheing is turned off */ + if (!gson.serializeNulls()) { + nullRespectingAttributes = gson.toJsonTree(attributes); } + /* Update the cache, and write to the writer */ + cache.updateCacheInfo(normalGroupPath, N5JsonCache.jsonFile, nullRespectingAttributes); } - return info; } @Override @@ -302,10 +216,7 @@ public void setAttributes( if (!exists(normalPath)) throw new N5Exception.N5IOException("" + normalPath + " is not a group or dataset."); - if (cacheMeta) - setCachedAttributes(normalPath, attributes); - else - writeAttributes(normalPath, attributes); + writeAttributes(normalPath, attributes); } @Override @@ -375,44 +286,24 @@ public boolean remove(final String path) throws IOException { final String normalPath = keyValueAccess.normalize(path); final String groupPath = groupPath(normalPath); + if (keyValueAccess.exists(groupPath)) + keyValueAccess.delete(groupPath); if (cacheMeta) { - removeCachedGroup(normalPath, groupPath); - } else { - if (keyValueAccess.exists(groupPath)) - keyValueAccess.delete(groupPath); + final String[] pathParts = keyValueAccess.components(normalPath); + final String parent; + if (pathParts.length <= 1) { + parent = keyValueAccess.normalize("/"); + } else { + final int parentPathLength = pathParts.length - 1; + parent = keyValueAccess.compose(Arrays.copyOf(pathParts, parentPathLength)); + } + cache.removeCache(parent, normalPath); } /* an IOException should have occurred if anything had failed midway */ return true; } - private void removeCachedGroup(final String normalPath, final String groupPath) throws IOException { - - synchronized (metaCache) { - if (keyValueAccess.exists(groupPath)) { - keyValueAccess.delete(groupPath); - - /* cache nonexistence for all prior children */ - for (final String key : metaCache.keySet()) { - if (key.startsWith(normalPath)) - metaCache.put(key, emptyGroupInfo); - } - - /* remove child from parent */ - final String parentPath = keyValueAccess.parent(normalPath); - final N5GroupInfo parent = metaCache.get(parentPath); - if (parent != null) { - final HashSet children = parent.children; - if (children != null) { - synchronized (children) { - children.remove(keyValueAccess.relativize(normalPath, parentPath)); - } - } - } - } - } - } - @Override public boolean deleteBlock( final String path, diff --git a/src/main/java/org/janelia/saalfeldlab/n5/cache/N5JsonCache.java b/src/main/java/org/janelia/saalfeldlab/n5/cache/N5JsonCache.java new file mode 100644 index 00000000..abf82f70 --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/cache/N5JsonCache.java @@ -0,0 +1,278 @@ +package org.janelia.saalfeldlab.n5.cache; + +import com.google.gson.JsonElement; + +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.function.BiFunction; +import java.util.function.Function; + +public class N5JsonCache { + + public static final N5CacheInfo emptyCacheInfo = new N5CacheInfo(); + public static final String jsonFile = "attributes.json"; + + + /** + * Data object for caching meta data. Elements that are null are not yet + * cached. + */ + private static class N5CacheInfo { + + private final HashSet children = new HashSet<>(); + + private final HashMap attributesCache = new HashMap<>(); + private Boolean isDataset = false; + private Boolean isGroup = false; + + private JsonElement getCache(final String normalCacheKey) { + + synchronized (attributesCache) { + return attributesCache.get(normalCacheKey); + } + } + + private boolean containsKey(final String normalCacheKey) { + + synchronized (attributesCache) { + return attributesCache.containsKey(normalCacheKey); + } + } + } + + private final BiFunction readBackingAttributes; + private final Function getExists; + private final Function getIsGroup; + private final Function getIsDataset; + private final Function listNormalChildren; + private final HashMap containerPathToCache = new HashMap<>(); + + public N5JsonCache( + final BiFunction readBackingAttributes, + final Function getExists, + final Function getIsGroup, + final Function getIsDataset, + final Function listNormalChildren + ) { + + this.readBackingAttributes = readBackingAttributes; + this.getExists = getExists; + this.getIsGroup = getIsGroup; + this.getIsDataset = getIsDataset; + this.listNormalChildren = listNormalChildren; + } + + /** + * Get cached attributes for the group identified by a normalPath + * + * @param cacheInfo + * @param normalCacheKey normalized cache key + * @return cached attributes + * null if the group exists but no attributes are set, the group does not exist, or if attributes have not been cached + */ + private JsonElement getCachedAttributes(final N5CacheInfo cacheInfo, final String normalCacheKey) { + return cacheInfo == null ? null : cacheInfo.getCache(normalCacheKey); + } + + public JsonElement getAttributes(final String normalPathKey, final String normalCacheKey) { + N5CacheInfo cacheInfo = getCacheInfo(normalPathKey); + if (cacheInfo == null) { + cacheAttributes(normalPathKey, normalCacheKey); + } + cacheInfo = getCacheInfo(normalPathKey); + synchronized (cacheInfo) { + if (!cacheInfo.containsKey(normalCacheKey)) { + updateCacheInfo(normalPathKey, normalCacheKey, null); + } + } + + return getCacheInfo(normalPathKey).getCache(normalCacheKey); + } + + public boolean isDataset(final String normalPathKey) { + + final N5CacheInfo cacheInfo = getCacheInfo(normalPathKey); + if (cacheInfo == null) { + addNewCacheInfo(normalPathKey); + } + return getCacheInfo(normalPathKey).isDataset; + } + + public boolean isGroup(final String normalPathKey) { + + final N5CacheInfo cacheInfo = getCacheInfo(normalPathKey); + if (cacheInfo == null) { + addNewCacheInfo(normalPathKey); + } + return getCacheInfo(normalPathKey).isGroup; + } + + public boolean exists(final String normalPathKey) { + + final N5CacheInfo cacheInfo = getCacheInfo(normalPathKey); + if (cacheInfo == null) { + addNewCacheInfo(normalPathKey); + } + return getCacheInfo(normalPathKey) != emptyCacheInfo; + } + + public String[] list(String normalPathKey) { + N5CacheInfo cacheInfo = getCacheInfo(normalPathKey); + if (cacheInfo == null) { + addNewCacheInfo(normalPathKey); + } + cacheInfo = getCacheInfo(normalPathKey); + if (cacheInfo == emptyCacheInfo) return null; + + final String[] children = new String[cacheInfo.children.size()]; + int i = 0; + for (String child : cacheInfo.children) { + children[i++] = child; + } + return children; + } + + private void cacheAttributes(final String normalPathKey, final String normalCacheKey) { + + final JsonElement uncachedValue = readBackingAttributes.apply(normalPathKey, normalCacheKey); + if (uncachedValue != null || getExists.apply(normalPathKey)) { + addNewCacheInfo(normalPathKey, normalCacheKey, uncachedValue); + } + } + private void addNewCacheInfo(String normalPathKey, String normalCacheKey, JsonElement uncachedAttributes) { + + final N5CacheInfo cacheInfo; + if (!getExists.apply(normalPathKey)) { + cacheInfo = emptyCacheInfo; + } else { + cacheInfo = new N5CacheInfo(); + synchronized (cacheInfo.attributesCache) { + cacheInfo.attributesCache.put(normalCacheKey, uncachedAttributes); + } + cacheInfo.isGroup = getIsGroup.apply(normalPathKey); + cacheInfo.isDataset = getIsDataset.apply(normalPathKey); + addChild(cacheInfo, normalPathKey); + } + synchronized (containerPathToCache) { + containerPathToCache.put(normalPathKey, cacheInfo); + } + } + + private void addChild(N5CacheInfo cacheInfo, String normalPathKey) { + + final String[] children = listNormalChildren.apply(normalPathKey); + Collections.addAll(cacheInfo.children, children); + } + + private void addNewCacheInfo(String normalPathKey) { + + final N5CacheInfo cacheInfo; + if (!getExists.apply(normalPathKey)) { + cacheInfo = emptyCacheInfo; + } else { + cacheInfo = new N5CacheInfo(); + cacheInfo.isGroup = getIsGroup.apply(normalPathKey); + cacheInfo.isDataset = getIsDataset.apply(normalPathKey); + addChild(cacheInfo, normalPathKey); + } + synchronized (containerPathToCache) { + containerPathToCache.put(normalPathKey, cacheInfo); + } + } + + public void updateCacheInfo(final String normalPathKey, final String normalCacheKey) { + + N5CacheInfo cacheInfo = getCacheInfo(normalPathKey); + if (cacheInfo == null ){ + addNewCacheInfo(normalPathKey); + return; + } else if (!getExists.apply(normalPathKey)) { + cacheInfo = emptyCacheInfo; + } else { + cacheInfo.isGroup = getIsGroup.apply(normalPathKey); + cacheInfo.isDataset = getIsDataset.apply(normalPathKey); + cacheInfo.children.clear(); + addChild(cacheInfo, normalPathKey); + } + synchronized (containerPathToCache) { + containerPathToCache.put(normalPathKey, cacheInfo); + } + } + + public void updateCacheInfo(final String normalPathKey, final String normalCacheKey, final JsonElement uncachedAttributes) { + + N5CacheInfo cacheInfo = getCacheInfo(normalPathKey); + if (cacheInfo == null ){ + addNewCacheInfo(normalPathKey); + return; + } else if (!getExists.apply(normalPathKey)) { + cacheInfo = emptyCacheInfo; + } else { + final JsonElement attributesToCache = uncachedAttributes == null ? readBackingAttributes.apply(normalPathKey, normalCacheKey) : uncachedAttributes; + synchronized (cacheInfo.attributesCache) { + cacheInfo.attributesCache.put(normalCacheKey, attributesToCache); + } + cacheInfo.isGroup = getIsGroup.apply(normalPathKey); + cacheInfo.isDataset = getIsDataset.apply(normalPathKey); + cacheInfo.children.clear(); + addChild(cacheInfo, normalPathKey); + } + synchronized (containerPathToCache) { + containerPathToCache.put(normalPathKey, cacheInfo); + } + } + + public void addChild(final String parent, final String child) { + + final N5CacheInfo cacheInfo = getCacheInfo(parent); + if (cacheInfo == null) return; + cacheInfo.children.add(child); + } + + public void removeCache(final String normalParentPathKey, final String normalPathKey) { + synchronized (containerPathToCache) { + containerPathToCache.remove(normalPathKey); + final N5CacheInfo parentCache = containerPathToCache.get(normalParentPathKey); + if (parentCache != null) { + parentCache.children.remove(normalPathKey); + } + } + } + public void clearCache(final String normalPathKey, final String normalCacheKey) { + + final N5CacheInfo cacheInfo = getCacheInfo(normalPathKey); + if (cacheInfo != null && cacheInfo != emptyCacheInfo) { + synchronized (cacheInfo.attributesCache) { + cacheInfo.attributesCache.remove(normalCacheKey); + } + } + } + + private void updateCache(String normalPathKey, String normalCacheKey, JsonElement uncachedAttributes) { + + final N5CacheInfo cacheInfo; + if (!getExists.apply(normalPathKey)) { + cacheInfo = emptyCacheInfo; + } else { + cacheInfo = new N5CacheInfo(); + synchronized (cacheInfo.attributesCache) { + cacheInfo.attributesCache.put(normalCacheKey, uncachedAttributes); + } + cacheInfo.isGroup = getIsGroup.apply(normalPathKey); + cacheInfo.isDataset = getIsDataset.apply(normalPathKey); + addChild(cacheInfo, normalPathKey); + } + synchronized (containerPathToCache) { + containerPathToCache.put(normalPathKey, cacheInfo); + } + } + + private N5CacheInfo getCacheInfo(String pathKey) { + + synchronized (containerPathToCache) { + return containerPathToCache.get(pathKey); + } + } + +} From 404649f320b12b3ffe4a130c29c845633b339361 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Thu, 6 Apr 2023 15:49:00 -0400 Subject: [PATCH 142/243] fix: use the corrent normalize in more places --- .../saalfeldlab/n5/N5KeyValueWriter.java | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriter.java b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriter.java index a6cc1164..4e2f7dd4 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriter.java @@ -111,12 +111,12 @@ protected void initializeGroup(final String normalPath) { @Override public void createGroup(final String path) throws IOException { - final String normalPath = keyValueAccess.normalize(path); + final String normalPath = N5URL.normalizeGroupPath(path); keyValueAccess.createDirectories(groupPath(normalPath)); if (cacheMeta) { final String[] pathParts = keyValueAccess.components(normalPath); if (pathParts.length > 1) { - String parent = keyValueAccess.normalize("/"); + String parent = N5URL.normalizeGroupPath("/"); for (String child : pathParts) { final String childPath = keyValueAccess.compose(parent, child); cache.addChild(parent, child); @@ -132,7 +132,7 @@ public void createDataset( final String path, final DatasetAttributes datasetAttributes) throws IOException { - final String normalPath = keyValueAccess.normalize(path); + final String normalPath = N5URL.normalizeGroupPath(path); createGroup(path); setDatasetAttributes(normalPath, datasetAttributes); } @@ -212,7 +212,7 @@ public void setAttributes( final String path, final Map attributes) throws IOException { - final String normalPath = keyValueAccess.normalize(path); + final String normalPath = N5URL.normalizeGroupPath(path); if (!exists(normalPath)) throw new N5Exception.N5IOException("" + normalPath + " is not a group or dataset."); @@ -222,7 +222,7 @@ public void setAttributes( @Override public boolean removeAttribute(final String pathName, final String key) throws IOException { - final String normalPath = keyValueAccess.normalize(pathName); + final String normalPath = N5URL.normalizeGroupPath(pathName); final String absoluteNormalPath = keyValueAccess.compose(basePath, normalPath); final String normalKey = N5URL.normalizeAttributePath(key); @@ -245,7 +245,7 @@ public boolean removeAttribute(final String pathName, final String key) throws I @Override public T removeAttribute(final String pathName, final String key, final Class cls) throws IOException { - final String normalPath = keyValueAccess.normalize(pathName); + final String normalPath = N5URL.normalizeGroupPath(pathName); final String normalKey = N5URL.normalizeAttributePath(key); final JsonElement attributes = getAttributes(normalPath); @@ -259,7 +259,7 @@ public T removeAttribute(final String pathName, final String key, final Clas @Override public boolean removeAttributes(final String pathName, final List attributes) throws IOException { - final String normalPath = keyValueAccess.normalize(pathName); + final String normalPath = N5URL.normalizeGroupPath(pathName); boolean removed = false; for (final String attribute : attributes) { final String normalKey = N5URL.normalizeAttributePath(attribute); @@ -274,7 +274,7 @@ public void writeBlock( final DatasetAttributes datasetAttributes, final DataBlock dataBlock) throws IOException { - final String blockPath = getDataBlockPath(keyValueAccess.normalize(path), dataBlock.getGridPosition()); + final String blockPath = getDataBlockPath(N5URL.normalizeGroupPath(path), dataBlock.getGridPosition()); try (final LockedChannel lock = keyValueAccess.lockForWriting(blockPath)) { DefaultBlockWriter.writeBlock(lock.newOutputStream(), datasetAttributes, dataBlock); @@ -284,7 +284,7 @@ public void writeBlock( @Override public boolean remove(final String path) throws IOException { - final String normalPath = keyValueAccess.normalize(path); + final String normalPath = N5URL.normalizeGroupPath(path); final String groupPath = groupPath(normalPath); if (keyValueAccess.exists(groupPath)) keyValueAccess.delete(groupPath); @@ -292,7 +292,7 @@ public boolean remove(final String path) throws IOException { final String[] pathParts = keyValueAccess.components(normalPath); final String parent; if (pathParts.length <= 1) { - parent = keyValueAccess.normalize("/"); + parent = N5URL.normalizeGroupPath("/"); } else { final int parentPathLength = pathParts.length - 1; parent = keyValueAccess.compose(Arrays.copyOf(pathParts, parentPathLength)); @@ -309,7 +309,7 @@ public boolean deleteBlock( final String path, final long... gridPosition) throws IOException { - final String blockPath = getDataBlockPath(keyValueAccess.normalize(path), gridPosition); + final String blockPath = getDataBlockPath(N5URL.normalizeGroupPath(path), gridPosition); if (keyValueAccess.exists(blockPath)) keyValueAccess.delete(blockPath); From ccda98e2bd9f1a490705d7c7d24ed74fe69904a7 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Fri, 7 Apr 2023 13:49:48 -0400 Subject: [PATCH 143/243] BREAKING: rm default groupPath impl in N5Reader * edit documentation * to reflect current impl in N5KeyValueReader --- .../org/janelia/saalfeldlab/n5/N5Reader.java | 22 +++---------------- 1 file changed, 3 insertions(+), 19 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java index a31d8078..639fce47 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java @@ -684,29 +684,13 @@ default String getGroupSeparator() { } /** - * Creates a group path by concatenating all nodes with the node separator - * defined by {@link #getGroupSeparator()}. The string will not have a - * leading or trailing node separator symbol. - * + * Returns an absolute path to a group by concatenating all nodes with the node separator + * defined by {@link #getGroupSeparator()}. * @param nodes * @return */ - default String groupPath(final String... nodes) { - - if (nodes == null || nodes.length == 0) - return ""; - - final String groupSeparator = getGroupSeparator(); - final StringBuilder builder = new StringBuilder(nodes[0]); + String groupPath(final String... nodes); - for (int i = 1; i < nodes.length; ++i) { - - builder.append(groupSeparator); - builder.append(nodes[i]); - } - - return builder.toString(); - } /** * Default implementation of {@link AutoCloseable#close()} for all From 530eee5ef56b6209046e005c76d68df1185bbd4a Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Fri, 7 Apr 2023 15:47:37 -0400 Subject: [PATCH 144/243] wip: separate KeyValue interfaces --- .../n5/CachedGsonKeyValueReader.java | 214 ++++++++++++++ .../n5/CachedGsonKeyValueWriter.java | 265 +++++++++++++++++ .../saalfeldlab/n5/GsonKeyValueReader.java | 277 ++++++++++++++++++ .../saalfeldlab/n5/GsonKeyValueWriter.java | 225 ++++++++++++++ .../n5/N5KeyValueReaderWithInterfaces.java | 118 ++++++++ .../saalfeldlab/n5/N5KeyValueWriter.java | 63 ++-- .../n5/N5KeyValueWriterWithInterfaces.java | 78 +++++ .../org/janelia/saalfeldlab/n5/N5Writer.java | 5 +- .../n5/N5KeyValueWithInterfacesTest.java | 257 ++++++++++++++++ 9 files changed, 1463 insertions(+), 39 deletions(-) create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueReader.java create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueWriter.java create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueReader.java create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueWriter.java create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReaderWithInterfaces.java create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriterWithInterfaces.java create mode 100644 src/test/java/org/janelia/saalfeldlab/n5/N5KeyValueWithInterfacesTest.java diff --git a/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueReader.java b/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueReader.java new file mode 100644 index 00000000..a75413ac --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueReader.java @@ -0,0 +1,214 @@ +/** + * Copyright (c) 2017--2021, Stephan Saalfeld + * All rights reserved. + *

    + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + *

    + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + *

    + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +package org.janelia.saalfeldlab.n5; + +import com.google.gson.Gson; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import org.janelia.saalfeldlab.n5.cache.N5JsonCache; + +import java.io.IOException; +import java.lang.reflect.Type; +import java.util.Arrays; + +/** + * {@link N5Reader} implementation through {@link KeyValueAccess} with JSON + * attributes parsed with {@link Gson}. + * + * @author Stephan Saalfeld + * @author Igor Pisarev + * @author Philipp Hanslovsky + */ +public interface CachedGsonKeyValueReader extends GsonKeyValueReader { + + default N5JsonCache newCache() { + + return new N5JsonCache( + (groupPath, cacheKey) -> GsonKeyValueReader.super.getAttributes(groupPath), + GsonKeyValueReader.super::exists, + GsonKeyValueReader.super::exists, + GsonKeyValueReader.super::datasetExists, + GsonKeyValueReader.super::list + ); + } + + boolean cacheMeta(); + + N5JsonCache getCache(); + + @Override + default DatasetAttributes getDatasetAttributes(final String pathName) { + + final String normalPath = N5URL.normalizeGroupPath(pathName); + final JsonElement attributes; + if (cacheMeta() && getCache().isDataset(normalPath)) { + attributes = getCache().getAttributes(normalPath, N5JsonCache.jsonFile); + } else { + attributes = GsonKeyValueReader.super.getAttributes(normalPath); + } + + return createDatasetAttributes(attributes); + } + + default DatasetAttributes normalGetDatasetAttributes(final String pathName) throws N5Exception.N5IOException { + + final String normalPath = N5URL.normalizeGroupPath(pathName); + final JsonElement attributes = GsonKeyValueReader.super.getAttributes(normalPath); + return createDatasetAttributes(attributes); + } + + @Override + default T getAttribute( + final String pathName, + final String key, + final Class clazz) throws IOException { + + final String normalPathName = N5URL.normalizeGroupPath(pathName); + final String normalizedAttributePath = N5URL.normalizeAttributePath(key); + + final JsonElement attributes; + if (cacheMeta()) { + attributes = getCache().getAttributes(normalPathName, N5JsonCache.jsonFile); + } else { + attributes = GsonKeyValueReader.super.getAttributes(normalPathName); + } + return GsonUtils.readAttribute(attributes, normalizedAttributePath, clazz, getGson()); + } + + @Override + default T getAttribute( + final String pathName, + final String key, + final Type type) throws IOException { + + final String normalPathName = N5URL.normalizeGroupPath(pathName); + final String normalizedAttributePath = N5URL.normalizeAttributePath(key); + JsonElement attributes; + if (cacheMeta()) { + attributes = getCache().getAttributes(normalPathName, N5JsonCache.jsonFile); + } else { + attributes = GsonKeyValueReader.super.getAttributes(normalPathName); + } + return GsonUtils.readAttribute(attributes, normalizedAttributePath, type, getGson()); + } + + @Override + default boolean exists(final String pathName) { + + final String normalPathName = N5URL.normalizeGroupPath(pathName); + if (cacheMeta()) + return getCache().exists(normalPathName); + else { + return GsonKeyValueReader.super.exists(normalPathName); + } + } + + @Override + default boolean datasetExists(final String pathName) throws N5Exception.N5IOException { + + if (cacheMeta()) { + final String normalPathName = N5URL.normalizeGroupPath(pathName); + return getCache().isDataset(normalPathName); + } + return GsonKeyValueReader.super.datasetExists(pathName); + } + + /** + * Reads or creates the attributes map of a group or dataset. + * + * @param pathName group path + * @return + * @throws IOException + */ + default JsonElement getAttributes(final String pathName) throws N5Exception.N5IOException { + + final String groupPath = N5URL.normalizeGroupPath(pathName); + + /* If cached, return the cache*/ + if (cacheMeta()) { + return getCache().getAttributes(groupPath, N5JsonCache.jsonFile); + } else { + return GsonKeyValueReader.super.getAttributes(groupPath); + } + + } + + + @Override + default String[] list(final String pathName) throws N5Exception.N5IOException { + + final String normalPath = N5URL.normalizeGroupPath(pathName); + if (cacheMeta()) { + return getCache().list(normalPath); + } else { + return GsonKeyValueReader.super.list(normalPath); + } + } + + /** + * Constructs the path for a data block in a dataset at a given grid position. + *

    + * The returned path is + * 

    +	 * $basePath/datasetPathName/$gridPosition[0]/$gridPosition[1]/.../$gridPosition[n]
+	 *
    + *

    + * This is the file into which the data block will be stored. + * + * @param normalPath normalized dataset path + * @param gridPosition + * @return + */ + default String getDataBlockPath( + final String normalPath, + final long... gridPosition) { + + final String[] components = new String[gridPosition.length + 2]; + components[0] = getBasePath(); + components[1] = normalPath; + int i = 1; + for (final long p : gridPosition) + components[++i] = Long.toString(p); + + return getKeyValueAccess().compose(components); + } + + /** + * Check for attributes that are required for a group to be a dataset. + * + * @param attributes to check for dataset attributes + * @return if {@link DatasetAttributes#DIMENSIONS_KEY} and {@link DatasetAttributes#DATA_TYPE_KEY} are present + */ + static boolean hasDatasetAttributes(final JsonElement attributes) { + + if (attributes == null || !attributes.isJsonObject()) { + return false; + } + + final JsonObject metadataCache = attributes.getAsJsonObject(); + return metadataCache.has(DatasetAttributes.DIMENSIONS_KEY) && metadataCache.has(DatasetAttributes.DATA_TYPE_KEY); + } +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueWriter.java b/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueWriter.java new file mode 100644 index 00000000..5d55485f --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueWriter.java @@ -0,0 +1,265 @@ +/** + * Copyright (c) 2017--2021, Stephan Saalfeld + * All rights reserved. + *

    + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + *

    + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + *

    + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +package org.janelia.saalfeldlab.n5; + +import com.google.gson.GsonBuilder; +import com.google.gson.JsonElement; +import com.google.gson.JsonNull; +import com.google.gson.JsonObject; +import org.janelia.saalfeldlab.n5.cache.N5JsonCache; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +/** + * Filesystem {@link N5Writer} implementation with version compatibility check. + * + * @author Stephan Saalfeld + */ +public interface CachedGsonKeyValueWriter extends CachedGsonKeyValueReader, N5Writer { + + default void setVersion(final String path) throws IOException { + + if (!VERSION.equals(getVersion())) + setAttribute("/", VERSION_KEY, VERSION.toString()); + } + + static String initializeContainer( + final KeyValueAccess keyValueAccess, + final String basePath) throws IOException { + + final String normBasePath = keyValueAccess.normalize(basePath); + keyValueAccess.createDirectories(normBasePath); + return normBasePath; + } + + /** + * Performs any necessary initialization to ensure the key given by the + * argument {@code normalPath} is a valid group after creation. Called by + * {@link #createGroup(String)}. + * + * @param normalPath the group path. + */ + default void initializeGroup(final String normalPath) { + + // Nothing to do here, but other implementations (e.g. zarr) use this. + } + + @Override + default void createGroup(final String path) throws IOException { + + final String normalPath = N5URL.normalizeGroupPath(path); + getKeyValueAccess().createDirectories(groupPath(normalPath)); + if (cacheMeta()) { + final String[] pathParts = getKeyValueAccess().components(normalPath); + if (pathParts.length > 1) { + String parent = N5URL.normalizeGroupPath("/"); + for (String child : pathParts) { + final String childPath = getKeyValueAccess().compose(parent, child); + getCache().addChild(parent, child); + parent = childPath; + } + } + } + initializeGroup(normalPath); + } + + /** + * Helper method that reads an existing JsonElement representing the root + * attributes for {@code normalGroupPath}, inserts and overrides the + * provided attributes, and writes them back into the attributes store. + * + * @param normalGroupPath to write the attributes to + * @param attributes to write + * @throws IOException if unable to read the attributes at {@code normalGroupPath} + *

    + * TODO consider cache (or you read the attributes twice?) + */ + default void writeAttributes( + final String normalGroupPath, + final JsonElement attributes) throws IOException { + + JsonElement root = getAttributes(normalGroupPath); + root = GsonUtils.insertAttribute(root, "/", attributes, getGson()); + writeAndCacheAttributes(normalGroupPath, root); + } + + /** + * Helper method that reads the existing map of attributes, JSON encodes, + * inserts and overrides the provided attributes, and writes them back into + * the attributes store. + * + * @param normalGroupPath to write the attributes to + * @param attributes to write + * @throws IOException if unable to read the attributes at {@code normalGroupPath} + */ + default void writeAttributes( + final String normalGroupPath, + final Map attributes) throws IOException { + + if (attributes != null && !attributes.isEmpty()) { + JsonElement existingAttributes = getAttributes(normalGroupPath); + existingAttributes = existingAttributes != null && existingAttributes.isJsonObject() + ? existingAttributes.getAsJsonObject() + : new JsonObject(); + JsonElement newAttributes = GsonUtils.insertAttributes(existingAttributes, attributes, getGson()); + writeAndCacheAttributes(normalGroupPath, newAttributes); + } + } + + default void writeAndCacheAttributes(final String normalGroupPath, final JsonElement attributes) throws IOException { + + try (final LockedChannel lock = getKeyValueAccess().lockForWriting(attributesPath(normalGroupPath))) { + GsonUtils.writeAttributes(lock.newWriter(), attributes, getGson()); + } + if (cacheMeta()) { + JsonElement nullRespectingAttributes = attributes; + /* Gson only filters out nulls when you write the JsonElement. This means it doesn't filter them out when caching. + * To handle this, we explicitly writer the existing JsonElement to a new JsonElement. + * The output is identical to the input if: + * - serializeNulls is true + * - no null values are present + * - cacheing is turned off */ + if (!getGson().serializeNulls()) { + nullRespectingAttributes = getGson().toJsonTree(attributes); + } + /* Update the cache, and write to the writer */ + getCache().updateCacheInfo(normalGroupPath, N5JsonCache.jsonFile, nullRespectingAttributes); + } + } + + @Override + default void setAttributes( + final String path, + final Map attributes) throws IOException { + + final String normalPath = N5URL.normalizeGroupPath(path); + if (!exists(normalPath)) + throw new N5Exception.N5IOException("" + normalPath + " is not a group or dataset."); + + writeAttributes(normalPath, attributes); + } + + @Override + default boolean removeAttribute(final String pathName, final String key) throws IOException { + + final String normalPath = N5URL.normalizeGroupPath(pathName); + final String absoluteNormalPath = getKeyValueAccess().compose(getBasePath(), normalPath); + final String normalKey = N5URL.normalizeAttributePath(key); + + if (!getKeyValueAccess().exists(absoluteNormalPath)) + return false; + + if (key.equals("/")) { + writeAttributes(normalPath, JsonNull.INSTANCE); + return true; + } + + final JsonElement attributes = getAttributes(normalPath); + if (GsonUtils.removeAttribute(attributes, normalKey) != null) { + writeAttributes(normalPath, attributes); + return true; + } + return false; + } + + @Override + default T removeAttribute(final String pathName, final String key, final Class cls) throws IOException { + + final String normalPath = N5URL.normalizeGroupPath(pathName); + final String normalKey = N5URL.normalizeAttributePath(key); + + final JsonElement attributes = getAttributes(normalPath); + final T obj = GsonUtils.removeAttribute(attributes, normalKey, cls, getGson()); + if (obj != null) { + writeAttributes(normalPath, attributes); + } + return obj; + } + + @Override + default boolean removeAttributes(final String pathName, final List attributes) throws IOException { + + final String normalPath = N5URL.normalizeGroupPath(pathName); + boolean removed = false; + for (final String attribute : attributes) { + final String normalKey = N5URL.normalizeAttributePath(attribute); + removed |= removeAttribute(normalPath, attribute); + } + return removed; + } + + @Override + default void writeBlock( + final String path, + final DatasetAttributes datasetAttributes, + final DataBlock dataBlock) throws IOException { + + final String blockPath = getDataBlockPath(N5URL.normalizeGroupPath(path), dataBlock.getGridPosition()); + try (final LockedChannel lock = getKeyValueAccess().lockForWriting(blockPath)) { + + DefaultBlockWriter.writeBlock(lock.newOutputStream(), datasetAttributes, dataBlock); + } + } + + @Override + default boolean remove(final String path) throws IOException { + + final String normalPath = N5URL.normalizeGroupPath(path); + final String groupPath = groupPath(normalPath); + if (getKeyValueAccess().exists(groupPath)) + getKeyValueAccess().delete(groupPath); + if (cacheMeta()) { + final String[] pathParts = getKeyValueAccess().components(normalPath); + final String parent; + if (pathParts.length <= 1) { + parent = N5URL.normalizeGroupPath("/"); + } else { + final int parentPathLength = pathParts.length - 1; + parent = getKeyValueAccess().compose(Arrays.copyOf(pathParts, parentPathLength)); + } + getCache().removeCache(parent, normalPath); + } + + /* an IOException should have occurred if anything had failed midway */ + return true; + } + + @Override + default boolean deleteBlock( + final String path, + final long... gridPosition) throws IOException { + + final String blockPath = getDataBlockPath(N5URL.normalizeGroupPath(path), gridPosition); + if (getKeyValueAccess().exists(blockPath)) + getKeyValueAccess().delete(blockPath); + + /* an IOException should have occurred if anything had failed midway */ + return true; + } +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueReader.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueReader.java new file mode 100644 index 00000000..1719918a --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueReader.java @@ -0,0 +1,277 @@ +/** + * Copyright (c) 2017--2021, Stephan Saalfeld + * All rights reserved. + *

    + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + *

    + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + *

    + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +package org.janelia.saalfeldlab.n5; + +import com.google.gson.Gson; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import org.janelia.saalfeldlab.n5.cache.N5JsonCache; + +import java.io.IOException; +import java.lang.reflect.Type; +import java.util.Arrays; +import java.util.Map; + +/** + * {@link N5Reader} implementation through {@link KeyValueAccess} with JSON + * attributes parsed with {@link Gson}. + * + * @author Stephan Saalfeld + * @author Igor Pisarev + * @author Philipp Hanslovsky + */ +public interface GsonKeyValueReader extends N5Reader { + + Gson getGson(); + + KeyValueAccess getKeyValueAccess(); + + @Override + default Map> listAttributes(final String pathName) throws IOException { + + return GsonUtils.listAttributes(getAttributes(pathName)); + } + + @Override + default DatasetAttributes getDatasetAttributes(final String pathName) throws N5Exception.N5IOException { + + final String normalPath = N5URL.normalizeGroupPath(pathName); + final JsonElement attributes = getAttributes(normalPath); + return createDatasetAttributes(attributes); + } + + default DatasetAttributes createDatasetAttributes(JsonElement attributes) { + + final long[] dimensions = GsonUtils.readAttribute(attributes, DatasetAttributes.DIMENSIONS_KEY, long[].class, getGson()); + if (dimensions == null) { + return null; + } + + final DataType dataType = GsonUtils.readAttribute(attributes, DatasetAttributes.DATA_TYPE_KEY, DataType.class, getGson()); + if (dataType == null) { + return null; + } + + final int[] blockSize = GsonUtils.readAttribute(attributes, DatasetAttributes.BLOCK_SIZE_KEY, int[].class, getGson()); + + final Compression compression = GsonUtils.readAttribute(attributes, DatasetAttributes.COMPRESSION_KEY, Compression.class, getGson()); + + /* version 0 */ + final String compressionVersion0Name = compression + == null + ? GsonUtils.readAttribute(attributes, DatasetAttributes.compressionTypeKey, String.class, getGson()) + : null; + + return createDatasetAttributes(dimensions, dataType, blockSize, compression, compressionVersion0Name); + } + + @Override + default T getAttribute(final String pathName, final String key, final Class clazz) throws IOException { + + final String normalPathName = N5URL.normalizeGroupPath(pathName); + final String normalizedAttributePath = N5URL.normalizeAttributePath(key); + + final JsonElement attributes = getAttributes(normalPathName); + return GsonUtils.readAttribute(attributes, normalizedAttributePath, clazz, getGson()); + } + + @Override + default T getAttribute(final String pathName, final String key, final Type type) throws IOException { + + final String normalPathName = N5URL.normalizeGroupPath(pathName); + final String normalizedAttributePath = N5URL.normalizeAttributePath(key); + JsonElement attributes = getAttributes(normalPathName); + return GsonUtils.readAttribute(attributes, normalizedAttributePath, type, getGson()); + } + + default boolean groupExists(final String absoluteNormalPath) { + + return getKeyValueAccess().exists(absoluteNormalPath) && getKeyValueAccess().isDirectory(absoluteNormalPath); + } + + @Override + default boolean exists(final String pathName) { + + final String normalPathName = N5URL.normalizeGroupPath(pathName); + return getKeyValueAccess().exists(groupPath(normalPathName)) || datasetExists(normalPathName); + } + + @Override + default boolean datasetExists(final String pathName) throws N5Exception.N5IOException { + // for n5, every dataset must be a group + return groupExists(groupPath(pathName)) && getDatasetAttributes(pathName) != null; + + } + + /** + * Reads or creates the attributes map of a group or dataset. + * + * @param pathName group path + * @return + * @throws IOException + */ + default JsonElement getAttributes(final String pathName) throws N5Exception.N5IOException { + + final String groupPath = N5URL.normalizeGroupPath(pathName); + final String attributesPath = attributesPath(groupPath); + if (!exists(groupPath)) + throw new N5Exception.N5IOException("Group " + groupPath + " does not exist"); + if (!getKeyValueAccess().exists(attributesPath)) + return null; + + try (final LockedChannel lockedChannel = getKeyValueAccess().lockForReading(attributesPath)) { + return GsonUtils.readAttributes(lockedChannel.newReader(), getGson()); + } catch (IOException e) { + throw new N5Exception.N5IOException("Cannot open lock for Reading", e); + } + + } + + @Override + default DataBlock readBlock( final String pathName, final DatasetAttributes datasetAttributes, final long... gridPosition) throws IOException { + + final String path = getDataBlockPath(N5URL.normalizeGroupPath(pathName), gridPosition); + if (!getKeyValueAccess().exists(path)) + return null; + + try (final LockedChannel lockedChannel = getKeyValueAccess().lockForReading(path)) { + return DefaultBlockReader.readBlock(lockedChannel.newInputStream(), datasetAttributes, gridPosition); + } + } + + @Override + default String[] list(final String pathName) throws N5Exception.N5IOException { + + try { + return getKeyValueAccess().listDirectories(groupPath(pathName)); + } catch (IOException e) { + throw new N5Exception.N5IOException("Cannot list directories for group " + pathName, e); + } + } + + /** + * Constructs the path for a data block in a dataset at a given grid position. + *

    + * The returned path is + * 

    +	 * $basePath/datasetPathName/$gridPosition[0]/$gridPosition[1]/.../$gridPosition[n]
+	 *
    + *

    + * This is the file into which the data block will be stored. + * + * @param normalPath normalized dataset path + * @param gridPosition + * @return + */ + default String getDataBlockPath( + final String normalPath, + final long... gridPosition) { + + final String[] components = new String[gridPosition.length + 2]; + components[0] = getBasePath(); + components[1] = normalPath; + int i = 1; + for (final long p : gridPosition) + components[++i] = Long.toString(p); + + return getKeyValueAccess().compose(components); + } + + /** + * Constructs the absolute path (in terms of this store) for the group or + * dataset. + * + * @param normalGroupPath normalized group path without leading slash + * @return the absolute path to the group + */ + default String groupPath(final String normalGroupPath) { + + return getKeyValueAccess().compose(getBasePath(), normalGroupPath); + } + + /** + * Constructs the absolute path (in terms of this store) for the attributes + * file of a group or dataset. + * + * @param normalPath normalized group path without leading slash + * @return the absolute path to the attributes + */ + default String attributesPath(final String normalPath) { + + return getKeyValueAccess().compose(getBasePath(), normalPath, N5JsonCache.jsonFile); + } + + static DatasetAttributes createDatasetAttributes( + final long[] dimensions, + final DataType dataType, + int[] blockSize, + Compression compression, + final String compressionVersion0Name + ) { + + if (blockSize == null) + blockSize = Arrays.stream(dimensions).mapToInt(a -> (int)a).toArray(); + + /* version 0 */ + if (compression == null) { + switch (compressionVersion0Name) { + case "raw": + compression = new RawCompression(); + break; + case "gzip": + compression = new GzipCompression(); + break; + case "bzip2": + compression = new Bzip2Compression(); + break; + case "lz4": + compression = new Lz4Compression(); + break; + case "xz": + compression = new XzCompression(); + break; + } + } + + return new DatasetAttributes(dimensions, blockSize, dataType, compression); + } + + /** + * Check for attributes that are required for a group to be a dataset. + * + * @param attributes to check for dataset attributes + * @return if {@link DatasetAttributes#DIMENSIONS_KEY} and {@link DatasetAttributes#DATA_TYPE_KEY} are present + */ + static boolean hasDatasetAttributes(final JsonElement attributes) { + + if (attributes == null || !attributes.isJsonObject()) { + return false; + } + + final JsonObject metadataCache = attributes.getAsJsonObject(); + return metadataCache.has(DatasetAttributes.DIMENSIONS_KEY) && metadataCache.has(DatasetAttributes.DATA_TYPE_KEY); + } +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueWriter.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueWriter.java new file mode 100644 index 00000000..a9b6f9a8 --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueWriter.java @@ -0,0 +1,225 @@ +/** + * Copyright (c) 2017--2021, Stephan Saalfeld + * All rights reserved. + *

    + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + *

    + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + *

    + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +package org.janelia.saalfeldlab.n5; + +import com.google.gson.JsonElement; +import com.google.gson.JsonNull; +import com.google.gson.JsonObject; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +/** + * Filesystem {@link N5Writer} implementation with version compatibility check. + * + * @author Stephan Saalfeld + */ +public interface GsonKeyValueWriter extends GsonKeyValueReader, N5Writer { + + default void setVersion(final String path) throws IOException { + + if (!VERSION.equals(getVersion())) + setAttribute("/", VERSION_KEY, VERSION.toString()); + } + + static String initializeContainer( + final KeyValueAccess keyValueAccess, + final String basePath) throws IOException { + + final String normBasePath = keyValueAccess.normalize(basePath); + keyValueAccess.createDirectories(normBasePath); + return normBasePath; + } + + default void initializeGroup(String normalPath) { + + } + + @Override + default void createGroup(final String path) throws IOException { + + final String normalPath = N5URL.normalizeGroupPath(path); + getKeyValueAccess().createDirectories(groupPath(normalPath)); + initializeGroup(normalPath); + } + + @Override + default void createDataset( + final String path, + final DatasetAttributes datasetAttributes) throws IOException { + + final String normalPath = N5URL.normalizeGroupPath(path); + createGroup(path); + setDatasetAttributes(normalPath, datasetAttributes); + } + + /** + * Helper method that reads an existing JsonElement representing the root + * attributes for {@code normalGroupPath}, inserts and overrides the + * provided attributes, and writes them back into the attributes store. + * + * @param normalGroupPath to write the attributes to + * @param attributes to write + * @throws IOException if unable to read the attributes at {@code normalGroupPath} + *

    + * TODO consider cache (or you read the attributes twice?) + */ + default void writeAttributes( + final String normalGroupPath, + final JsonElement attributes) throws IOException { + + JsonElement root = getAttributes(normalGroupPath); + root = GsonUtils.insertAttribute(root, "/", attributes, getGson()); + try (final LockedChannel lock = getKeyValueAccess().lockForWriting(attributesPath(normalGroupPath))) { + GsonUtils.writeAttributes(lock.newWriter(), root, getGson()); + } + } + + /** + * Helper method that reads the existing map of attributes, JSON encodes, + * inserts and overrides the provided attributes, and writes them back into + * the attributes store. + * + * @param normalGroupPath to write the attributes to + * @param attributes to write + * @throws IOException if unable to read the attributes at {@code normalGroupPath} + */ + default void writeAttributes( + final String normalGroupPath, + final Map attributes) throws IOException { + + if (attributes != null && !attributes.isEmpty()) { + JsonElement existingAttributes = getAttributes(normalGroupPath); + existingAttributes = existingAttributes != null && existingAttributes.isJsonObject() + ? existingAttributes.getAsJsonObject() + : new JsonObject(); + JsonElement newAttributes = GsonUtils.insertAttributes(existingAttributes, attributes, getGson()); + try (final LockedChannel lock = getKeyValueAccess().lockForWriting(attributesPath(normalGroupPath))) { + GsonUtils.writeAttributes(lock.newWriter(), newAttributes, getGson()); + } + } + } + + @Override + default void setAttributes( + final String path, + final Map attributes) throws IOException { + + final String normalPath = N5URL.normalizeGroupPath(path); + if (!exists(normalPath)) + throw new N5Exception.N5IOException("" + normalPath + " is not a group or dataset."); + + writeAttributes(normalPath, attributes); + } + + @Override + default boolean removeAttribute(final String pathName, final String key) throws IOException { + + final String normalPath = N5URL.normalizeGroupPath(pathName); + final String absoluteNormalPath = getKeyValueAccess().compose(getBasePath(), normalPath); + final String normalKey = N5URL.normalizeAttributePath(key); + + if (!getKeyValueAccess().exists(absoluteNormalPath)) + return false; + + if (key.equals("/")) { + writeAttributes(normalPath, JsonNull.INSTANCE); + return true; + } + + final JsonElement attributes = getAttributes(normalPath); + if (GsonUtils.removeAttribute(attributes, normalKey) != null) { + writeAttributes(normalPath, attributes); + return true; + } + return false; + } + + @Override + default T removeAttribute(final String pathName, final String key, final Class cls) throws IOException { + + final String normalPath = N5URL.normalizeGroupPath(pathName); + final String normalKey = N5URL.normalizeAttributePath(key); + + final JsonElement attributes = getAttributes(normalPath); + final T obj = GsonUtils.removeAttribute(attributes, normalKey, cls, getGson()); + if (obj != null) { + writeAttributes(normalPath, attributes); + } + return obj; + } + + @Override + default boolean removeAttributes(final String pathName, final List attributes) throws IOException { + + final String normalPath = N5URL.normalizeGroupPath(pathName); + boolean removed = false; + for (final String attribute : attributes) { + final String normalKey = N5URL.normalizeAttributePath(attribute); + removed |= removeAttribute(normalPath, attribute); + } + return removed; + } + + @Override + default void writeBlock( + final String path, + final DatasetAttributes datasetAttributes, + final DataBlock dataBlock) throws IOException { + + final String blockPath = getDataBlockPath(N5URL.normalizeGroupPath(path), dataBlock.getGridPosition()); + try (final LockedChannel lock = getKeyValueAccess().lockForWriting(blockPath)) { + + DefaultBlockWriter.writeBlock(lock.newOutputStream(), datasetAttributes, dataBlock); + } + } + + @Override + default boolean remove(final String path) throws IOException { + + final String normalPath = N5URL.normalizeGroupPath(path); + final String groupPath = groupPath(normalPath); + if (getKeyValueAccess().exists(groupPath)) + getKeyValueAccess().delete(groupPath); + + /* an IOException should have occurred if anything had failed midway */ + return true; + } + + @Override + default boolean deleteBlock( + final String path, + final long... gridPosition) throws IOException { + + final String blockPath = getDataBlockPath(N5URL.normalizeGroupPath(path), gridPosition); + if (getKeyValueAccess().exists(blockPath)) + getKeyValueAccess().delete(blockPath); + + /* an IOException should have occurred if anything had failed midway */ + return true; + } +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReaderWithInterfaces.java b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReaderWithInterfaces.java new file mode 100644 index 00000000..97564b7f --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReaderWithInterfaces.java @@ -0,0 +1,118 @@ +/** + * Copyright (c) 2017--2021, Stephan Saalfeld + * All rights reserved. + *

    + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + *

    + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + *

    + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +package org.janelia.saalfeldlab.n5; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import org.janelia.saalfeldlab.n5.cache.N5JsonCache; + +import java.io.IOException; +import java.lang.reflect.Type; +import java.util.Arrays; +import java.util.Map; + +/** + * {@link N5Reader} implementation through {@link KeyValueAccess} with JSON + * attributes parsed with {@link Gson}. + * + * @author Stephan Saalfeld + * @author Igor Pisarev + * @author Philipp Hanslovsky + */ +public class N5KeyValueReaderWithInterfaces implements CachedGsonKeyValueReader { + + protected final KeyValueAccess keyValueAccess; + + protected final Gson gson; + protected final boolean cacheMeta; + + protected final String basePath; + private final N5JsonCache cache; + + /** + * Opens an {@link N5KeyValueReaderWithInterfaces} at a given base path with a custom + * {@link GsonBuilder} to support custom attributes. + * + * @param keyValueAccess + * @param basePath N5 base path + * @param gsonBuilder + * @param cacheMeta cache attributes and meta data + * Setting this to true avoids frequent reading and parsing of JSON + * encoded attributes and other meta data that requires accessing the + * store. This is most interesting for high latency backends. Changes + * of cached attributes and meta data by an independent writer will + * not be tracked. + * + * @throws IOException + * if the base path cannot be read or does not exist, + * if the N5 version of the container is not compatible with this + * implementation. + */ + public N5KeyValueReaderWithInterfaces( + final KeyValueAccess keyValueAccess, + final String basePath, + final GsonBuilder gsonBuilder, + final boolean cacheMeta) throws IOException { + + this.keyValueAccess = keyValueAccess; + this.basePath = keyValueAccess.normalize(basePath); + this.gson = GsonUtils.registerGson(gsonBuilder); + this.cacheMeta = cacheMeta; + this.cache = getCache(); + + /* Existence checks, if any, go in subclasses */ + /* Check that version (if there is one) is compatible. */ + final Version version = getVersion(); + if (!VERSION.isCompatible(version)) + throw new N5Exception.N5IOException("Incompatible version " + version + " (this is " + VERSION + ")."); + } + + public Gson getGson() { + + return gson; + } + public KeyValueAccess getKeyValueAccess() { + return keyValueAccess; + } + + @Override + public String getBasePath() { + + return this.basePath; + } + + @Override public boolean cacheMeta() { + + return cacheMeta; + } + + @Override public N5JsonCache getCache() { + + return this.cache; + } +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriter.java b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriter.java index 4e2f7dd4..d7c11696 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriter.java @@ -25,16 +25,16 @@ */ package org.janelia.saalfeldlab.n5; -import java.io.IOException; -import java.util.List; -import java.util.Map; - import com.google.gson.GsonBuilder; import com.google.gson.JsonElement; import com.google.gson.JsonNull; import com.google.gson.JsonObject; import org.janelia.saalfeldlab.n5.cache.N5JsonCache; + +import java.io.IOException; import java.util.Arrays; +import java.util.List; +import java.util.Map; /** * Filesystem {@link N5Writer} implementation with version compatibility check. @@ -54,19 +54,15 @@ public class N5KeyValueWriter extends N5KeyValueReader implements N5Writer { * will be set to the current N5 version of this implementation. * * @param keyValueAccess - * @param basePath - * n5 base path + * @param basePath n5 base path * @param gsonBuilder - * @param cacheAttributes - * Setting this to true avoids frequent reading and parsing of - * JSON encoded attributes, this is most interesting for high - * latency file systems. Changes of attributes by an independent - * writer will not be tracked. - * - * @throws IOException - * if the base path cannot be written to or cannot be created, - * if the N5 version of the container is not compatible with - * this implementation. + * @param cacheAttributes Setting this to true avoids frequent reading and parsing of + * JSON encoded attributes, this is most interesting for high + * latency file systems. Changes of attributes by an independent + * writer will not be tracked. + * @throws IOException if the base path cannot be written to or cannot be created, + * if the N5 version of the container is not compatible with + * this implementation. */ public N5KeyValueWriter( final KeyValueAccess keyValueAccess, @@ -100,8 +96,7 @@ protected static String initializeContainer( * argument {@code normalPath} is a valid group after creation. Called by * {@link #createGroup(String)}. * - * @param normalPath - * the group path. + * @param normalPath the group path. */ protected void initializeGroup(final String normalPath) { @@ -142,14 +137,11 @@ public void createDataset( * attributes for {@code normalGroupPath}, inserts and overrides the * provided attributes, and writes them back into the attributes store. * - * @param normalGroupPath - * to write the attributes to - * @param attributes - * to write - * @throws IOException - * if unable to read the attributes at {@code normalGroupPath} - * - * TODO consider cache (or you read the attributes twice?) + * @param normalGroupPath to write the attributes to + * @param attributes to write + * @throws IOException if unable to read the attributes at {@code normalGroupPath} + *

    + * TODO consider cache (or you read the attributes twice?) */ protected void writeAttributes( final String normalGroupPath, @@ -165,12 +157,9 @@ protected void writeAttributes( * inserts and overrides the provided attributes, and writes them back into * the attributes store. * - * @param normalGroupPath - * to write the attributes to - * @param attributes - * to write - * @throws IOException - * if unable to read the attributes at {@code normalGroupPath} + * @param normalGroupPath to write the attributes to + * @param attributes to write + * @throws IOException if unable to read the attributes at {@code normalGroupPath} */ protected void writeAttributes( final String normalGroupPath, @@ -194,11 +183,11 @@ private void writeAndCacheAttributes(final String normalGroupPath, final JsonEle if (cacheMeta) { JsonElement nullRespectingAttributes = attributes; /* Gson only filters out nulls when you write the JsonElement. This means it doesn't filter them out when caching. - * To handle this, we explicitly writer the existing JsonElement to a new JsonElement. - * The output is identical to the input if: - * - serializeNulls is true - * - no null values are present - * - cacheing is turned off */ + * To handle this, we explicitly writer the existing JsonElement to a new JsonElement. + * The output is identical to the input if: + * - serializeNulls is true + * - no null values are present + * - cacheing is turned off */ if (!gson.serializeNulls()) { nullRespectingAttributes = gson.toJsonTree(attributes); } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriterWithInterfaces.java b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriterWithInterfaces.java new file mode 100644 index 00000000..ded92913 --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriterWithInterfaces.java @@ -0,0 +1,78 @@ +/** + * Copyright (c) 2017--2021, Stephan Saalfeld + * All rights reserved. + *

    + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + *

    + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + *

    + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +package org.janelia.saalfeldlab.n5; + +import com.google.gson.GsonBuilder; +import com.google.gson.JsonElement; +import com.google.gson.JsonNull; +import com.google.gson.JsonObject; +import org.janelia.saalfeldlab.n5.cache.N5JsonCache; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +/** + * Filesystem {@link N5Writer} implementation with version compatibility check. + * + * @author Stephan Saalfeld + */ +public class N5KeyValueWriterWithInterfaces extends N5KeyValueReaderWithInterfaces implements CachedGsonKeyValueWriter { + + /** + * Opens an {@link N5KeyValueWriterWithInterfaces} at a given base path with a custom + * {@link GsonBuilder} to support custom attributes. + *

    + * If the base path does not exist, it will be created. + *

    + * If the base path exists and if the N5 version of the container is + * compatible with this implementation, the N5 version of this container + * will be set to the current N5 version of this implementation. + * + * @param keyValueAccess + * @param basePath n5 base path + * @param gsonBuilder + * @param cacheAttributes Setting this to true avoids frequent reading and parsing of + * JSON encoded attributes, this is most interesting for high + * latency file systems. Changes of attributes by an independent + * writer will not be tracked. + * @throws IOException if the base path cannot be written to or cannot be created, + * if the N5 version of the container is not compatible with + * this implementation. + */ + public N5KeyValueWriterWithInterfaces( + final KeyValueAccess keyValueAccess, + final String basePath, + final GsonBuilder gsonBuilder, + final boolean cacheAttributes) + throws IOException { + + super(keyValueAccess, GsonKeyValueWriter.initializeContainer(keyValueAccess, basePath), gsonBuilder, cacheAttributes); + createGroup("/"); + setVersion("/"); + } +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java index ae9c8bb3..3c5941de 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java @@ -169,8 +169,9 @@ default void createDataset( final String pathName, final DatasetAttributes datasetAttributes) throws IOException { - createGroup(pathName); - setDatasetAttributes(pathName, datasetAttributes); + final String normalPath = N5URL.normalizeGroupPath(pathName); + createGroup(normalPath); + setDatasetAttributes(normalPath, datasetAttributes); } /** diff --git a/src/test/java/org/janelia/saalfeldlab/n5/N5KeyValueWithInterfacesTest.java b/src/test/java/org/janelia/saalfeldlab/n5/N5KeyValueWithInterfacesTest.java new file mode 100644 index 00000000..8cef539c --- /dev/null +++ b/src/test/java/org/janelia/saalfeldlab/n5/N5KeyValueWithInterfacesTest.java @@ -0,0 +1,257 @@ +/** + * Copyright (c) 2017--2021, Stephan Saalfeld + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +package org.janelia.saalfeldlab.n5; + +import com.google.gson.GsonBuilder; +import org.apache.commons.io.FileUtils; +import org.janelia.saalfeldlab.n5.url.UrlAttributeTest; +import org.junit.AfterClass; +import org.junit.Test; + +import java.io.File; +import java.io.IOException; +import java.nio.file.FileSystems; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +import static org.junit.Assert.fail; + +/** + * Initiates testing of the filesystem-based N5 implementation. + * + * @author Stephan Saalfeld + * @author Igor Pisarev + */ +public class N5KeyValueWithInterfacesTest extends AbstractN5Test { + + private static FileSystemKeyValueAccess access = new FileSystemKeyValueAccess(FileSystems.getDefault()); + private static String testDirPath = tempN5PathName(); + + @AfterClass + public static void cleanup() { + + for (String tmpFile : tmpFiles) { + try{ + FileUtils.deleteDirectory(new File(tmpFile)); + } catch (Exception e) { } + } + } + + /** + * @throws IOException + */ + @Override + protected N5Writer createN5Writer() throws IOException { + return new N5KeyValueWriterWithInterfaces(access, tempN5PathName(), new GsonBuilder(), true); + } + + @Override + protected N5Writer createN5Writer(String location, GsonBuilder gson) throws IOException { + if (!new File(location).exists()) { + tmpFiles.add(location); + } + return new N5KeyValueWriterWithInterfaces(access, location, gson, true); + } + + @Override + protected N5Reader createN5Reader(String location, GsonBuilder gson) throws IOException { + return new N5FSReader(location, gson); + } + + @Test + public void customObjectTest() throws IOException { + + final String testGroup = "test"; + final ArrayList> existingTests = new ArrayList<>(); + + final UrlAttributeTest.TestDoubles doubles1 = new UrlAttributeTest.TestDoubles("doubles", "doubles1", new double[]{5.7, 4.5, 3.4}); + final UrlAttributeTest.TestDoubles doubles2 = new UrlAttributeTest.TestDoubles("doubles", "doubles2", new double[]{5.8, 4.6, 3.5}); + final UrlAttributeTest.TestDoubles doubles3 = new UrlAttributeTest.TestDoubles("doubles", "doubles3", new double[]{5.9, 4.7, 3.6}); + final UrlAttributeTest.TestDoubles doubles4 = new UrlAttributeTest.TestDoubles("doubles", "doubles4", new double[]{5.10, 4.8, 3.7}); + n5.createGroup(testGroup); + addAndTest(n5, existingTests, new TestData<>(testGroup, "/doubles[1]", doubles1)); + addAndTest(n5, existingTests, new TestData<>(testGroup, "/doubles[2]", doubles2)); + addAndTest(n5, existingTests, new TestData<>(testGroup, "/doubles[3]", doubles3)); + addAndTest(n5, existingTests, new TestData<>(testGroup, "/doubles[4]", doubles4)); + + /* Test overwrite custom */ + addAndTest(n5, existingTests, new TestData<>(testGroup, "/doubles[1]", doubles4)); + } + + +// @Test + public void testReadLock() throws IOException, InterruptedException { + + final Path path = Paths.get(testDirPath, "lock"); + try { + Files.delete(path); + } catch (final IOException e) {} + + LockedChannel lock = access.lockForWriting(path); + lock.close(); + lock = access.lockForReading(path); + System.out.println("locked"); + + final ExecutorService exec = Executors.newSingleThreadExecutor(); + final Future future = exec.submit(() -> { + access.lockForWriting(path).close(); + return null; + }); + + try { + System.out.println("Trying to acquire locked readable channel..."); + System.out.println(future.get(3, TimeUnit.SECONDS)); + fail("Lock broken!"); + } catch (final TimeoutException e) { + System.out.println("Lock held!"); + future.cancel(true); + } catch (final InterruptedException | ExecutionException e) { + future.cancel(true); + System.out.println("Test was interrupted!"); + } finally { + lock.close(); + Files.delete(path); + } + + exec.shutdownNow(); + } + + +// @Test + public void testWriteLock() throws IOException { + + final Path path = Paths.get(testDirPath, "lock"); + try { + Files.delete(path); + } catch (final IOException e) {} + + final LockedChannel lock = access.lockForWriting(path); + System.out.println("locked"); + + final ExecutorService exec = Executors.newSingleThreadExecutor(); + final Future future = exec.submit(() -> { + access.lockForReading(path).close(); + return null; + }); + + try { + System.out.println("Trying to acquire locked writable channel..."); + System.out.println(future.get(3, TimeUnit.SECONDS)); + fail("Lock broken!"); + } catch (final TimeoutException e) { + System.out.println("Lock held!"); + future.cancel(true); + } catch (final InterruptedException | ExecutionException e) { + future.cancel(true); + System.out.println("Test was interrupted!"); + } finally { + lock.close(); + Files.delete(path); + } + + exec.shutdownNow(); + } + + @Test + public void testLockReleaseByReader() throws IOException { + + System.out.println("Testing lock release by Reader."); + + final Path path = Paths.get(testDirPath, "lock"); + try { + Files.delete(path); + } catch (final IOException e) {} + + final LockedChannel lock = access.lockForWriting(path); + + lock.newReader().close(); + + final ExecutorService exec = Executors.newSingleThreadExecutor(); + final Future future = exec.submit(() -> { + access.lockForWriting(path).close(); + return null; + }); + + try { + future.get(3, TimeUnit.SECONDS); + } catch (final TimeoutException e) { + fail("... lock not released!"); + future.cancel(true); + } catch (final InterruptedException | ExecutionException e) { + future.cancel(true); + System.out.println("... test interrupted!"); + } finally { + lock.close(); + Files.delete(path); + } + + exec.shutdownNow(); + } + + @Test + public void testLockReleaseByInputStream() throws IOException { + + System.out.println("Testing lock release by InputStream."); + + final Path path = Paths.get(testDirPath, "lock"); + try { + Files.delete(path); + } catch (final IOException e) {} + + final LockedChannel lock = access.lockForWriting(path); + + lock.newInputStream().close(); + + final ExecutorService exec = Executors.newSingleThreadExecutor(); + final Future future = exec.submit(() -> { + access.lockForWriting(path).close(); + return null; + }); + + try { + future.get(3, TimeUnit.SECONDS); + } catch (final TimeoutException e) { + fail("... lock not released!"); + future.cancel(true); + } catch (final InterruptedException | ExecutionException e) { + future.cancel(true); + System.out.println("... test interrupted!"); + } finally { + lock.close(); + Files.delete(path); + } + + exec.shutdownNow(); + } +} From a687bee65eac8a68590536759079b988aad2a2d1 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Tue, 11 Apr 2023 16:47:48 -0400 Subject: [PATCH 145/243] fix: add groupPath(String...) method * implement N5Reader interface method --- .../org/janelia/saalfeldlab/n5/N5KeyValueReader.java | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java index b294cd36..072ba05f 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java @@ -29,6 +29,7 @@ import java.lang.reflect.Type; import java.util.Arrays; import java.util.Map; +import java.util.stream.Stream; import com.google.gson.Gson; import com.google.gson.GsonBuilder; @@ -355,6 +356,14 @@ protected String groupPath(final String normalGroupPath) { return keyValueAccess.compose(basePath, normalGroupPath); } + @Override + public String groupPath(String... nodes) { + + // alternatively call compose twice, once with this functions inputs, then pass the result to the other groupPath method + // this impl assumes streams and array building are less expensive than keyValueAccess composition (may not always be true) + return keyValueAccess.compose(Stream.concat(Stream.of(basePath), Arrays.stream(nodes)).toArray(String[]::new)); + } + /** * Constructs the absolute path (in terms of this store) for the attributes * file of a group or dataset. From 7c37cf9f6b6f05df143eb00ae58f97dc04e3dd25 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Tue, 11 Apr 2023 16:49:31 -0400 Subject: [PATCH 146/243] feat: add setter methods to N5JsonCache * helpful for zarr impl and perhaps generally --- .../saalfeldlab/n5/cache/N5JsonCache.java | 31 ++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/cache/N5JsonCache.java b/src/main/java/org/janelia/saalfeldlab/n5/cache/N5JsonCache.java index abf82f70..ea456308 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/cache/N5JsonCache.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/cache/N5JsonCache.java @@ -152,6 +152,7 @@ private void addNewCacheInfo(String normalPathKey, String normalCacheKey, JsonEl } cacheInfo.isGroup = getIsGroup.apply(normalPathKey); cacheInfo.isDataset = getIsDataset.apply(normalPathKey); + cacheInfo.attributesCache.put(normalCacheKey, readBackingAttributes.apply(normalPathKey, normalCacheKey)); addChild(cacheInfo, normalPathKey); } synchronized (containerPathToCache) { @@ -165,7 +166,7 @@ private void addChild(N5CacheInfo cacheInfo, String normalPathKey) { Collections.addAll(cacheInfo.children, children); } - private void addNewCacheInfo(String normalPathKey) { + private N5CacheInfo addNewCacheInfo(String normalPathKey) { final N5CacheInfo cacheInfo; if (!getExists.apply(normalPathKey)) { @@ -179,6 +180,7 @@ private void addNewCacheInfo(String normalPathKey) { synchronized (containerPathToCache) { containerPathToCache.put(normalPathKey, cacheInfo); } + return cacheInfo; } public void updateCacheInfo(final String normalPathKey, final String normalCacheKey) { @@ -218,6 +220,33 @@ public void updateCacheInfo(final String normalPathKey, final String normalCache cacheInfo.children.clear(); addChild(cacheInfo, normalPathKey); } + + synchronized (containerPathToCache) { + containerPathToCache.put(normalPathKey, cacheInfo); + } + } + + public void setCacheInfo( final String normalPathKey, final String normalCacheKey, final JsonElement attributes, final boolean isGroup, final boolean isDataset ) { + + N5CacheInfo cacheInfo = getCacheInfo(normalPathKey); + if (cacheInfo == null ) + cacheInfo = addNewCacheInfo(normalPathKey); + + cacheInfo.isGroup = isGroup; + cacheInfo.isDataset = isDataset; + cacheInfo.attributesCache.put( normalCacheKey, attributes ); + synchronized (containerPathToCache) { + containerPathToCache.put(normalPathKey, cacheInfo); + } + } + + public void setCacheAttributes( final String normalPathKey, final String normalCacheKey, final JsonElement attributes ) { + + N5CacheInfo cacheInfo = getCacheInfo(normalPathKey); + if (cacheInfo == null ) + cacheInfo = addNewCacheInfo(normalPathKey); + + cacheInfo.attributesCache.put( normalCacheKey, attributes ); synchronized (containerPathToCache) { containerPathToCache.put(normalPathKey, cacheInfo); } From 4ddcc3b3dbfb52aa2f23d4599c7c196fe1424699 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Thu, 13 Apr 2023 10:34:49 -0400 Subject: [PATCH 147/243] doc: update description of N5KeyValueWriter --- .../java/org/janelia/saalfeldlab/n5/N5KeyValueWriter.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriter.java b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriter.java index 4e2f7dd4..5362a1f6 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriter.java @@ -29,6 +29,7 @@ import java.util.List; import java.util.Map; +import com.google.gson.Gson; import com.google.gson.GsonBuilder; import com.google.gson.JsonElement; import com.google.gson.JsonNull; @@ -37,7 +38,8 @@ import java.util.Arrays; /** - * Filesystem {@link N5Writer} implementation with version compatibility check. + * {@link N5Writer} implementation through a {@link KeyValueAccess} with version compatibility check. + * JSON attributes are parsed and written with {@link Gson}. * * @author Stephan Saalfeld */ From 0715550f0b5384ae869b35ee4a2434abbcfcf085 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Thu, 13 Apr 2023 10:39:51 -0400 Subject: [PATCH 148/243] test(style): caching of groups and datasets --- .../saalfeldlab/n5/N5CachedFSTest.java | 60 +++++++++++++++++-- 1 file changed, 56 insertions(+), 4 deletions(-) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/N5CachedFSTest.java b/src/test/java/org/janelia/saalfeldlab/n5/N5CachedFSTest.java index 98e36531..e1f73591 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/N5CachedFSTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/N5CachedFSTest.java @@ -4,6 +4,11 @@ import org.junit.Assert; import org.junit.Test; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + import java.io.File; import java.io.IOException; import java.nio.file.Files; @@ -12,12 +17,20 @@ public class N5CachedFSTest extends N5FSTest { - @Override protected N5Writer createN5Writer() throws IOException { + @Override + protected N5Writer createN5Writer() throws IOException { return new N5FSWriter(tempN5PathName(), true); } - @Override protected N5Writer createN5Writer(String location, GsonBuilder gson) throws IOException { + @Override + protected N5Writer createN5Writer(final String location) throws IOException { + + return new N5FSWriter(location, true); + } + + @Override + protected N5Writer createN5Writer(final String location, final GsonBuilder gson) throws IOException { if (!new File(location).exists()) { tmpFiles.add(location); @@ -25,7 +38,8 @@ public class N5CachedFSTest extends N5FSTest { return new N5FSWriter(location, gson, true); } - @Override protected N5Reader createN5Reader(String location, GsonBuilder gson) throws IOException { + @Override + protected N5Reader createN5Reader(final String location, final GsonBuilder gson) throws IOException { return new N5FSReader(location, gson, true); } @@ -40,7 +54,6 @@ public void cacheTest() throws IOException { final String attributesPath = n5.attributesPath(cachedGroup); - final ArrayList> tests = new ArrayList<>(); n5.createGroup(cachedGroup); addAndTest(n5, tests, new TestData<>(cachedGroup, "a/b/c", 100)); @@ -65,4 +78,43 @@ public void cacheTest() throws IOException { Assert.assertThrows(AssertionError.class, () -> runTests(n5, tests)); } } + + @Test + public void cacheGroupDatasetTest() throws IOException { + + final String datasetName = "dd"; + final String groupName = "gg"; + + final String tmpPath = tempN5PathName(); + try (N5KeyValueWriter w1 = (N5KeyValueWriter) createN5Writer(tmpPath); + N5KeyValueWriter w2 = (N5KeyValueWriter) createN5Writer(tmpPath);) { + + // create a group, both writers know it exists + w1.createGroup(groupName); + assertTrue(w1.exists(groupName)); + assertTrue(w2.exists(groupName)); + + // one writer removes the group + w2.remove(groupName); + assertTrue(w1.exists(groupName)); // w1's cache thinks group still exists + assertFalse(w2.exists(groupName)); // w2 knows group has been removed + + // create a dataset + w1.createDataset(datasetName, dimensions, blockSize, DataType.UINT8, new RawCompression()); + assertTrue(w1.exists(datasetName)); + assertTrue(w2.exists(datasetName)); + + assertNotNull(w1.getDatasetAttributes(datasetName)); + assertNotNull(w2.getDatasetAttributes(datasetName)); + + // one writer removes the data + w2.remove(datasetName); + assertTrue(w1.exists(datasetName)); // w1's cache thinks group still exists + assertFalse(w2.exists(datasetName)); // w2 knows group has been removed + + assertNotNull(w1.getDatasetAttributes(datasetName)); + assertNull(w2.getDatasetAttributes(datasetName)); + } + } + } From dd9a1bde0d9f0640528c785b5c01f94fb2d4fc87 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Thu, 13 Apr 2023 10:59:38 -0400 Subject: [PATCH 149/243] feat!: reader can skip version check, rm initializeContainer * add protected reader constructor --- .../saalfeldlab/n5/N5KeyValueReader.java | 22 ++++++++++++++----- .../saalfeldlab/n5/N5KeyValueWriter.java | 17 +++++--------- 2 files changed, 23 insertions(+), 16 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java index 072ba05f..c19518f2 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java @@ -88,16 +88,28 @@ public N5KeyValueReader( final GsonBuilder gsonBuilder, final boolean cacheMeta) throws IOException { + this(true, keyValueAccess, basePath, gsonBuilder, cacheMeta); + } + + protected N5KeyValueReader( + final boolean checkVersion, + final KeyValueAccess keyValueAccess, + final String basePath, + final GsonBuilder gsonBuilder, + final boolean cacheMeta) throws IOException { + this.keyValueAccess = keyValueAccess; this.basePath = keyValueAccess.normalize(basePath); this.gson = GsonUtils.registerGson(gsonBuilder); this.cacheMeta = cacheMeta; - /* Existence checks, if any, go in subclasses */ - /* Check that version (if there is one) is compatible. */ - final Version version = getVersion(); - if (!VERSION.isCompatible(version)) - throw new N5Exception.N5IOException("Incompatible version " + version + " (this is " + VERSION + ")."); + if( checkVersion ) { + /* Existence checks, if any, go in subclasses */ + /* Check that version (if there is one) is compatible. */ + final Version version = getVersion(); + if (!VERSION.isCompatible(version)) + throw new N5Exception.N5IOException("Incompatible version " + version + " (this is " + VERSION + ")."); + } } public Gson getGson() { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriter.java b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriter.java index 5362a1f6..6cb4e140 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriter.java @@ -77,24 +77,19 @@ public N5KeyValueWriter( final boolean cacheAttributes) throws IOException { - super(keyValueAccess, initializeContainer(keyValueAccess, basePath), gsonBuilder, cacheAttributes); + super(false, keyValueAccess, basePath, gsonBuilder, cacheAttributes); createGroup("/"); setVersion("/"); } protected void setVersion(final String path) throws IOException { - if (!VERSION.equals(getVersion())) - setAttribute("/", VERSION_KEY, VERSION.toString()); - } + final Version version = getVersion(); + if (!VERSION.isCompatible(version)) + throw new N5Exception.N5IOException("Incompatible version " + version + " (this is " + VERSION + ")."); - protected static String initializeContainer( - final KeyValueAccess keyValueAccess, - final String basePath) throws IOException { - - final String normBasePath = keyValueAccess.normalize(basePath); - keyValueAccess.createDirectories(normBasePath); - return normBasePath; + if (!VERSION.equals(version)) + setAttribute("/", VERSION_KEY, VERSION.toString()); } /** From 01dc8681d5cb50b986ce6d6707a28544a467b0d6 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Thu, 13 Apr 2023 12:17:03 -0400 Subject: [PATCH 150/243] doc: address javadoc warnings --- .../org/janelia/saalfeldlab/n5/DataBlock.java | 6 +- .../org/janelia/saalfeldlab/n5/DataType.java | 14 +- .../saalfeldlab/n5/DefaultBlockReader.java | 10 +- .../saalfeldlab/n5/GsonAttributesParser.java | 51 +++-- .../org/janelia/saalfeldlab/n5/GsonUtils.java | 35 +-- .../saalfeldlab/n5/KeyValueAccess.java | 46 ++-- .../n5/LinkedAttributePathToken.java | 4 +- .../janelia/saalfeldlab/n5/LockedChannel.java | 12 +- .../saalfeldlab/n5/N5KeyValueReader.java | 8 +- .../org/janelia/saalfeldlab/n5/N5Reader.java | 213 +++++++++--------- .../org/janelia/saalfeldlab/n5/N5Writer.java | 69 +++--- .../saalfeldlab/n5/cache/N5JsonCache.java | 1 - 12 files changed, 236 insertions(+), 233 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/DataBlock.java index 2e82ec90..64c7ad5b 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DataBlock.java @@ -98,15 +98,15 @@ public interface DataBlock { * not necessarily equal {@link #getNumElements(int[]) * getNumElements(getSize())}. * - * @return + * @return the number of elements */ public int getNumElements(); /** * Returns the number of elements in a box of given size. * - * @param size - * @return + * @param size the size + * @return the number of elements */ public static int getNumElements(final int[] size) { int n = size[0]; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DataType.java b/src/main/java/org/janelia/saalfeldlab/n5/DataType.java index 6a87e390..76c906fd 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DataType.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DataType.java @@ -81,10 +81,10 @@ public static DataType fromString(final String string) { /** * Factory for {@link DataBlock DataBlocks}. * - * @param blockSize - * @param gridPosition - * @param numElements not necessarily one element per block element - * @return + * @param blockSize the block size + * @param gridPosition the grid position + * @param numElements the number of elements (not necessarily one element per block element) + * @return the data block */ public DataBlock createDataBlock(final int[] blockSize, final long[] gridPosition, final int numElements) { @@ -95,9 +95,9 @@ public DataBlock createDataBlock(final int[] blockSize, final long[] gridPosi * Factory for {@link DataBlock DataBlocks} with one data element for each * block element (e.g. pixel image). * - * @param blockSize - * @param gridPosition - * @return + * @param blockSize the block size + * @param gridPosition the grid position + * @return the data block */ public DataBlock createDataBlock(final int[] blockSize, final long[] gridPosition) { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockReader.java b/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockReader.java index 258fffcc..d103ddf2 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockReader.java @@ -56,11 +56,11 @@ public default > void read( /** * Reads a {@link DataBlock} from an {@link InputStream}. * - * @param in - * @param datasetAttributes - * @param gridPosition - * @return - * @throws IOException + * @param in the input stream + * @param datasetAttributes the dataset attributes + * @param gridPosition the grid position + * @return the block + * @throws IOException the exception */ public static DataBlock readBlock( final InputStream in, diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonAttributesParser.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonAttributesParser.java index 474e93c5..7a4cc411 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonAttributesParser.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonAttributesParser.java @@ -55,8 +55,8 @@ public interface GsonAttributesParser extends N5Reader { * Reads or creates the attributes map of a group or dataset. * * @param pathName group path - * @return - * @throws IOException + * @return the attributes + * @throws IOException the exception */ @Deprecated public HashMap getAttributes(final String pathName) throws IOException; @@ -64,11 +64,11 @@ public interface GsonAttributesParser extends N5Reader { /** * Parses an attribute from the given attributes map. * - * @param map - * @param key - * @param clazz - * @return - * @throws IOException + * @param map the attributes map + * @param key the key + * @param clazz the class + * @return the attribute + * @throws IOException the exception */ @Deprecated public static T parseAttribute( @@ -87,11 +87,11 @@ public static T parseAttribute( /** * Parses an attribute from the given attributes map. * - * @param map - * @param key - * @param type - * @return - * @throws IOException + * @param map the attributes map + * @param key the key + * @param type the type + * @return the attribute + * @throws IOException the exception */ @Deprecated public static T parseAttribute( @@ -110,9 +110,9 @@ public static T parseAttribute( /** * Reads the attributes map from a given {@link Reader}. * - * @param reader - * @return - * @throws IOException + * @param reader the reader + * @return the attributes map + * @throws IOException the exception */ @Deprecated public static HashMap readAttributes(final Reader reader, final Gson gson) throws IOException { @@ -125,10 +125,10 @@ public static HashMap readAttributes(final Reader reader, f /** * Inserts new the JSON export of attributes into the given attributes map. * - * @param map - * @param attributes - * @param gson - * @throws IOException + * @param map the target attributes map to be edited + * @param attributes the source attributes map + * @param gson the gson + * @throws IOException the exception */ @Deprecated public static void insertAttributes( @@ -143,9 +143,9 @@ public static void insertAttributes( /** * Writes the attributes map to a given {@link Writer}. * - * @param writer - * @param map - * @throws IOException + * @param writer the writer + * @param map the attributes map + * @throws IOException the exception */ @Deprecated public static void writeAttributes( @@ -168,8 +168,8 @@ public static void writeAttributes( *

  • Object
    • * * - * @param jsonPrimitive - * @return + * @param jsonPrimitive the json + * @return the class */ @Deprecated public static Class classForJsonPrimitive(final JsonPrimitive jsonPrimitive) { @@ -201,6 +201,9 @@ else if (jsonPrimitive.isNumber()) { *
  • String[]
    • *
  • Object[]
    • * + * + * @param pathName the path + * @return the attributes map */ @Override @Deprecated diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonUtils.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonUtils.java index e84a09bc..51d2cefe 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonUtils.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonUtils.java @@ -61,9 +61,9 @@ static Gson registerGson(final GsonBuilder gsonBuilder) { /** * Reads the attributes json from a given {@link Reader}. * - * @param reader + * @param reader the reader * @return the root {@link JsonObject} of the attributes - * @throws IOException + * @throws IOException the exception */ static JsonElement readAttributes(final Reader reader, final Gson gson) throws IOException { @@ -83,13 +83,13 @@ static T readAttribute(final JsonElement root, final String normalizedAttrib } /** - * Deserialize the {@code attribute} as {@link Type type} {@link T}. + * Deserialize the {@code attribute} as {@link Type type} {@code T}. * * @param attribute to deserialize as {@link Type type} * @param gson used to deserialize {@code attribute} * @param type to desrialize {@code attribute} as * @param return type represented by {@link Type type} - * @return the deserialized attribute object, or {@code null} if {@code attribute} cannot deserialize to {@link T} + * @return the deserialized attribute object, or {@code null} if {@code attribute} cannot deserialize to {@code T} */ static T parseAttributeElement(final JsonElement attribute, final Gson gson, final Type type) { @@ -131,10 +131,11 @@ static T parseAttributeElement(final JsonElement attribute, final Gson gson, } /** - * Return the attribute at {@code normalizedAttributePath} as a {@link JsonElement}. - * Does not attempt to parse the attribute. + * Return the attribute at {@code normalizedAttributePath} as a + * {@link JsonElement}. Does not attempt to parse the attribute. * - * @param root to search for the {@link JsonElement} at location {@code normalizedAttributePath} + * @param root to search for the {@link JsonElement} at + * location {@code normalizedAttributePath} * @param normalizedAttributePath to the attribute * @return the attribute as a {@link JsonElement}. */ @@ -319,8 +320,8 @@ static T getJsonAsArray(final Gson gson, final JsonArray array, final Type t *
  • Object
    • * * - * @param jsonPrimitive - * @return + * @param jsonPrimitive the json primitive + * @return the class */ static Class classForJsonPrimitive(final JsonPrimitive jsonPrimitive) { @@ -339,12 +340,12 @@ else if (jsonPrimitive.isNumber()) { } /** - * If there is an attribute in {@code root} such that it can be parsed and desrialized as {@link T}, + * If there is an attribute in {@code root} such that it can be parsed and deserialized as {@code T}, * then remove it from {@code root}, write {@code root} to the {@code writer}, and return the removed attribute. *

    - * If there is an attribute at the location specified by {@code normalizedAttributePath} but it cannot be deserialized to {@link T}, then it is not removed. + * If there is an attribute at the location specified by {@code normalizedAttributePath} but it cannot be deserialized to {@code T}, then it is not removed. *

    - * If nothing is removed, then {@code root} is not writen to the {@code writer}. + * If nothing is removed, then {@code root} is not written to the {@code writer}. * * @param writer to write the modified {@code root} to after removal of the attribute * @param root to remove the attribute from @@ -353,7 +354,7 @@ else if (jsonPrimitive.isNumber()) { * @param gson to deserialize the attribute with * @param of the removed attribute * @return the removed attribute, or null if nothing removed - * @throws IOException + * @throws IOException the exception */ static T removeAttribute( final Writer writer, @@ -393,10 +394,10 @@ static boolean removeAttribute( } /** - * If there is an attribute in {@code root} such that it can be parsed and desrialized as {@link T}, + * If there is an attribute in {@code root} such that it can be parsed and desrialized as {@code T}, * then remove it from {@code root} and return the removed attribute. *

    - * If there is an attribute at the location specified by {@code normalizedAttributePath} but it cannot be deserialized to {@link T}, then it is not removed. + * If there is an attribute at the location specified by {@code normalizedAttributePath} but it cannot be deserialized to {@code T}, then it is not removed. * * @param root to remove the attribute from * @param normalizedAttributePath to the attribute location @@ -468,7 +469,7 @@ static JsonElement removeAttribute(JsonElement root, final String normalizedAttr * @param normalizedAttributePath * @param attribute * @param gson - * @throws IOException + * @throws IOException the exception */ static void writeAttribute( final Writer writer, @@ -487,7 +488,7 @@ static void writeAttribute( * * @param writer * @param root - * @throws IOException + * @throws IOException the exception */ static void writeAttributes( final Writer writer, diff --git a/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccess.java index 4e904855..d17f5d22 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccess.java @@ -41,33 +41,33 @@ public interface KeyValueAccess { /** * Split a path string into its components. * - * @param path - * @return + * @param path the path + * @return the path components */ public String[] components(final String path); /** * Compose a path from components. * - * @param components - * @return + * @param components the path components + * @return the path */ public String compose(final String... components); /** * Get the parent of a path string. * - * @param path - * @return null if the path has no parent + * @param path the path + * @return the parent path or null if the path has no parent */ public String parent(final String path); /** * Relativize path relative to base. * - * @param path - * @param base - * @return null if the path has no parent + * @param path the path + * @param base the base path + * @return the result or null if the path has no parent */ public String relativize(final String path, final String base); @@ -76,8 +76,8 @@ public interface KeyValueAccess { * location return the same output. This is most important for cached * data pointing at the same location getting the same key. * - * @param path - * @return + * @param path the path + * @return the normalized path */ public String normalize(final String path); @@ -86,7 +86,7 @@ public interface KeyValueAccess { * * @param normalPath is expected to be in normalized form, no further * efforts are made to normalize it. - * @return + * @return true if the path exists */ public boolean exists(final String normalPath); @@ -95,7 +95,7 @@ public interface KeyValueAccess { * * @param normalPath is expected to be in normalized form, no further * efforts are made to normalize it. - * @return + * @return true if the path is a directory */ public boolean isDirectory(String normalPath); @@ -104,7 +104,7 @@ public interface KeyValueAccess { * * @param normalPath is expected to be in normalized form, no further * efforts are made to normalize it. - * @return + * @return true if the path is a file */ public boolean isFile(String normalPath); @@ -119,8 +119,8 @@ public interface KeyValueAccess { * * @param normalPath is expected to be in normalized form, no further * efforts are made to normalize it. - * @return - * @throws IOException + * @return the locked channel + * @throws IOException the exception */ public LockedChannel lockForReading(final String normalPath) throws IOException; @@ -136,8 +136,8 @@ public interface KeyValueAccess { * * @param normalPath is expected to be in normalized form, no further * efforts are made to normalize it. - * @return - * @throws IOException + * @return the locked channel + * @throws IOException the exception */ public LockedChannel lockForWriting(final String normalPath) throws IOException; @@ -146,8 +146,8 @@ public interface KeyValueAccess { * * @param normalPath is expected to be in normalized form, no further * efforts are made to normalize it. - * @return - * @throws IOException + * @return the directories + * @throws IOException the exception */ public String[] listDirectories(final String normalPath) throws IOException; @@ -156,8 +156,8 @@ public interface KeyValueAccess { * * @param normalPath is expected to be in normalized form, no further * efforts are made to normalize it. - * @return - * @throws IOException + * @return the the child paths + * @throws IOException the exception */ public String[] list(final String normalPath) throws IOException; @@ -169,7 +169,7 @@ public interface KeyValueAccess { * * @param normalPath is expected to be in normalized form, no further * efforts are made to normalize it. - * @throws IOException + * @throws IOException the exception */ public void createDirectories(final String normalPath) throws IOException; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/LinkedAttributePathToken.java b/src/main/java/org/janelia/saalfeldlab/n5/LinkedAttributePathToken.java index df803151..104b6cb9 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/LinkedAttributePathToken.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/LinkedAttributePathToken.java @@ -22,7 +22,7 @@ public abstract class LinkedAttributePathToken implements protected LinkedAttributePathToken childToken; /** - * @return a reference object of type {@link T} + * @return a reference object of type {@code T} */ public abstract T getJsonType(); @@ -103,7 +103,7 @@ public boolean jsonCompatible(JsonElement json) { * If {@code json} is compatible with the type or token that we are, then set {@link #parentJson} to {@code json}. *
    * However, if {@code json} is either {@code null} or not {@link #jsonCompatible(JsonElement)} then - * {@link #parentJson} will be set to a new instance of {@link T}. + * {@link #parentJson} will be set to a new instance of {@code T}. * * @param json to attempt to set to {@link #parentJson} * @return the value set to {@link #parentJson}. diff --git a/src/main/java/org/janelia/saalfeldlab/n5/LockedChannel.java b/src/main/java/org/janelia/saalfeldlab/n5/LockedChannel.java index d5a97d62..a2ae6f4c 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/LockedChannel.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/LockedChannel.java @@ -43,28 +43,28 @@ public interface LockedChannel extends Closeable { /** * Create a UTF-8 {@link Reader}. * - * @return - * @throws IOException + * @return the reader + * @throws IOException the exception */ public Reader newReader() throws IOException; /** * Create a new {@link InputStream}. * - * @return - * @throws IOException + * @return the reader + * @throws IOException the exception */ public InputStream newInputStream() throws IOException; /** * Create a new UTF-8 {@link Writer}. - * @return + * @return the writer */ public Writer newWriter() throws IOException; /** * Create a new {@link OutputStream}. - * @return + * @return the output stream */ public OutputStream newOutputStream() throws IOException; } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java index 072ba05f..36acdcdf 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java @@ -244,8 +244,8 @@ private boolean normalDatasetExists(String pathName) throws N5Exception.N5IOExce * Reads or creates the attributes map of a group or dataset. * * @param pathName group path - * @return - * @throws IOException + * @return the json element + * @throws IOException the exception */ public JsonElement getAttributes(final String pathName) throws IOException { @@ -327,8 +327,8 @@ public String[] list(final String pathName) throws IOException { * This is the file into which the data block will be stored. * * @param normalPath normalized dataset path - * @param gridPosition - * @return + * @param gridPosition the grid position + * @return the block path */ protected String getDataBlockPath( final String normalPath, diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java index 639fce47..8299ab74 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java @@ -173,8 +173,8 @@ public boolean equals(final Object other) { * Currently, this means that the version is less than or equal to * 1.X.X. * - * @param version - * @return + * @param version the version + * @return true if this version is compatible */ public boolean isCompatible(final Version version) { @@ -201,8 +201,8 @@ public boolean isCompatible(final Version version) { * such as 1.2, the missing elements are filled with 0, i.e. 1.2.0 in this * case. * - * @return - * @throws IOException + * @return the version + * @throws IOException the exception */ default Version getVersion() throws IOException { @@ -219,13 +219,12 @@ default Version getVersion() throws IOException { /** * Reads an attribute. * - * @param pathName - * group path - * @param key - * @param clazz - * attribute class - * @return - * @throws IOException + * @param pathName group path + * @param key the key + * @param clazz attribute class + * @param the attribute type + * @return the attribute + * @throws IOException the exception */ T getAttribute( final String pathName, @@ -237,11 +236,12 @@ T getAttribute( * * @param pathName * group path - * @param key + * @param key the key * @param type * attribute Type (use this for specifying generic types) - * @return - * @throws IOException + * @param the attribute type + * @return the attribute + * @throws IOException the exception */ T getAttribute( final String pathName, @@ -251,23 +251,21 @@ T getAttribute( /** * Get mandatory dataset attributes. * - * @param pathName - * dataset path - * @return dataset attributes or null if either dimensions or dataType are - * not set - * @throws IOException + * @param pathName dataset path + * @return dataset attributes or null if either dimensions or dataType are not + * set + * @throws IOException the exception */ DatasetAttributes getDatasetAttributes(final String pathName) throws IOException; /** * Reads a {@link DataBlock}. * - * @param pathName - * dataset path - * @param datasetAttributes - * @param gridPosition - * @return - * @throws IOException + * @param pathName dataset path + * @param datasetAttributes the dataset attributes + * @param gridPosition the grid position + * @return the data block + * @throws IOException the exception */ DataBlock readBlock( final String pathName, @@ -275,14 +273,14 @@ DataBlock readBlock( final long... gridPosition) throws IOException; /** - * Load a {@link DataBlock} as a {@link Serializable}. The offset is given - * in {@link DataBlock} grid coordinates. + * Load a {@link DataBlock} as a {@link Serializable}. The offset is given in + * {@link DataBlock} grid coordinates. * - * @param dataset - * @param attributes - * @param gridPosition - * @throws IOException - * @throws ClassNotFoundException + * @param dataset the dataset path + * @param attributes the dataset attributes + * @param gridPosition the grid position + * @throws IOException the io exception + * @throws ClassNotFoundException the class not found exception */ @SuppressWarnings("unchecked") default T readSerializedBlock( @@ -301,21 +299,19 @@ default T readSerializedBlock( } /** - * Test whether a group or dataset exists. + * Test whether a group or dataset exists at a given path. * - * @param pathName - * group path - * @return + * @param pathName group path + * @return true if the path exists */ boolean exists(final String pathName); /** * Test whether a dataset exists. * - * @param pathName - * dataset path - * @return - * @throws IOException + * @param pathName dataset path + * @return true if a dataset exists + * @throws IOException the exception */ default boolean datasetExists(final String pathName) throws IOException { @@ -325,10 +321,9 @@ default boolean datasetExists(final String pathName) throws IOException { /** * List all groups (including datasets) in a group. * - * @param pathName - * group path - * @return - * @throws IOException + * @param pathName group path + * @return list of children + * @throws IOException the exception */ String[] list(final String pathName) throws IOException; @@ -344,7 +339,7 @@ default boolean datasetExists(final String pathName) throws IOException { * @param filter * filter for children to be included * @return list of groups - * @throws IOException + * @throws IOException the exception */ default String[] deepList( final String pathName, @@ -365,10 +360,9 @@ default String[] deepList( /** * Recursively list all groups (including datasets) in the given group. * - * @param pathName - * base group path + * @param pathName base group path * @return list of groups - * @throws IOException + * @throws IOException the exception */ default String[] deepList(final String pathName) throws IOException { @@ -376,30 +370,32 @@ default String[] deepList(final String pathName) throws IOException { } /** - * Recursively list all datasets in the given group. - * Only paths that satisfy the provided filter will be included, but the - * children of paths that were excluded may be included (filter does not - * apply to the subtree). + * Recursively list all datasets in the given group. Only paths that satisfy the + * provided filter will be included, but the children of paths that were + * excluded may be included (filter does not apply to the subtree). * - *

    This method delivers the same results as

    + *

    + * This method delivers the same results as + *

    + * * 
    {@code
-	 * n5.deepList(
-	 *   prefix,
-	 *   a -> {
-	 *     try { return n5.datasetExists(a) && filter.test(a); }
-	 *     catch (final IOException e) { return false; }
-	 *   });
+	 * n5.deepList(prefix, a -> {
+	 * 	try {
+	 * 		return n5.datasetExists(a) && filter.test(a);
+	 * 	} catch (final IOException e) {
+	 * 		return false;
+	 * 	}
+	 * });
 	 * }
    - *

    but will execute {@link #datasetExists(String)} only once per node. - * This can be relevant for performance on high latency backends such as - * cloud stores.

    + *

    + * but will execute {@link #datasetExists(String)} only once per node. This can + * be relevant for performance on high latency backends such as cloud stores. + *

    * - * @param pathName - * base group path - * @param filter - * filter for datasets to be included + * @param pathName base group path + * @param filter filter for datasets to be included * @return list of groups - * @throws IOException + * @throws IOException the exception */ default String[] deepListDatasets( final String pathName, @@ -420,23 +416,27 @@ default String[] deepListDatasets( /** * Recursively list all including datasets in the given group. * - *

    This method delivers the same results as

    + *

    + * This method delivers the same results as + *

    + * * 
    {@code
-	 * n5.deepList(
-	 *   prefix,
-	 *   a -> {
-	 *     try { return n5.datasetExists(a); }
-	 *     catch (final IOException e) { return false; }
-	 *   });
+	 * n5.deepList(prefix, a -> {
+	 * 	try {
+	 * 		return n5.datasetExists(a);
+	 * 	} catch (final IOException e) {
+	 * 		return false;
+	 * 	}
+	 * });
 	 * }
    - *

    but will execute {@link #datasetExists(String)} only once per node. - * This can be relevant for performance on high latency backends such as - * cloud stores.

    + *

    + * but will execute {@link #datasetExists(String)} only once per node. This can + * be relevant for performance on high latency backends such as cloud stores. + *

    * - * @param pathName - * base group path + * @param pathName base group path * @return list of groups - * @throws IOException + * @throws IOException the exception */ default String[] deepListDatasets(final String pathName) throws IOException { @@ -449,6 +449,12 @@ default String[] deepListDatasets(final String pathName) throws IOException { * private interface methods yet. * * TODO make private when committing to Java versions newer than 8 + * + * @param n5 the n5 reader + * @param pathName the base group path + * @param datasetsOnly true if only dataset paths should be returned + * @param filter a dataset filter + * @return the list of all children */ static ArrayList deepList( final N5Reader n5, @@ -474,21 +480,17 @@ static ArrayList deepList( /** * Recursively list all groups (including datasets) in the given group, in - * parallel, using the given {@link ExecutorService}. Only paths that - * satisfy the provided filter will be included, but the children of paths - * that were excluded may be included (filter does not apply to the - * subtree). + * parallel, using the given {@link ExecutorService}. Only paths that satisfy + * the provided filter will be included, but the children of paths that were + * excluded may be included (filter does not apply to the subtree). * - * @param pathName - * base group path - * @param filter - * filter for children to be included - * @param executor - * executor service + * @param pathName base group path + * @param filter filter for children to be included + * @param executor executor service * @return list of datasets - * @throws IOException - * @throws ExecutionException - * @throws InterruptedException + * @throws IOException the io exception + * @throws ExecutionException the execution exception + * @throws InterruptedException the interrupted exception */ default String[] deepList( final String pathName, @@ -515,14 +517,12 @@ default String[] deepList( * Recursively list all groups (including datasets) in the given group, in * parallel, using the given {@link ExecutorService}. * - * @param pathName - * base group path - * @param executor - * executor service + * @param pathName base group path + * @param executor executor service * @return list of groups - * @throws IOException - * @throws ExecutionException - * @throws InterruptedException + * @throws IOException the io exception + * @throws ExecutionException the execution exception + * @throws InterruptedException the interrupted exception */ default String[] deepList( final String pathName, @@ -666,17 +666,16 @@ static void deepListHelper( /** * List all attributes and their class of a group. * - * @param pathName - * group path - * @return - * @throws IOException + * @param pathName group path + * @return the attribute map + * @throws IOException the exception */ Map> listAttributes(final String pathName) throws IOException; /** * Returns the symbol that is used to separate nodes in a group path. * - * @return + * @return the group separator */ default String getGroupSeparator() { @@ -686,8 +685,8 @@ default String getGroupSeparator() { /** * Returns an absolute path to a group by concatenating all nodes with the node separator * defined by {@link #getGroupSeparator()}. - * @param nodes - * @return + * @param nodes the group components + * @return the absolute group path */ String groupPath(final String... nodes); diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java index ae9c8bb3..f6616c21 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java @@ -46,9 +46,10 @@ public interface N5Writer extends N5Reader { * Sets an attribute. * * @param pathName group path - * @param key - * @param attribute - * @throws IOException + * @param key the key + * @param attribute the attribute + * @param the attribute type type + * @throws IOException the exception */ default void setAttribute( final String pathName, @@ -62,8 +63,8 @@ default void setAttribute( * Sets a map of attributes. * * @param pathName group path - * @param attributes - * @throws IOException + * @param attributes the attribute map + * @throws IOException the exception */ void setAttributes( final String pathName, @@ -75,35 +76,35 @@ void setAttributes( * @param pathName group path * @param key of attribute to remove * @return true if attribute removed, else false - * @throws IOException + * @throws IOException the exception */ boolean removeAttribute(String pathName, String key) throws IOException; /** - * Remove the attribute from group {@code pathName} with key {@code key} and type {@link T}. + * Remove the attribute from group {@code pathName} with key {@code key} and type {@code T}. *

    - * If an attribute at {@code pathName} and {@code key} exists, but is not of type {@link T}, it is not removed. + * If an attribute at {@code pathName} and {@code key} exists, but is not of type {@code T}, it is not removed. * * @param pathName group path * @param key of attribute to remove * @param clazz of the attribute to remove * @param of the attribute - * @return the removed attribute, as {@link T}, or {@code null} if no matching attribute - * @throws IOException + * @return the removed attribute, as {@code T}, or {@code null} if no matching attribute + * @throws IOException the exception */ T removeAttribute(String pathName, String key, Class clazz) throws IOException; /** * Remove attributes as provided by {@code attributes}. *

    - * If any element of {@code attributes} does not exist, it wil be ignored. + * If any element of {@code attributes} does not exist, it will be ignored. * If at least one attribute from {@code attributes} is removed, this will return {@code true}. * * * @param pathName group path * @param attributes to remove * @return true if any of the listed attributes were removed - * @throws IOException + * @throws IOException the exception */ boolean removeAttributes( String pathName, List attributes) throws IOException; @@ -111,8 +112,8 @@ void setAttributes( * Sets mandatory dataset attributes. * * @param pathName dataset path - * @param datasetAttributes - * @throws IOException + * @param datasetAttributes the dataset attributse + * @throws IOException the exception */ default void setDatasetAttributes( final String pathName, @@ -124,8 +125,8 @@ default void setDatasetAttributes( /** * Creates a group (directory) * - * @param pathName - * @throws IOException + * @param pathName the path + * @throws IOException the exception */ void createGroup(final String pathName) throws IOException; @@ -142,7 +143,7 @@ default void setDatasetAttributes( * * @param pathName group path * @return true if removal was successful, false otherwise - * @throws IOException + * @throws IOException the exception */ boolean remove(final String pathName) throws IOException; @@ -150,7 +151,7 @@ default void setDatasetAttributes( * Removes the N5 container. * * @return true if removal was successful, false otherwise - * @throws IOException + * @throws IOException the exception */ default boolean remove() throws IOException { @@ -162,8 +163,8 @@ default boolean remove() throws IOException { * mandatory attributes only. * * @param pathName dataset path - * @param datasetAttributes - * @throws IOException + * @param datasetAttributes the dataset attributes + * @throws IOException the exception */ default void createDataset( final String pathName, @@ -174,14 +175,14 @@ default void createDataset( } /** - * Creates a dataset. This does not create any data but the path and + * Creates a dataset. This does not create any data but the path and * mandatory attributes only. * * @param pathName dataset path - * @param dimensions - * @param blockSize - * @param dataType - * @throws IOException + * @param dimensions the dataset dimensions + * @param blockSize the block size + * @param dataType the data type + * @throws IOException the exception */ default void createDataset( final String pathName, @@ -197,9 +198,9 @@ default void createDataset( * Writes a {@link DataBlock}. * * @param pathName dataset path - * @param datasetAttributes - * @param dataBlock - * @throws IOException + * @param datasetAttributes the dataset attributes + * @param dataBlock the data block + * @throws IOException the exception */ void writeBlock( final String pathName, @@ -212,7 +213,7 @@ void writeBlock( * * @param pathName dataset path * @param gridPosition position of block to be deleted - * @throws IOException + * @throws IOException the exception * * @return {@code true} if the block at {@code gridPosition} is "empty" after deletion. The meaning of "empty" is * implementation dependent. For example "empty" means that no file exists on the file system for the deleted block @@ -227,11 +228,11 @@ boolean deleteBlock( * Save a {@link Serializable} as an N5 {@link DataBlock} at a given * offset. The offset is given in {@link DataBlock} grid coordinates. * - * @param object - * @param dataset - * @param datasetAttributes - * @param gridPosition - * @throws IOException + * @param object the object to serialize + * @param dataset the dataset path + * @param datasetAttributes the dataset attributes + * @param gridPosition the grid position + * @throws IOException the exception */ default void writeSerializedBlock( final Serializable object, diff --git a/src/main/java/org/janelia/saalfeldlab/n5/cache/N5JsonCache.java b/src/main/java/org/janelia/saalfeldlab/n5/cache/N5JsonCache.java index ea456308..e67d9fb2 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/cache/N5JsonCache.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/cache/N5JsonCache.java @@ -13,7 +13,6 @@ public class N5JsonCache { public static final N5CacheInfo emptyCacheInfo = new N5CacheInfo(); public static final String jsonFile = "attributes.json"; - /** * Data object for caching meta data. Elements that are null are not yet * cached. From 983d5d83fd374ef2f3adea200bbbc48b99b68506 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Thu, 13 Apr 2023 12:51:49 -0400 Subject: [PATCH 151/243] doc: address more javadoc warnings --- .../saalfeldlab/n5/AbstractDataBlock.java | 2 +- .../org/janelia/saalfeldlab/n5/DataBlock.java | 2 +- .../saalfeldlab/n5/GsonAttributesParser.java | 6 ++ .../janelia/saalfeldlab/n5/N5FSWriter.java | 4 +- .../saalfeldlab/n5/N5KeyValueReader.java | 7 +- .../saalfeldlab/n5/N5KeyValueWriter.java | 7 +- .../org/janelia/saalfeldlab/n5/N5Reader.java | 97 ++++++++++--------- .../org/janelia/saalfeldlab/n5/N5URL.java | 9 +- 8 files changed, 74 insertions(+), 60 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/AbstractDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/AbstractDataBlock.java index 55f8884b..ca85eb7b 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/AbstractDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/AbstractDataBlock.java @@ -28,7 +28,7 @@ /** * Abstract base class for {@link DataBlock} implementations. * - * @param + * @param the block data type * * @author Stephan Saalfeld */ diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/DataBlock.java index 64c7ad5b..dc6366f4 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DataBlock.java @@ -89,7 +89,7 @@ public interface DataBlock { * modifying the data object of this data block after calling this method * may or may not change the content of the {@link ByteBuffer}. * - * @param buffer + * @param buffer the byte buffer */ public void readData(final ByteBuffer buffer); diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonAttributesParser.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonAttributesParser.java index 7a4cc411..876426f2 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonAttributesParser.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonAttributesParser.java @@ -67,6 +67,8 @@ public interface GsonAttributesParser extends N5Reader { * @param map the attributes map * @param key the key * @param clazz the class + * @param gson the gson + * @param the attribute type * @return the attribute * @throws IOException the exception */ @@ -90,6 +92,8 @@ public static T parseAttribute( * @param map the attributes map * @param key the key * @param type the type + * @param gson the gson + * @param the attribute type * @return the attribute * @throws IOException the exception */ @@ -111,6 +115,7 @@ public static T parseAttribute( * Reads the attributes map from a given {@link Reader}. * * @param reader the reader + * @param gson the gson * @return the attributes map * @throws IOException the exception */ @@ -145,6 +150,7 @@ public static void insertAttributes( * * @param writer the writer * @param map the attributes map + * @param gson the gson * @throws IOException the exception */ @Deprecated diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java b/src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java index ea7f1cc5..244dee3a 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java @@ -48,7 +48,7 @@ public class N5FSWriter extends N5KeyValueWriter implements N5Writer { * will be set to the current N5 version of this implementation. * * @param basePath n5 base path - * @param gsonBuilder + * @param gsonBuilder the gson builder * @param cacheAttributes cache attributes and meta data * Setting this to true avoids frequent reading and parsing of JSON * encoded attributes and other meta data that requires accessing the @@ -110,7 +110,7 @@ public N5FSWriter(final String basePath, final boolean cacheAttributes) throws I *

    * * @param basePath n5 base path - * @param gsonBuilder + * @param gsonBuilder the gson builder * * @throws IOException * if the base path cannot be written to or cannot be created, diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java index 36acdcdf..529a390a 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java @@ -65,11 +65,12 @@ public class N5KeyValueReader implements N5Reader { /** * Opens an {@link N5KeyValueReader} at a given base path with a custom - * {@link GsonBuilder} to support custom attributes. + * {@link GsonBuilder} to support custom attributes. Storage is managed by + * the given {@link KeyValueAccess}. * - * @param keyValueAccess + * @param keyValueAccess the key value access * @param basePath N5 base path - * @param gsonBuilder + * @param gsonBuilder the gson builder * @param cacheMeta cache attributes and meta data * Setting this to true avoids frequent reading and parsing of JSON * encoded attributes and other meta data that requires accessing the diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriter.java b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriter.java index 5362a1f6..698775c4 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriter.java @@ -47,7 +47,8 @@ public class N5KeyValueWriter extends N5KeyValueReader implements N5Writer { /** * Opens an {@link N5KeyValueWriter} at a given base path with a custom - * {@link GsonBuilder} to support custom attributes. + * {@link GsonBuilder} to support custom attributes. Storage is managed by + * the given {@link KeyValueAccess}. *

    * If the base path does not exist, it will be created. *

    @@ -55,10 +56,10 @@ public class N5KeyValueWriter extends N5KeyValueReader implements N5Writer { * compatible with this implementation, the N5 version of this container * will be set to the current N5 version of this implementation. * - * @param keyValueAccess + * @param keyValueAccess the key value access * @param basePath * n5 base path - * @param gsonBuilder + * @param gsonBuilder the gson builder * @param cacheAttributes * Setting this to true avoids frequent reading and parsing of * JSON encoded attributes, this is most interesting for high diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java index 8299ab74..56bf4c62 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java @@ -83,7 +83,7 @@ public Version( * If the version string is null or not a SemVer version, this * version will be "0.0.0" *

    - * @param versionString + * @param versionString the string representation of the version */ public Version(final String versionString) { @@ -278,7 +278,9 @@ DataBlock readBlock( * * @param dataset the dataset path * @param attributes the dataset attributes + * @param the data block type * @param gridPosition the grid position + * @return the data block * @throws IOException the io exception * @throws ClassNotFoundException the class not found exception */ @@ -455,6 +457,7 @@ default String[] deepListDatasets(final String pathName) throws IOException { * @param datasetsOnly true if only dataset paths should be returned * @param filter a dataset filter * @return the list of all children + * @throws IOException the exception */ static ArrayList deepList( final N5Reader n5, @@ -532,36 +535,36 @@ default String[] deepList( } /** - * Recursively list all datasets in the given group, in - * parallel, using the given {@link ExecutorService}. Only paths that - * satisfy the provided filter will be included, but the children of paths - * that were excluded may be included (filter does not apply to the - * subtree). + * Recursively list all datasets in the given group, in parallel, using the + * given {@link ExecutorService}. Only paths that satisfy the provided filter + * will be included, but the children of paths that were excluded may be + * included (filter does not apply to the subtree). * - *

    This method delivers the same results as

    + *

    + * This method delivers the same results as + *

    + * * 
    {@code
-	 * n5.deepList(
-	 *   prefix,
-	 *   a -> {
-	 *     try { return n5.datasetExists(a) && filter.test(a); }
-	 *     catch (final IOException e) { return false; }
-	 *   },
-	 *   exec);
+	 * n5.deepList(prefix, a -> {
+	 * 	try {
+	 * 		return n5.datasetExists(a) && filter.test(a);
+	 * 	} catch (final IOException e) {
+	 * 		return false;
+	 * 	}
+	 * }, exec);
 	 * }
    - *

    but will execute {@link #datasetExists(String)} only once per node. - * This can be relevant for performance on high latency backends such as - * cloud stores.

    + *

    + * but will execute {@link #datasetExists(String)} only once per node. This can + * be relevant for performance on high latency backends such as cloud stores. + *

    * - * @param pathName - * base group path - * @param filter - * filter for datasets to be included - * @param executor - * executor service + * @param pathName base group path + * @param filter filter for datasets to be included + * @param executor executor service * @return list of datasets - * @throws IOException - * @throws ExecutionException - * @throws InterruptedException + * @throws IOException the io exception + * @throws ExecutionException the execution exception + * @throws InterruptedException the interrupted exception */ default String[] deepListDatasets( final String pathName, @@ -585,31 +588,33 @@ default String[] deepListDatasets( } /** - * Recursively list all datasets in the given group, in - * parallel, using the given {@link ExecutorService}. + * Recursively list all datasets in the given group, in parallel, using the + * given {@link ExecutorService}. * - *

    This method delivers the same results as

    + *

    + * This method delivers the same results as + *

    + * * 
    {@code
-	 * n5.deepList(
-	 *   prefix,
-	 *   a -> {
-	 *     try { return n5.datasetExists(a); }
-	 *     catch (final IOException e) { return false; }
-	 *   },
-	 *   exec);
+	 * n5.deepList(prefix, a -> {
+	 * 	try {
+	 * 		return n5.datasetExists(a);
+	 * 	} catch (final IOException e) {
+	 * 		return false;
+	 * 	}
+	 * }, exec);
 	 * }
    - *

    but will execute {@link #datasetExists(String)} only once per node. - * This can be relevant for performance on high latency backends such as - * cloud stores.

    + *

    + * but will execute {@link #datasetExists(String)} only once per node. This can + * be relevant for performance on high latency backends such as cloud stores. + *

    * - * @param pathName - * base group path - * @param executor - * executor service + * @param pathName base group path + * @param executor executor service * @return list of groups - * @throws IOException - * @throws ExecutionException - * @throws InterruptedException + * @throws IOException the io exception + * @throws ExecutionException the execution exception + * @throws InterruptedException the interrupted exception */ default String[] deepListDatasets( final String pathName, diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java b/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java index 36fbecf8..1437c98e 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java @@ -98,6 +98,7 @@ public String normalizeAttributePath() { * Parse this {@link N5URL} as a {@link LinkedAttributePathToken}. * * @see N5URL#getAttributePathTokens(String) + * @return the linked attribute path token */ public LinkedAttributePathToken getAttributePathTokens() { @@ -203,7 +204,7 @@ public boolean isAbsolute() { * * @param relativeN5Url N5URL to resolve against ourselves * @return the result of the resolution. - * @throws URISyntaxException + * @throws URISyntaxException if the uri is malformed */ public N5URL resolve(final N5URL relativeN5Url) throws URISyntaxException { @@ -280,7 +281,7 @@ public N5URL resolve(final N5URL relativeN5Url) throws URISyntaxException { * * @param relativeUri URI to resolve against ourselves * @return the result of the resolution. - * @throws URISyntaxException + * @throws URISyntaxException if the uri is malformed */ public N5URL resolve(final URI relativeUri) throws URISyntaxException { @@ -293,7 +294,7 @@ public N5URL resolve(final URI relativeUri) throws URISyntaxException { * * @param relativeString String to resolve against ourselves * @return the result of the resolution. - * @throws URISyntaxException + * @throws URISyntaxException if the uri is malformed */ public N5URL resolve(final String relativeString) throws URISyntaxException { @@ -511,7 +512,7 @@ public static URI encodeAsUri(final String uri) throws URISyntaxException { * @param group of the N5Url * @param attribute of the N5Url * @return the {@link N5URL} - * @throws URISyntaxException + * @throws URISyntaxException if the uri is malformed */ public static N5URL from(final String container, final String group, final String attribute) throws URISyntaxException { From 6b36c8ca131122f487759eea6f6eced27b270a85 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Thu, 13 Apr 2023 13:13:45 -0400 Subject: [PATCH 152/243] doc: even more javadoc --- .../saalfeldlab/n5/AbstractGsonReader.java | 2 +- .../janelia/saalfeldlab/n5/BlockReader.java | 8 ++-- .../janelia/saalfeldlab/n5/BlockWriter.java | 7 ++-- .../saalfeldlab/n5/DefaultBlockWriter.java | 9 +++-- .../org/janelia/saalfeldlab/n5/GsonUtils.java | 20 ++++++---- .../janelia/saalfeldlab/n5/N5FSReader.java | 4 +- .../saalfeldlab/n5/N5KeyValueReader.java | 2 +- .../org/janelia/saalfeldlab/n5/N5Reader.java | 17 ++++---- .../org/janelia/saalfeldlab/n5/N5Writer.java | 39 ++++++++++--------- 9 files changed, 61 insertions(+), 47 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/AbstractGsonReader.java b/src/main/java/org/janelia/saalfeldlab/n5/AbstractGsonReader.java index f4e660b7..d1029bb1 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/AbstractGsonReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/AbstractGsonReader.java @@ -51,7 +51,7 @@ public abstract class AbstractGsonReader implements GsonAttributesParser, N5Read * Constructs an {@link AbstractGsonReader} with a custom * {@link GsonBuilder} to support custom attributes. * - * @param gsonBuilder + * @param gsonBuilder the gson builder */ @Deprecated public AbstractGsonReader(final GsonBuilder gsonBuilder) { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/BlockReader.java b/src/main/java/org/janelia/saalfeldlab/n5/BlockReader.java index d0dcd43b..8f63ddd2 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/BlockReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/BlockReader.java @@ -39,9 +39,11 @@ public interface BlockReader { /** * Reads a {@link DataBlock} from an {@link InputStream}. * - * @param dataBlock - * @param in - * @throws IOException + * @param dataBlock the data block + * @param in the input stream + * @param the block data type + * @param the block type + * @throws IOException the exception */ public > void read(final B dataBlock, final InputStream in) throws IOException; } \ No newline at end of file diff --git a/src/main/java/org/janelia/saalfeldlab/n5/BlockWriter.java b/src/main/java/org/janelia/saalfeldlab/n5/BlockWriter.java index 21bbee8a..8bb384d3 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/BlockWriter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/BlockWriter.java @@ -39,9 +39,10 @@ public interface BlockWriter { /** * Writes a {@link DataBlock} into an {@link OutputStream}. * - * @param dataBlock - * @param out - * @throws IOException + * @param dataBlock the data block + * @param out the output stream + * @param the block data type + * @throws IOException the exception */ public void write(final DataBlock dataBlock, final OutputStream out) throws IOException; } \ No newline at end of file diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockWriter.java b/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockWriter.java index 95c99b8f..aa9c5fc4 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockWriter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockWriter.java @@ -55,10 +55,11 @@ public default void write( /** * Writes a {@link DataBlock} into an {@link OutputStream}. * - * @param out - * @param datasetAttributes - * @param dataBlock - * @throws IOException + * @param out the output stream + * @param datasetAttributes the dataset attributes + * @param dataBlock the data block + * @param the block data type + * @throws IOException the exception */ public static void writeBlock( final OutputStream out, diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonUtils.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonUtils.java index 51d2cefe..af055778 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonUtils.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonUtils.java @@ -183,6 +183,9 @@ static JsonElement getAttribute(JsonElement root, final String normalizedAttribu *
  • String[]
    • *
  • Object[]
    • * + * + * @param root the json element + * @return the attribute map */ static Map> listAttributes(final JsonElement root) throws N5Exception.N5IOException { @@ -464,11 +467,12 @@ static JsonElement removeAttribute(JsonElement root, final String normalizedAttr *

    * If {@code root} is not a {@link JsonObject}, then it is overwritten with an object containing {@code "normalizedAttributePath": attribute } * - * @param writer - * @param root - * @param normalizedAttributePath - * @param attribute - * @param gson + * @param writer the writer + * @param root the root json element + * @param normalizedAttributePath the attribute path + * @param attribute the attribute + * @param gson the gson + * @param the attribute type * @throws IOException the exception */ static void writeAttribute( @@ -486,8 +490,10 @@ static void writeAttribute( * Writes the attributes JsonElemnt to a given {@link Writer}. * This will overwrite any existing attributes. * - * @param writer - * @param root + * @param writer the writer + * @param root the root json element + * @param gson the gson + * @param the attribute type * @throws IOException the exception */ static void writeAttributes( diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5FSReader.java b/src/main/java/org/janelia/saalfeldlab/n5/N5FSReader.java index 083c92b7..9f377f24 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5FSReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5FSReader.java @@ -46,7 +46,7 @@ public class N5FSReader extends N5KeyValueReader { * {@link GsonBuilder} to support custom attributes. * * @param basePath N5 base path - * @param gsonBuilder + * @param gsonBuilder the gson builder * @param cacheMeta cache attributes and meta data * Setting this to true avoids frequent reading and parsing of JSON * encoded attributes and other meta data that requires accessing the @@ -97,7 +97,7 @@ public N5FSReader(final String basePath, final boolean cacheMeta) throws IOExcep * {@link GsonBuilder} to support custom attributes. * * @param basePath N5 base path - * @param gsonBuilder + * @param gsonBuilder the gson builder * @throws IOException * if the base path cannot be read or does not exist, * if the N5 version of the container is not compatible with this diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java index 529a390a..f23356a2 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java @@ -294,7 +294,7 @@ public DataBlock readBlock( /** * @param normalPath normalized path name * @return the normalized groups in {@code normalPath} - * @throws IOException + * @throws N5Exception.N5IOException the exception */ protected String[] normalList(final String normalPath) throws N5Exception.N5IOException { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java index 56bf4c62..4b67a12d 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java @@ -624,17 +624,18 @@ default String[] deepListDatasets( } /** - * Helper method for parallel deep listing. This method is not part of the - * public API and is accessible only because Java 8 does not support - * private interface methods yet. + * Helper method for parallel deep listing. This method is not part of the + * public API and is accessible only because Java 8 does not support private + * interface methods yet. * * TODO make private when committing to Java versions newer than 8 * - * @param n5 - * @param path - * @param filter - * @param executor - * @param datasetFutures + * @param n5 the n5 reader + * @param path the base path + * @param datasetsOnly true if only dataset paths should be returned + * @param filter filter for datasets to be included + * @param executor the executor service + * @param datasetFutures result futures */ static void deepListHelper( final N5Reader n5, diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java index f6616c21..f240d48c 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java @@ -175,13 +175,14 @@ default void createDataset( } /** - * Creates a dataset. This does not create any data but the path and - * mandatory attributes only. - * - * @param pathName dataset path - * @param dimensions the dataset dimensions - * @param blockSize the block size - * @param dataType the data type + * Creates a dataset. This does not create any data but the path and mandatory + * attributes only. + * + * @param pathName dataset path + * @param dimensions the dataset dimensions + * @param blockSize the block size + * @param dataType the data type + * @param compression the compression * @throws IOException the exception */ default void createDataset( @@ -197,9 +198,10 @@ default void createDataset( /** * Writes a {@link DataBlock}. * - * @param pathName dataset path + * @param pathName dataset path * @param datasetAttributes the dataset attributes - * @param dataBlock the data block + * @param dataBlock the data block + * @param the data block data type * @throws IOException the exception */ void writeBlock( @@ -211,13 +213,14 @@ void writeBlock( /** * Deletes the block at {@code gridPosition} * - * @param pathName dataset path + * @param pathName dataset path * @param gridPosition position of block to be deleted * @throws IOException the exception * - * @return {@code true} if the block at {@code gridPosition} is "empty" after deletion. The meaning of "empty" is - * implementation dependent. For example "empty" means that no file exists on the file system for the deleted block - * in case of the file system implementation. + * @return {@code true} if the block at {@code gridPosition} is "empty" after + * deletion. The meaning of "empty" is implementation dependent. For + * example "empty" means that no file exists on the file system for the + * deleted block in case of the file system implementation. * */ boolean deleteBlock( @@ -225,13 +228,13 @@ boolean deleteBlock( final long... gridPosition) throws IOException; /** - * Save a {@link Serializable} as an N5 {@link DataBlock} at a given - * offset. The offset is given in {@link DataBlock} grid coordinates. + * Save a {@link Serializable} as an N5 {@link DataBlock} at a given offset. The + * offset is given in {@link DataBlock} grid coordinates. * - * @param object the object to serialize - * @param dataset the dataset path + * @param object the object to serialize + * @param dataset the dataset path * @param datasetAttributes the dataset attributes - * @param gridPosition the grid position + * @param gridPosition the grid position * @throws IOException the exception */ default void writeSerializedBlock( From e7cdd33b78d5396f908cfd1b422f9b793181f7bd Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Thu, 13 Apr 2023 14:39:27 -0400 Subject: [PATCH 153/243] test: an explicit test of N5CacheTest --- .../saalfeldlab/n5/cache/N5CacheTest.java | 121 ++++++++++++++++++ 1 file changed, 121 insertions(+) create mode 100644 src/test/java/org/janelia/saalfeldlab/n5/cache/N5CacheTest.java diff --git a/src/test/java/org/janelia/saalfeldlab/n5/cache/N5CacheTest.java b/src/test/java/org/janelia/saalfeldlab/n5/cache/N5CacheTest.java new file mode 100644 index 00000000..4cbb9d15 --- /dev/null +++ b/src/test/java/org/janelia/saalfeldlab/n5/cache/N5CacheTest.java @@ -0,0 +1,121 @@ +package org.janelia.saalfeldlab.n5.cache; + +import static org.junit.Assert.assertEquals; + +import org.junit.Test; + +import com.google.gson.JsonElement; + +public class N5CacheTest { + + @Test + public void cacheBackingTest() { + + final DummyBackingStorage backingStorage = new DummyBackingStorage(); + final N5JsonCache cache = new N5JsonCache(backingStorage::backingAttrs, backingStorage::exists, + backingStorage::isGroup, backingStorage::isDataset, backingStorage::list); + + // check existance, ensure backing storage is only called once + assertEquals(0, backingStorage.existsCallCount); + cache.exists("a"); + assertEquals(1, backingStorage.existsCallCount); + cache.exists("a"); + assertEquals(1, backingStorage.existsCallCount); + + // check existance of new group, ensure backing storage is only called one more time + cache.exists("b"); + assertEquals(2, backingStorage.existsCallCount); + cache.exists("b"); + assertEquals(2, backingStorage.existsCallCount); + + // check isDataset, ensure backing storage is only called when expected + // isDataset is called by exists, so should have been called twice here + assertEquals(2, backingStorage.isDatasetCallCount); + cache.isDataset("a"); + assertEquals(2, backingStorage.isDatasetCallCount); + + assertEquals(2, backingStorage.isDatasetCallCount); + cache.isDataset("b"); + assertEquals(2, backingStorage.isDatasetCallCount); + + // check isGroup, ensure backing storage is only called when expected + // isGroup is called by exists, so should have been called twice here + assertEquals(2, backingStorage.isGroupCallCount); + cache.isDataset("a"); + assertEquals(2, backingStorage.isGroupCallCount); + + assertEquals(2, backingStorage.isGroupCallCount); + cache.isDataset("b"); + assertEquals(2, backingStorage.isGroupCallCount); + + // similarly check list, ensure backing storage is only called when expected + // list is called by exists, so should have been called twice here + assertEquals(2, backingStorage.listCallCount); + cache.list("a"); + assertEquals(2, backingStorage.listCallCount); + + assertEquals(2, backingStorage.listCallCount); + cache.list("b"); + assertEquals(2, backingStorage.listCallCount); + + // finally check getAttributes + // it is not called by exists (since it needs the cache key) + assertEquals(0, backingStorage.attrCallCount); + cache.getAttributes("a", "foo"); + assertEquals(1, backingStorage.attrCallCount); + cache.getAttributes("a", "foo"); + assertEquals(1, backingStorage.attrCallCount); + cache.getAttributes("a", "bar"); + assertEquals(2, backingStorage.attrCallCount); + cache.getAttributes("a", "bar"); + assertEquals(2, backingStorage.attrCallCount); + + cache.getAttributes("b", "foo"); + assertEquals(3, backingStorage.attrCallCount); + cache.getAttributes("b", "foo"); + assertEquals(3, backingStorage.attrCallCount); + cache.getAttributes("b", "bar"); + assertEquals(4, backingStorage.attrCallCount); + cache.getAttributes("b", "bar"); + assertEquals(4, backingStorage.attrCallCount); + + } + + protected static class DummyBackingStorage { + + int attrCallCount = 0; + int existsCallCount = 0; + int isGroupCallCount = 0; + int isDatasetCallCount = 0; + int listCallCount = 0; + + public DummyBackingStorage() { + } + + public JsonElement backingAttrs(final String key, final String cacheKey) { + attrCallCount++; + return null; + } + + public boolean exists(final String key) { + existsCallCount++; + return true; + } + + public boolean isGroup(final String key) { + isGroupCallCount++; + return true; + } + + public boolean isDataset(final String key) { + isDatasetCallCount++; + return true; + } + + public String[] list(final String key) { + listCallCount++; + return new String[] { "list" }; + } + } + +} From 14707ec3ab770bd2e645d38f87eabc00107f3707 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Thu, 13 Apr 2023 15:10:32 -0400 Subject: [PATCH 154/243] fix(style): implement groupPath --- .../n5/N5KeyValueReaderWithInterfaces.java | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReaderWithInterfaces.java b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReaderWithInterfaces.java index 97564b7f..2ab95aaf 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReaderWithInterfaces.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReaderWithInterfaces.java @@ -35,6 +35,7 @@ import java.lang.reflect.Type; import java.util.Arrays; import java.util.Map; +import java.util.stream.Stream; /** * {@link N5Reader} implementation through {@link KeyValueAccess} with JSON @@ -106,13 +107,20 @@ public String getBasePath() { return this.basePath; } - @Override public boolean cacheMeta() { + @Override + public boolean cacheMeta() { return cacheMeta; } - @Override public N5JsonCache getCache() { + @Override + public N5JsonCache getCache() { return this.cache; } + + @Override + public String groupPath(String... nodes) { + return keyValueAccess.compose(Stream.concat(Stream.of(basePath), Arrays.stream(nodes)).toArray(String[]::new)); + } } From 973e5a190aac1fcae6a70d241a03930bc38007e1 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Thu, 13 Apr 2023 16:32:39 -0400 Subject: [PATCH 155/243] feat: wip toward cache interfaces * CachedGsonKVR: add more "normal" methods --- .../n5/CachedGsonKeyValueReader.java | 42 +++++++++++++--- .../n5/CachedGsonKeyValueWriter.java | 10 ++-- .../saalfeldlab/n5/GsonKeyValueReader.java | 6 ++- .../janelia/saalfeldlab/n5/N5FSReader.java | 3 +- .../janelia/saalfeldlab/n5/N5FSWriter.java | 3 +- .../n5/N5KeyValueReaderWithInterfaces.java | 50 +++++++++++++++---- .../n5/N5KeyValueWriterWithInterfaces.java | 9 +--- .../saalfeldlab/n5/N5CachedFSTest.java | 26 ++++++---- 8 files changed, 107 insertions(+), 42 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueReader.java b/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueReader.java index a75413ac..62a5e22e 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueReader.java @@ -32,7 +32,6 @@ import java.io.IOException; import java.lang.reflect.Type; -import java.util.Arrays; /** * {@link N5Reader} implementation through {@link KeyValueAccess} with JSON @@ -48,9 +47,9 @@ default N5JsonCache newCache() { return new N5JsonCache( (groupPath, cacheKey) -> GsonKeyValueReader.super.getAttributes(groupPath), - GsonKeyValueReader.super::exists, - GsonKeyValueReader.super::exists, - GsonKeyValueReader.super::datasetExists, + this::normalExists, + this::normalGroupExists, + this::normalDatasetExists, GsonKeyValueReader.super::list ); } @@ -64,7 +63,10 @@ default DatasetAttributes getDatasetAttributes(final String pathName) { final String normalPath = N5URL.normalizeGroupPath(pathName); final JsonElement attributes; - if (cacheMeta() && getCache().isDataset(normalPath)) { + + // TODO the isDataset check results in unexpected behavior: breaks the cacheGroupDatasetTest test +// if (cacheMeta() && getCache().isDataset(normalPath)) { + if (cacheMeta()) { attributes = getCache().getAttributes(normalPath, N5JsonCache.jsonFile); } else { attributes = GsonKeyValueReader.super.getAttributes(normalPath); @@ -122,10 +124,31 @@ default boolean exists(final String pathName) { if (cacheMeta()) return getCache().exists(normalPathName); else { - return GsonKeyValueReader.super.exists(normalPathName); + return normalExists(normalPathName); } } + default boolean normalExists(final String pathName) { + + final String normalPathName = N5URL.normalizeGroupPath(pathName); + return getKeyValueAccess().exists(groupPath(normalPathName)) || normalDatasetExists(normalPathName); + } + + default boolean groupExists(final String pathName) { + + final String normalPathName = N5URL.normalizeGroupPath(pathName); + if (cacheMeta()) + return getCache().isGroup(normalPathName); + else { + return normalGroupExists(normalPathName); + } + } + + default boolean normalGroupExists(final String pathName) { + + return GsonKeyValueReader.super.groupExists(pathName); + } + @Override default boolean datasetExists(final String pathName) throws N5Exception.N5IOException { @@ -133,7 +156,12 @@ default boolean datasetExists(final String pathName) throws N5Exception.N5IOExce final String normalPathName = N5URL.normalizeGroupPath(pathName); return getCache().isDataset(normalPathName); } - return GsonKeyValueReader.super.datasetExists(pathName); + return normalDatasetExists(pathName); + } + + default boolean normalDatasetExists(final String pathName) throws N5Exception.N5IOException { + + return normalGroupExists(groupPath(pathName)) && normalGetDatasetAttributes(pathName) != null; } /** diff --git a/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueWriter.java b/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueWriter.java index 5d55485f..b9b062a1 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueWriter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueWriter.java @@ -25,10 +25,10 @@ */ package org.janelia.saalfeldlab.n5; -import com.google.gson.GsonBuilder; import com.google.gson.JsonElement; import com.google.gson.JsonNull; import com.google.gson.JsonObject; + import org.janelia.saalfeldlab.n5.cache.N5JsonCache; import java.io.IOException; @@ -45,8 +45,12 @@ public interface CachedGsonKeyValueWriter extends CachedGsonKeyValueReader, N5Wr default void setVersion(final String path) throws IOException { - if (!VERSION.equals(getVersion())) - setAttribute("/", VERSION_KEY, VERSION.toString()); + final Version version = getVersion(); + if (!VERSION.isCompatible(version)) + throw new N5Exception.N5IOException("Incompatible version " + version + " (this is " + VERSION + ")."); + + if (!VERSION.equals(version)) + setAttribute("/", VERSION_KEY, VERSION.toString());; } static String initializeContainer( diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueReader.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueReader.java index 1719918a..07931029 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueReader.java @@ -136,9 +136,13 @@ default boolean datasetExists(final String pathName) throws N5Exception.N5IOExce default JsonElement getAttributes(final String pathName) throws N5Exception.N5IOException { final String groupPath = N5URL.normalizeGroupPath(pathName); + final String absoluteGroupPath = getKeyValueAccess().compose( getBasePath(), groupPath ); final String attributesPath = attributesPath(groupPath); - if (!exists(groupPath)) + + // calling exists can result in a stackoverflowerror when caching + if (!getKeyValueAccess().exists(absoluteGroupPath)) throw new N5Exception.N5IOException("Group " + groupPath + " does not exist"); + if (!getKeyValueAccess().exists(attributesPath)) return null; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5FSReader.java b/src/main/java/org/janelia/saalfeldlab/n5/N5FSReader.java index 9f377f24..2817f221 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5FSReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5FSReader.java @@ -39,7 +39,8 @@ * @author Igor Pisarev * @author Philipp Hanslovsky */ -public class N5FSReader extends N5KeyValueReader { +//public class N5FSReader extends N5KeyValueReader { +public class N5FSReader extends N5KeyValueReaderWithInterfaces { /** * Opens an {@link N5FSReader} at a given base path with a custom diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java b/src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java index 244dee3a..8cbd8acc 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java @@ -35,7 +35,8 @@ * * @author Stephan Saalfeld */ -public class N5FSWriter extends N5KeyValueWriter implements N5Writer { +//public class N5FSWriter extends N5KeyValueWriter implements N5Writer { +public class N5FSWriter extends N5KeyValueWriterWithInterfaces { /** * Opens an {@link N5FSWriter} at a given base path with a custom diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReaderWithInterfaces.java b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReaderWithInterfaces.java index 2ab95aaf..a8ee9d1b 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReaderWithInterfaces.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReaderWithInterfaces.java @@ -27,14 +27,10 @@ import com.google.gson.Gson; import com.google.gson.GsonBuilder; -import com.google.gson.JsonElement; -import com.google.gson.JsonObject; import org.janelia.saalfeldlab.n5.cache.N5JsonCache; import java.io.IOException; -import java.lang.reflect.Type; import java.util.Arrays; -import java.util.Map; import java.util.stream.Stream; /** @@ -80,17 +76,49 @@ public N5KeyValueReaderWithInterfaces( final GsonBuilder gsonBuilder, final boolean cacheMeta) throws IOException { + this( true, keyValueAccess, basePath, gsonBuilder, cacheMeta ); + } + + /** + * Opens an {@link N5KeyValueReaderWithInterfaces} at a given base path with a custom + * {@link GsonBuilder} to support custom attributes. + * + * @param checkVersion do the version check + * @param keyValueAccess + * @param basePath N5 base path + * @param gsonBuilder + * @param cacheMeta cache attributes and meta data + * Setting this to true avoids frequent reading and parsing of JSON + * encoded attributes and other meta data that requires accessing the + * store. This is most interesting for high latency backends. Changes + * of cached attributes and meta data by an independent writer will + * not be tracked. + * + * @throws IOException + * if the base path cannot be read or does not exist, + * if the N5 version of the container is not compatible with this + * implementation. + */ + protected N5KeyValueReaderWithInterfaces( + final boolean checkVersion, + final KeyValueAccess keyValueAccess, + final String basePath, + final GsonBuilder gsonBuilder, + final boolean cacheMeta) throws IOException { + this.keyValueAccess = keyValueAccess; this.basePath = keyValueAccess.normalize(basePath); this.gson = GsonUtils.registerGson(gsonBuilder); this.cacheMeta = cacheMeta; - this.cache = getCache(); - - /* Existence checks, if any, go in subclasses */ - /* Check that version (if there is one) is compatible. */ - final Version version = getVersion(); - if (!VERSION.isCompatible(version)) - throw new N5Exception.N5IOException("Incompatible version " + version + " (this is " + VERSION + ")."); + this.cache = newCache(); + + if (checkVersion) { + /* Existence checks, if any, go in subclasses */ + /* Check that version (if there is one) is compatible. */ + final Version version = getVersion(); + if (!VERSION.isCompatible(version)) + throw new N5Exception.N5IOException("Incompatible version " + version + " (this is " + VERSION + ")."); + } } public Gson getGson() { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriterWithInterfaces.java b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriterWithInterfaces.java index ded92913..8fcf6ab2 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriterWithInterfaces.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriterWithInterfaces.java @@ -26,15 +26,8 @@ package org.janelia.saalfeldlab.n5; import com.google.gson.GsonBuilder; -import com.google.gson.JsonElement; -import com.google.gson.JsonNull; -import com.google.gson.JsonObject; -import org.janelia.saalfeldlab.n5.cache.N5JsonCache; import java.io.IOException; -import java.util.Arrays; -import java.util.List; -import java.util.Map; /** * Filesystem {@link N5Writer} implementation with version compatibility check. @@ -71,7 +64,7 @@ public N5KeyValueWriterWithInterfaces( final boolean cacheAttributes) throws IOException { - super(keyValueAccess, GsonKeyValueWriter.initializeContainer(keyValueAccess, basePath), gsonBuilder, cacheAttributes); + super(false, keyValueAccess, basePath, gsonBuilder, cacheAttributes); createGroup("/"); setVersion("/"); } diff --git a/src/test/java/org/janelia/saalfeldlab/n5/N5CachedFSTest.java b/src/test/java/org/janelia/saalfeldlab/n5/N5CachedFSTest.java index e1f73591..1b25e5bd 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/N5CachedFSTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/N5CachedFSTest.java @@ -1,12 +1,12 @@ package org.janelia.saalfeldlab.n5; import com.google.gson.GsonBuilder; -import org.junit.Assert; import org.junit.Test; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertThrows; import static org.junit.Assert.assertTrue; import java.io.File; @@ -26,7 +26,12 @@ protected N5Writer createN5Writer() throws IOException { @Override protected N5Writer createN5Writer(final String location) throws IOException { - return new N5FSWriter(location, true); + return createN5Writer(location, true); + } + + protected N5Writer createN5Writer(final String location, final boolean cache ) throws IOException { + + return new N5FSWriter(location, cache); } @Override @@ -38,10 +43,9 @@ protected N5Writer createN5Writer(final String location, final GsonBuilder gson) return new N5FSWriter(location, gson, true); } - @Override - protected N5Reader createN5Reader(final String location, final GsonBuilder gson) throws IOException { + protected N5Reader createN5Reader(final String location, final GsonBuilder gson, final boolean cache) throws IOException { - return new N5FSReader(location, gson, true); + return new N5FSReader(location, gson, cache); } @Test @@ -49,7 +53,8 @@ public void cacheTest() throws IOException { /* Test the cache by setting many attributes, then manually deleting the underlying file. * The only possible way for the test to succeed is if it never again attempts to read the file, and relies on the cache. */ - try (N5KeyValueWriter n5 = (N5KeyValueWriter) createN5Writer()) { +// try (N5KeyValueWriter n5 = (N5KeyValueWriter) createN5Writer()) { + try (N5KeyValueWriterWithInterfaces n5 = (N5KeyValueWriterWithInterfaces) createN5Writer()) { final String cachedGroup = "cachedGroup"; final String attributesPath = n5.attributesPath(cachedGroup); @@ -64,7 +69,8 @@ public void cacheTest() throws IOException { runTests(n5, tests); } - try (N5KeyValueWriter n5 = new N5FSWriter(tempN5PathName(), false)) { +// try (N5KeyValueWriter n5 = new N5FSWriter(tempN5PathName(), false)) { + try (N5KeyValueWriterWithInterfaces n5 = (N5KeyValueWriterWithInterfaces)createN5Writer(tempN5PathName(), false)) { final String cachedGroup = "cachedGroup"; final String attributesPath = n5.attributesPath(cachedGroup); @@ -75,7 +81,7 @@ public void cacheTest() throws IOException { addAndTest(n5, tests, new TestData<>(cachedGroup, "a/a[2]", 0)); Files.delete(Paths.get(attributesPath)); - Assert.assertThrows(AssertionError.class, () -> runTests(n5, tests)); + assertThrows(AssertionError.class, () -> runTests(n5, tests)); } } @@ -86,8 +92,8 @@ public void cacheGroupDatasetTest() throws IOException { final String groupName = "gg"; final String tmpPath = tempN5PathName(); - try (N5KeyValueWriter w1 = (N5KeyValueWriter) createN5Writer(tmpPath); - N5KeyValueWriter w2 = (N5KeyValueWriter) createN5Writer(tmpPath);) { + try (N5KeyValueWriterWithInterfaces w1 = (N5KeyValueWriterWithInterfaces) createN5Writer(tmpPath); + N5KeyValueWriterWithInterfaces w2 = (N5KeyValueWriterWithInterfaces) createN5Writer(tmpPath);) { // create a group, both writers know it exists w1.createGroup(groupName); From 63380bf1de225263f72a992301f8967d2f1b29a2 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Fri, 14 Apr 2023 16:01:46 -0400 Subject: [PATCH 156/243] feat!: add cacheKey to public cache methods, attrs always updated * isDataset, isGroup, exists * attributes are always added when new cacheInfo is added --- .../n5/CachedGsonKeyValueReader.java | 14 ++-- .../saalfeldlab/n5/GsonKeyValueReader.java | 5 -- .../saalfeldlab/n5/N5KeyValueReader.java | 6 +- .../saalfeldlab/n5/cache/N5JsonCache.java | 66 ++++--------------- .../saalfeldlab/n5/cache/N5CacheTest.java | 39 ++++++----- 5 files changed, 45 insertions(+), 85 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueReader.java b/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueReader.java index 62a5e22e..962c7c0b 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueReader.java @@ -41,8 +41,8 @@ * @author Igor Pisarev * @author Philipp Hanslovsky */ -public interface CachedGsonKeyValueReader extends GsonKeyValueReader { +public interface CachedGsonKeyValueReader extends GsonKeyValueReader { default N5JsonCache newCache() { return new N5JsonCache( @@ -64,8 +64,9 @@ default DatasetAttributes getDatasetAttributes(final String pathName) { final String normalPath = N5URL.normalizeGroupPath(pathName); final JsonElement attributes; - // TODO the isDataset check results in unexpected behavior: breaks the cacheGroupDatasetTest test -// if (cacheMeta() && getCache().isDataset(normalPath)) { + if (!datasetExists(pathName)) + return null; + if (cacheMeta()) { attributes = getCache().getAttributes(normalPath, N5JsonCache.jsonFile); } else { @@ -122,7 +123,7 @@ default boolean exists(final String pathName) { final String normalPathName = N5URL.normalizeGroupPath(pathName); if (cacheMeta()) - return getCache().exists(normalPathName); + return getCache().exists(normalPathName, N5JsonCache.jsonFile); else { return normalExists(normalPathName); } @@ -138,7 +139,7 @@ default boolean groupExists(final String pathName) { final String normalPathName = N5URL.normalizeGroupPath(pathName); if (cacheMeta()) - return getCache().isGroup(normalPathName); + return getCache().isGroup(normalPathName, N5JsonCache.jsonFile); else { return normalGroupExists(normalPathName); } @@ -154,7 +155,7 @@ default boolean datasetExists(final String pathName) throws N5Exception.N5IOExce if (cacheMeta()) { final String normalPathName = N5URL.normalizeGroupPath(pathName); - return getCache().isDataset(normalPathName); + return getCache().isDataset(normalPathName, N5JsonCache.jsonFile); } return normalDatasetExists(pathName); } @@ -184,7 +185,6 @@ default JsonElement getAttributes(final String pathName) throws N5Exception.N5IO } - @Override default String[] list(final String pathName) throws N5Exception.N5IOException { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueReader.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueReader.java index 07931029..8231617b 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueReader.java @@ -136,13 +136,8 @@ default boolean datasetExists(final String pathName) throws N5Exception.N5IOExce default JsonElement getAttributes(final String pathName) throws N5Exception.N5IOException { final String groupPath = N5URL.normalizeGroupPath(pathName); - final String absoluteGroupPath = getKeyValueAccess().compose( getBasePath(), groupPath ); final String attributesPath = attributesPath(groupPath); - // calling exists can result in a stackoverflowerror when caching - if (!getKeyValueAccess().exists(absoluteGroupPath)) - throw new N5Exception.N5IOException("Group " + groupPath + " does not exist"); - if (!getKeyValueAccess().exists(attributesPath)) return null; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java index ffcec6bd..e8df0ea2 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java @@ -139,7 +139,7 @@ public DatasetAttributes getDatasetAttributes(final String pathName) throws IOEx final String normalPath = N5URL.normalizeGroupPath(pathName); final JsonElement attributes; - if (cacheMeta && cache.isDataset(normalPath)) { + if (cacheMeta && cache.isDataset(normalPath, N5JsonCache.jsonFile)) { attributes = cache.getAttributes(normalPath, N5JsonCache.jsonFile); } else { attributes = getAttributes(normalPath); @@ -225,7 +225,7 @@ public boolean exists(final String pathName) { final String normalPathName = N5URL.normalizeGroupPath(pathName); if (cacheMeta) - return cache.exists(normalPathName); + return cache.exists(normalPathName, N5JsonCache.jsonFile); else { return normalExists(normalPathName); } @@ -241,7 +241,7 @@ public boolean datasetExists(final String pathName) throws N5Exception.N5IOExcep if (cacheMeta) { final String normalPathName = N5URL.normalizeGroupPath(pathName); - return cache.isDataset(normalPathName); + return cache.isDataset(normalPathName, N5JsonCache.jsonFile); } return normalDatasetExists(pathName); } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/cache/N5JsonCache.java b/src/main/java/org/janelia/saalfeldlab/n5/cache/N5JsonCache.java index e67d9fb2..bb0b2cd0 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/cache/N5JsonCache.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/cache/N5JsonCache.java @@ -89,29 +89,29 @@ public JsonElement getAttributes(final String normalPathKey, final String normal return getCacheInfo(normalPathKey).getCache(normalCacheKey); } - public boolean isDataset(final String normalPathKey) { + public boolean isDataset(final String normalPathKey, final String normalCacheKey) { final N5CacheInfo cacheInfo = getCacheInfo(normalPathKey); if (cacheInfo == null) { - addNewCacheInfo(normalPathKey); + addNewCacheInfo(normalPathKey, normalCacheKey); } return getCacheInfo(normalPathKey).isDataset; } - public boolean isGroup(final String normalPathKey) { + public boolean isGroup(final String normalPathKey, final String normalCacheKey) { final N5CacheInfo cacheInfo = getCacheInfo(normalPathKey); if (cacheInfo == null) { - addNewCacheInfo(normalPathKey); + addNewCacheInfo(normalPathKey, normalCacheKey); } return getCacheInfo(normalPathKey).isGroup; } - public boolean exists(final String normalPathKey) { + public boolean exists(final String normalPathKey, final String normalCacheKey ) { final N5CacheInfo cacheInfo = getCacheInfo(normalPathKey); if (cacheInfo == null) { - addNewCacheInfo(normalPathKey); + addNewCacheInfo(normalPathKey, normalCacheKey ); } return getCacheInfo(normalPathKey) != emptyCacheInfo; } @@ -139,6 +139,12 @@ private void cacheAttributes(final String normalPathKey, final String normalCach addNewCacheInfo(normalPathKey, normalCacheKey, uncachedValue); } } + + private void addNewCacheInfo(String normalPathKey, String normalCacheKey) { + + addNewCacheInfo( normalPathKey, normalCacheKey, readBackingAttributes.apply( normalPathKey, normalCacheKey)); + } + private void addNewCacheInfo(String normalPathKey, String normalCacheKey, JsonElement uncachedAttributes) { final N5CacheInfo cacheInfo; @@ -151,7 +157,6 @@ private void addNewCacheInfo(String normalPathKey, String normalCacheKey, JsonEl } cacheInfo.isGroup = getIsGroup.apply(normalPathKey); cacheInfo.isDataset = getIsDataset.apply(normalPathKey); - cacheInfo.attributesCache.put(normalCacheKey, readBackingAttributes.apply(normalPathKey, normalCacheKey)); addChild(cacheInfo, normalPathKey); } synchronized (containerPathToCache) { @@ -205,7 +210,7 @@ public void updateCacheInfo(final String normalPathKey, final String normalCache N5CacheInfo cacheInfo = getCacheInfo(normalPathKey); if (cacheInfo == null ){ - addNewCacheInfo(normalPathKey); + addNewCacheInfo(normalPathKey, normalCacheKey, uncachedAttributes ); return; } else if (!getExists.apply(normalPathKey)) { cacheInfo = emptyCacheInfo; @@ -225,32 +230,6 @@ public void updateCacheInfo(final String normalPathKey, final String normalCache } } - public void setCacheInfo( final String normalPathKey, final String normalCacheKey, final JsonElement attributes, final boolean isGroup, final boolean isDataset ) { - - N5CacheInfo cacheInfo = getCacheInfo(normalPathKey); - if (cacheInfo == null ) - cacheInfo = addNewCacheInfo(normalPathKey); - - cacheInfo.isGroup = isGroup; - cacheInfo.isDataset = isDataset; - cacheInfo.attributesCache.put( normalCacheKey, attributes ); - synchronized (containerPathToCache) { - containerPathToCache.put(normalPathKey, cacheInfo); - } - } - - public void setCacheAttributes( final String normalPathKey, final String normalCacheKey, final JsonElement attributes ) { - - N5CacheInfo cacheInfo = getCacheInfo(normalPathKey); - if (cacheInfo == null ) - cacheInfo = addNewCacheInfo(normalPathKey); - - cacheInfo.attributesCache.put( normalCacheKey, attributes ); - synchronized (containerPathToCache) { - containerPathToCache.put(normalPathKey, cacheInfo); - } - } - public void addChild(final String parent, final String child) { final N5CacheInfo cacheInfo = getCacheInfo(parent); @@ -277,25 +256,6 @@ public void clearCache(final String normalPathKey, final String normalCacheKey) } } - private void updateCache(String normalPathKey, String normalCacheKey, JsonElement uncachedAttributes) { - - final N5CacheInfo cacheInfo; - if (!getExists.apply(normalPathKey)) { - cacheInfo = emptyCacheInfo; - } else { - cacheInfo = new N5CacheInfo(); - synchronized (cacheInfo.attributesCache) { - cacheInfo.attributesCache.put(normalCacheKey, uncachedAttributes); - } - cacheInfo.isGroup = getIsGroup.apply(normalPathKey); - cacheInfo.isDataset = getIsDataset.apply(normalPathKey); - addChild(cacheInfo, normalPathKey); - } - synchronized (containerPathToCache) { - containerPathToCache.put(normalPathKey, cacheInfo); - } - } - private N5CacheInfo getCacheInfo(String pathKey) { synchronized (containerPathToCache) { diff --git a/src/test/java/org/janelia/saalfeldlab/n5/cache/N5CacheTest.java b/src/test/java/org/janelia/saalfeldlab/n5/cache/N5CacheTest.java index 4cbb9d15..b8b12b72 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/cache/N5CacheTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/cache/N5CacheTest.java @@ -12,40 +12,41 @@ public class N5CacheTest { public void cacheBackingTest() { final DummyBackingStorage backingStorage = new DummyBackingStorage(); + final N5JsonCache cache = new N5JsonCache(backingStorage::backingAttrs, backingStorage::exists, backingStorage::isGroup, backingStorage::isDataset, backingStorage::list); // check existance, ensure backing storage is only called once assertEquals(0, backingStorage.existsCallCount); - cache.exists("a"); + cache.exists("a","face"); assertEquals(1, backingStorage.existsCallCount); - cache.exists("a"); + cache.exists("a","face"); assertEquals(1, backingStorage.existsCallCount); // check existance of new group, ensure backing storage is only called one more time - cache.exists("b"); + cache.exists("b","face"); assertEquals(2, backingStorage.existsCallCount); - cache.exists("b"); + cache.exists("b","face"); assertEquals(2, backingStorage.existsCallCount); // check isDataset, ensure backing storage is only called when expected // isDataset is called by exists, so should have been called twice here assertEquals(2, backingStorage.isDatasetCallCount); - cache.isDataset("a"); + cache.isDataset("a", "face"); assertEquals(2, backingStorage.isDatasetCallCount); assertEquals(2, backingStorage.isDatasetCallCount); - cache.isDataset("b"); + cache.isDataset("b", "face"); assertEquals(2, backingStorage.isDatasetCallCount); // check isGroup, ensure backing storage is only called when expected // isGroup is called by exists, so should have been called twice here assertEquals(2, backingStorage.isGroupCallCount); - cache.isDataset("a"); + cache.isDataset("a", "face"); assertEquals(2, backingStorage.isGroupCallCount); assertEquals(2, backingStorage.isGroupCallCount); - cache.isDataset("b"); + cache.isDataset("b", "face"); assertEquals(2, backingStorage.isGroupCallCount); // similarly check list, ensure backing storage is only called when expected @@ -60,24 +61,28 @@ public void cacheBackingTest() { // finally check getAttributes // it is not called by exists (since it needs the cache key) - assertEquals(0, backingStorage.attrCallCount); + assertEquals(2, backingStorage.attrCallCount); cache.getAttributes("a", "foo"); - assertEquals(1, backingStorage.attrCallCount); + assertEquals(3, backingStorage.attrCallCount); cache.getAttributes("a", "foo"); - assertEquals(1, backingStorage.attrCallCount); + assertEquals(3, backingStorage.attrCallCount); cache.getAttributes("a", "bar"); - assertEquals(2, backingStorage.attrCallCount); + assertEquals(4, backingStorage.attrCallCount); cache.getAttributes("a", "bar"); - assertEquals(2, backingStorage.attrCallCount); + assertEquals(4, backingStorage.attrCallCount); + cache.getAttributes("a", "face"); + assertEquals(4, backingStorage.attrCallCount); cache.getAttributes("b", "foo"); - assertEquals(3, backingStorage.attrCallCount); + assertEquals(5, backingStorage.attrCallCount); cache.getAttributes("b", "foo"); - assertEquals(3, backingStorage.attrCallCount); + assertEquals(5, backingStorage.attrCallCount); cache.getAttributes("b", "bar"); - assertEquals(4, backingStorage.attrCallCount); + assertEquals(6, backingStorage.attrCallCount); cache.getAttributes("b", "bar"); - assertEquals(4, backingStorage.attrCallCount); + assertEquals(6, backingStorage.attrCallCount); + cache.getAttributes("b", "face"); + assertEquals(6, backingStorage.attrCallCount); } From a94c9cd71aa3c0a3c12b8e4e58ecb1f3605c00c9 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Mon, 17 Apr 2023 15:32:09 -0400 Subject: [PATCH 157/243] chore!: remove unnecessary method --- .../janelia/saalfeldlab/n5/CachedGsonKeyValueWriter.java | 9 --------- 1 file changed, 9 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueWriter.java b/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueWriter.java index b9b062a1..95026b84 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueWriter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueWriter.java @@ -53,15 +53,6 @@ default void setVersion(final String path) throws IOException { setAttribute("/", VERSION_KEY, VERSION.toString());; } - static String initializeContainer( - final KeyValueAccess keyValueAccess, - final String basePath) throws IOException { - - final String normBasePath = keyValueAccess.normalize(basePath); - keyValueAccess.createDirectories(normBasePath); - return normBasePath; - } - /** * Performs any necessary initialization to ensure the key given by the * argument {@code normalPath} is a valid group after creation. Called by From f9105eb43ece7b97687885dd9a71ff80e93f87a5 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Mon, 17 Apr 2023 15:33:08 -0400 Subject: [PATCH 158/243] fix: cache always updates attributes with new elem --- .../java/org/janelia/saalfeldlab/n5/cache/N5JsonCache.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/cache/N5JsonCache.java b/src/main/java/org/janelia/saalfeldlab/n5/cache/N5JsonCache.java index bb0b2cd0..e02ba4ee 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/cache/N5JsonCache.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/cache/N5JsonCache.java @@ -196,6 +196,10 @@ public void updateCacheInfo(final String normalPathKey, final String normalCache } else if (!getExists.apply(normalPathKey)) { cacheInfo = emptyCacheInfo; } else { + final JsonElement attributesToCache = readBackingAttributes.apply(normalPathKey, normalCacheKey); + synchronized (cacheInfo.attributesCache) { + cacheInfo.attributesCache.put(normalCacheKey, attributesToCache); + } cacheInfo.isGroup = getIsGroup.apply(normalPathKey); cacheInfo.isDataset = getIsDataset.apply(normalPathKey); cacheInfo.children.clear(); From 069ceda0a745c1f343499ae3904a7982c66f3155 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Tue, 18 Apr 2023 11:13:07 -0400 Subject: [PATCH 159/243] feat!: add N5JsonCachableContainer interface * use this interface instead of functions for N5JsonCache --- .../n5/CachedGsonKeyValueReader.java | 41 +++++++------ .../saalfeldlab/n5/N5KeyValueReader.java | 43 +++++++------ .../saalfeldlab/n5/cache/N5JsonCache.java | 60 +++++++----------- .../n5/cache/N5JsonCacheableContainer.java | 61 +++++++++++++++++++ .../saalfeldlab/n5/cache/N5CacheTest.java | 15 +++-- 5 files changed, 135 insertions(+), 85 deletions(-) create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/cache/N5JsonCacheableContainer.java diff --git a/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueReader.java b/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueReader.java index 962c7c0b..0397460a 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueReader.java @@ -29,6 +29,7 @@ import com.google.gson.JsonElement; import com.google.gson.JsonObject; import org.janelia.saalfeldlab.n5.cache.N5JsonCache; +import org.janelia.saalfeldlab.n5.cache.N5JsonCacheableContainer; import java.io.IOException; import java.lang.reflect.Type; @@ -42,22 +43,22 @@ * @author Philipp Hanslovsky */ -public interface CachedGsonKeyValueReader extends GsonKeyValueReader { +public interface CachedGsonKeyValueReader extends GsonKeyValueReader, N5JsonCacheableContainer { default N5JsonCache newCache() { - - return new N5JsonCache( - (groupPath, cacheKey) -> GsonKeyValueReader.super.getAttributes(groupPath), - this::normalExists, - this::normalGroupExists, - this::normalDatasetExists, - GsonKeyValueReader.super::list - ); + return new N5JsonCache( this ); } boolean cacheMeta(); N5JsonCache getCache(); + default JsonElement getAttributesFromContainer(final String normalPathName, final String normalCacheKey) { + + // this implementation doesn't use cache key, but rather depends on + // attributesPath being implemented + return GsonKeyValueReader.super.getAttributes(normalPathName); + } + @Override default DatasetAttributes getDatasetAttributes(final String pathName) { @@ -125,14 +126,14 @@ default boolean exists(final String pathName) { if (cacheMeta()) return getCache().exists(normalPathName, N5JsonCache.jsonFile); else { - return normalExists(normalPathName); + return existsFromContainer(normalPathName); } } - default boolean normalExists(final String pathName) { + default boolean existsFromContainer(final String pathName) { final String normalPathName = N5URL.normalizeGroupPath(pathName); - return getKeyValueAccess().exists(groupPath(normalPathName)) || normalDatasetExists(normalPathName); + return getKeyValueAccess().exists(groupPath(normalPathName)) || isDatasetFromContainer(normalPathName); } default boolean groupExists(final String pathName) { @@ -141,11 +142,11 @@ default boolean groupExists(final String pathName) { if (cacheMeta()) return getCache().isGroup(normalPathName, N5JsonCache.jsonFile); else { - return normalGroupExists(normalPathName); + return isGroupFromContainer(normalPathName); } } - default boolean normalGroupExists(final String pathName) { + default boolean isGroupFromContainer(final String pathName) { return GsonKeyValueReader.super.groupExists(pathName); } @@ -157,12 +158,12 @@ default boolean datasetExists(final String pathName) throws N5Exception.N5IOExce final String normalPathName = N5URL.normalizeGroupPath(pathName); return getCache().isDataset(normalPathName, N5JsonCache.jsonFile); } - return normalDatasetExists(pathName); + return isDatasetFromContainer(pathName); } - default boolean normalDatasetExists(final String pathName) throws N5Exception.N5IOException { + default boolean isDatasetFromContainer(final String pathName) throws N5Exception.N5IOException { - return normalGroupExists(groupPath(pathName)) && normalGetDatasetAttributes(pathName) != null; + return isGroupFromContainer(groupPath(pathName)) && normalGetDatasetAttributes(pathName) != null; } /** @@ -196,6 +197,12 @@ default String[] list(final String pathName) throws N5Exception.N5IOException { } } + default String[] listFromContainer(final String normalPathName) { + + // this implementation doesn't use cache key, but rather depends on + return GsonKeyValueReader.super.list(normalPathName); + } + /** * Constructs the path for a data block in a dataset at a given grid position. *

    diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java index e8df0ea2..9cdcb118 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java @@ -36,6 +36,7 @@ import com.google.gson.JsonElement; import com.google.gson.JsonObject; import org.janelia.saalfeldlab.n5.cache.N5JsonCache; +import org.janelia.saalfeldlab.n5.cache.N5JsonCacheableContainer; /** * {@link N5Reader} implementation through {@link KeyValueAccess} with JSON @@ -45,19 +46,13 @@ * @author Igor Pisarev * @author Philipp Hanslovsky */ -public class N5KeyValueReader implements N5Reader { +public class N5KeyValueReader implements N5Reader, N5JsonCacheableContainer { protected final KeyValueAccess keyValueAccess; protected final Gson gson; - protected final N5JsonCache cache = new N5JsonCache( - (groupPath,cacheKey) -> normalGetAttributes(groupPath), - this::normalExists, - this::normalExists, - this::normalDatasetExists, - this::normalList - ); + protected final N5JsonCache cache = new N5JsonCache(this); protected final boolean cacheMeta; @@ -151,7 +146,7 @@ public DatasetAttributes getDatasetAttributes(final String pathName) throws IOEx private DatasetAttributes normalGetDatasetAttributes(final String pathName) throws N5Exception.N5IOException { final String normalPath = N5URL.normalizeGroupPath(pathName); - final JsonElement attributes = normalGetAttributes(normalPath); + final JsonElement attributes = getAttributesFromContainer(normalPath); return createDatasetAttributes(attributes); } @@ -215,7 +210,7 @@ public T getAttribute( return GsonUtils.readAttribute(attributes, normalizedAttributePath, type, gson); } - protected boolean groupExists(final String absoluteNormalPath) { + public boolean isGroupFromContainer(final String absoluteNormalPath) { return keyValueAccess.exists(absoluteNormalPath) && keyValueAccess.isDirectory(absoluteNormalPath); } @@ -227,13 +222,13 @@ public boolean exists(final String pathName) { if (cacheMeta) return cache.exists(normalPathName, N5JsonCache.jsonFile); else { - return normalExists(normalPathName); + return existsFromContainer(normalPathName); } } - private boolean normalExists(String normalPathName) { + public boolean existsFromContainer(String normalPathName) { - return keyValueAccess.exists(groupPath(normalPathName)) || normalDatasetExists(normalPathName); + return keyValueAccess.exists(groupPath(normalPathName)); } @Override @@ -243,14 +238,13 @@ public boolean datasetExists(final String pathName) throws N5Exception.N5IOExcep final String normalPathName = N5URL.normalizeGroupPath(pathName); return cache.isDataset(normalPathName, N5JsonCache.jsonFile); } - return normalDatasetExists(pathName); + return isDatasetFromContainer(pathName); } - private boolean normalDatasetExists(String pathName) throws N5Exception.N5IOException { + public boolean isDatasetFromContainer(String pathName) throws N5Exception.N5IOException { // for n5, every dataset must be a group - return groupExists(groupPath(pathName)) && normalGetDatasetAttributes(pathName) != null; - + return isGroupFromContainer(groupPath(pathName)) && normalGetDatasetAttributes(pathName) != null; } /** @@ -268,15 +262,20 @@ public JsonElement getAttributes(final String pathName) throws IOException { if (cacheMeta) { return cache.getAttributes(groupPath, N5JsonCache.jsonFile); } else { - return normalGetAttributes(groupPath); + return getAttributesFromContainer(groupPath); } } - private JsonElement normalGetAttributes(String groupPath) throws N5Exception.N5IOException { + public JsonElement getAttributesFromContainer(String groupPath ) throws N5Exception.N5IOException { + + return getAttributesFromContainer(groupPath, N5JsonCache.jsonFile); + } + + public JsonElement getAttributesFromContainer(String groupPath, String cachePath) throws N5Exception.N5IOException { final String attributesPath = attributesPath(groupPath); - if (!normalExists(groupPath)) + if (!existsFromContainer(groupPath)) throw new N5Exception.N5IOException("Group " + groupPath + " does not exist"); if (!keyValueAccess.exists(attributesPath)) return null; @@ -308,7 +307,7 @@ public DataBlock readBlock( * @return the normalized groups in {@code normalPath} * @throws N5Exception.N5IOException the exception */ - protected String[] normalList(final String normalPath) throws N5Exception.N5IOException { + public String[] listFromContainer(final String normalPath) throws N5Exception.N5IOException { try { return keyValueAccess.listDirectories(groupPath(normalPath)); @@ -324,7 +323,7 @@ public String[] list(final String pathName) throws IOException { if (cacheMeta) { return cache.list(normalPath); } else { - return normalList(normalPath); + return listFromContainer(normalPath); } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/cache/N5JsonCache.java b/src/main/java/org/janelia/saalfeldlab/n5/cache/N5JsonCache.java index e02ba4ee..d4aeabfa 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/cache/N5JsonCache.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/cache/N5JsonCache.java @@ -5,14 +5,14 @@ import java.util.Collections; import java.util.HashMap; import java.util.HashSet; -import java.util.function.BiFunction; -import java.util.function.Function; public class N5JsonCache { public static final N5CacheInfo emptyCacheInfo = new N5CacheInfo(); public static final String jsonFile = "attributes.json"; + private final N5JsonCacheableContainer container; + /** * Data object for caching meta data. Elements that are null are not yet * cached. @@ -40,26 +40,10 @@ private boolean containsKey(final String normalCacheKey) { } } - private final BiFunction readBackingAttributes; - private final Function getExists; - private final Function getIsGroup; - private final Function getIsDataset; - private final Function listNormalChildren; private final HashMap containerPathToCache = new HashMap<>(); - public N5JsonCache( - final BiFunction readBackingAttributes, - final Function getExists, - final Function getIsGroup, - final Function getIsDataset, - final Function listNormalChildren - ) { - - this.readBackingAttributes = readBackingAttributes; - this.getExists = getExists; - this.getIsGroup = getIsGroup; - this.getIsDataset = getIsDataset; - this.listNormalChildren = listNormalChildren; + public N5JsonCache( final N5JsonCacheableContainer container ) { + this.container = container; } /** @@ -134,29 +118,29 @@ public String[] list(String normalPathKey) { private void cacheAttributes(final String normalPathKey, final String normalCacheKey) { - final JsonElement uncachedValue = readBackingAttributes.apply(normalPathKey, normalCacheKey); - if (uncachedValue != null || getExists.apply(normalPathKey)) { + final JsonElement uncachedValue = container.getAttributesFromContainer(normalPathKey, normalCacheKey); + if (uncachedValue != null || container.existsFromContainer(normalPathKey)) { addNewCacheInfo(normalPathKey, normalCacheKey, uncachedValue); } } private void addNewCacheInfo(String normalPathKey, String normalCacheKey) { - addNewCacheInfo( normalPathKey, normalCacheKey, readBackingAttributes.apply( normalPathKey, normalCacheKey)); + addNewCacheInfo( normalPathKey, normalCacheKey, container.getAttributesFromContainer( normalPathKey, normalCacheKey)); } private void addNewCacheInfo(String normalPathKey, String normalCacheKey, JsonElement uncachedAttributes) { final N5CacheInfo cacheInfo; - if (!getExists.apply(normalPathKey)) { + if (!container.existsFromContainer(normalPathKey)) { cacheInfo = emptyCacheInfo; } else { cacheInfo = new N5CacheInfo(); synchronized (cacheInfo.attributesCache) { cacheInfo.attributesCache.put(normalCacheKey, uncachedAttributes); } - cacheInfo.isGroup = getIsGroup.apply(normalPathKey); - cacheInfo.isDataset = getIsDataset.apply(normalPathKey); + cacheInfo.isGroup = container.isGroupFromContainer(normalPathKey); + cacheInfo.isDataset = container.isDatasetFromContainer(normalPathKey); addChild(cacheInfo, normalPathKey); } synchronized (containerPathToCache) { @@ -166,19 +150,19 @@ private void addNewCacheInfo(String normalPathKey, String normalCacheKey, JsonEl private void addChild(N5CacheInfo cacheInfo, String normalPathKey) { - final String[] children = listNormalChildren.apply(normalPathKey); + final String[] children = container.listFromContainer(normalPathKey); Collections.addAll(cacheInfo.children, children); } private N5CacheInfo addNewCacheInfo(String normalPathKey) { final N5CacheInfo cacheInfo; - if (!getExists.apply(normalPathKey)) { + if (!container.existsFromContainer(normalPathKey)) { cacheInfo = emptyCacheInfo; } else { cacheInfo = new N5CacheInfo(); - cacheInfo.isGroup = getIsGroup.apply(normalPathKey); - cacheInfo.isDataset = getIsDataset.apply(normalPathKey); + cacheInfo.isGroup = container.isGroupFromContainer(normalPathKey); + cacheInfo.isDataset = container.isDatasetFromContainer(normalPathKey); addChild(cacheInfo, normalPathKey); } synchronized (containerPathToCache) { @@ -193,15 +177,15 @@ public void updateCacheInfo(final String normalPathKey, final String normalCache if (cacheInfo == null ){ addNewCacheInfo(normalPathKey); return; - } else if (!getExists.apply(normalPathKey)) { + } else if (!container.existsFromContainer(normalPathKey)) { cacheInfo = emptyCacheInfo; } else { - final JsonElement attributesToCache = readBackingAttributes.apply(normalPathKey, normalCacheKey); + final JsonElement attributesToCache = container.getAttributesFromContainer(normalPathKey, normalCacheKey); synchronized (cacheInfo.attributesCache) { cacheInfo.attributesCache.put(normalCacheKey, attributesToCache); } - cacheInfo.isGroup = getIsGroup.apply(normalPathKey); - cacheInfo.isDataset = getIsDataset.apply(normalPathKey); + cacheInfo.isGroup = container.isGroupFromContainer(normalPathKey); + cacheInfo.isDataset = container.isDatasetFromContainer(normalPathKey); cacheInfo.children.clear(); addChild(cacheInfo, normalPathKey); } @@ -216,15 +200,15 @@ public void updateCacheInfo(final String normalPathKey, final String normalCache if (cacheInfo == null ){ addNewCacheInfo(normalPathKey, normalCacheKey, uncachedAttributes ); return; - } else if (!getExists.apply(normalPathKey)) { + } else if (!container.existsFromContainer(normalPathKey)) { cacheInfo = emptyCacheInfo; } else { - final JsonElement attributesToCache = uncachedAttributes == null ? readBackingAttributes.apply(normalPathKey, normalCacheKey) : uncachedAttributes; + final JsonElement attributesToCache = uncachedAttributes == null ? container.getAttributesFromContainer(normalPathKey, normalCacheKey) : uncachedAttributes; synchronized (cacheInfo.attributesCache) { cacheInfo.attributesCache.put(normalCacheKey, attributesToCache); } - cacheInfo.isGroup = getIsGroup.apply(normalPathKey); - cacheInfo.isDataset = getIsDataset.apply(normalPathKey); + cacheInfo.isGroup = container.isGroupFromContainer(normalPathKey); + cacheInfo.isDataset = container.isDatasetFromContainer(normalPathKey); cacheInfo.children.clear(); addChild(cacheInfo, normalPathKey); } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/cache/N5JsonCacheableContainer.java b/src/main/java/org/janelia/saalfeldlab/n5/cache/N5JsonCacheableContainer.java new file mode 100644 index 00000000..2179003d --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/cache/N5JsonCacheableContainer.java @@ -0,0 +1,61 @@ +package org.janelia.saalfeldlab.n5.cache; + +import com.google.gson.JsonElement; + +/** + * An N5 container whose structure and attributes can be cached. + *

    + * Implementations of interface methods must explicitly query the backing + * storage. Cached implmentations (e.g {@link CachedGsonKeyValueReader}) call + * these methods to update their {@link N5JsonCache}. Corresponding + * {@link N5Reader} methods should use the cache, if present. + */ +public interface N5JsonCacheableContainer { + + /** + * Returns a {@link JsonElement} containing attributes at a given path, + * for a given cache key. + * + * @param normalPathName the normalized path name + * @param normalCacheKey the cache key + * @return the attributes as a json element. + * @see GsonKeyValueReader#getAttributes + */ + JsonElement getAttributesFromContainer(final String normalPathName, final String normalCacheKey); + + /** + * Query whether a path exists in this container. + * + * @param normalPathName the normalized path name + * @return true if the path is a group or dataset + * @see N5Reader#exists + */ + boolean existsFromContainer(final String normalPathName); + + /** + * Query whether a path in this container is a group. + * + * @param normalPathName the normalized path name + * @return true if the path is a group + */ + boolean isGroupFromContainer(final String normalPathName); + + /** + * Query whether a path in this container is a dataset. + * + * @param normalPathName the normalized path name + * @return true if the path is a dataset + * @see N5Reader#datasetExists + */ + boolean isDatasetFromContainer(final String normalPathName); + + /** + * List the children of a path for this container. + * + * @param normalPathName the normalized path name + * @return list of children + * @see N5Reader#list + */ + String[] listFromContainer(final String normalPathName); + +} diff --git a/src/test/java/org/janelia/saalfeldlab/n5/cache/N5CacheTest.java b/src/test/java/org/janelia/saalfeldlab/n5/cache/N5CacheTest.java index b8b12b72..adec4a39 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/cache/N5CacheTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/cache/N5CacheTest.java @@ -13,8 +13,7 @@ public void cacheBackingTest() { final DummyBackingStorage backingStorage = new DummyBackingStorage(); - final N5JsonCache cache = new N5JsonCache(backingStorage::backingAttrs, backingStorage::exists, - backingStorage::isGroup, backingStorage::isDataset, backingStorage::list); + final N5JsonCache cache = new N5JsonCache(backingStorage); // check existance, ensure backing storage is only called once assertEquals(0, backingStorage.existsCallCount); @@ -86,7 +85,7 @@ public void cacheBackingTest() { } - protected static class DummyBackingStorage { + protected static class DummyBackingStorage implements N5JsonCacheableContainer { int attrCallCount = 0; int existsCallCount = 0; @@ -97,27 +96,27 @@ protected static class DummyBackingStorage { public DummyBackingStorage() { } - public JsonElement backingAttrs(final String key, final String cacheKey) { + public JsonElement getAttributesFromContainer(final String key, final String cacheKey) { attrCallCount++; return null; } - public boolean exists(final String key) { + public boolean existsFromContainer(final String key) { existsCallCount++; return true; } - public boolean isGroup(final String key) { + public boolean isGroupFromContainer(final String key) { isGroupCallCount++; return true; } - public boolean isDataset(final String key) { + public boolean isDatasetFromContainer(final String key) { isDatasetCallCount++; return true; } - public String[] list(final String key) { + public String[] listFromContainer(final String key) { listCallCount++; return new String[] { "list" }; } From 239b3d910d7a5fec3d1dcf0f377feb25c8e1c341 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Tue, 18 Apr 2023 11:19:28 -0400 Subject: [PATCH 160/243] feat: add N5JsonCache methods useful to N5Writers * avoid reading immediately after writing to update cache --- .../n5/CachedGsonKeyValueWriter.java | 30 ++++++++++++++-- .../saalfeldlab/n5/cache/N5JsonCache.java | 34 +++++++++++++++++++ 2 files changed, 62 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueWriter.java b/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueWriter.java index 95026b84..5a9468aa 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueWriter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueWriter.java @@ -61,8 +61,8 @@ default void setVersion(final String path) throws IOException { * @param normalPath the group path. */ default void initializeGroup(final String normalPath) { - // Nothing to do here, but other implementations (e.g. zarr) use this. + // TODO remove if not used by zarr } @Override @@ -76,6 +76,8 @@ default void createGroup(final String path) throws IOException { String parent = N5URL.normalizeGroupPath("/"); for (String child : pathParts) { final String childPath = getKeyValueAccess().compose(parent, child); + // add the group to the cache + getCache().addNewCacheInfo(childPath, N5JsonCache.jsonFile, null, true, false); getCache().addChild(parent, child); parent = childPath; } @@ -84,6 +86,30 @@ default void createGroup(final String path) throws IOException { initializeGroup(normalPath); } + /** + * Creates a dataset. This does not create any data but the path and + * mandatory attributes only. + * + * @param pathName dataset path + * @param datasetAttributes the dataset attributes + * @throws IOException the exception + */ + @Override + default void createDataset( + final String pathName, + final DatasetAttributes datasetAttributes) throws IOException { + + // N5Writer.super.createDataset(pathName, datasetAttributes); + /* the three lines below duplicate the single line above but would have to call + * normalizeGroupPath again the below duplicates code, but avoids extra work */ + final String normalPath = N5URL.normalizeGroupPath(pathName); + createGroup(normalPath); + setDatasetAttributes(normalPath, datasetAttributes); + + if (cacheMeta()) + getCache().setIsDataset(normalPath, true); + } + /** * Helper method that reads an existing JsonElement representing the root * attributes for {@code normalGroupPath}, inserts and overrides the @@ -144,7 +170,7 @@ default void writeAndCacheAttributes(final String normalGroupPath, final JsonEle nullRespectingAttributes = getGson().toJsonTree(attributes); } /* Update the cache, and write to the writer */ - getCache().updateCacheInfo(normalGroupPath, N5JsonCache.jsonFile, nullRespectingAttributes); + getCache().setAttributes(normalGroupPath, N5JsonCache.jsonFile, nullRespectingAttributes); } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/cache/N5JsonCache.java b/src/main/java/org/janelia/saalfeldlab/n5/cache/N5JsonCache.java index d4aeabfa..311e8348 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/cache/N5JsonCache.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/cache/N5JsonCache.java @@ -148,6 +148,20 @@ private void addNewCacheInfo(String normalPathKey, String normalCacheKey, JsonEl } } + public void addNewCacheInfo(String normalPathKey, String normalCacheKey, JsonElement uncachedAttributes, boolean isGroup, boolean isDataset) { + + final N5CacheInfo cacheInfo = new N5CacheInfo(); + synchronized (cacheInfo.attributesCache) { + cacheInfo.attributesCache.put(normalCacheKey, uncachedAttributes); + } + cacheInfo.isGroup = isGroup; + cacheInfo.isDataset = isDataset; + addChild(cacheInfo, normalPathKey); + synchronized (containerPathToCache) { + containerPathToCache.put(normalPathKey, cacheInfo); + } + } + private void addChild(N5CacheInfo cacheInfo, String normalPathKey) { final String[] children = container.listFromContainer(normalPathKey); @@ -218,6 +232,26 @@ public void updateCacheInfo(final String normalPathKey, final String normalCache } } + public void setIsDataset(final String normalPathKey, final boolean isDataset ) { + final N5CacheInfo cacheInfo = getCacheInfo(normalPathKey); + if (cacheInfo == null ){ + return; + } + synchronized (cacheInfo) { + cacheInfo.isDataset = isDataset; + } + } + + public void setAttributes(final String normalPathKey, final String normalCacheKey, final JsonElement attributes ) { + final N5CacheInfo cacheInfo = getCacheInfo(normalPathKey); + if (cacheInfo == null ){ + return; + } + synchronized (cacheInfo.attributesCache) { + cacheInfo.attributesCache.put(normalCacheKey, attributes); + } + } + public void addChild(final String parent, final String child) { final N5CacheInfo cacheInfo = getCacheInfo(parent); From df285ee43b60bc890fd1937a7c8e4a4666ffca4e Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Tue, 18 Apr 2023 11:26:11 -0400 Subject: [PATCH 161/243] refactor: use new interfaces * remove temporary 'WithInterfaces' classes --- .../janelia/saalfeldlab/n5/N5FSReader.java | 2 +- .../janelia/saalfeldlab/n5/N5FSWriter.java | 2 +- .../saalfeldlab/n5/N5KeyValueReader.java | 365 ++---------------- .../n5/N5KeyValueReaderWithInterfaces.java | 154 -------- .../saalfeldlab/n5/N5KeyValueWriter.java | 268 +------------ .../n5/N5KeyValueWriterWithInterfaces.java | 71 ---- .../saalfeldlab/n5/AbstractN5Test.java | 4 +- .../saalfeldlab/n5/N5CachedFSTest.java | 8 +- .../n5/N5KeyValueWithInterfacesTest.java | 4 +- 9 files changed, 61 insertions(+), 817 deletions(-) delete mode 100644 src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReaderWithInterfaces.java delete mode 100644 src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriterWithInterfaces.java diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5FSReader.java b/src/main/java/org/janelia/saalfeldlab/n5/N5FSReader.java index 2817f221..93311358 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5FSReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5FSReader.java @@ -40,7 +40,7 @@ * @author Philipp Hanslovsky */ //public class N5FSReader extends N5KeyValueReader { -public class N5FSReader extends N5KeyValueReaderWithInterfaces { +public class N5FSReader extends N5KeyValueReader { /** * Opens an {@link N5FSReader} at a given base path with a custom diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java b/src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java index 8cbd8acc..e53db44c 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java @@ -36,7 +36,7 @@ * @author Stephan Saalfeld */ //public class N5FSWriter extends N5KeyValueWriter implements N5Writer { -public class N5FSWriter extends N5KeyValueWriterWithInterfaces { +public class N5FSWriter extends N5KeyValueWriter { /** * Opens an {@link N5FSWriter} at a given base path with a custom diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java index 9cdcb118..d866f009 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java @@ -25,18 +25,13 @@ */ package org.janelia.saalfeldlab.n5; -import java.io.IOException; -import java.lang.reflect.Type; -import java.util.Arrays; -import java.util.Map; -import java.util.stream.Stream; - import com.google.gson.Gson; import com.google.gson.GsonBuilder; -import com.google.gson.JsonElement; -import com.google.gson.JsonObject; import org.janelia.saalfeldlab.n5.cache.N5JsonCache; -import org.janelia.saalfeldlab.n5.cache.N5JsonCacheableContainer; + +import java.io.IOException; +import java.util.Arrays; +import java.util.stream.Stream; /** * {@link N5Reader} implementation through {@link KeyValueAccess} with JSON @@ -46,26 +41,23 @@ * @author Igor Pisarev * @author Philipp Hanslovsky */ -public class N5KeyValueReader implements N5Reader, N5JsonCacheableContainer { +public class N5KeyValueReader implements CachedGsonKeyValueReader { protected final KeyValueAccess keyValueAccess; protected final Gson gson; - - protected final N5JsonCache cache = new N5JsonCache(this); - protected final boolean cacheMeta; protected final String basePath; + private final N5JsonCache cache; /** * Opens an {@link N5KeyValueReader} at a given base path with a custom - * {@link GsonBuilder} to support custom attributes. Storage is managed by - * the given {@link KeyValueAccess}. + * {@link GsonBuilder} to support custom attributes. * - * @param keyValueAccess the key value access + * @param keyValueAccess * @param basePath N5 base path - * @param gsonBuilder the gson builder + * @param gsonBuilder * @param cacheMeta cache attributes and meta data * Setting this to true avoids frequent reading and parsing of JSON * encoded attributes and other meta data that requires accessing the @@ -84,9 +76,29 @@ public N5KeyValueReader( final GsonBuilder gsonBuilder, final boolean cacheMeta) throws IOException { - this(true, keyValueAccess, basePath, gsonBuilder, cacheMeta); + this( true, keyValueAccess, basePath, gsonBuilder, cacheMeta ); } + /** + * Opens an {@link N5KeyValueReader} at a given base path with a custom + * {@link GsonBuilder} to support custom attributes. + * + * @param checkVersion do the version check + * @param keyValueAccess + * @param basePath N5 base path + * @param gsonBuilder + * @param cacheMeta cache attributes and meta data + * Setting this to true avoids frequent reading and parsing of JSON + * encoded attributes and other meta data that requires accessing the + * store. This is most interesting for high latency backends. Changes + * of cached attributes and meta data by an independent writer will + * not be tracked. + * + * @throws IOException + * if the base path cannot be read or does not exist, + * if the N5 version of the container is not compatible with this + * implementation. + */ protected N5KeyValueReader( final boolean checkVersion, final KeyValueAccess keyValueAccess, @@ -98,8 +110,9 @@ protected N5KeyValueReader( this.basePath = keyValueAccess.normalize(basePath); this.gson = GsonUtils.registerGson(gsonBuilder); this.cacheMeta = cacheMeta; + this.cache = newCache(); - if( checkVersion ) { + if (checkVersion) { /* Existence checks, if any, go in subclasses */ /* Check that version (if there is one) is compatible. */ final Version version = getVersion(); @@ -112,13 +125,6 @@ public Gson getGson() { return gson; } - - @Override - public Map> listAttributes(final String pathName) throws IOException { - - return GsonUtils.listAttributes(getAttributes(pathName)); - } - public KeyValueAccess getKeyValueAccess() { return keyValueAccess; } @@ -130,318 +136,19 @@ public String getBasePath() { } @Override - public DatasetAttributes getDatasetAttributes(final String pathName) throws IOException { - - final String normalPath = N5URL.normalizeGroupPath(pathName); - final JsonElement attributes; - if (cacheMeta && cache.isDataset(normalPath, N5JsonCache.jsonFile)) { - attributes = cache.getAttributes(normalPath, N5JsonCache.jsonFile); - } else { - attributes = getAttributes(normalPath); - } + public boolean cacheMeta() { - return createDatasetAttributes(attributes); - } - - private DatasetAttributes normalGetDatasetAttributes(final String pathName) throws N5Exception.N5IOException { - - final String normalPath = N5URL.normalizeGroupPath(pathName); - final JsonElement attributes = getAttributesFromContainer(normalPath); - return createDatasetAttributes(attributes); - } - - private DatasetAttributes createDatasetAttributes(JsonElement attributes) { - - final long[] dimensions = GsonUtils.readAttribute(attributes, DatasetAttributes.DIMENSIONS_KEY, long[].class, gson); - if (dimensions == null) { - return null; - } - - final DataType dataType = GsonUtils.readAttribute(attributes, DatasetAttributes.DATA_TYPE_KEY, DataType.class, gson); - if (dataType == null) { - return null; - } - - final int[] blockSize = GsonUtils.readAttribute(attributes, DatasetAttributes.BLOCK_SIZE_KEY, int[].class, gson); - - final Compression compression = GsonUtils.readAttribute(attributes, DatasetAttributes.COMPRESSION_KEY, Compression.class, gson); - - /* version 0 */ - final String compressionVersion0Name = compression - == null - ? GsonUtils.readAttribute(attributes, DatasetAttributes.compressionTypeKey, String.class, gson) - : null; - - return createDatasetAttributes(dimensions, dataType, blockSize, compression, compressionVersion0Name); - } - - @Override - public T getAttribute( - final String pathName, - final String key, - final Class clazz) throws IOException { - - final String normalPathName = N5URL.normalizeGroupPath(pathName); - final String normalizedAttributePath = N5URL.normalizeAttributePath(key); - - final JsonElement attributes; - if (cacheMeta) { - attributes = cache.getAttributes(normalPathName, N5JsonCache.jsonFile); - } else { - attributes = getAttributes(normalPathName); - } - return GsonUtils.readAttribute(attributes, normalizedAttributePath, clazz, gson); - } - - @Override - public T getAttribute( - final String pathName, - final String key, - final Type type) throws IOException { - - final String normalPathName = N5URL.normalizeGroupPath(pathName); - final String normalizedAttributePath = N5URL.normalizeAttributePath(key); - JsonElement attributes; - if (cacheMeta) { - attributes = cache.getAttributes(normalPathName, N5JsonCache.jsonFile); - } else { - attributes = getAttributes(normalPathName); - } - return GsonUtils.readAttribute(attributes, normalizedAttributePath, type, gson); - } - - public boolean isGroupFromContainer(final String absoluteNormalPath) { - - return keyValueAccess.exists(absoluteNormalPath) && keyValueAccess.isDirectory(absoluteNormalPath); + return cacheMeta; } @Override - public boolean exists(final String pathName) { - - final String normalPathName = N5URL.normalizeGroupPath(pathName); - if (cacheMeta) - return cache.exists(normalPathName, N5JsonCache.jsonFile); - else { - return existsFromContainer(normalPathName); - } - } - - public boolean existsFromContainer(String normalPathName) { + public N5JsonCache getCache() { - return keyValueAccess.exists(groupPath(normalPathName)); - } - - @Override - public boolean datasetExists(final String pathName) throws N5Exception.N5IOException { - - if (cacheMeta) { - final String normalPathName = N5URL.normalizeGroupPath(pathName); - return cache.isDataset(normalPathName, N5JsonCache.jsonFile); - } - return isDatasetFromContainer(pathName); - } - - public boolean isDatasetFromContainer(String pathName) throws N5Exception.N5IOException { - - // for n5, every dataset must be a group - return isGroupFromContainer(groupPath(pathName)) && normalGetDatasetAttributes(pathName) != null; - } - - /** - * Reads or creates the attributes map of a group or dataset. - * - * @param pathName group path - * @return the json element - * @throws IOException the exception - */ - public JsonElement getAttributes(final String pathName) throws IOException { - - final String groupPath = N5URL.normalizeGroupPath(pathName); - - /* If cached, return the cache*/ - if (cacheMeta) { - return cache.getAttributes(groupPath, N5JsonCache.jsonFile); - } else { - return getAttributesFromContainer(groupPath); - } - - } - - public JsonElement getAttributesFromContainer(String groupPath ) throws N5Exception.N5IOException { - - return getAttributesFromContainer(groupPath, N5JsonCache.jsonFile); - } - - public JsonElement getAttributesFromContainer(String groupPath, String cachePath) throws N5Exception.N5IOException { - - final String attributesPath = attributesPath(groupPath); - if (!existsFromContainer(groupPath)) - throw new N5Exception.N5IOException("Group " + groupPath + " does not exist"); - if (!keyValueAccess.exists(attributesPath)) - return null; - - try (final LockedChannel lockedChannel = keyValueAccess.lockForReading(attributesPath)) { - return GsonUtils.readAttributes(lockedChannel.newReader(), gson); - } catch (IOException e) { - throw new N5Exception.N5IOException("Cannot open lock for Reading", e); - } - } - - @Override - public DataBlock readBlock( - final String pathName, - final DatasetAttributes datasetAttributes, - final long... gridPosition) throws IOException { - - final String path = getDataBlockPath(N5URL.normalizeGroupPath(pathName), gridPosition); - if (!keyValueAccess.exists(path)) - return null; - - try (final LockedChannel lockedChannel = keyValueAccess.lockForReading(path)) { - return DefaultBlockReader.readBlock(lockedChannel.newInputStream(), datasetAttributes, gridPosition); - } - } - - /** - * @param normalPath normalized path name - * @return the normalized groups in {@code normalPath} - * @throws N5Exception.N5IOException the exception - */ - public String[] listFromContainer(final String normalPath) throws N5Exception.N5IOException { - - try { - return keyValueAccess.listDirectories(groupPath(normalPath)); - } catch (IOException e) { - throw new N5Exception.N5IOException("Cannot list directories for group " + normalPath, e); - } - } - - @Override - public String[] list(final String pathName) throws IOException { - - final String normalPath = N5URL.normalizeGroupPath(pathName); - if (cacheMeta) { - return cache.list(normalPath); - } else { - return listFromContainer(normalPath); - } - } - - - /** - * Constructs the path for a data block in a dataset at a given grid position. - *

    - * The returned path is - * 

    -	 * $basePath/datasetPathName/$gridPosition[0]/$gridPosition[1]/.../$gridPosition[n]
-	 *
    - *

    - * This is the file into which the data block will be stored. - * - * @param normalPath normalized dataset path - * @param gridPosition the grid position - * @return the block path - */ - protected String getDataBlockPath( - final String normalPath, - final long... gridPosition) { - - final String[] components = new String[gridPosition.length + 2]; - components[0] = basePath; - components[1] = normalPath; - int i = 1; - for (final long p : gridPosition) - components[++i] = Long.toString(p); - - return keyValueAccess.compose(components); - } - - /** - * Constructs the absolute path (in terms of this store) for the group or - * dataset. - * - * @param normalGroupPath normalized group path without leading slash - * @return the absolute path to the group - */ - protected String groupPath(final String normalGroupPath) { - - return keyValueAccess.compose(basePath, normalGroupPath); + return this.cache; } @Override public String groupPath(String... nodes) { - - // alternatively call compose twice, once with this functions inputs, then pass the result to the other groupPath method - // this impl assumes streams and array building are less expensive than keyValueAccess composition (may not always be true) return keyValueAccess.compose(Stream.concat(Stream.of(basePath), Arrays.stream(nodes)).toArray(String[]::new)); } - - /** - * Constructs the absolute path (in terms of this store) for the attributes - * file of a group or dataset. - * - * @param normalPath normalized group path without leading slash - * @return the absolute path to the attributes - */ - protected String attributesPath(final String normalPath) { - - return keyValueAccess.compose(basePath, normalPath, N5JsonCache.jsonFile); - } - - @Override - public String toString() { - - return String.format("%s[access=%s, basePath=%s]", getClass().getSimpleName(), keyValueAccess, basePath); - } - - private static DatasetAttributes createDatasetAttributes( - final long[] dimensions, - final DataType dataType, - int[] blockSize, - Compression compression, - final String compressionVersion0Name - ) { - - if (blockSize == null) - blockSize = Arrays.stream(dimensions).mapToInt(a -> (int)a).toArray(); - - /* version 0 */ - if (compression == null) { - switch (compressionVersion0Name) { - case "raw": - compression = new RawCompression(); - break; - case "gzip": - compression = new GzipCompression(); - break; - case "bzip2": - compression = new Bzip2Compression(); - break; - case "lz4": - compression = new Lz4Compression(); - break; - case "xz": - compression = new XzCompression(); - break; - } - } - - return new DatasetAttributes(dimensions, blockSize, dataType, compression); - } - - /** - * Check for attributes that are required for a group to be a dataset. - * - * @param attributes to check for dataset attributes - * @return if {@link DatasetAttributes#DIMENSIONS_KEY} and {@link DatasetAttributes#DATA_TYPE_KEY} are present - */ - protected static boolean hasDatasetAttributes(final JsonElement attributes) { - - if (attributes == null || !attributes.isJsonObject()) { - return false; - } - - final JsonObject metadataCache = attributes.getAsJsonObject(); - return metadataCache.has(DatasetAttributes.DIMENSIONS_KEY) && metadataCache.has(DatasetAttributes.DATA_TYPE_KEY); - } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReaderWithInterfaces.java b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReaderWithInterfaces.java deleted file mode 100644 index a8ee9d1b..00000000 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReaderWithInterfaces.java +++ /dev/null @@ -1,154 +0,0 @@ -/** - * Copyright (c) 2017--2021, Stephan Saalfeld - * All rights reserved. - *

    - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - *

    - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - *

    - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ -package org.janelia.saalfeldlab.n5; - -import com.google.gson.Gson; -import com.google.gson.GsonBuilder; -import org.janelia.saalfeldlab.n5.cache.N5JsonCache; - -import java.io.IOException; -import java.util.Arrays; -import java.util.stream.Stream; - -/** - * {@link N5Reader} implementation through {@link KeyValueAccess} with JSON - * attributes parsed with {@link Gson}. - * - * @author Stephan Saalfeld - * @author Igor Pisarev - * @author Philipp Hanslovsky - */ -public class N5KeyValueReaderWithInterfaces implements CachedGsonKeyValueReader { - - protected final KeyValueAccess keyValueAccess; - - protected final Gson gson; - protected final boolean cacheMeta; - - protected final String basePath; - private final N5JsonCache cache; - - /** - * Opens an {@link N5KeyValueReaderWithInterfaces} at a given base path with a custom - * {@link GsonBuilder} to support custom attributes. - * - * @param keyValueAccess - * @param basePath N5 base path - * @param gsonBuilder - * @param cacheMeta cache attributes and meta data - * Setting this to true avoids frequent reading and parsing of JSON - * encoded attributes and other meta data that requires accessing the - * store. This is most interesting for high latency backends. Changes - * of cached attributes and meta data by an independent writer will - * not be tracked. - * - * @throws IOException - * if the base path cannot be read or does not exist, - * if the N5 version of the container is not compatible with this - * implementation. - */ - public N5KeyValueReaderWithInterfaces( - final KeyValueAccess keyValueAccess, - final String basePath, - final GsonBuilder gsonBuilder, - final boolean cacheMeta) throws IOException { - - this( true, keyValueAccess, basePath, gsonBuilder, cacheMeta ); - } - - /** - * Opens an {@link N5KeyValueReaderWithInterfaces} at a given base path with a custom - * {@link GsonBuilder} to support custom attributes. - * - * @param checkVersion do the version check - * @param keyValueAccess - * @param basePath N5 base path - * @param gsonBuilder - * @param cacheMeta cache attributes and meta data - * Setting this to true avoids frequent reading and parsing of JSON - * encoded attributes and other meta data that requires accessing the - * store. This is most interesting for high latency backends. Changes - * of cached attributes and meta data by an independent writer will - * not be tracked. - * - * @throws IOException - * if the base path cannot be read or does not exist, - * if the N5 version of the container is not compatible with this - * implementation. - */ - protected N5KeyValueReaderWithInterfaces( - final boolean checkVersion, - final KeyValueAccess keyValueAccess, - final String basePath, - final GsonBuilder gsonBuilder, - final boolean cacheMeta) throws IOException { - - this.keyValueAccess = keyValueAccess; - this.basePath = keyValueAccess.normalize(basePath); - this.gson = GsonUtils.registerGson(gsonBuilder); - this.cacheMeta = cacheMeta; - this.cache = newCache(); - - if (checkVersion) { - /* Existence checks, if any, go in subclasses */ - /* Check that version (if there is one) is compatible. */ - final Version version = getVersion(); - if (!VERSION.isCompatible(version)) - throw new N5Exception.N5IOException("Incompatible version " + version + " (this is " + VERSION + ")."); - } - } - - public Gson getGson() { - - return gson; - } - public KeyValueAccess getKeyValueAccess() { - return keyValueAccess; - } - - @Override - public String getBasePath() { - - return this.basePath; - } - - @Override - public boolean cacheMeta() { - - return cacheMeta; - } - - @Override - public N5JsonCache getCache() { - - return this.cache; - } - - @Override - public String groupPath(String... nodes) { - return keyValueAccess.compose(Stream.concat(Stream.of(basePath), Arrays.stream(nodes)).toArray(String[]::new)); - } -} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriter.java b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriter.java index 7ceb5714..01f18a82 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriter.java @@ -25,31 +25,20 @@ */ package org.janelia.saalfeldlab.n5; -import java.io.IOException; -import java.util.Arrays; -import java.util.List; -import java.util.Map; - -import com.google.gson.Gson; import com.google.gson.GsonBuilder; -import com.google.gson.JsonElement; -import com.google.gson.JsonNull; -import com.google.gson.JsonObject; -import org.janelia.saalfeldlab.n5.cache.N5JsonCache; +import java.io.IOException; /** - * {@link N5Writer} implementation through a {@link KeyValueAccess} with version compatibility check. - * JSON attributes are parsed and written with {@link Gson}. + * Filesystem {@link N5Writer} implementation with version compatibility check. * * @author Stephan Saalfeld */ -public class N5KeyValueWriter extends N5KeyValueReader implements N5Writer { +public class N5KeyValueWriter extends N5KeyValueReader implements CachedGsonKeyValueWriter { /** * Opens an {@link N5KeyValueWriter} at a given base path with a custom - * {@link GsonBuilder} to support custom attributes. Storage is managed by - * the given {@link KeyValueAccess}. + * {@link GsonBuilder} to support custom attributes. *

    * If the base path does not exist, it will be created. *

    @@ -57,20 +46,16 @@ public class N5KeyValueWriter extends N5KeyValueReader implements N5Writer { * compatible with this implementation, the N5 version of this container * will be set to the current N5 version of this implementation. * - * @param keyValueAccess the key value access - * @param basePath - * n5 base path - * @param gsonBuilder the gson builder - * @param cacheAttributes - * Setting this to true avoids frequent reading and parsing of - * JSON encoded attributes, this is most interesting for high - * latency file systems. Changes of attributes by an independent - * writer will not be tracked. - * - * @throws IOException - * if the base path cannot be written to or cannot be created, - * if the N5 version of the container is not compatible with - * this implementation. + * @param keyValueAccess + * @param basePath n5 base path + * @param gsonBuilder + * @param cacheAttributes Setting this to true avoids frequent reading and parsing of + * JSON encoded attributes, this is most interesting for high + * latency file systems. Changes of attributes by an independent + * writer will not be tracked. + * @throws IOException if the base path cannot be written to or cannot be created, + * if the N5 version of the container is not compatible with + * this implementation. */ public N5KeyValueWriter( final KeyValueAccess keyValueAccess, @@ -83,229 +68,4 @@ public N5KeyValueWriter( createGroup("/"); setVersion("/"); } - - protected void setVersion(final String path) throws IOException { - - final Version version = getVersion(); - if (!VERSION.isCompatible(version)) - throw new N5Exception.N5IOException("Incompatible version " + version + " (this is " + VERSION + ")."); - - if (!VERSION.equals(version)) - setAttribute("/", VERSION_KEY, VERSION.toString()); - } - - /** - * Performs any necessary initialization to ensure the key given by the - * argument {@code normalPath} is a valid group after creation. Called by - * {@link #createGroup(String)}. - * - * @param normalPath the group path. - */ - protected void initializeGroup(final String normalPath) { - - // Nothing to do here, but other implementations (e.g. zarr) use this. - } - - @Override - public void createGroup(final String path) throws IOException { - - final String normalPath = N5URL.normalizeGroupPath(path); - keyValueAccess.createDirectories(groupPath(normalPath)); - if (cacheMeta) { - final String[] pathParts = keyValueAccess.components(normalPath); - if (pathParts.length > 1) { - String parent = N5URL.normalizeGroupPath("/"); - for (String child : pathParts) { - final String childPath = keyValueAccess.compose(parent, child); - cache.addChild(parent, child); - parent = childPath; - } - } - } - initializeGroup(normalPath); - } - - @Override - public void createDataset( - final String path, - final DatasetAttributes datasetAttributes) throws IOException { - - final String normalPath = N5URL.normalizeGroupPath(path); - createGroup(path); - setDatasetAttributes(normalPath, datasetAttributes); - } - - /** - * Helper method that reads an existing JsonElement representing the root - * attributes for {@code normalGroupPath}, inserts and overrides the - * provided attributes, and writes them back into the attributes store. - * - * @param normalGroupPath to write the attributes to - * @param attributes to write - * @throws IOException if unable to read the attributes at {@code normalGroupPath} - *

    - * TODO consider cache (or you read the attributes twice?) - */ - protected void writeAttributes( - final String normalGroupPath, - final JsonElement attributes) throws IOException { - - JsonElement root = getAttributes(normalGroupPath); - root = GsonUtils.insertAttribute(root, "/", attributes, gson); - writeAndCacheAttributes(normalGroupPath, root); - } - - /** - * Helper method that reads the existing map of attributes, JSON encodes, - * inserts and overrides the provided attributes, and writes them back into - * the attributes store. - * - * @param normalGroupPath to write the attributes to - * @param attributes to write - * @throws IOException if unable to read the attributes at {@code normalGroupPath} - */ - protected void writeAttributes( - final String normalGroupPath, - final Map attributes) throws IOException { - - if (attributes != null && !attributes.isEmpty()) { - JsonElement existingAttributes = getAttributes(normalGroupPath); - existingAttributes = existingAttributes != null && existingAttributes.isJsonObject() - ? existingAttributes.getAsJsonObject() - : new JsonObject(); - JsonElement newAttributes = GsonUtils.insertAttributes(existingAttributes, attributes, gson); - writeAndCacheAttributes(normalGroupPath, newAttributes); - } - } - - private void writeAndCacheAttributes(final String normalGroupPath, final JsonElement attributes) throws IOException { - - try (final LockedChannel lock = keyValueAccess.lockForWriting(attributesPath(normalGroupPath))) { - GsonUtils.writeAttributes(lock.newWriter(), attributes, gson); - } - if (cacheMeta) { - JsonElement nullRespectingAttributes = attributes; - /* Gson only filters out nulls when you write the JsonElement. This means it doesn't filter them out when caching. - * To handle this, we explicitly writer the existing JsonElement to a new JsonElement. - * The output is identical to the input if: - * - serializeNulls is true - * - no null values are present - * - cacheing is turned off */ - if (!gson.serializeNulls()) { - nullRespectingAttributes = gson.toJsonTree(attributes); - } - /* Update the cache, and write to the writer */ - cache.updateCacheInfo(normalGroupPath, N5JsonCache.jsonFile, nullRespectingAttributes); - } - } - - @Override - public void setAttributes( - final String path, - final Map attributes) throws IOException { - - final String normalPath = N5URL.normalizeGroupPath(path); - if (!exists(normalPath)) - throw new N5Exception.N5IOException("" + normalPath + " is not a group or dataset."); - - writeAttributes(normalPath, attributes); - } - - @Override - public boolean removeAttribute(final String pathName, final String key) throws IOException { - - final String normalPath = N5URL.normalizeGroupPath(pathName); - final String absoluteNormalPath = keyValueAccess.compose(basePath, normalPath); - final String normalKey = N5URL.normalizeAttributePath(key); - - if (!keyValueAccess.exists(absoluteNormalPath)) - return false; - - if (key.equals("/")) { - writeAttributes(normalPath, JsonNull.INSTANCE); - return true; - } - - final JsonElement attributes = getAttributes(normalPath); - if (GsonUtils.removeAttribute(attributes, normalKey) != null) { - writeAttributes(normalPath, attributes); - return true; - } - return false; - } - - @Override - public T removeAttribute(final String pathName, final String key, final Class cls) throws IOException { - - final String normalPath = N5URL.normalizeGroupPath(pathName); - final String normalKey = N5URL.normalizeAttributePath(key); - - final JsonElement attributes = getAttributes(normalPath); - final T obj = GsonUtils.removeAttribute(attributes, normalKey, cls, gson); - if (obj != null) { - writeAttributes(normalPath, attributes); - } - return obj; - } - - @Override - public boolean removeAttributes(final String pathName, final List attributes) throws IOException { - - final String normalPath = N5URL.normalizeGroupPath(pathName); - boolean removed = false; - for (final String attribute : attributes) { - final String normalKey = N5URL.normalizeAttributePath(attribute); - removed |= removeAttribute(normalPath, attribute); - } - return removed; - } - - @Override - public void writeBlock( - final String path, - final DatasetAttributes datasetAttributes, - final DataBlock dataBlock) throws IOException { - - final String blockPath = getDataBlockPath(N5URL.normalizeGroupPath(path), dataBlock.getGridPosition()); - try (final LockedChannel lock = keyValueAccess.lockForWriting(blockPath)) { - - DefaultBlockWriter.writeBlock(lock.newOutputStream(), datasetAttributes, dataBlock); - } - } - - @Override - public boolean remove(final String path) throws IOException { - - final String normalPath = N5URL.normalizeGroupPath(path); - final String groupPath = groupPath(normalPath); - if (keyValueAccess.exists(groupPath)) - keyValueAccess.delete(groupPath); - if (cacheMeta) { - final String[] pathParts = keyValueAccess.components(normalPath); - final String parent; - if (pathParts.length <= 1) { - parent = N5URL.normalizeGroupPath("/"); - } else { - final int parentPathLength = pathParts.length - 1; - parent = keyValueAccess.compose(Arrays.copyOf(pathParts, parentPathLength)); - } - cache.removeCache(parent, normalPath); - } - - /* an IOException should have occurred if anything had failed midway */ - return true; - } - - @Override - public boolean deleteBlock( - final String path, - final long... gridPosition) throws IOException { - - final String blockPath = getDataBlockPath(N5URL.normalizeGroupPath(path), gridPosition); - if (keyValueAccess.exists(blockPath)) - keyValueAccess.delete(blockPath); - - /* an IOException should have occurred if anything had failed midway */ - return true; - } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriterWithInterfaces.java b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriterWithInterfaces.java deleted file mode 100644 index 8fcf6ab2..00000000 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriterWithInterfaces.java +++ /dev/null @@ -1,71 +0,0 @@ -/** - * Copyright (c) 2017--2021, Stephan Saalfeld - * All rights reserved. - *

    - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - *

    - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - *

    - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ -package org.janelia.saalfeldlab.n5; - -import com.google.gson.GsonBuilder; - -import java.io.IOException; - -/** - * Filesystem {@link N5Writer} implementation with version compatibility check. - * - * @author Stephan Saalfeld - */ -public class N5KeyValueWriterWithInterfaces extends N5KeyValueReaderWithInterfaces implements CachedGsonKeyValueWriter { - - /** - * Opens an {@link N5KeyValueWriterWithInterfaces} at a given base path with a custom - * {@link GsonBuilder} to support custom attributes. - *

    - * If the base path does not exist, it will be created. - *

    - * If the base path exists and if the N5 version of the container is - * compatible with this implementation, the N5 version of this container - * will be set to the current N5 version of this implementation. - * - * @param keyValueAccess - * @param basePath n5 base path - * @param gsonBuilder - * @param cacheAttributes Setting this to true avoids frequent reading and parsing of - * JSON encoded attributes, this is most interesting for high - * latency file systems. Changes of attributes by an independent - * writer will not be tracked. - * @throws IOException if the base path cannot be written to or cannot be created, - * if the N5 version of the container is not compatible with - * this implementation. - */ - public N5KeyValueWriterWithInterfaces( - final KeyValueAccess keyValueAccess, - final String basePath, - final GsonBuilder gsonBuilder, - final boolean cacheAttributes) - throws IOException { - - super(false, keyValueAccess, basePath, gsonBuilder, cacheAttributes); - createGroup("/"); - setVersion("/"); - } -} diff --git a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java index e0a29f89..72c93789 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java @@ -173,7 +173,9 @@ public void setUpOnce() throws IOException { public static void rampDownAfterClass() throws IOException { if (n5 != null) { - Assert.assertTrue(n5.remove()); + try { + Assert.assertTrue(n5.remove()); + } catch ( Exception e ) {} n5 = null; } } diff --git a/src/test/java/org/janelia/saalfeldlab/n5/N5CachedFSTest.java b/src/test/java/org/janelia/saalfeldlab/n5/N5CachedFSTest.java index 1b25e5bd..396f82a7 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/N5CachedFSTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/N5CachedFSTest.java @@ -54,7 +54,7 @@ public void cacheTest() throws IOException { * The only possible way for the test to succeed is if it never again attempts to read the file, and relies on the cache. */ // try (N5KeyValueWriter n5 = (N5KeyValueWriter) createN5Writer()) { - try (N5KeyValueWriterWithInterfaces n5 = (N5KeyValueWriterWithInterfaces) createN5Writer()) { + try (N5KeyValueWriter n5 = (N5KeyValueWriter) createN5Writer()) { final String cachedGroup = "cachedGroup"; final String attributesPath = n5.attributesPath(cachedGroup); @@ -70,7 +70,7 @@ public void cacheTest() throws IOException { } // try (N5KeyValueWriter n5 = new N5FSWriter(tempN5PathName(), false)) { - try (N5KeyValueWriterWithInterfaces n5 = (N5KeyValueWriterWithInterfaces)createN5Writer(tempN5PathName(), false)) { + try (N5KeyValueWriter n5 = (N5KeyValueWriter)createN5Writer(tempN5PathName(), false)) { final String cachedGroup = "cachedGroup"; final String attributesPath = n5.attributesPath(cachedGroup); @@ -92,8 +92,8 @@ public void cacheGroupDatasetTest() throws IOException { final String groupName = "gg"; final String tmpPath = tempN5PathName(); - try (N5KeyValueWriterWithInterfaces w1 = (N5KeyValueWriterWithInterfaces) createN5Writer(tmpPath); - N5KeyValueWriterWithInterfaces w2 = (N5KeyValueWriterWithInterfaces) createN5Writer(tmpPath);) { + try (N5KeyValueWriter w1 = (N5KeyValueWriter) createN5Writer(tmpPath); + N5KeyValueWriter w2 = (N5KeyValueWriter) createN5Writer(tmpPath);) { // create a group, both writers know it exists w1.createGroup(groupName); diff --git a/src/test/java/org/janelia/saalfeldlab/n5/N5KeyValueWithInterfacesTest.java b/src/test/java/org/janelia/saalfeldlab/n5/N5KeyValueWithInterfacesTest.java index 8cef539c..43a7e0d3 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/N5KeyValueWithInterfacesTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/N5KeyValueWithInterfacesTest.java @@ -73,7 +73,7 @@ public static void cleanup() { */ @Override protected N5Writer createN5Writer() throws IOException { - return new N5KeyValueWriterWithInterfaces(access, tempN5PathName(), new GsonBuilder(), true); + return new N5KeyValueWriter(access, tempN5PathName(), new GsonBuilder(), true); } @Override @@ -81,7 +81,7 @@ protected N5Writer createN5Writer(String location, GsonBuilder gson) throws IOEx if (!new File(location).exists()) { tmpFiles.add(location); } - return new N5KeyValueWriterWithInterfaces(access, location, gson, true); + return new N5KeyValueWriter(access, location, gson, true); } @Override From 07945bd763607f4aa2a0fa46767dbf7d81372d41 Mon Sep 17 00:00:00 2001 From: Stephan Saalfeld Date: Fri, 21 Apr 2023 15:34:03 -0400 Subject: [PATCH 162/243] fix: remove superfluous version --- pom.xml | 1 - 1 file changed, 1 deletion(-) diff --git a/pom.xml b/pom.xml index 843a8f85..a9e818da 100644 --- a/pom.xml +++ b/pom.xml @@ -150,7 +150,6 @@ cisd jhdf5 - ${cisd.jhdf5.version} test From 66a61e7572c1b8ac211d1ea8ba4f992d78e59891 Mon Sep 17 00:00:00 2001 From: Stephan Saalfeld Date: Fri, 21 Apr 2023 15:42:12 -0400 Subject: [PATCH 163/243] style: LinkedAttributePathToken --- .../n5/LinkedAttributePathToken.java | 154 +++++++++++------- 1 file changed, 99 insertions(+), 55 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/LinkedAttributePathToken.java b/src/main/java/org/janelia/saalfeldlab/n5/LinkedAttributePathToken.java index 104b6cb9..d1df5f40 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/LinkedAttributePathToken.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/LinkedAttributePathToken.java @@ -1,5 +1,7 @@ package org.janelia.saalfeldlab.n5; +import java.util.Iterator; + import com.google.gson.Gson; import com.google.gson.JsonArray; import com.google.gson.JsonElement; @@ -7,8 +9,6 @@ import com.google.gson.JsonObject; import com.google.gson.JsonPrimitive; -import java.util.Iterator; - public abstract class LinkedAttributePathToken implements Iterator> { /** @@ -27,10 +27,14 @@ public abstract class LinkedAttributePathToken implements public abstract T getJsonType(); /** - * This method will retgurn the child element, if the {@link #parentJson} contains a child that - * is valid for this token. If the {@link #parentJson} does not have a child, this method will - * create it. If the {@link #parentJson} does have a child, but it is not {@link #jsonCompatible(JsonElement)} - * with our {@link #childToken}, then this method will also create a new (compatible) child, and replace + * This method will retgurn the child element, if the {@link #parentJson} + * contains a child that + * is valid for this token. If the {@link #parentJson} does not have a + * child, this method will + * create it. If the {@link #parentJson} does have a child, but it is not + * {@link #jsonCompatible(JsonElement)} + * with our {@link #childToken}, then this method will also create a new + * (compatible) child, and replace * the existing incompatible child. * * @return the resulting child element @@ -38,22 +42,30 @@ public abstract class LinkedAttributePathToken implements public abstract JsonElement getOrCreateChildElement(); /** - * This method will write into {@link #parentJson} the subsequent {@link JsonElement}. + * This method will write into {@link #parentJson} the subsequent + * {@link JsonElement}. *
    * The written JsonElement will EITHER be: *

      - *
    • The result of serializing {@code value} ( if {@link #childToken} is {@code null} )
      • - *
    • The {@link JsonElement} which represents our {@link #childToken} (See {@link #getOrCreateChildElement()} )
      • + *
    • The result of serializing {@code value} ( if {@link #childToken} is + * {@code null} )
      • + *
    • The {@link JsonElement} which represents our {@link #childToken} (See + * {@link #getOrCreateChildElement()} )
      • *
    * - * @param gson instance used to serialize {@code value } - * @param value to write + * @param gson + * instance used to serialize {@code value } + * @param value + * to write * @return the object that was written. */ - public JsonElement writeChild(Gson gson, Object value) { + public JsonElement writeChild(final Gson gson, final Object value) { if (childToken != null) - /* If we have a child, get/create it and set current json element to it */ + /* + * If we have a child, get/create it and set current json element to + * it + */ return getOrCreateChildElement(); else { /* We are done, no token remaining */ @@ -67,14 +79,16 @@ public JsonElement writeChild(Gson gson, Object value) { *
    * Compatibility means thath the provided {@code JsonElement} does not * explicitly conflict with this token. That means that {@code null} is - * always compatible. The only real incompatibility is when the {@code JsonElement} + * always compatible. The only real incompatibility is when the + * {@code JsonElement} * that we receive is different that we expect. * - * @param json element that we are testing for compatibility. + * @param json + * element that we are testing for compatibility. * @return false if {@code json} is not null and is not of type {@code T} */ @SuppressWarnings("BooleanMethodIsAlwaysInverted") - public boolean jsonCompatible(JsonElement json) { + public boolean jsonCompatible(final JsonElement json) { return json == null || json.getClass() == getJsonType().getClass(); } @@ -82,39 +96,47 @@ public boolean jsonCompatible(JsonElement json) { /** * Write {@code value} into {@link #parentJson}. *
    - * Should only be called when {@link #childToken} is null (i.e. this is the last token in the attribute path). + * Should only be called when {@link #childToken} is null (i.e. this is the + * last token in the attribute path). * - * @param gson instance used to serialize {@code value} - * @param value to serialize + * @param gson + * instance used to serialize {@code value} + * @param value + * to serialize */ protected abstract void writeValue(Gson gson, Object value); /** - * Write the {@link JsonElement} the corresponds to {@link #childToken} into {@link #parentJson}. + * Write the {@link JsonElement} the corresponds to {@link #childToken} into + * {@link #parentJson}. */ protected abstract void writeChildElement(); /** - * @return the element that is represented by {@link #childToken} if present. + * @return the element that is represented by {@link #childToken} if + * present. */ protected abstract JsonElement getChildElement(); /** - * If {@code json} is compatible with the type or token that we are, then set {@link #parentJson} to {@code json}. + * If {@code json} is compatible with the type or token that we are, then + * set {@link #parentJson} to {@code json}. *
    - * However, if {@code json} is either {@code null} or not {@link #jsonCompatible(JsonElement)} then + * However, if {@code json} is either {@code null} or not + * {@link #jsonCompatible(JsonElement)} then * {@link #parentJson} will be set to a new instance of {@code T}. * - * @param json to attempt to set to {@link #parentJson} + * @param json + * to attempt to set to {@link #parentJson} * @return the value set to {@link #parentJson}. */ - protected JsonElement setAndCreateParentElement(JsonElement json) { + protected JsonElement setAndCreateParentElement(final JsonElement json) { if (json == null || !jsonCompatible(json)) { - //noinspection unchecked + // noinspection unchecked parentJson = (T)getJsonType().deepCopy(); } else { - //noinspection unchecked + // noinspection unchecked parentJson = (T)json; } return parentJson; @@ -123,7 +145,8 @@ protected JsonElement setAndCreateParentElement(JsonElement json) { /** * @return if we have a {@link #childToken}. */ - @Override public boolean hasNext() { + @Override + public boolean hasNext() { return childToken != null; } @@ -131,7 +154,8 @@ protected JsonElement setAndCreateParentElement(JsonElement json) { /** * @return {@link #childToken} */ - @Override public LinkedAttributePathToken next() { + @Override + public LinkedAttributePathToken next() { return childToken; } @@ -142,30 +166,34 @@ public static class ObjectAttributeToken extends LinkedAttributePathToken= parentJson.size() || childToken != null && !childToken.jsonCompatible(parentJson.get(index))) writeChildElement(); return parentJson.get(index); } - @Override protected void writeChildElement() { + @Override + protected void writeChildElement() { if (childToken != null) { fillArrayToIndex(parentJson, index, null); @@ -237,26 +274,32 @@ public int getIndex() { } } - @Override protected JsonElement getChildElement() { + @Override + protected JsonElement getChildElement() { return parentJson.get(index); } - @Override protected void writeValue(Gson gson, Object value) { + @Override + protected void writeValue(final Gson gson, final Object value) { fillArrayToIndex(parentJson, index, value); parentJson.set(index, gson.toJsonTree(value)); } /** - * Fill {@code array} up to, and including, {@code index} with a default value, determined by the type of {@code value}. + * Fill {@code array} up to, and including, {@code index} with a default + * value, determined by the type of {@code value}. * Importantly, this does NOT set {@code array[index]=value}. * - * @param array to fill - * @param index to fill the array to (inclusive) - * @param value used to determine the default array fill value + * @param array + * to fill + * @param index + * to fill the array to (inclusive) + * @param value + * used to determine the default array fill value */ - private static void fillArrayToIndex(JsonArray array, int index, Object value) { + private static void fillArrayToIndex(final JsonArray array, final int index, final Object value) { final JsonElement fillValue; if (valueRepresentsANumber(value)) { @@ -274,17 +317,18 @@ private static void fillArrayToIndex(JsonArray array, int index, Object value) { * Check if {@value} represents a {@link Number}. * True if {@code value} either: *
      - *
    • is a {@code Number}, or;
      • - *
    • is a {@link JsonPrimitive} where {@link JsonPrimitive#isNumber()}
      • + *
    • is a {@code Number}, or;
      • + *
    • is a {@link JsonPrimitive} where + * {@link JsonPrimitive#isNumber()}
      • *
    * - * @param value we wish to check if represents a {@link Number} or not. + * @param value + * we wish to check if represents a {@link Number} or not. * @return true if {@code value} represents a {@link Number} */ - private static boolean valueRepresentsANumber(Object value) { + private static boolean valueRepresentsANumber(final Object value) { return value instanceof Number || (value instanceof JsonPrimitive && ((JsonPrimitive)value).isNumber()); } } } - From c9f0f307cacdb87a9f7ab54b7e0461d0c39e4d53 Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Tue, 4 Apr 2023 09:47:07 +0200 Subject: [PATCH 164/243] gitignore IntelliJ stuff --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index f60441e5..4c6539a7 100644 --- a/.gitignore +++ b/.gitignore @@ -16,3 +16,6 @@ hs_err_pid* .classpath .project .settings + +.idea +*.iml From 3033a3c652e3759e53f33a30f0cd3c4741700ee9 Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Tue, 4 Apr 2023 10:35:05 +0200 Subject: [PATCH 165/243] fix javadoc --- src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java index 6d5cbc80..5bf62555 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java @@ -133,7 +133,7 @@ default void setDatasetAttributes( /** * Removes a group or dataset (directory and all contained files). * - *

    {@link #remove(String) remove("")} or + *

    {@link #remove(String) remove("/")} or * {@link #remove(String) remove("")} will delete this N5 * container. Please note that no checks for safety will be performed, * e.g. {@link #remove(String) remove("..")} will try to From 33f575982a6f12427ec863248cf34dcbf3628558 Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Tue, 11 Apr 2023 12:58:14 +0200 Subject: [PATCH 166/243] TODO --- src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccess.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccess.java index d17f5d22..9e92f9fe 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccess.java @@ -106,7 +106,7 @@ public interface KeyValueAccess { * efforts are made to normalize it. * @return true if the path is a file */ - public boolean isFile(String normalPath); + public boolean isFile(String normalPath); // TODO: Looks un-used. Remove? /** * Create a lock on a path for reading. This isn't meant to be kept @@ -159,7 +159,7 @@ public interface KeyValueAccess { * @return the the child paths * @throws IOException the exception */ - public String[] list(final String normalPath) throws IOException; + public String[] list(final String normalPath) throws IOException; // TODO: Looks un-used. Remove? /** * Create a directory and all parent paths along the way. The directory From 80e5f88addd40256317757df0974842db4db9ca3 Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Fri, 12 May 2023 17:24:55 +0200 Subject: [PATCH 167/243] POM: Bump parent to pom-scijava-34.0.0 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index a9e818da..55aca2eb 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ org.scijava pom-scijava - 31.1.0 + 34.0.0 From 8f2bfbd2c1dc56bc31afb69c6c00a5df6a3b26fa Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Fri, 12 May 2023 17:29:26 +0200 Subject: [PATCH 168/243] Add N5Reader.getContainerURI() which gives the full URL of the container For example: "file://path/to/container.n5" "s3://bucket/path/tocontainer.n5" --- .../saalfeldlab/n5/FileSystemKeyValueAccess.java | 7 +++++++ .../org/janelia/saalfeldlab/n5/KeyValueAccess.java | 10 ++++++++++ .../janelia/saalfeldlab/n5/N5KeyValueReader.java | 14 +++++++++++++- .../java/org/janelia/saalfeldlab/n5/N5Reader.java | 9 +++++++++ 4 files changed, 39 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java index 3dc753eb..15e1afd4 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java @@ -30,6 +30,8 @@ import java.io.OutputStream; import java.io.Reader; import java.io.Writer; +import java.net.URI; +import java.net.URISyntaxException; import java.nio.channels.Channels; import java.nio.channels.FileChannel; import java.nio.channels.OverlappingFileLockException; @@ -270,6 +272,11 @@ public String normalize(final String path) { return fileSystem.getPath(path).normalize().toString(); } + @Override + public String absoluteURI(final String normalPath) throws URISyntaxException { + return new URI("file", null, normalPath, null).toString(); + } + @Override public String compose(final String... components) { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccess.java index 9e92f9fe..e003a02c 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccess.java @@ -26,6 +26,7 @@ package org.janelia.saalfeldlab.n5; import java.io.IOException; +import java.net.URISyntaxException; import java.nio.file.FileSystem; /** @@ -81,6 +82,15 @@ public interface KeyValueAccess { */ public String normalize(final String path); + /** + * Get the absolute (including scheme) URI of the given path + * + * @param normalPath is expected to be in normalized form, no further + * efforts are made to normalize it. + * @return absolute URI + */ + public String absoluteURI(final String normalPath) throws URISyntaxException; + /** * Test whether the path exists. * diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java index d866f009..1443c294 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java @@ -27,11 +27,12 @@ import com.google.gson.Gson; import com.google.gson.GsonBuilder; +import java.net.URISyntaxException; +import java.util.stream.Stream; import org.janelia.saalfeldlab.n5.cache.N5JsonCache; import java.io.IOException; import java.util.Arrays; -import java.util.stream.Stream; /** * {@link N5Reader} implementation through {@link KeyValueAccess} with JSON @@ -125,6 +126,7 @@ public Gson getGson() { return gson; } + public KeyValueAccess getKeyValueAccess() { return keyValueAccess; } @@ -135,6 +137,16 @@ public String getBasePath() { return this.basePath; } + @Override + public String getContainerURI() { + + try { + return keyValueAccess.absoluteURI(basePath); + } catch (URISyntaxException e) { + throw new RuntimeException(e); + } + } + @Override public boolean cacheMeta() { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java index 4b67a12d..89b7d0ef 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java @@ -216,6 +216,15 @@ default Version getVersion() throws IOException { */ String getBasePath(); + /** + * Returns a String representation of the absolute URI of the container. + * + * @return the absolute URI of the container + */ + // TODO: should this throw URISyntaxException or can we assume that this is + // never possible if we were able to instantiate this N5Reader? + String getContainerURI(); + /** * Reads an attribute. * From 67552360f8dfc5432e8b868682d2e7ffd8964d17 Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Thu, 11 May 2023 13:18:59 +0200 Subject: [PATCH 169/243] Fix tests to use tempN5Location() (full container URI) and disentangle N5FS specifics --- .../org/janelia/saalfeldlab/n5/N5URL.java | 1 + .../saalfeldlab/n5/AbstractN5Test.java | 121 ++++++++---------- .../saalfeldlab/n5/N5CachedFSTest.java | 42 +++--- .../org/janelia/saalfeldlab/n5/N5FSTest.java | 54 +++++--- .../n5/N5KeyValueWithInterfacesTest.java | 52 +++++--- 5 files changed, 139 insertions(+), 131 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java b/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java index 1437c98e..ffb393e4 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java @@ -614,4 +614,5 @@ private static String decodeFragment(final String rawFragment) { return sb.toString(); } + } diff --git a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java index 72c93789..6be51faf 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java @@ -37,7 +37,6 @@ import org.junit.Before; import org.junit.Test; -import java.io.File; import java.io.IOException; import java.net.URISyntaxException; import java.nio.file.Files; @@ -46,11 +45,9 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Random; -import java.util.Set; import java.util.concurrent.ExecutionException; import java.util.concurrent.Executors; import java.util.function.Predicate; @@ -90,28 +87,24 @@ public abstract class AbstractN5Test { static protected N5Writer n5; - protected abstract N5Writer createN5Writer() throws IOException; + protected abstract String tempN5Location() throws URISyntaxException; - protected N5Writer createN5Writer(String location) throws IOException { + protected N5Writer createN5Writer() throws IOException, URISyntaxException { + return createN5Writer(tempN5Location()); + } - final Path testN5Path = Paths.get(location); - final boolean existsBefore = testN5Path.toFile().exists(); - final N5Writer n5Writer = createN5Writer(location, new GsonBuilder()); - final boolean existsAfter = testN5Path.toFile().exists(); - if (!existsBefore && existsAfter) { - tmpFiles.add(location); - } - return n5Writer; + protected N5Writer createN5Writer(String location) throws IOException, URISyntaxException { + return createN5Writer(location, new GsonBuilder()); } - protected abstract N5Writer createN5Writer(String location, GsonBuilder gson) throws IOException; + protected abstract N5Writer createN5Writer(String location, GsonBuilder gson) throws IOException, URISyntaxException; - protected N5Reader createN5Reader(String location) throws IOException { + protected N5Reader createN5Reader(String location) throws IOException, URISyntaxException { return createN5Reader(location, new GsonBuilder()); } - protected abstract N5Reader createN5Reader(String location, GsonBuilder gson) throws IOException; + protected abstract N5Reader createN5Reader(String location, GsonBuilder gson) throws IOException, URISyntaxException; protected Compression[] getCompressions() { @@ -125,24 +118,11 @@ protected Compression[] getCompressions() { }; } - protected static Set tmpFiles = new HashSet<>(); - protected static String tempN5PathName() { - try { - final File tmpFile = Files.createTempDirectory("n5-test-").toFile(); - tmpFile.deleteOnExit(); - final String tmpPath = tmpFile.getCanonicalPath(); - tmpFiles.add(tmpPath); - return tmpPath; - } catch (Exception e) { - throw new RuntimeException(e); - } - } - /** * @throws IOException */ @Before - public void setUpOnce() throws IOException { + public void setUpOnce() throws IOException, URISyntaxException { if (n5 != null) return; @@ -196,7 +176,7 @@ public void testCreateGroup() { } @Test - public void testSetAttributeDoesntCreateGroup() throws IOException { + public void testSetAttributeDoesntCreateGroup() throws IOException, URISyntaxException { try (final N5Writer writer = createN5Writer()) { final String testGroup = "/group/should/not/exit"; @@ -207,7 +187,7 @@ public void testSetAttributeDoesntCreateGroup() throws IOException { } @Test - public void testCreateDataset() throws IOException { + public void testCreateDataset() throws IOException, URISyntaxException { final DatasetAttributes info; try (N5Writer writer = createN5Writer()) { @@ -254,7 +234,7 @@ public void testWriteReadByteBlock() { } @Test - public void testWriteReadShortBlock() { + public void testWriteReadShortBlock() throws URISyntaxException { for (final Compression compression : getCompressions()) { for (final DataType dataType : new DataType[]{ @@ -283,7 +263,7 @@ public void testWriteReadShortBlock() { } @Test - public void testWriteReadIntBlock() { + public void testWriteReadIntBlock() throws URISyntaxException { for (final Compression compression : getCompressions()) { for (final DataType dataType : new DataType[]{ @@ -312,7 +292,7 @@ public void testWriteReadIntBlock() { } @Test - public void testWriteReadLongBlock() { + public void testWriteReadLongBlock() throws URISyntaxException { for (final Compression compression : getCompressions()) { for (final DataType dataType : new DataType[]{ @@ -341,7 +321,7 @@ public void testWriteReadLongBlock() { } @Test - public void testWriteReadFloatBlock() { + public void testWriteReadFloatBlock() throws URISyntaxException { for (final Compression compression : getCompressions()) { System.out.println("Testing " + compression.getType() + " float32"); @@ -365,7 +345,7 @@ public void testWriteReadFloatBlock() { } @Test - public void testWriteReadDoubleBlock() { + public void testWriteReadDoubleBlock() throws URISyntaxException { for (final Compression compression : getCompressions()) { System.out.println("Testing " + compression.getType() + " float64"); @@ -389,7 +369,7 @@ public void testWriteReadDoubleBlock() { } @Test - public void testMode1WriteReadByteBlock() { + public void testMode1WriteReadByteBlock() throws URISyntaxException { final int[] differentBlockSize = new int[]{5, 10, 15}; @@ -420,7 +400,7 @@ public void testMode1WriteReadByteBlock() { } @Test - public void testWriteReadSerializableBlock() throws ClassNotFoundException { + public void testWriteReadSerializableBlock() throws ClassNotFoundException, URISyntaxException { for (final Compression compression : getCompressions()) { @@ -452,7 +432,7 @@ public void testWriteReadSerializableBlock() throws ClassNotFoundException { } @Test - public void testOverwriteBlock() { + public void testOverwriteBlock() throws URISyntaxException { try (final N5Writer n5 = createN5Writer()) { n5.createDataset(datasetName, dimensions, blockSize, DataType.INT32, new GzipCompression()); @@ -478,7 +458,7 @@ public void testOverwriteBlock() { } @Test - public void testAttributeParsingPrimitive() throws IOException { + public void testAttributeParsingPrimitive() throws IOException, URISyntaxException { try (final N5Writer n5 = createN5Writer()) { @@ -554,7 +534,7 @@ public void testAttributeParsingPrimitive() throws IOException { } @Test - public void testAttributes() throws IOException { + public void testAttributes() throws IOException, URISyntaxException { try (final N5Writer n5 = createN5Writer()) { n5.createGroup(groupName); @@ -616,10 +596,10 @@ public void testAttributes() throws IOException { } @Test - public void testNullAttributes() throws IOException { + public void testNullAttributes() throws IOException, URISyntaxException { /* serializeNulls*/ - try (N5Writer writer = createN5Writer(tempN5PathName(), new GsonBuilder().serializeNulls())) { + try (N5Writer writer = createN5Writer(tempN5Location(), new GsonBuilder().serializeNulls())) { writer.createGroup(groupName); writer.setAttribute(groupName, "nullValue", null); @@ -701,9 +681,9 @@ public void testNullAttributes() throws IOException { } @Test - public void testRemoveAttributes() throws IOException { + public void testRemoveAttributes() throws IOException, URISyntaxException { - try (N5Writer writer = createN5Writer(tempN5PathName(), new GsonBuilder().serializeNulls())) { + try (N5Writer writer = createN5Writer(tempN5Location(), new GsonBuilder().serializeNulls())) { writer.setAttribute("", "a/b/c", 100); assertEquals((Integer)100, writer.getAttribute("", "a/b/c", Integer.class)); @@ -814,10 +794,11 @@ public void testRemoveAttributes() throws IOException { } @Test - public void testRemoveContainer() throws IOException { + public void testRemoveContainer() throws IOException, URISyntaxException { - final String location = tempN5PathName(); - try (final N5Writer n5 = createN5Writer(location)) { + String location; + try (final N5Writer n5 = createN5Writer()) { + location = n5.getContainerURI(); assertNotNull(createN5Reader(location)); n5.remove(); assertThrows(Exception.class, () -> createN5Reader(location)); @@ -826,7 +807,7 @@ public void testRemoveContainer() throws IOException { } @Test - public void testRemoveGroup() throws IOException { + public void testRemoveGroup() throws IOException, URISyntaxException { try (final N5Writer n5 = createN5Writer()) { n5.createDataset(datasetName, dimensions, blockSize, DataType.UINT64, new RawCompression()); @@ -839,7 +820,7 @@ public void testRemoveGroup() throws IOException { } @Test - public void testList() throws IOException { + public void testList() throws IOException, URISyntaxException { try (final N5Writer listN5 = createN5Writer()) { listN5.createGroup(groupName); @@ -861,12 +842,9 @@ public void testList() throws IOException { } @Test - public void testDeepList() throws IOException, ExecutionException, InterruptedException { + public void testDeepList() throws IOException, URISyntaxException, ExecutionException, InterruptedException { - final File tmpFile = Files.createTempDirectory("deeplist-test-").toFile(); - tmpFile.delete(); - final String canonicalPath = tmpFile.getCanonicalPath(); - try (final N5Writer n5 = createN5Writer(canonicalPath)) { + try (final N5Writer n5 = createN5Writer()) { n5.createGroup(groupName); for (final String subGroup : subGroupNames) @@ -1019,7 +997,7 @@ public void testDeepList() throws IOException, ExecutionException, InterruptedEx } @Test - public void testExists() throws IOException { + public void testExists() throws IOException, URISyntaxException { final String groupName2 = groupName + "-2"; final String datasetName2 = datasetName + "-2"; @@ -1090,7 +1068,7 @@ public void testListAttributes() { } @Test - public void testVersion() throws NumberFormatException, IOException { + public void testVersion() throws NumberFormatException, IOException, URISyntaxException { try (final N5Writer writer = createN5Writer()) { @@ -1103,7 +1081,10 @@ public void testVersion() throws NumberFormatException, IOException { final Version version = writer.getVersion(); assertFalse(N5Reader.VERSION.isCompatible(version)); - assertThrows(N5Exception.N5IOException.class, () -> createN5Writer( writer.getBasePath() )); + assertThrows(N5Exception.N5IOException.class, () -> { + final String containerPath = writer.getContainerURI(); + createN5Writer(containerPath); + }); final Version compatibleVersion = new Version(N5Reader.VERSION.getMajor(), N5Reader.VERSION.getMinor(), N5Reader.VERSION.getPatch()); writer.setAttribute("/", N5Reader.VERSION_KEY, compatibleVersion.toString()); @@ -1112,24 +1093,24 @@ public void testVersion() throws NumberFormatException, IOException { } @Test - public void testReaderCreation() throws IOException { + public void testReaderCreation() throws IOException, URISyntaxException { - final String canonicalPath = tempN5PathName(); - try (N5Writer writer = createN5Writer(canonicalPath)) { + final String location = tempN5Location(); + try (N5Writer writer = createN5Writer(location)) { - final N5Reader n5r = createN5Reader(canonicalPath); + final N5Reader n5r = createN5Reader(location); assertNotNull(n5r); // existing directory without attributes is okay; // Remove and create to remove attributes store writer.removeAttribute("/", "/"); - final N5Reader na = createN5Reader(canonicalPath); + final N5Reader na = createN5Reader(location); assertNotNull(na); // existing location with attributes, but no version writer.removeAttribute("/", "/"); writer.setAttribute("/", "mystring", "ms"); - final N5Reader wa = createN5Reader(canonicalPath); + final N5Reader wa = createN5Reader(location); assertNotNull(wa); // existing directory with incompatible version should fail @@ -1138,14 +1119,14 @@ public void testReaderCreation() throws IOException { new Version(N5Reader.VERSION.getMajor() + 1, N5Reader.VERSION.getMinor(), N5Reader.VERSION.getPatch()).toString()); assertThrows("Incompatible version throws error", N5Exception.N5IOException.class, () -> { - createN5Reader(canonicalPath); + createN5Reader(location); }); writer.remove(); } // non-existent group should fail assertThrows("Non-existant location throws error", N5Exception.N5IOException.class, () -> { - final N5Reader test = createN5Reader(canonicalPath); + final N5Reader test = createN5Reader(location); test.list("/"); }); } @@ -1234,7 +1215,7 @@ protected static void runTests(N5Writer writer, ArrayList> existingT } @Test - public void testAttributePaths() throws IOException { + public void testAttributePaths() throws IOException, URISyntaxException { try (final N5Writer writer = createN5Writer()) { @@ -1339,7 +1320,7 @@ public void testAttributePaths() throws IOException { } @Test - public void testAttributePathEscaping() throws IOException { + public void testAttributePathEscaping() throws IOException, URISyntaxException { final JsonObject emptyObj = new JsonObject(); @@ -1436,7 +1417,7 @@ private String readAttributesAsString(final String group) { @Test public void - testRootLeaves() throws IOException { + testRootLeaves() throws IOException, URISyntaxException { /* Test retrieving non-JsonObject root leaves */ try (final N5Writer n5 = createN5Writer()) { diff --git a/src/test/java/org/janelia/saalfeldlab/n5/N5CachedFSTest.java b/src/test/java/org/janelia/saalfeldlab/n5/N5CachedFSTest.java index 396f82a7..dece78db 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/N5CachedFSTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/N5CachedFSTest.java @@ -1,6 +1,8 @@ package org.janelia.saalfeldlab.n5; import com.google.gson.GsonBuilder; +import java.net.URI; +import java.net.URISyntaxException; import org.junit.Test; import static org.junit.Assert.assertFalse; @@ -18,42 +20,39 @@ public class N5CachedFSTest extends N5FSTest { @Override - protected N5Writer createN5Writer() throws IOException { + protected N5Writer createN5Writer(final String location, final GsonBuilder gson) throws IOException, URISyntaxException { - return new N5FSWriter(tempN5PathName(), true); + return createN5Writer(location, gson, true); } @Override - protected N5Writer createN5Writer(final String location) throws IOException { + protected N5Reader createN5Reader(final String location, final GsonBuilder gson) throws IOException, URISyntaxException { - return createN5Writer(location, true); + return createN5Reader(location, gson, true); } - protected N5Writer createN5Writer(final String location, final boolean cache ) throws IOException { + protected N5Writer createN5Writer(final String location, final GsonBuilder gson, final boolean cache) throws IOException, URISyntaxException { - return new N5FSWriter(location, cache); + final String basePath = new File(new URI(location)).getCanonicalPath(); + return new N5FSWriter(basePath, gson, cache); } - @Override - protected N5Writer createN5Writer(final String location, final GsonBuilder gson) throws IOException { + protected N5Writer createN5Writer(final String location, final boolean cache) throws IOException, URISyntaxException { - if (!new File(location).exists()) { - tmpFiles.add(location); - } - return new N5FSWriter(location, gson, true); + return createN5Writer(location, new GsonBuilder(), cache); } - protected N5Reader createN5Reader(final String location, final GsonBuilder gson, final boolean cache) throws IOException { + protected N5Reader createN5Reader(final String location, final GsonBuilder gson, final boolean cache) throws IOException, URISyntaxException { - return new N5FSReader(location, gson, cache); + final String basePath = new File(new URI(location)).getCanonicalPath(); + return new N5FSReader(basePath, gson, cache); } @Test - public void cacheTest() throws IOException { + public void cacheTest() throws IOException, URISyntaxException { /* Test the cache by setting many attributes, then manually deleting the underlying file. * The only possible way for the test to succeed is if it never again attempts to read the file, and relies on the cache. */ -// try (N5KeyValueWriter n5 = (N5KeyValueWriter) createN5Writer()) { try (N5KeyValueWriter n5 = (N5KeyValueWriter) createN5Writer()) { final String cachedGroup = "cachedGroup"; final String attributesPath = n5.attributesPath(cachedGroup); @@ -69,8 +68,7 @@ public void cacheTest() throws IOException { runTests(n5, tests); } -// try (N5KeyValueWriter n5 = new N5FSWriter(tempN5PathName(), false)) { - try (N5KeyValueWriter n5 = (N5KeyValueWriter)createN5Writer(tempN5PathName(), false)) { + try (N5KeyValueWriter n5 = (N5KeyValueWriter)createN5Writer(tempN5Location(), false)) { final String cachedGroup = "cachedGroup"; final String attributesPath = n5.attributesPath(cachedGroup); @@ -86,14 +84,14 @@ public void cacheTest() throws IOException { } @Test - public void cacheGroupDatasetTest() throws IOException { + public void cacheGroupDatasetTest() throws IOException, URISyntaxException { final String datasetName = "dd"; final String groupName = "gg"; - final String tmpPath = tempN5PathName(); - try (N5KeyValueWriter w1 = (N5KeyValueWriter) createN5Writer(tmpPath); - N5KeyValueWriter w2 = (N5KeyValueWriter) createN5Writer(tmpPath);) { + final String tmpLocation = tempN5Location(); + try (N5KeyValueWriter w1 = (N5KeyValueWriter) createN5Writer(tmpLocation); + N5KeyValueWriter w2 = (N5KeyValueWriter) createN5Writer(tmpLocation);) { // create a group, both writers know it exists w1.createGroup(groupName); diff --git a/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java b/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java index fc87a229..a59fe339 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java @@ -29,10 +29,14 @@ import java.io.File; import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; import java.nio.file.FileSystems; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.HashSet; +import java.util.Set; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -59,38 +63,46 @@ public class N5FSTest extends AbstractN5Test { private static FileSystemKeyValueAccess access = new FileSystemKeyValueAccess(FileSystems.getDefault()); private static String testDirPath = tempN5PathName(); - @AfterClass - public static void cleanup() { + private static Set tmpFiles = new HashSet<>(); - for (String tmpFile : tmpFiles) { - try{ - FileUtils.deleteDirectory(new File(tmpFile)); - } catch (Exception e) { } + private static String tempN5PathName() { + try { + final File tmpFile = Files.createTempDirectory("n5-test-").toFile(); + tmpFile.deleteOnExit(); + final String tmpPath = tmpFile.getCanonicalPath(); + tmpFiles.add(tmpPath); + return tmpPath; + } catch (Exception e) { + throw new RuntimeException(e); } } - /** - * @throws IOException - */ @Override - protected N5Writer createN5Writer() throws IOException { - return new N5FSWriter(tempN5PathName(), false); + protected String tempN5Location() throws URISyntaxException { + final String basePath = tempN5PathName(); + return new URI("file", null, basePath, null).toString(); } - - - @Override - protected N5Writer createN5Writer(String location, GsonBuilder gson) throws IOException { - if (!new File(location).exists()) { - tmpFiles.add(location); - } - return new N5FSWriter(location, gson); + protected N5Writer createN5Writer(final String location, final GsonBuilder gson) throws IOException, URISyntaxException { + final String basePath = new File(new URI(location)).getCanonicalPath(); + return new N5FSWriter(basePath, gson); } @Override - protected N5Reader createN5Reader(String location, GsonBuilder gson) throws IOException { - return new N5FSReader(location, gson); + protected N5Reader createN5Reader(final String location, final GsonBuilder gson) throws IOException, URISyntaxException { + final String basePath = new File(new URI(location)).getCanonicalPath(); + return new N5FSReader(basePath, gson); + } + + @AfterClass + public static void cleanup() { + + for (String tmpFile : tmpFiles) { + try{ + FileUtils.deleteDirectory(new File(tmpFile)); + } catch (Exception e) { } + } } @Test diff --git a/src/test/java/org/janelia/saalfeldlab/n5/N5KeyValueWithInterfacesTest.java b/src/test/java/org/janelia/saalfeldlab/n5/N5KeyValueWithInterfacesTest.java index 43a7e0d3..aa85547e 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/N5KeyValueWithInterfacesTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/N5KeyValueWithInterfacesTest.java @@ -26,6 +26,10 @@ package org.janelia.saalfeldlab.n5; import com.google.gson.GsonBuilder; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.HashSet; +import java.util.Set; import org.apache.commons.io.FileUtils; import org.janelia.saalfeldlab.n5.url.UrlAttributeTest; import org.junit.AfterClass; @@ -53,40 +57,52 @@ * @author Stephan Saalfeld * @author Igor Pisarev */ +// TODO Remove? This looks like a near-duplicate of N5FSTest. Leftover from some intermediate state and not cleaned up? public class N5KeyValueWithInterfacesTest extends AbstractN5Test { private static FileSystemKeyValueAccess access = new FileSystemKeyValueAccess(FileSystems.getDefault()); private static String testDirPath = tempN5PathName(); - @AfterClass - public static void cleanup() { + private static Set tmpFiles = new HashSet<>(); - for (String tmpFile : tmpFiles) { - try{ - FileUtils.deleteDirectory(new File(tmpFile)); - } catch (Exception e) { } + private static String tempN5PathName() { + try { + final File tmpFile = Files.createTempDirectory("n5-test-").toFile(); + tmpFile.deleteOnExit(); + final String tmpPath = tmpFile.getCanonicalPath(); + tmpFiles.add(tmpPath); + return tmpPath; + } catch (Exception e) { + throw new RuntimeException(e); } } - /** - * @throws IOException - */ @Override - protected N5Writer createN5Writer() throws IOException { - return new N5KeyValueWriter(access, tempN5PathName(), new GsonBuilder(), true); + protected String tempN5Location() throws URISyntaxException { + final String basePath = tempN5PathName(); + return new URI("file", null, basePath, null).toString(); } @Override - protected N5Writer createN5Writer(String location, GsonBuilder gson) throws IOException { - if (!new File(location).exists()) { - tmpFiles.add(location); - } - return new N5KeyValueWriter(access, location, gson, true); + protected N5Writer createN5Writer(final String location, final GsonBuilder gson) throws IOException, URISyntaxException { + final String basePath = new File(new URI(location)).getCanonicalPath(); + return new N5KeyValueWriter(access, basePath, gson, true); } @Override - protected N5Reader createN5Reader(String location, GsonBuilder gson) throws IOException { - return new N5FSReader(location, gson); + protected N5Reader createN5Reader(final String location, final GsonBuilder gson) throws IOException, URISyntaxException { + final String basePath = new File(new URI(location)).getCanonicalPath(); + return new N5FSReader(basePath, gson); + } + + @AfterClass + public static void cleanup() { + + for (String tmpFile : tmpFiles) { + try{ + FileUtils.deleteDirectory(new File(tmpFile)); + } catch (Exception e) { } + } } @Test From 75b36f09d66aa891ab0a0deb8a39ee784ef670b7 Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Wed, 17 May 2023 11:38:34 +0200 Subject: [PATCH 170/243] fix: initialize static member tmpFiles before using it --- src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java b/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java index a59fe339..a86d625d 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java @@ -61,9 +61,8 @@ public class N5FSTest extends AbstractN5Test { private static FileSystemKeyValueAccess access = new FileSystemKeyValueAccess(FileSystems.getDefault()); - private static String testDirPath = tempN5PathName(); - private static Set tmpFiles = new HashSet<>(); + private static String testDirPath = tempN5PathName(); private static String tempN5PathName() { try { From 62a18f2e3dd4d2849be0ad578ee00529533393e7 Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Wed, 17 May 2023 11:07:51 +0200 Subject: [PATCH 171/243] Use Integer.valueOf instead of Integer::new (Integer::new is deprecated and marked for removal) --- .../org/janelia/saalfeldlab/n5/AbstractN5Test.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java index 6be51faf..2e2a6671 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java @@ -570,18 +570,18 @@ public void testAttributes() throws IOException, URISyntaxException { }.getType())); // test the case where the resulting file becomes shorter - n5.setAttribute(groupName, "key1", new Integer(1)); - n5.setAttribute(groupName, "key2", new Integer(2)); + n5.setAttribute(groupName, "key1", Integer.valueOf(1)); + n5.setAttribute(groupName, "key2", Integer.valueOf(2)); Assert.assertEquals(3, n5.listAttributes(groupName).size()); /* class interface */ - Assert.assertEquals(new Integer(1), n5.getAttribute(groupName, "key1", Integer.class)); - Assert.assertEquals(new Integer(2), n5.getAttribute(groupName, "key2", Integer.class)); + Assert.assertEquals(Integer.valueOf(1), n5.getAttribute(groupName, "key1", Integer.class)); + Assert.assertEquals(Integer.valueOf(2), n5.getAttribute(groupName, "key2", Integer.class)); Assert.assertEquals("value3", n5.getAttribute(groupName, "key3", String.class)); /* type interface */ - Assert.assertEquals(new Integer(1), n5.getAttribute(groupName, "key1", new TypeToken() { + Assert.assertEquals(Integer.valueOf(1), n5.getAttribute(groupName, "key1", new TypeToken() { }.getType())); - Assert.assertEquals(new Integer(2), n5.getAttribute(groupName, "key2", new TypeToken() { + Assert.assertEquals(Integer.valueOf(2), n5.getAttribute(groupName, "key2", new TypeToken() { }.getType())); Assert.assertEquals("value3", n5.getAttribute(groupName, "key3", new TypeToken() { From 8eb819e354895433e2fdae8eb36f0c6070a56682 Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Wed, 17 May 2023 11:09:00 +0200 Subject: [PATCH 172/243] Add back assertion that was (I guess) lost in merge commit ea2201c1 --- src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java index 2e2a6671..6ffc5c24 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java @@ -865,6 +865,7 @@ public void testDeepList() throws IOException, URISyntaxException, ExecutionExce final List datasetList = Arrays.asList(n5.deepList("/")); for (final String subGroup : subGroupNames) Assert.assertTrue("deepList contents", datasetList.contains(groupName.replaceFirst("/", "") + "/" + subGroup)); + Assert.assertTrue("deepList contents", datasetList.contains(datasetName.replaceFirst("/", ""))); assertFalse("deepList stops at datasets", datasetList.contains(datasetName + "/0")); final List datasetList2 = Arrays.asList(n5.deepList("")); From 50c7b32a6e8651fc4650471f794575015756f711 Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Wed, 17 May 2023 11:22:14 +0200 Subject: [PATCH 173/243] Use static imports of Assert methods --- .../saalfeldlab/n5/AbstractN5Test.java | 316 +++++++++--------- 1 file changed, 158 insertions(+), 158 deletions(-) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java index 6ffc5c24..a10b11d1 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java @@ -33,7 +33,6 @@ import com.google.gson.reflect.TypeToken; import org.janelia.saalfeldlab.n5.N5Reader.Version; import org.junit.AfterClass; -import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -52,6 +51,7 @@ import java.util.concurrent.Executors; import java.util.function.Predicate; +import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; @@ -154,7 +154,7 @@ public static void rampDownAfterClass() throws IOException { if (n5 != null) { try { - Assert.assertTrue(n5.remove()); + assertTrue(n5.remove()); } catch ( Exception e ) {} n5 = null; } @@ -198,10 +198,10 @@ public void testCreateDataset() throws IOException, URISyntaxException { info = writer.getDatasetAttributes(datasetName); } - Assert.assertArrayEquals(dimensions, info.getDimensions()); - Assert.assertArrayEquals(blockSize, info.getBlockSize()); - Assert.assertEquals(DataType.UINT64, info.getDataType()); - Assert.assertTrue(info.getCompression() instanceof RawCompression); + assertArrayEquals(dimensions, info.getDimensions()); + assertArrayEquals(blockSize, info.getBlockSize()); + assertEquals(DataType.UINT64, info.getDataType()); + assertTrue(info.getCompression() instanceof RawCompression); } @Test @@ -221,9 +221,9 @@ public void testWriteReadByteBlock() { final DataBlock loadedDataBlock = n5.readBlock(datasetName, attributes, new long[]{0, 0, 0}); - Assert.assertArrayEquals(byteBlock, (byte[])loadedDataBlock.getData()); + assertArrayEquals(byteBlock, (byte[])loadedDataBlock.getData()); - Assert.assertTrue(n5.remove(datasetName)); + assertTrue(n5.remove(datasetName)); } catch (final IOException e) { e.printStackTrace(); @@ -250,9 +250,9 @@ public void testWriteReadShortBlock() throws URISyntaxException { final DataBlock loadedDataBlock = n5.readBlock(datasetName, attributes, new long[]{0, 0, 0}); - Assert.assertArrayEquals(shortBlock, (short[])loadedDataBlock.getData()); + assertArrayEquals(shortBlock, (short[])loadedDataBlock.getData()); - Assert.assertTrue(n5.remove(datasetName)); + assertTrue(n5.remove(datasetName)); } catch (final IOException e) { e.printStackTrace(); @@ -279,9 +279,9 @@ public void testWriteReadIntBlock() throws URISyntaxException { final DataBlock loadedDataBlock = n5.readBlock(datasetName, attributes, new long[]{0, 0, 0}); - Assert.assertArrayEquals(intBlock, (int[])loadedDataBlock.getData()); + assertArrayEquals(intBlock, (int[])loadedDataBlock.getData()); - Assert.assertTrue(n5.remove(datasetName)); + assertTrue(n5.remove(datasetName)); } catch (final IOException e) { e.printStackTrace(); @@ -308,9 +308,9 @@ public void testWriteReadLongBlock() throws URISyntaxException { final DataBlock loadedDataBlock = n5.readBlock(datasetName, attributes, new long[]{0, 0, 0}); - Assert.assertArrayEquals(longBlock, (long[])loadedDataBlock.getData()); + assertArrayEquals(longBlock, (long[])loadedDataBlock.getData()); - Assert.assertTrue(n5.remove(datasetName)); + assertTrue(n5.remove(datasetName)); } catch (final IOException e) { e.printStackTrace(); @@ -333,9 +333,9 @@ public void testWriteReadFloatBlock() throws URISyntaxException { final DataBlock loadedDataBlock = n5.readBlock(datasetName, attributes, new long[]{0, 0, 0}); - Assert.assertArrayEquals(floatBlock, (float[])loadedDataBlock.getData(), 0.001f); + assertArrayEquals(floatBlock, (float[])loadedDataBlock.getData(), 0.001f); - Assert.assertTrue(n5.remove(datasetName)); + assertTrue(n5.remove(datasetName)); } catch (final IOException e) { e.printStackTrace(); @@ -357,9 +357,9 @@ public void testWriteReadDoubleBlock() throws URISyntaxException { final DataBlock loadedDataBlock = n5.readBlock(datasetName, attributes, new long[]{0, 0, 0}); - Assert.assertArrayEquals(doubleBlock, (double[])loadedDataBlock.getData(), 0.001); + assertArrayEquals(doubleBlock, (double[])loadedDataBlock.getData(), 0.001); - Assert.assertTrue(n5.remove(datasetName)); + assertTrue(n5.remove(datasetName)); } catch (final IOException e) { e.printStackTrace(); @@ -387,9 +387,9 @@ public void testMode1WriteReadByteBlock() throws URISyntaxException { final DataBlock loadedDataBlock = n5.readBlock(datasetName, attributes, new long[]{0, 0, 0}); - Assert.assertArrayEquals(byteBlock, (byte[])loadedDataBlock.getData()); + assertArrayEquals(byteBlock, (byte[])loadedDataBlock.getData()); - Assert.assertTrue(n5.remove(datasetName)); + assertTrue(n5.remove(datasetName)); } catch (final IOException e) { e.printStackTrace(); @@ -420,9 +420,9 @@ public void testWriteReadSerializableBlock() throws ClassNotFoundException, URIS final HashMap> loadedObject = n5.readSerializedBlock(datasetName, attributes, new long[]{0, 0, 0}); - object.entrySet().stream().forEach(e -> Assert.assertArrayEquals(e.getValue().get(0), loadedObject.get(e.getKey()).get(0), 0.01)); + object.entrySet().stream().forEach(e -> assertArrayEquals(e.getValue().get(0), loadedObject.get(e.getKey()).get(0), 0.01)); - Assert.assertTrue(n5.remove(datasetName)); + assertTrue(n5.remove(datasetName)); } catch (final IOException e) { e.printStackTrace(); @@ -441,15 +441,15 @@ public void testOverwriteBlock() throws URISyntaxException { final IntArrayDataBlock randomDataBlock = new IntArrayDataBlock(blockSize, new long[]{0, 0, 0}, intBlock); n5.writeBlock(datasetName, attributes, randomDataBlock); final DataBlock loadedRandomDataBlock = n5.readBlock(datasetName, attributes, new long[]{0, 0, 0}); - Assert.assertArrayEquals(intBlock, (int[])loadedRandomDataBlock.getData()); + assertArrayEquals(intBlock, (int[])loadedRandomDataBlock.getData()); // test the case where the resulting file becomes shorter final IntArrayDataBlock emptyDataBlock = new IntArrayDataBlock(blockSize, new long[]{0, 0, 0}, new int[DataBlock.getNumElements(blockSize)]); n5.writeBlock(datasetName, attributes, emptyDataBlock); final DataBlock loadedEmptyDataBlock = n5.readBlock(datasetName, attributes, new long[]{0, 0, 0}); - Assert.assertArrayEquals(new int[DataBlock.getNumElements(blockSize)], (int[])loadedEmptyDataBlock.getData()); + assertArrayEquals(new int[DataBlock.getNumElements(blockSize)], (int[])loadedEmptyDataBlock.getData()); - Assert.assertTrue(n5.remove(datasetName)); + assertTrue(n5.remove(datasetName)); } catch (final IOException e) { e.printStackTrace(); @@ -478,58 +478,58 @@ public void testAttributeParsingPrimitive() throws IOException, URISyntaxExcepti */ n5.setAttribute(groupName, "key", "value"); - Assert.assertNull(n5.getAttribute(groupName, "key", Integer.class )); - Assert.assertNull(n5.getAttribute(groupName, "key", int[].class )); - Assert.assertNull(n5.getAttribute(groupName, "key", Double.class )); - Assert.assertNull(n5.getAttribute(groupName, "key", double[].class )); - Assert.assertNotNull(n5.getAttribute(groupName, "key", String.class )); - Assert.assertNull(n5.getAttribute(groupName, "key", String[].class )); - Assert.assertNotNull(n5.getAttribute(groupName, "key", JsonElement.class )); + assertNull(n5.getAttribute(groupName, "key", Integer.class )); + assertNull(n5.getAttribute(groupName, "key", int[].class )); + assertNull(n5.getAttribute(groupName, "key", Double.class )); + assertNull(n5.getAttribute(groupName, "key", double[].class )); + assertNotNull(n5.getAttribute(groupName, "key", String.class )); + assertNull(n5.getAttribute(groupName, "key", String[].class )); + assertNotNull(n5.getAttribute(groupName, "key", JsonElement.class )); n5.setAttribute(groupName, "key", new String[]{"value" }); - Assert.assertNull(n5.getAttribute(groupName, "key", Integer.class )); - Assert.assertNull(n5.getAttribute(groupName, "key", int[].class )); - Assert.assertNull(n5.getAttribute(groupName, "key", Double.class )); - Assert.assertNull(n5.getAttribute(groupName, "key", double[].class )); - Assert.assertNotNull(n5.getAttribute(groupName, "key", String.class )); - Assert.assertNotNull(n5.getAttribute(groupName, "key", String[].class )); - Assert.assertNotNull(n5.getAttribute(groupName, "key", JsonElement.class )); + assertNull(n5.getAttribute(groupName, "key", Integer.class )); + assertNull(n5.getAttribute(groupName, "key", int[].class )); + assertNull(n5.getAttribute(groupName, "key", Double.class )); + assertNull(n5.getAttribute(groupName, "key", double[].class )); + assertNotNull(n5.getAttribute(groupName, "key", String.class )); + assertNotNull(n5.getAttribute(groupName, "key", String[].class )); + assertNotNull(n5.getAttribute(groupName, "key", JsonElement.class )); n5.setAttribute(groupName, "key", 1); - Assert.assertNotNull(n5.getAttribute(groupName, "key", Integer.class )); - Assert.assertNull(n5.getAttribute(groupName, "key", int[].class )); - Assert.assertNotNull(n5.getAttribute(groupName, "key", Double.class )); - Assert.assertNull(n5.getAttribute(groupName, "key", double[].class )); - Assert.assertNotNull(n5.getAttribute(groupName, "key", String.class )); - Assert.assertNull(n5.getAttribute(groupName, "key", String[].class )); - Assert.assertNotNull(n5.getAttribute(groupName, "key", JsonElement.class )); + assertNotNull(n5.getAttribute(groupName, "key", Integer.class )); + assertNull(n5.getAttribute(groupName, "key", int[].class )); + assertNotNull(n5.getAttribute(groupName, "key", Double.class )); + assertNull(n5.getAttribute(groupName, "key", double[].class )); + assertNotNull(n5.getAttribute(groupName, "key", String.class )); + assertNull(n5.getAttribute(groupName, "key", String[].class )); + assertNotNull(n5.getAttribute(groupName, "key", JsonElement.class )); n5.setAttribute(groupName, "key", new int[]{2, 3}); - Assert.assertNull(n5.getAttribute(groupName, "key", Integer.class )); - Assert.assertNotNull(n5.getAttribute(groupName, "key", int[].class )); - Assert.assertNull(n5.getAttribute(groupName, "key", Double.class )); - Assert.assertNotNull(n5.getAttribute(groupName, "key", double[].class )); - Assert.assertNotNull(n5.getAttribute(groupName, "key", String.class )); - Assert.assertNotNull(n5.getAttribute(groupName, "key", String[].class )); - Assert.assertNotNull(n5.getAttribute(groupName, "key", JsonElement.class )); + assertNull(n5.getAttribute(groupName, "key", Integer.class )); + assertNotNull(n5.getAttribute(groupName, "key", int[].class )); + assertNull(n5.getAttribute(groupName, "key", Double.class )); + assertNotNull(n5.getAttribute(groupName, "key", double[].class )); + assertNotNull(n5.getAttribute(groupName, "key", String.class )); + assertNotNull(n5.getAttribute(groupName, "key", String[].class )); + assertNotNull(n5.getAttribute(groupName, "key", JsonElement.class )); n5.setAttribute(groupName, "key", 0.1); -// Assert.assertNull(n5.getAttribute(groupName, "key", Integer.class )); // TODO returns 0, is this right - Assert.assertNull(n5.getAttribute(groupName, "key", int[].class )); - Assert.assertNotNull(n5.getAttribute(groupName, "key", Double.class )); - Assert.assertNull(n5.getAttribute(groupName, "key", double[].class )); - Assert.assertNotNull(n5.getAttribute(groupName, "key", String.class )); - Assert.assertNull(n5.getAttribute(groupName, "key", String[].class )); - Assert.assertNotNull(n5.getAttribute(groupName, "key", JsonElement.class )); +// assertNull(n5.getAttribute(groupName, "key", Integer.class )); // TODO returns 0, is this right + assertNull(n5.getAttribute(groupName, "key", int[].class )); + assertNotNull(n5.getAttribute(groupName, "key", Double.class )); + assertNull(n5.getAttribute(groupName, "key", double[].class )); + assertNotNull(n5.getAttribute(groupName, "key", String.class )); + assertNull(n5.getAttribute(groupName, "key", String[].class )); + assertNotNull(n5.getAttribute(groupName, "key", JsonElement.class )); n5.setAttribute(groupName, "key", new double[]{0.2, 0.3}); - Assert.assertNull(n5.getAttribute(groupName, "key", Integer.class )); -// Assert.assertNull(n5.getAttribute(groupName, "key", int[].class )); // TODO returns not null, is this right? - Assert.assertNull(n5.getAttribute(groupName, "key", Double.class )); - Assert.assertNotNull(n5.getAttribute(groupName, "key", double[].class )); - Assert.assertNotNull(n5.getAttribute(groupName, "key", String.class )); - Assert.assertNotNull(n5.getAttribute(groupName, "key", String[].class )); - Assert.assertNotNull(n5.getAttribute(groupName, "key", JsonElement.class )); + assertNull(n5.getAttribute(groupName, "key", Integer.class )); +// assertNull(n5.getAttribute(groupName, "key", int[].class )); // TODO returns not null, is this right? + assertNull(n5.getAttribute(groupName, "key", Double.class )); + assertNotNull(n5.getAttribute(groupName, "key", double[].class )); + assertNotNull(n5.getAttribute(groupName, "key", String.class )); + assertNotNull(n5.getAttribute(groupName, "key", String[].class )); + assertNotNull(n5.getAttribute(groupName, "key", JsonElement.class )); } } @@ -540,12 +540,12 @@ public void testAttributes() throws IOException, URISyntaxException { n5.createGroup(groupName); n5.setAttribute(groupName, "key1", "value1"); - Assert.assertEquals(1, n5.listAttributes(groupName).size()); + assertEquals(1, n5.listAttributes(groupName).size()); /* class interface */ - Assert.assertEquals("value1", n5.getAttribute(groupName, "key1", String.class)); + assertEquals("value1", n5.getAttribute(groupName, "key1", String.class)); /* type interface */ - Assert.assertEquals("value1", n5.getAttribute(groupName, "key1", new TypeToken() { + assertEquals("value1", n5.getAttribute(groupName, "key1", new TypeToken() { }.getType())); @@ -553,45 +553,45 @@ public void testAttributes() throws IOException, URISyntaxException { newAttributes.put("key2", "value2"); newAttributes.put("key3", "value3"); n5.setAttributes(groupName, newAttributes); - Assert.assertEquals(3, n5.listAttributes(groupName).size()); + assertEquals(3, n5.listAttributes(groupName).size()); /* class interface */ - Assert.assertEquals("value1", n5.getAttribute(groupName, "key1", String.class)); - Assert.assertEquals("value2", n5.getAttribute(groupName, "key2", String.class)); - Assert.assertEquals("value3", n5.getAttribute(groupName, "key3", String.class)); + assertEquals("value1", n5.getAttribute(groupName, "key1", String.class)); + assertEquals("value2", n5.getAttribute(groupName, "key2", String.class)); + assertEquals("value3", n5.getAttribute(groupName, "key3", String.class)); /* type interface */ - Assert.assertEquals("value1", n5.getAttribute(groupName, "key1", new TypeToken() { + assertEquals("value1", n5.getAttribute(groupName, "key1", new TypeToken() { }.getType())); - Assert.assertEquals("value2", n5.getAttribute(groupName, "key2", new TypeToken() { + assertEquals("value2", n5.getAttribute(groupName, "key2", new TypeToken() { }.getType())); - Assert.assertEquals("value3", n5.getAttribute(groupName, "key3", new TypeToken() { + assertEquals("value3", n5.getAttribute(groupName, "key3", new TypeToken() { }.getType())); // test the case where the resulting file becomes shorter n5.setAttribute(groupName, "key1", Integer.valueOf(1)); n5.setAttribute(groupName, "key2", Integer.valueOf(2)); - Assert.assertEquals(3, n5.listAttributes(groupName).size()); + assertEquals(3, n5.listAttributes(groupName).size()); /* class interface */ - Assert.assertEquals(Integer.valueOf(1), n5.getAttribute(groupName, "key1", Integer.class)); - Assert.assertEquals(Integer.valueOf(2), n5.getAttribute(groupName, "key2", Integer.class)); - Assert.assertEquals("value3", n5.getAttribute(groupName, "key3", String.class)); + assertEquals(Integer.valueOf(1), n5.getAttribute(groupName, "key1", Integer.class)); + assertEquals(Integer.valueOf(2), n5.getAttribute(groupName, "key2", Integer.class)); + assertEquals("value3", n5.getAttribute(groupName, "key3", String.class)); /* type interface */ - Assert.assertEquals(Integer.valueOf(1), n5.getAttribute(groupName, "key1", new TypeToken() { + assertEquals(Integer.valueOf(1), n5.getAttribute(groupName, "key1", new TypeToken() { }.getType())); - Assert.assertEquals(Integer.valueOf(2), n5.getAttribute(groupName, "key2", new TypeToken() { + assertEquals(Integer.valueOf(2), n5.getAttribute(groupName, "key2", new TypeToken() { }.getType())); - Assert.assertEquals("value3", n5.getAttribute(groupName, "key3", new TypeToken() { + assertEquals("value3", n5.getAttribute(groupName, "key3", new TypeToken() { }.getType())); n5.setAttribute(groupName, "key1", null); n5.setAttribute(groupName, "key2", null); n5.setAttribute(groupName, "key3", null); - Assert.assertEquals(0, n5.listAttributes(groupName).size()); + assertEquals(0, n5.listAttributes(groupName).size()); } } @@ -830,11 +830,11 @@ public void testList() throws IOException, URISyntaxException { final String[] groupsList = listN5.list(groupName); Arrays.sort(groupsList); - Assert.assertArrayEquals(subGroupNames, groupsList); + assertArrayEquals(subGroupNames, groupsList); // test listing the root group ("" and "/" should give identical results) - Assert.assertArrayEquals(new String[]{"test"}, listN5.list("")); - Assert.assertArrayEquals(new String[]{"test"}, listN5.list("/")); + assertArrayEquals(new String[]{"test"}, listN5.list("")); + assertArrayEquals(new String[]{"test"}, listN5.list("/")); } catch (final IOException e) { fail(e.getMessage()); @@ -852,10 +852,10 @@ public void testDeepList() throws IOException, URISyntaxException, ExecutionExce final List groupsList = Arrays.asList(n5.deepList("/")); for (final String subGroup : subGroupNames) - Assert.assertTrue("deepList contents", groupsList.contains(groupName.replaceFirst("/", "") + "/" + subGroup)); + assertTrue("deepList contents", groupsList.contains(groupName.replaceFirst("/", "") + "/" + subGroup)); for (final String subGroup : subGroupNames) - Assert.assertTrue("deepList contents", Arrays.asList(n5.deepList("")).contains(groupName.replaceFirst("/", "") + "/" + subGroup)); + assertTrue("deepList contents", Arrays.asList(n5.deepList("")).contains(groupName.replaceFirst("/", "") + "/" + subGroup)); final DatasetAttributes datasetAttributes = new DatasetAttributes(dimensions, blockSize, DataType.UINT64, new RawCompression()); final LongArrayDataBlock dataBlock = new LongArrayDataBlock(blockSize, new long[]{0, 0, 0}, new long[blockNumElements]); @@ -864,40 +864,40 @@ public void testDeepList() throws IOException, URISyntaxException, ExecutionExce final List datasetList = Arrays.asList(n5.deepList("/")); for (final String subGroup : subGroupNames) - Assert.assertTrue("deepList contents", datasetList.contains(groupName.replaceFirst("/", "") + "/" + subGroup)); - Assert.assertTrue("deepList contents", datasetList.contains(datasetName.replaceFirst("/", ""))); + assertTrue("deepList contents", datasetList.contains(groupName.replaceFirst("/", "") + "/" + subGroup)); + assertTrue("deepList contents", datasetList.contains(datasetName.replaceFirst("/", ""))); assertFalse("deepList stops at datasets", datasetList.contains(datasetName + "/0")); final List datasetList2 = Arrays.asList(n5.deepList("")); for (final String subGroup : subGroupNames) - Assert.assertTrue("deepList contents", datasetList2.contains(groupName.replaceFirst("/", "") + "/" + subGroup)); - Assert.assertTrue("deepList contents", datasetList2.contains(datasetName.replaceFirst("/", ""))); + assertTrue("deepList contents", datasetList2.contains(groupName.replaceFirst("/", "") + "/" + subGroup)); + assertTrue("deepList contents", datasetList2.contains(datasetName.replaceFirst("/", ""))); assertFalse("deepList stops at datasets", datasetList2.contains(datasetName + "/0")); final String prefix = "/test"; final String datasetSuffix = "group/dataset"; final List datasetList3 = Arrays.asList(n5.deepList(prefix)); for (final String subGroup : subGroupNames) - Assert.assertTrue("deepList contents", datasetList3.contains("group/" + subGroup)); - Assert.assertTrue("deepList contents", datasetList3.contains(datasetName.replaceFirst(prefix + "/", ""))); + assertTrue("deepList contents", datasetList3.contains("group/" + subGroup)); + assertTrue("deepList contents", datasetList3.contains(datasetName.replaceFirst(prefix + "/", ""))); // parallel deepList tests final List datasetListP = Arrays.asList(n5.deepList("/", Executors.newFixedThreadPool(2))); for (final String subGroup : subGroupNames) - Assert.assertTrue("deepList contents", datasetListP.contains(groupName.replaceFirst("/", "") + "/" + subGroup)); - Assert.assertTrue("deepList contents", datasetListP.contains(datasetName.replaceFirst("/", ""))); + assertTrue("deepList contents", datasetListP.contains(groupName.replaceFirst("/", "") + "/" + subGroup)); + assertTrue("deepList contents", datasetListP.contains(datasetName.replaceFirst("/", ""))); assertFalse("deepList stops at datasets", datasetListP.contains(datasetName + "/0")); final List datasetListP2 = Arrays.asList(n5.deepList("", Executors.newFixedThreadPool(2))); for (final String subGroup : subGroupNames) - Assert.assertTrue("deepList contents", datasetListP2.contains(groupName.replaceFirst("/", "") + "/" + subGroup)); - Assert.assertTrue("deepList contents", datasetListP2.contains(datasetName.replaceFirst("/", ""))); + assertTrue("deepList contents", datasetListP2.contains(groupName.replaceFirst("/", "") + "/" + subGroup)); + assertTrue("deepList contents", datasetListP2.contains(datasetName.replaceFirst("/", ""))); assertFalse("deepList stops at datasets", datasetListP2.contains(datasetName + "/0")); final List datasetListP3 = Arrays.asList(n5.deepList(prefix, Executors.newFixedThreadPool(2))); for (final String subGroup : subGroupNames) - Assert.assertTrue("deepList contents", datasetListP3.contains("group/" + subGroup)); - Assert.assertTrue("deepList contents", datasetListP3.contains(datasetName.replaceFirst(prefix + "/", ""))); + assertTrue("deepList contents", datasetListP3.contains("group/" + subGroup)); + assertTrue("deepList contents", datasetListP3.contains(datasetName.replaceFirst(prefix + "/", ""))); assertFalse("deepList stops at datasets", datasetListP3.contains(datasetName + "/0")); // test filtering @@ -909,33 +909,33 @@ public void testDeepList() throws IOException, URISyntaxException, ExecutionExce }; final List datasetListFilter1 = Arrays.asList(n5.deepList(prefix, isCalledDataset)); - Assert.assertTrue( + assertTrue( "deepList filter \"dataset\"", datasetListFilter1.stream().map(x -> prefix + x).allMatch(isCalledDataset)); final List datasetListFilter2 = Arrays.asList(n5.deepList(prefix, isBorC)); - Assert.assertTrue( + assertTrue( "deepList filter \"b or c\"", datasetListFilter2.stream().map(x -> prefix + x).allMatch(isBorC)); final List datasetListFilterP1 = Arrays.asList(n5.deepList(prefix, isCalledDataset, Executors.newFixedThreadPool(2))); - Assert.assertTrue( + assertTrue( "deepList filter \"dataset\"", datasetListFilterP1.stream().map(x -> prefix + x).allMatch(isCalledDataset)); final List datasetListFilterP2 = Arrays.asList(n5.deepList(prefix, isBorC, Executors.newFixedThreadPool(2))); - Assert.assertTrue( + assertTrue( "deepList filter \"b or c\"", datasetListFilterP2.stream().map(x -> prefix + x).allMatch(isBorC)); // test dataset filtering final List datasetListFilterD = Arrays.asList(n5.deepListDatasets(prefix)); - Assert.assertTrue( + assertTrue( "deepListDataset", datasetListFilterD.size() == 1 && (prefix + "/" + datasetListFilterD.get(0)).equals(datasetName)); - Assert.assertArrayEquals( + assertArrayEquals( datasetListFilterD.toArray(), n5.deepList( prefix, @@ -948,8 +948,8 @@ public void testDeepList() throws IOException, URISyntaxException, ExecutionExce })); final List datasetListFilterDandBC = Arrays.asList(n5.deepListDatasets(prefix, isBorC)); - Assert.assertTrue("deepListDatasetFilter", datasetListFilterDandBC.size() == 0); - Assert.assertArrayEquals( + assertTrue("deepListDatasetFilter", datasetListFilterDandBC.size() == 0); + assertArrayEquals( datasetListFilterDandBC.toArray(), n5.deepList( prefix, @@ -963,10 +963,10 @@ public void testDeepList() throws IOException, URISyntaxException, ExecutionExce final List datasetListFilterDP = Arrays.asList(n5.deepListDatasets(prefix, Executors.newFixedThreadPool(2))); - Assert.assertTrue( + assertTrue( "deepListDataset Parallel", datasetListFilterDP.size() == 1 && (prefix + "/" + datasetListFilterDP.get(0)).equals(datasetName)); - Assert.assertArrayEquals( + assertArrayEquals( datasetListFilterDP.toArray(), n5.deepList( prefix, @@ -981,8 +981,8 @@ public void testDeepList() throws IOException, URISyntaxException, ExecutionExce final List datasetListFilterDandBCP = Arrays.asList(n5.deepListDatasets(prefix, isBorC, Executors.newFixedThreadPool(2))); - Assert.assertTrue("deepListDatasetFilter Parallel", datasetListFilterDandBCP.size() == 0); - Assert.assertArrayEquals( + assertTrue("deepListDatasetFilter Parallel", datasetListFilterDandBCP.size() == 0); + assertArrayEquals( datasetListFilterDandBCP.toArray(), n5.deepList( prefix, @@ -1005,11 +1005,11 @@ public void testExists() throws IOException, URISyntaxException { final String notExists = groupName + "-notexists"; try (N5Writer n5 = createN5Writer()){ n5.createDataset(datasetName2, dimensions, blockSize, DataType.UINT64, new RawCompression()); - Assert.assertTrue(n5.exists(datasetName2)); - Assert.assertTrue(n5.datasetExists(datasetName2)); + assertTrue(n5.exists(datasetName2)); + assertTrue(n5.datasetExists(datasetName2)); n5.createGroup(groupName2); - Assert.assertTrue(n5.exists(groupName2)); + assertTrue(n5.exists(groupName2)); assertFalse(n5.datasetExists(groupName2)); assertFalse(n5.exists(notExists)); @@ -1035,14 +1035,14 @@ public void testListAttributes() { n5.setAttribute(datasetName2, "attr8", new Object[]{"1", 2, 3.1}); Map> attributesMap = n5.listAttributes(datasetName2); - Assert.assertTrue(attributesMap.get("attr1") == double[].class); - Assert.assertTrue(attributesMap.get("attr2") == String[].class); - Assert.assertTrue(attributesMap.get("attr3") == double.class); - Assert.assertTrue(attributesMap.get("attr4") == String.class); - Assert.assertTrue(attributesMap.get("attr5") == long[].class); - Assert.assertTrue(attributesMap.get("attr6") == long.class); - Assert.assertTrue(attributesMap.get("attr7") == double[].class); - Assert.assertTrue(attributesMap.get("attr8") == Object[].class); + assertTrue(attributesMap.get("attr1") == double[].class); + assertTrue(attributesMap.get("attr2") == String[].class); + assertTrue(attributesMap.get("attr3") == double.class); + assertTrue(attributesMap.get("attr4") == String.class); + assertTrue(attributesMap.get("attr5") == long[].class); + assertTrue(attributesMap.get("attr6") == long.class); + assertTrue(attributesMap.get("attr7") == double[].class); + assertTrue(attributesMap.get("attr8") == Object[].class); n5.createGroup(groupName2); n5.setAttribute(groupName2, "attr1", new double[]{1.1, 2.1, 3.1}); @@ -1055,14 +1055,14 @@ public void testListAttributes() { n5.setAttribute(groupName2, "attr8", new Object[]{"1", 2, 3.1}); attributesMap = n5.listAttributes(groupName2); - Assert.assertTrue(attributesMap.get("attr1") == double[].class); - Assert.assertTrue(attributesMap.get("attr2") == String[].class); - Assert.assertTrue(attributesMap.get("attr3") == double.class); - Assert.assertTrue(attributesMap.get("attr4") == String.class); - Assert.assertTrue(attributesMap.get("attr5") == long[].class); - Assert.assertTrue(attributesMap.get("attr6") == long.class); - Assert.assertTrue(attributesMap.get("attr7") == double[].class); - Assert.assertTrue(attributesMap.get("attr8") == Object[].class); + assertTrue(attributesMap.get("attr1") == double[].class); + assertTrue(attributesMap.get("attr2") == String[].class); + assertTrue(attributesMap.get("attr3") == double.class); + assertTrue(attributesMap.get("attr4") == String.class); + assertTrue(attributesMap.get("attr5") == long[].class); + assertTrue(attributesMap.get("attr6") == long.class); + assertTrue(attributesMap.get("attr7") == double[].class); + assertTrue(attributesMap.get("attr8") == Object[].class); } catch (final IOException e) { fail(e.getMessage()); } @@ -1075,7 +1075,7 @@ public void testVersion() throws NumberFormatException, IOException, URISyntaxEx final Version n5Version = writer.getVersion(); - Assert.assertTrue(n5Version.equals(N5Reader.VERSION)); + assertTrue(n5Version.equals(N5Reader.VERSION)); final Version incompatibleVersion = new Version(N5Reader.VERSION.getMajor() + 1, N5Reader.VERSION.getMinor(), N5Reader.VERSION.getPatch()); writer.setAttribute("/", N5Reader.VERSION_KEY, incompatibleVersion.toString()); @@ -1142,27 +1142,27 @@ public void testDelete() throws IOException { final long[] position2 = {0, 1, 2}; // no blocks should exist to begin with - Assert.assertTrue(testDeleteIsBlockDeleted(n5.readBlock(datasetName, attributes, position1))); - Assert.assertTrue(testDeleteIsBlockDeleted(n5.readBlock(datasetName, attributes, position2))); + assertTrue(testDeleteIsBlockDeleted(n5.readBlock(datasetName, attributes, position1))); + assertTrue(testDeleteIsBlockDeleted(n5.readBlock(datasetName, attributes, position2))); final ByteArrayDataBlock dataBlock = new ByteArrayDataBlock(blockSize, position1, byteBlock); n5.writeBlock(datasetName, attributes, dataBlock); // block should exist at position1 but not at position2 final DataBlock readBlock = n5.readBlock(datasetName, attributes, position1); - Assert.assertNotNull(readBlock); - Assert.assertTrue(readBlock instanceof ByteArrayDataBlock); - Assert.assertArrayEquals(byteBlock, ((ByteArrayDataBlock)readBlock).getData()); - Assert.assertTrue(testDeleteIsBlockDeleted(n5.readBlock(datasetName, attributes, position2))); + assertNotNull(readBlock); + assertTrue(readBlock instanceof ByteArrayDataBlock); + assertArrayEquals(byteBlock, ((ByteArrayDataBlock)readBlock).getData()); + assertTrue(testDeleteIsBlockDeleted(n5.readBlock(datasetName, attributes, position2))); // deletion should report true in all cases - Assert.assertTrue(n5.deleteBlock(datasetName, position1)); - Assert.assertTrue(n5.deleteBlock(datasetName, position1)); - Assert.assertTrue(n5.deleteBlock(datasetName, position2)); + assertTrue(n5.deleteBlock(datasetName, position1)); + assertTrue(n5.deleteBlock(datasetName, position1)); + assertTrue(n5.deleteBlock(datasetName, position2)); // no block should exist anymore - Assert.assertTrue(testDeleteIsBlockDeleted(n5.readBlock(datasetName, attributes, position1))); - Assert.assertTrue(testDeleteIsBlockDeleted(n5.readBlock(datasetName, attributes, position2))); + assertTrue(testDeleteIsBlockDeleted(n5.readBlock(datasetName, attributes, position1))); + assertTrue(testDeleteIsBlockDeleted(n5.readBlock(datasetName, attributes, position2))); } protected boolean testDeleteIsBlockDeleted(final DataBlock dataBlock) { @@ -1189,8 +1189,8 @@ public TestData(String groupPath, String key, T attributeValue) { protected static void addAndTest(N5Writer writer, ArrayList> existingTests, TestData testData) throws IOException { /* test a new value on existing path */ writer.setAttribute(testData.groupPath, testData.attributePath, testData.attributeValue); - Assert.assertEquals(testData.attributeValue, writer.getAttribute(testData.groupPath, testData.attributePath, testData.attributeClass)); - Assert.assertEquals(testData.attributeValue, + assertEquals(testData.attributeValue, writer.getAttribute(testData.groupPath, testData.attributePath, testData.attributeClass)); + assertEquals(testData.attributeValue, writer.getAttribute(testData.groupPath, testData.attributePath, TypeToken.get(testData.attributeClass).getType())); /* previous values should still be there, but we remove first if the test we just added overwrites. */ @@ -1210,8 +1210,8 @@ protected static void addAndTest(N5Writer writer, ArrayList> existin protected static void runTests(N5Writer writer, ArrayList> existingTests) throws IOException { for (TestData test : existingTests) { - Assert.assertEquals(test.attributeValue, writer.getAttribute(test.groupPath, test.attributePath, test.attributeClass)); - Assert.assertEquals(test.attributeValue, writer.getAttribute(test.groupPath, test.attributePath, TypeToken.get(test.attributeClass).getType())); + assertEquals(test.attributeValue, writer.getAttribute(test.groupPath, test.attributePath, test.attributeClass)); + assertEquals(test.attributeValue, writer.getAttribute(test.groupPath, test.attributePath, TypeToken.get(test.attributeClass).getType())); } } @@ -1280,7 +1280,7 @@ public void testAttributePaths() throws IOException, URISyntaxException { addAndTest(writer, existingTests, new TestData<>(testGroup, "/filled/double_array[0]", 0.0)); /* We intentionally skipped index 3, it should be `0` */ - Assert.assertEquals((Integer)0, writer.getAttribute(testGroup, "/filled/double_array[3]", Integer.class)); + assertEquals((Integer)0, writer.getAttribute(testGroup, "/filled/double_array[3]", Integer.class)); /* Fill an array with Object */ addAndTest(writer, existingTests, new TestData<>(testGroup, "/filled/string_array[5]", "f")); @@ -1291,7 +1291,7 @@ public void testAttributePaths() throws IOException, URISyntaxException { addAndTest(writer, existingTests, new TestData<>(testGroup, "/filled/string_array[0]", "a")); /* We intentionally skipped index 3, it should be null */ - Assert.assertNull(writer.getAttribute(testGroup, "/filled/double_array[3]", JsonNull.class)); + assertNull(writer.getAttribute(testGroup, "/filled/double_array[3]", JsonNull.class)); /* Ensure that escaping does NOT interpret the json path structure, but rather it adds the keys opaquely*/ final HashMap testAttributes = new HashMap<>(); @@ -1459,8 +1459,8 @@ private String readAttributesAsString(final String group) { try (final N5Writer writer = createN5Writer()) { writer.createGroup(testData.groupPath); writer.setAttribute(testData.groupPath, testData.attributePath, testData.attributeValue); - Assert.assertEquals(testData.attributeValue, writer.getAttribute(testData.groupPath, testData.attributePath, testData.attributeClass)); - Assert.assertEquals(testData.attributeValue, + assertEquals(testData.attributeValue, writer.getAttribute(testData.groupPath, testData.attributePath, testData.attributeClass)); + assertEquals(testData.attributeValue, writer.getAttribute(testData.groupPath, testData.attributePath, TypeToken.get(testData.attributeClass).getType())); } } @@ -1475,8 +1475,8 @@ private String readAttributesAsString(final String group) { writer.createGroup(groupName); for (TestData testData : tests) { writer.setAttribute(testData.groupPath, testData.attributePath, testData.attributeValue); - Assert.assertEquals(testData.attributeValue, writer.getAttribute(testData.groupPath, testData.attributePath, testData.attributeClass)); - Assert.assertEquals(testData.attributeValue, + assertEquals(testData.attributeValue, writer.getAttribute(testData.groupPath, testData.attributePath, testData.attributeClass)); + assertEquals(testData.attributeValue, writer.getAttribute(testData.groupPath, testData.attributePath, TypeToken.get(testData.attributeClass).getType())); } } @@ -1493,15 +1493,15 @@ private String readAttributesAsString(final String group) { for (TestData test : tests) { /* Set the root as Object*/ writer.setAttribute(rootAsObject.groupPath, rootAsObject.attributePath, rootAsObject.attributeValue); - Assert.assertEquals(rootAsObject.attributeValue, + assertEquals(rootAsObject.attributeValue, writer.getAttribute(rootAsObject.groupPath, rootAsObject.attributePath, rootAsObject.attributeClass)); /* Override the root with something else */ writer.setAttribute(test.groupPath, test.attributePath, test.attributeValue); /* Verify original root is gone */ - Assert.assertNull(writer.getAttribute(rootAsObject.groupPath, rootAsObject.attributePath, rootAsObject.attributeClass)); + assertNull(writer.getAttribute(rootAsObject.groupPath, rootAsObject.attributePath, rootAsObject.attributeClass)); /* verify new root exists */ - Assert.assertEquals(test.attributeValue, writer.getAttribute(test.groupPath, test.attributePath, test.attributeClass)); + assertEquals(test.attributeValue, writer.getAttribute(test.groupPath, test.attributePath, test.attributeClass)); } } } From 4263cefd8c931ad21758350c382319dd0b85611c Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Wed, 17 May 2023 11:26:15 +0200 Subject: [PATCH 174/243] Simplify assertEquals(null, ...) to assertNull(...) --- .../saalfeldlab/n5/AbstractN5Test.java | 48 +++++++++---------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java index a10b11d1..81e235a1 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java @@ -603,7 +603,7 @@ public void testNullAttributes() throws IOException, URISyntaxException { writer.createGroup(groupName); writer.setAttribute(groupName, "nullValue", null); - assertEquals(null, writer.getAttribute(groupName, "nullValue", Object.class)); + assertNull(writer.getAttribute(groupName, "nullValue", Object.class)); assertEquals(JsonNull.INSTANCE, writer.getAttribute(groupName, "nullValue", JsonElement.class)); final HashMap nulls = new HashMap<>(); nulls.put("anotherNullValue", null); @@ -611,30 +611,30 @@ public void testNullAttributes() throws IOException, URISyntaxException { nulls.put("implicitNulls[3]", null); writer.setAttributes(groupName, nulls); - assertEquals(null, writer.getAttribute(groupName, "anotherNullValue", Object.class)); + assertNull(writer.getAttribute(groupName, "anotherNullValue", Object.class)); assertEquals(JsonNull.INSTANCE, writer.getAttribute(groupName, "anotherNullValue", JsonElement.class)); - assertEquals(null, writer.getAttribute(groupName, "structured/nullValue", Object.class)); + assertNull(writer.getAttribute(groupName, "structured/nullValue", Object.class)); assertEquals(JsonNull.INSTANCE, writer.getAttribute(groupName, "structured/nullValue", JsonElement.class)); - assertEquals(null, writer.getAttribute(groupName, "implicitNulls[3]", Object.class)); + assertNull(writer.getAttribute(groupName, "implicitNulls[3]", Object.class)); assertEquals(JsonNull.INSTANCE, writer.getAttribute(groupName, "implicitNulls[3]", JsonElement.class)); - assertEquals(null, writer.getAttribute(groupName, "implicitNulls[1]", Object.class)); + assertNull(writer.getAttribute(groupName, "implicitNulls[1]", Object.class)); assertEquals(JsonNull.INSTANCE, writer.getAttribute(groupName, "implicitNulls[1]", JsonElement.class)); /* Negative test; a value that truly doesn't exist will still return `null` but will also return `null` when querying as a `JsonElement` */ - assertEquals(null, writer.getAttribute(groupName, "implicitNulls[10]", Object.class)); - assertEquals(null, writer.getAttribute(groupName, "implicitNulls[10]", JsonElement.class)); + assertNull(writer.getAttribute(groupName, "implicitNulls[10]", Object.class)); + assertNull(writer.getAttribute(groupName, "implicitNulls[10]", JsonElement.class)); - assertEquals(null, writer.getAttribute(groupName, "keyDoesn'tExist", Object.class)); - assertEquals(null, writer.getAttribute(groupName, "keyDoesn'tExist", JsonElement.class)); + assertNull(writer.getAttribute(groupName, "keyDoesn'tExist", Object.class)); + assertNull(writer.getAttribute(groupName, "keyDoesn'tExist", JsonElement.class)); /* check existing value gets overwritten */ writer.setAttribute(groupName, "existingValue", 1); assertEquals((Integer)1, writer.getAttribute(groupName, "existingValue", Integer.class)); writer.setAttribute(groupName, "existingValue", null); - assertEquals(null, writer.getAttribute(groupName, "existingValue", Integer.class)); + assertNull(writer.getAttribute(groupName, "existingValue", Integer.class)); assertEquals(JsonNull.INSTANCE, writer.getAttribute(groupName, "existingValue", JsonElement.class)); } @@ -643,40 +643,40 @@ public void testNullAttributes() throws IOException, URISyntaxException { writer.createGroup(groupName); writer.setAttribute(groupName, "nullValue", null); - assertEquals(null, writer.getAttribute(groupName, "nullValue", Object.class)); - assertEquals(null, writer.getAttribute(groupName, "nullValue", JsonElement.class)); + assertNull(writer.getAttribute(groupName, "nullValue", Object.class)); + assertNull(writer.getAttribute(groupName, "nullValue", JsonElement.class)); final HashMap nulls = new HashMap<>(); nulls.put("anotherNullValue", null); nulls.put("structured/nullValue", null); nulls.put("implicitNulls[3]", null); writer.setAttributes(groupName, nulls); - assertEquals(null, writer.getAttribute(groupName, "anotherNullValue", Object.class)); - assertEquals(null, writer.getAttribute(groupName, "anotherNullValue", JsonElement.class)); + assertNull(writer.getAttribute(groupName, "anotherNullValue", Object.class)); + assertNull(writer.getAttribute(groupName, "anotherNullValue", JsonElement.class)); - assertEquals(null, writer.getAttribute(groupName, "structured/nullValue", Object.class)); - assertEquals(null, writer.getAttribute(groupName, "structured/nullValue", JsonElement.class)); + assertNull(writer.getAttribute(groupName, "structured/nullValue", Object.class)); + assertNull(writer.getAttribute(groupName, "structured/nullValue", JsonElement.class)); /* Arrays are still filled with `null`, regardless of `serializeNulls()`*/ - assertEquals(null, writer.getAttribute(groupName, "implicitNulls[3]", Object.class)); + assertNull(writer.getAttribute(groupName, "implicitNulls[3]", Object.class)); assertEquals(JsonNull.INSTANCE, writer.getAttribute(groupName, "implicitNulls[3]", JsonElement.class)); - assertEquals(null, writer.getAttribute(groupName, "implicitNulls[1]", Object.class)); + assertNull(writer.getAttribute(groupName, "implicitNulls[1]", Object.class)); assertEquals(JsonNull.INSTANCE, writer.getAttribute(groupName, "implicitNulls[1]", JsonElement.class)); /* Negative test; a value that truly doesn't exist will still return `null` but will also return `null` when querying as a `JsonElement` */ - assertEquals(null, writer.getAttribute(groupName, "implicitNulls[10]", Object.class)); - assertEquals(null, writer.getAttribute(groupName, "implicitNulls[10]", JsonElement.class)); + assertNull(writer.getAttribute(groupName, "implicitNulls[10]", Object.class)); + assertNull(writer.getAttribute(groupName, "implicitNulls[10]", JsonElement.class)); - assertEquals(null, writer.getAttribute(groupName, "keyDoesn'tExist", Object.class)); - assertEquals(null, writer.getAttribute(groupName, "keyDoesn'tExist", JsonElement.class)); + assertNull(writer.getAttribute(groupName, "keyDoesn'tExist", Object.class)); + assertNull(writer.getAttribute(groupName, "keyDoesn'tExist", JsonElement.class)); /* check existing value gets overwritten */ writer.setAttribute(groupName, "existingValue", 1); assertEquals((Integer)1, writer.getAttribute(groupName, "existingValue", Integer.class)); writer.setAttribute(groupName, "existingValue", null); - assertEquals(null, writer.getAttribute(groupName, "existingValue", Integer.class)); - assertEquals(null, writer.getAttribute(groupName, "existingValue", JsonElement.class)); + assertNull(writer.getAttribute(groupName, "existingValue", Integer.class)); + assertNull(writer.getAttribute(groupName, "existingValue", JsonElement.class)); } } From 7bcdd5d2f57bf0eda981a98a65c375f3eb55672e Mon Sep 17 00:00:00 2001 From: tpietzsch Date: Wed, 17 May 2023 11:27:27 +0200 Subject: [PATCH 175/243] Remove N5KeyValueWithInterfacesTest which is a near-duplicate of N5FSTest --- .../n5/N5KeyValueWithInterfacesTest.java | 273 ------------------ 1 file changed, 273 deletions(-) delete mode 100644 src/test/java/org/janelia/saalfeldlab/n5/N5KeyValueWithInterfacesTest.java diff --git a/src/test/java/org/janelia/saalfeldlab/n5/N5KeyValueWithInterfacesTest.java b/src/test/java/org/janelia/saalfeldlab/n5/N5KeyValueWithInterfacesTest.java deleted file mode 100644 index aa85547e..00000000 --- a/src/test/java/org/janelia/saalfeldlab/n5/N5KeyValueWithInterfacesTest.java +++ /dev/null @@ -1,273 +0,0 @@ -/** - * Copyright (c) 2017--2021, Stephan Saalfeld - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ -package org.janelia.saalfeldlab.n5; - -import com.google.gson.GsonBuilder; -import java.net.URI; -import java.net.URISyntaxException; -import java.util.HashSet; -import java.util.Set; -import org.apache.commons.io.FileUtils; -import org.janelia.saalfeldlab.n5.url.UrlAttributeTest; -import org.junit.AfterClass; -import org.junit.Test; - -import java.io.File; -import java.io.IOException; -import java.nio.file.FileSystems; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; - -import static org.junit.Assert.fail; - -/** - * Initiates testing of the filesystem-based N5 implementation. - * - * @author Stephan Saalfeld - * @author Igor Pisarev - */ -// TODO Remove? This looks like a near-duplicate of N5FSTest. Leftover from some intermediate state and not cleaned up? -public class N5KeyValueWithInterfacesTest extends AbstractN5Test { - - private static FileSystemKeyValueAccess access = new FileSystemKeyValueAccess(FileSystems.getDefault()); - private static String testDirPath = tempN5PathName(); - - private static Set tmpFiles = new HashSet<>(); - - private static String tempN5PathName() { - try { - final File tmpFile = Files.createTempDirectory("n5-test-").toFile(); - tmpFile.deleteOnExit(); - final String tmpPath = tmpFile.getCanonicalPath(); - tmpFiles.add(tmpPath); - return tmpPath; - } catch (Exception e) { - throw new RuntimeException(e); - } - } - - @Override - protected String tempN5Location() throws URISyntaxException { - final String basePath = tempN5PathName(); - return new URI("file", null, basePath, null).toString(); - } - - @Override - protected N5Writer createN5Writer(final String location, final GsonBuilder gson) throws IOException, URISyntaxException { - final String basePath = new File(new URI(location)).getCanonicalPath(); - return new N5KeyValueWriter(access, basePath, gson, true); - } - - @Override - protected N5Reader createN5Reader(final String location, final GsonBuilder gson) throws IOException, URISyntaxException { - final String basePath = new File(new URI(location)).getCanonicalPath(); - return new N5FSReader(basePath, gson); - } - - @AfterClass - public static void cleanup() { - - for (String tmpFile : tmpFiles) { - try{ - FileUtils.deleteDirectory(new File(tmpFile)); - } catch (Exception e) { } - } - } - - @Test - public void customObjectTest() throws IOException { - - final String testGroup = "test"; - final ArrayList> existingTests = new ArrayList<>(); - - final UrlAttributeTest.TestDoubles doubles1 = new UrlAttributeTest.TestDoubles("doubles", "doubles1", new double[]{5.7, 4.5, 3.4}); - final UrlAttributeTest.TestDoubles doubles2 = new UrlAttributeTest.TestDoubles("doubles", "doubles2", new double[]{5.8, 4.6, 3.5}); - final UrlAttributeTest.TestDoubles doubles3 = new UrlAttributeTest.TestDoubles("doubles", "doubles3", new double[]{5.9, 4.7, 3.6}); - final UrlAttributeTest.TestDoubles doubles4 = new UrlAttributeTest.TestDoubles("doubles", "doubles4", new double[]{5.10, 4.8, 3.7}); - n5.createGroup(testGroup); - addAndTest(n5, existingTests, new TestData<>(testGroup, "/doubles[1]", doubles1)); - addAndTest(n5, existingTests, new TestData<>(testGroup, "/doubles[2]", doubles2)); - addAndTest(n5, existingTests, new TestData<>(testGroup, "/doubles[3]", doubles3)); - addAndTest(n5, existingTests, new TestData<>(testGroup, "/doubles[4]", doubles4)); - - /* Test overwrite custom */ - addAndTest(n5, existingTests, new TestData<>(testGroup, "/doubles[1]", doubles4)); - } - - -// @Test - public void testReadLock() throws IOException, InterruptedException { - - final Path path = Paths.get(testDirPath, "lock"); - try { - Files.delete(path); - } catch (final IOException e) {} - - LockedChannel lock = access.lockForWriting(path); - lock.close(); - lock = access.lockForReading(path); - System.out.println("locked"); - - final ExecutorService exec = Executors.newSingleThreadExecutor(); - final Future future = exec.submit(() -> { - access.lockForWriting(path).close(); - return null; - }); - - try { - System.out.println("Trying to acquire locked readable channel..."); - System.out.println(future.get(3, TimeUnit.SECONDS)); - fail("Lock broken!"); - } catch (final TimeoutException e) { - System.out.println("Lock held!"); - future.cancel(true); - } catch (final InterruptedException | ExecutionException e) { - future.cancel(true); - System.out.println("Test was interrupted!"); - } finally { - lock.close(); - Files.delete(path); - } - - exec.shutdownNow(); - } - - -// @Test - public void testWriteLock() throws IOException { - - final Path path = Paths.get(testDirPath, "lock"); - try { - Files.delete(path); - } catch (final IOException e) {} - - final LockedChannel lock = access.lockForWriting(path); - System.out.println("locked"); - - final ExecutorService exec = Executors.newSingleThreadExecutor(); - final Future future = exec.submit(() -> { - access.lockForReading(path).close(); - return null; - }); - - try { - System.out.println("Trying to acquire locked writable channel..."); - System.out.println(future.get(3, TimeUnit.SECONDS)); - fail("Lock broken!"); - } catch (final TimeoutException e) { - System.out.println("Lock held!"); - future.cancel(true); - } catch (final InterruptedException | ExecutionException e) { - future.cancel(true); - System.out.println("Test was interrupted!"); - } finally { - lock.close(); - Files.delete(path); - } - - exec.shutdownNow(); - } - - @Test - public void testLockReleaseByReader() throws IOException { - - System.out.println("Testing lock release by Reader."); - - final Path path = Paths.get(testDirPath, "lock"); - try { - Files.delete(path); - } catch (final IOException e) {} - - final LockedChannel lock = access.lockForWriting(path); - - lock.newReader().close(); - - final ExecutorService exec = Executors.newSingleThreadExecutor(); - final Future future = exec.submit(() -> { - access.lockForWriting(path).close(); - return null; - }); - - try { - future.get(3, TimeUnit.SECONDS); - } catch (final TimeoutException e) { - fail("... lock not released!"); - future.cancel(true); - } catch (final InterruptedException | ExecutionException e) { - future.cancel(true); - System.out.println("... test interrupted!"); - } finally { - lock.close(); - Files.delete(path); - } - - exec.shutdownNow(); - } - - @Test - public void testLockReleaseByInputStream() throws IOException { - - System.out.println("Testing lock release by InputStream."); - - final Path path = Paths.get(testDirPath, "lock"); - try { - Files.delete(path); - } catch (final IOException e) {} - - final LockedChannel lock = access.lockForWriting(path); - - lock.newInputStream().close(); - - final ExecutorService exec = Executors.newSingleThreadExecutor(); - final Future future = exec.submit(() -> { - access.lockForWriting(path).close(); - return null; - }); - - try { - future.get(3, TimeUnit.SECONDS); - } catch (final TimeoutException e) { - fail("... lock not released!"); - future.cancel(true); - } catch (final InterruptedException | ExecutionException e) { - future.cancel(true); - System.out.println("... test interrupted!"); - } finally { - lock.close(); - Files.delete(path); - } - - exec.shutdownNow(); - } -} From a3d348df65b487f56ca23f7a37e4a5177089ef02 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Wed, 17 May 2023 15:59:21 -0400 Subject: [PATCH 176/243] fix/test: avoid ExceptionInInitializerError --- src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java | 5 +++-- .../janelia/saalfeldlab/n5/N5KeyValueWithInterfacesTest.java | 3 ++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java b/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java index a59fe339..abb9f713 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java @@ -61,11 +61,12 @@ public class N5FSTest extends AbstractN5Test { private static FileSystemKeyValueAccess access = new FileSystemKeyValueAccess(FileSystems.getDefault()); - private static String testDirPath = tempN5PathName(); private static Set tmpFiles = new HashSet<>(); - private static String tempN5PathName() { + private static String testDirPath = tempN5PathName(); + + private static String tempN5PathName() { try { final File tmpFile = Files.createTempDirectory("n5-test-").toFile(); tmpFile.deleteOnExit(); diff --git a/src/test/java/org/janelia/saalfeldlab/n5/N5KeyValueWithInterfacesTest.java b/src/test/java/org/janelia/saalfeldlab/n5/N5KeyValueWithInterfacesTest.java index aa85547e..15d96b46 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/N5KeyValueWithInterfacesTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/N5KeyValueWithInterfacesTest.java @@ -61,10 +61,11 @@ public class N5KeyValueWithInterfacesTest extends AbstractN5Test { private static FileSystemKeyValueAccess access = new FileSystemKeyValueAccess(FileSystems.getDefault()); - private static String testDirPath = tempN5PathName(); private static Set tmpFiles = new HashSet<>(); + private static String testDirPath = tempN5PathName(); + private static String tempN5PathName() { try { final File tmpFile = Files.createTempDirectory("n5-test-").toFile(); From a9d12d479c541c8d15a574cb5e5ef0ed1e1760db Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Wed, 17 May 2023 16:00:11 -0400 Subject: [PATCH 177/243] fix/test: ensure testReaderCreation throws N5IOException --- .../org/janelia/saalfeldlab/n5/N5KeyValueReader.java | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java index 1443c294..d8a713a4 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java @@ -116,9 +116,15 @@ protected N5KeyValueReader( if (checkVersion) { /* Existence checks, if any, go in subclasses */ /* Check that version (if there is one) is compatible. */ - final Version version = getVersion(); - if (!VERSION.isCompatible(version)) - throw new N5Exception.N5IOException("Incompatible version " + version + " (this is " + VERSION + ")."); + try { + final Version version = getVersion(); + if (!VERSION.isCompatible(version)) + throw new N5Exception.N5IOException("Incompatible version " + version + " (this is " + VERSION + ")."); + } catch( NullPointerException e ) + { + throw new N5Exception.N5IOException("Could not read version from " + basePath); + } + } } From 8eb3c0c04ac06d57e3d4e491cf7be20f0ca9779e Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Thu, 18 May 2023 17:30:42 -0400 Subject: [PATCH 178/243] fix: ensure the static empty cache instance is not modified * this caused issues in the zarr impl * check this with caleb --- .../saalfeldlab/n5/cache/N5JsonCache.java | 33 +++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/cache/N5JsonCache.java b/src/main/java/org/janelia/saalfeldlab/n5/cache/N5JsonCache.java index 311e8348..89b52413 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/cache/N5JsonCache.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/cache/N5JsonCache.java @@ -194,6 +194,9 @@ public void updateCacheInfo(final String normalPathKey, final String normalCache } else if (!container.existsFromContainer(normalPathKey)) { cacheInfo = emptyCacheInfo; } else { + if( cacheInfo == emptyCacheInfo ) + cacheInfo = new N5CacheInfo(); + final JsonElement attributesToCache = container.getAttributesFromContainer(normalPathKey, normalCacheKey); synchronized (cacheInfo.attributesCache) { cacheInfo.attributesCache.put(normalCacheKey, attributesToCache); @@ -217,6 +220,9 @@ public void updateCacheInfo(final String normalPathKey, final String normalCache } else if (!container.existsFromContainer(normalPathKey)) { cacheInfo = emptyCacheInfo; } else { + if( cacheInfo == emptyCacheInfo ) + cacheInfo = new N5CacheInfo(); + final JsonElement attributesToCache = uncachedAttributes == null ? container.getAttributesFromContainer(normalPathKey, normalCacheKey) : uncachedAttributes; synchronized (cacheInfo.attributesCache) { cacheInfo.attributesCache.put(normalCacheKey, attributesToCache); @@ -233,23 +239,46 @@ public void updateCacheInfo(final String normalPathKey, final String normalCache } public void setIsDataset(final String normalPathKey, final boolean isDataset ) { - final N5CacheInfo cacheInfo = getCacheInfo(normalPathKey); + N5CacheInfo cacheInfo = getCacheInfo(normalPathKey); + boolean update = false; if (cacheInfo == null ){ return; } + if( cacheInfo == emptyCacheInfo ) { + cacheInfo = new N5CacheInfo(); + update = true; + } + synchronized (cacheInfo) { cacheInfo.isDataset = isDataset; } + + if( update ) + synchronized (containerPathToCache) { + containerPathToCache.put(normalPathKey, cacheInfo); + } } public void setAttributes(final String normalPathKey, final String normalCacheKey, final JsonElement attributes ) { - final N5CacheInfo cacheInfo = getCacheInfo(normalPathKey); + N5CacheInfo cacheInfo = getCacheInfo(normalPathKey); + boolean update = false; if (cacheInfo == null ){ return; } + + if( cacheInfo == emptyCacheInfo ) { + cacheInfo = new N5CacheInfo(); + update = true; + } + synchronized (cacheInfo.attributesCache) { cacheInfo.attributesCache.put(normalCacheKey, attributes); } + + if( update ) + synchronized (containerPathToCache) { + containerPathToCache.put(normalPathKey, cacheInfo); + } } public void addChild(final String parent, final String child) { From 2d0d45fd2d7dfc4a4f551beb143887e1c9536e4e Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Mon, 22 May 2023 10:50:44 -0400 Subject: [PATCH 179/243] feat: cache lazier * better separate listing from attribute caching * fewer backend calls --- .../n5/CachedGsonKeyValueReader.java | 24 ++- .../n5/CachedGsonKeyValueWriter.java | 26 +-- .../saalfeldlab/n5/N5KeyValueWriter.java | 17 +- .../saalfeldlab/n5/cache/N5JsonCache.java | 170 +++++++----------- .../n5/cache/N5JsonCacheableContainer.java | 28 ++- .../saalfeldlab/n5/cache/N5CacheTest.java | 64 ++++--- 6 files changed, 179 insertions(+), 150 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueReader.java b/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueReader.java index 0397460a..ef619526 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueReader.java @@ -124,23 +124,25 @@ default boolean exists(final String pathName) { final String normalPathName = N5URL.normalizeGroupPath(pathName); if (cacheMeta()) - return getCache().exists(normalPathName, N5JsonCache.jsonFile); + return getCache().isGroup(normalPathName, N5JsonCache.jsonFile); else { - return existsFromContainer(normalPathName); + return existsFromContainer(normalPathName, null); } } - default boolean existsFromContainer(final String pathName) { + default boolean existsFromContainer(final String normalPathName, final String normalCacheKey) { - final String normalPathName = N5URL.normalizeGroupPath(pathName); - return getKeyValueAccess().exists(groupPath(normalPathName)) || isDatasetFromContainer(normalPathName); + if( normalCacheKey == null ) + return getKeyValueAccess().exists(groupPath(normalPathName)); + else + return getKeyValueAccess().exists(getKeyValueAccess().compose(groupPath(normalPathName), normalCacheKey)); } default boolean groupExists(final String pathName) { final String normalPathName = N5URL.normalizeGroupPath(pathName); if (cacheMeta()) - return getCache().isGroup(normalPathName, N5JsonCache.jsonFile); + return getCache().isGroup(normalPathName, null); else { return isGroupFromContainer(normalPathName); } @@ -151,12 +153,16 @@ default boolean isGroupFromContainer(final String pathName) { return GsonKeyValueReader.super.groupExists(pathName); } + default boolean isGroupFromAttributes(final String normalCacheKey, final JsonElement attributes) { + return true; + } + @Override default boolean datasetExists(final String pathName) throws N5Exception.N5IOException { if (cacheMeta()) { final String normalPathName = N5URL.normalizeGroupPath(pathName); - return getCache().isDataset(normalPathName, N5JsonCache.jsonFile); + return getCache().isDataset(normalPathName, null); } return isDatasetFromContainer(pathName); } @@ -166,6 +172,10 @@ default boolean isDatasetFromContainer(final String pathName) throws N5Exception return isGroupFromContainer(groupPath(pathName)) && normalGetDatasetAttributes(pathName) != null; } + default boolean isDatasetFromAttributes(final String normalCacheKey, final JsonElement attributes) { + return isGroupFromAttributes(normalCacheKey, attributes) && createDatasetAttributes(attributes) != null; + } + /** * Reads or creates the attributes map of a group or dataset. * diff --git a/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueWriter.java b/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueWriter.java index 5a9468aa..2d29fe3d 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueWriter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueWriter.java @@ -71,16 +71,19 @@ default void createGroup(final String path) throws IOException { final String normalPath = N5URL.normalizeGroupPath(path); getKeyValueAccess().createDirectories(groupPath(normalPath)); if (cacheMeta()) { - final String[] pathParts = getKeyValueAccess().components(normalPath); - if (pathParts.length > 1) { - String parent = N5URL.normalizeGroupPath("/"); - for (String child : pathParts) { - final String childPath = getKeyValueAccess().compose(parent, child); - // add the group to the cache - getCache().addNewCacheInfo(childPath, N5JsonCache.jsonFile, null, true, false); + String[] pathParts = getKeyValueAccess().components(normalPath); + String parent = N5URL.normalizeGroupPath("/"); + if (pathParts.length == 0) { + pathParts = new String[]{""}; + } + for (String child : pathParts) { + final String childPath = getKeyValueAccess().compose(parent, child); + // add the group to the cache + getCache().forceAddNewCacheInfo(childPath, N5JsonCache.jsonFile, null, true, false ); + if( parent != null && !child.isEmpty()) getCache().addChild(parent, child); - parent = childPath; - } + + parent = childPath; } } initializeGroup(normalPath); @@ -105,9 +108,6 @@ default void createDataset( final String normalPath = N5URL.normalizeGroupPath(pathName); createGroup(normalPath); setDatasetAttributes(normalPath, datasetAttributes); - - if (cacheMeta()) - getCache().setIsDataset(normalPath, true); } /** @@ -170,7 +170,7 @@ default void writeAndCacheAttributes(final String normalGroupPath, final JsonEle nullRespectingAttributes = getGson().toJsonTree(attributes); } /* Update the cache, and write to the writer */ - getCache().setAttributes(normalGroupPath, N5JsonCache.jsonFile, nullRespectingAttributes); + getCache().updateCacheInfo(normalGroupPath, N5JsonCache.jsonFile, nullRespectingAttributes); } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriter.java b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriter.java index 01f18a82..8763a567 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriter.java @@ -29,6 +29,8 @@ import java.io.IOException; +import org.janelia.saalfeldlab.n5.N5Reader.Version; + /** * Filesystem {@link N5Writer} implementation with version compatibility check. * @@ -65,7 +67,18 @@ public N5KeyValueWriter( throws IOException { super(false, keyValueAccess, basePath, gsonBuilder, cacheAttributes); - createGroup("/"); - setVersion("/"); + + Version version = null; + try { + version = getVersion(); + if (!VERSION.isCompatible(version)) + throw new N5Exception.N5IOException("Incompatible version " + version + " (this is " + VERSION + ")."); + } catch (NullPointerException e) { + } + + if (version == null || version.equals(new Version(0, 0, 0, ""))) { + createGroup("/"); + setVersion("/"); + } } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/cache/N5JsonCache.java b/src/main/java/org/janelia/saalfeldlab/n5/cache/N5JsonCache.java index 89b52413..30c4ea96 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/cache/N5JsonCache.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/cache/N5JsonCache.java @@ -19,9 +19,8 @@ public class N5JsonCache { */ private static class N5CacheInfo { - private final HashSet children = new HashSet<>(); - private final HashMap attributesCache = new HashMap<>(); + private HashSet children = null; private Boolean isDataset = false; private Boolean isGroup = false; @@ -73,29 +72,44 @@ public JsonElement getAttributes(final String normalPathKey, final String normal return getCacheInfo(normalPathKey).getCache(normalCacheKey); } - public boolean isDataset(final String normalPathKey, final String normalCacheKey) { + private void cacheAttributes(final String normalPathKey, final String normalCacheKey) { + + final JsonElement uncachedValue = container.getAttributesFromContainer(normalPathKey, normalCacheKey); + if (uncachedValue != null || container.existsFromContainer(normalPathKey, normalCacheKey)) { + addNewCacheInfo(normalPathKey, normalCacheKey, uncachedValue); + } + } + + public boolean isDataset(final String normalPathKey, final String normalCacheKey ) { final N5CacheInfo cacheInfo = getCacheInfo(normalPathKey); if (cacheInfo == null) { - addNewCacheInfo(normalPathKey, normalCacheKey); + addNewCacheInfo(normalPathKey, normalCacheKey, null); } return getCacheInfo(normalPathKey).isDataset; } - public boolean isGroup(final String normalPathKey, final String normalCacheKey) { + public boolean isGroup(final String normalPathKey, String cacheKey) { final N5CacheInfo cacheInfo = getCacheInfo(normalPathKey); if (cacheInfo == null) { - addNewCacheInfo(normalPathKey, normalCacheKey); + addNewCacheInfo(normalPathKey, cacheKey, null); } return getCacheInfo(normalPathKey).isGroup; } - public boolean exists(final String normalPathKey, final String normalCacheKey ) { + /** + * Returns true if a resource exists. + * + * @param normalPathKey the container path + * @param normalCacheKey the cache key / resource (may be null) + * @return true if exists + */ + public boolean exists(final String normalPathKey, final String normalCacheKey) { final N5CacheInfo cacheInfo = getCacheInfo(normalPathKey); if (cacheInfo == null) { - addNewCacheInfo(normalPathKey, normalCacheKey ); + addNewCacheInfo( normalPathKey, normalCacheKey, null); } return getCacheInfo(normalPathKey) != emptyCacheInfo; } @@ -108,6 +122,9 @@ public String[] list(String normalPathKey) { cacheInfo = getCacheInfo(normalPathKey); if (cacheInfo == emptyCacheInfo) return null; + if( cacheInfo.children == null ) + addChild( cacheInfo, normalPathKey ); + final String[] children = new String[cacheInfo.children.size()]; int i = 0; for (String child : cacheInfo.children) { @@ -116,43 +133,43 @@ public String[] list(String normalPathKey) { return children; } - private void cacheAttributes(final String normalPathKey, final String normalCacheKey) { + public N5CacheInfo addNewCacheInfo(String normalPathKey, String normalCacheKey, JsonElement uncachedAttributes) { - final JsonElement uncachedValue = container.getAttributesFromContainer(normalPathKey, normalCacheKey); - if (uncachedValue != null || container.existsFromContainer(normalPathKey)) { - addNewCacheInfo(normalPathKey, normalCacheKey, uncachedValue); + final N5CacheInfo cacheInfo; + if ( container.existsFromContainer(normalPathKey, null)) { + cacheInfo = new N5CacheInfo(); + } else { + cacheInfo = emptyCacheInfo; } - } - - private void addNewCacheInfo(String normalPathKey, String normalCacheKey) { - addNewCacheInfo( normalPathKey, normalCacheKey, container.getAttributesFromContainer( normalPathKey, normalCacheKey)); - } - - private void addNewCacheInfo(String normalPathKey, String normalCacheKey, JsonElement uncachedAttributes) { + if (cacheInfo != emptyCacheInfo && normalCacheKey != null) { + final JsonElement attributes = (uncachedAttributes == null) + ? container.getAttributesFromContainer(normalPathKey, normalCacheKey) + : uncachedAttributes; - final N5CacheInfo cacheInfo; - if (!container.existsFromContainer(normalPathKey)) { - cacheInfo = emptyCacheInfo; - } else { - cacheInfo = new N5CacheInfo(); synchronized (cacheInfo.attributesCache) { - cacheInfo.attributesCache.put(normalCacheKey, uncachedAttributes); + cacheInfo.attributesCache.put(normalCacheKey, attributes); } + cacheInfo.isGroup = container.isGroupFromAttributes(normalCacheKey, attributes); + cacheInfo.isDataset = container.isDatasetFromAttributes(normalCacheKey, attributes); + } else { cacheInfo.isGroup = container.isGroupFromContainer(normalPathKey); cacheInfo.isDataset = container.isDatasetFromContainer(normalPathKey); - addChild(cacheInfo, normalPathKey); } + synchronized (containerPathToCache) { containerPathToCache.put(normalPathKey, cacheInfo); } + return cacheInfo; } - public void addNewCacheInfo(String normalPathKey, String normalCacheKey, JsonElement uncachedAttributes, boolean isGroup, boolean isDataset) { + public N5CacheInfo forceAddNewCacheInfo(String normalPathKey, String normalCacheKey, JsonElement uncachedAttributes, boolean isGroup, boolean isDataset) { final N5CacheInfo cacheInfo = new N5CacheInfo(); - synchronized (cacheInfo.attributesCache) { - cacheInfo.attributesCache.put(normalCacheKey, uncachedAttributes); + if (normalCacheKey != null) { + synchronized (cacheInfo.attributesCache) { + cacheInfo.attributesCache.put(normalCacheKey, uncachedAttributes); + } } cacheInfo.isGroup = isGroup; cacheInfo.isDataset = isDataset; @@ -160,55 +177,21 @@ public void addNewCacheInfo(String normalPathKey, String normalCacheKey, JsonEle synchronized (containerPathToCache) { containerPathToCache.put(normalPathKey, cacheInfo); } - } - - private void addChild(N5CacheInfo cacheInfo, String normalPathKey) { - - final String[] children = container.listFromContainer(normalPathKey); - Collections.addAll(cacheInfo.children, children); + return cacheInfo; } private N5CacheInfo addNewCacheInfo(String normalPathKey) { - final N5CacheInfo cacheInfo; - if (!container.existsFromContainer(normalPathKey)) { - cacheInfo = emptyCacheInfo; - } else { - cacheInfo = new N5CacheInfo(); - cacheInfo.isGroup = container.isGroupFromContainer(normalPathKey); - cacheInfo.isDataset = container.isDatasetFromContainer(normalPathKey); - addChild(cacheInfo, normalPathKey); - } - synchronized (containerPathToCache) { - containerPathToCache.put(normalPathKey, cacheInfo); - } - return cacheInfo; + return addNewCacheInfo(normalPathKey, null, null); } - public void updateCacheInfo(final String normalPathKey, final String normalCacheKey) { + private void addChild(N5CacheInfo cacheInfo, String normalPathKey) { - N5CacheInfo cacheInfo = getCacheInfo(normalPathKey); - if (cacheInfo == null ){ - addNewCacheInfo(normalPathKey); - return; - } else if (!container.existsFromContainer(normalPathKey)) { - cacheInfo = emptyCacheInfo; - } else { - if( cacheInfo == emptyCacheInfo ) - cacheInfo = new N5CacheInfo(); + if( cacheInfo.children == null ) + cacheInfo.children = new HashSet<>(); - final JsonElement attributesToCache = container.getAttributesFromContainer(normalPathKey, normalCacheKey); - synchronized (cacheInfo.attributesCache) { - cacheInfo.attributesCache.put(normalCacheKey, attributesToCache); - } - cacheInfo.isGroup = container.isGroupFromContainer(normalPathKey); - cacheInfo.isDataset = container.isDatasetFromContainer(normalPathKey); - cacheInfo.children.clear(); - addChild(cacheInfo, normalPathKey); - } - synchronized (containerPathToCache) { - containerPathToCache.put(normalPathKey, cacheInfo); - } + final String[] children = container.listFromContainer(normalPathKey); + Collections.addAll(cacheInfo.children, children); } public void updateCacheInfo(final String normalPathKey, final String normalCacheKey, final JsonElement uncachedAttributes) { @@ -217,20 +200,26 @@ public void updateCacheInfo(final String normalPathKey, final String normalCache if (cacheInfo == null ){ addNewCacheInfo(normalPathKey, normalCacheKey, uncachedAttributes ); return; - } else if (!container.existsFromContainer(normalPathKey)) { + } else if (!container.existsFromContainer(normalPathKey, normalCacheKey)) { cacheInfo = emptyCacheInfo; } else { if( cacheInfo == emptyCacheInfo ) cacheInfo = new N5CacheInfo(); - final JsonElement attributesToCache = uncachedAttributes == null ? container.getAttributesFromContainer(normalPathKey, normalCacheKey) : uncachedAttributes; - synchronized (cacheInfo.attributesCache) { - cacheInfo.attributesCache.put(normalCacheKey, attributesToCache); + if (normalCacheKey != null) { + final JsonElement attributesToCache = uncachedAttributes == null + ? container.getAttributesFromContainer(normalPathKey, normalCacheKey) + : uncachedAttributes; + synchronized (cacheInfo.attributesCache) { + cacheInfo.attributesCache.put(normalCacheKey, attributesToCache); + } + cacheInfo.isGroup = container.isGroupFromAttributes(normalCacheKey, attributesToCache); + cacheInfo.isDataset = container.isDatasetFromAttributes(normalCacheKey, attributesToCache); + } + else { + cacheInfo.isGroup = container.isGroupFromContainer(normalPathKey); + cacheInfo.isDataset = container.isDatasetFromContainer(normalPathKey); } - cacheInfo.isGroup = container.isGroupFromContainer(normalPathKey); - cacheInfo.isDataset = container.isDatasetFromContainer(normalPathKey); - cacheInfo.children.clear(); - addChild(cacheInfo, normalPathKey); } synchronized (containerPathToCache) { @@ -238,27 +227,6 @@ public void updateCacheInfo(final String normalPathKey, final String normalCache } } - public void setIsDataset(final String normalPathKey, final boolean isDataset ) { - N5CacheInfo cacheInfo = getCacheInfo(normalPathKey); - boolean update = false; - if (cacheInfo == null ){ - return; - } - if( cacheInfo == emptyCacheInfo ) { - cacheInfo = new N5CacheInfo(); - update = true; - } - - synchronized (cacheInfo) { - cacheInfo.isDataset = isDataset; - } - - if( update ) - synchronized (containerPathToCache) { - containerPathToCache.put(normalPathKey, cacheInfo); - } - } - public void setAttributes(final String normalPathKey, final String normalCacheKey, final JsonElement attributes ) { N5CacheInfo cacheInfo = getCacheInfo(normalPathKey); boolean update = false; @@ -290,9 +258,9 @@ public void addChild(final String parent, final String child) { public void removeCache(final String normalParentPathKey, final String normalPathKey) { synchronized (containerPathToCache) { - containerPathToCache.remove(normalPathKey); + containerPathToCache.put(normalPathKey, emptyCacheInfo); final N5CacheInfo parentCache = containerPathToCache.get(normalParentPathKey); - if (parentCache != null) { + if (parentCache != null && parentCache.children != null ) { parentCache.children.remove(normalPathKey); } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/cache/N5JsonCacheableContainer.java b/src/main/java/org/janelia/saalfeldlab/n5/cache/N5JsonCacheableContainer.java index 2179003d..0a20ab60 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/cache/N5JsonCacheableContainer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/cache/N5JsonCacheableContainer.java @@ -24,13 +24,13 @@ public interface N5JsonCacheableContainer { JsonElement getAttributesFromContainer(final String normalPathName, final String normalCacheKey); /** - * Query whether a path exists in this container. + * Query whether a resource exists in this container. * * @param normalPathName the normalized path name - * @return true if the path is a group or dataset - * @see N5Reader#exists + * @param normalCacheKey the normalized resource name (may be null). + * @return true if the resouce exists */ - boolean existsFromContainer(final String normalPathName); + boolean existsFromContainer(final String normalPathName, final String normalCacheKey); /** * Query whether a path in this container is a group. @@ -49,6 +49,26 @@ public interface N5JsonCacheableContainer { */ boolean isDatasetFromContainer(final String normalPathName); + /** + * If this method is called, its parent path must exist, + * and normalCacheKey must exist, with contents given by attributes. + * + * @param normalCacheKey the cache key + * @param attributes the attributes + * @return true if the path is a group + */ + boolean isGroupFromAttributes(final String normalCacheKey, final JsonElement attributes); + + /** + * If this method is called, its parent path must exist, + * and normalCacheKey must exist, with contents given by attributes. + * + * @param normalCacheKey the cache key + * @param attributes the attributes + * @return true if the path is a dataset + */ + boolean isDatasetFromAttributes(final String normalCacheKey, final JsonElement attributes); + /** * List the children of a path for this container. * diff --git a/src/test/java/org/janelia/saalfeldlab/n5/cache/N5CacheTest.java b/src/test/java/org/janelia/saalfeldlab/n5/cache/N5CacheTest.java index adec4a39..2f768c6f 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/cache/N5CacheTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/cache/N5CacheTest.java @@ -5,9 +5,11 @@ import org.junit.Test; import com.google.gson.JsonElement; +import com.google.gson.JsonObject; public class N5CacheTest { + @Test public void cacheBackingTest() { @@ -17,69 +19,69 @@ public void cacheBackingTest() { // check existance, ensure backing storage is only called once assertEquals(0, backingStorage.existsCallCount); - cache.exists("a","face"); + cache.exists("a", null); assertEquals(1, backingStorage.existsCallCount); - cache.exists("a","face"); + cache.exists("a", null); assertEquals(1, backingStorage.existsCallCount); // check existance of new group, ensure backing storage is only called one more time - cache.exists("b","face"); + cache.exists("b", null); assertEquals(2, backingStorage.existsCallCount); - cache.exists("b","face"); + cache.exists("b", null); assertEquals(2, backingStorage.existsCallCount); // check isDataset, ensure backing storage is only called when expected // isDataset is called by exists, so should have been called twice here assertEquals(2, backingStorage.isDatasetCallCount); - cache.isDataset("a", "face"); + cache.isDataset("a", null); assertEquals(2, backingStorage.isDatasetCallCount); assertEquals(2, backingStorage.isDatasetCallCount); - cache.isDataset("b", "face"); + cache.isDataset("b", null); assertEquals(2, backingStorage.isDatasetCallCount); // check isGroup, ensure backing storage is only called when expected // isGroup is called by exists, so should have been called twice here assertEquals(2, backingStorage.isGroupCallCount); - cache.isDataset("a", "face"); + cache.isDataset("a", null); assertEquals(2, backingStorage.isGroupCallCount); assertEquals(2, backingStorage.isGroupCallCount); - cache.isDataset("b", "face"); + cache.isDataset("b", null); assertEquals(2, backingStorage.isGroupCallCount); // similarly check list, ensure backing storage is only called when expected // list is called by exists, so should have been called twice here - assertEquals(2, backingStorage.listCallCount); + assertEquals(0, backingStorage.listCallCount); cache.list("a"); - assertEquals(2, backingStorage.listCallCount); + assertEquals(1, backingStorage.listCallCount); - assertEquals(2, backingStorage.listCallCount); + assertEquals(1, backingStorage.listCallCount); cache.list("b"); assertEquals(2, backingStorage.listCallCount); // finally check getAttributes // it is not called by exists (since it needs the cache key) - assertEquals(2, backingStorage.attrCallCount); + assertEquals(0, backingStorage.attrCallCount); cache.getAttributes("a", "foo"); - assertEquals(3, backingStorage.attrCallCount); + assertEquals(1, backingStorage.attrCallCount); cache.getAttributes("a", "foo"); - assertEquals(3, backingStorage.attrCallCount); + assertEquals(1, backingStorage.attrCallCount); cache.getAttributes("a", "bar"); - assertEquals(4, backingStorage.attrCallCount); + assertEquals(2, backingStorage.attrCallCount); cache.getAttributes("a", "bar"); - assertEquals(4, backingStorage.attrCallCount); + assertEquals(2, backingStorage.attrCallCount); cache.getAttributes("a", "face"); - assertEquals(4, backingStorage.attrCallCount); + assertEquals(3, backingStorage.attrCallCount); cache.getAttributes("b", "foo"); - assertEquals(5, backingStorage.attrCallCount); + assertEquals(4, backingStorage.attrCallCount); cache.getAttributes("b", "foo"); - assertEquals(5, backingStorage.attrCallCount); + assertEquals(4, backingStorage.attrCallCount); cache.getAttributes("b", "bar"); - assertEquals(6, backingStorage.attrCallCount); + assertEquals(5, backingStorage.attrCallCount); cache.getAttributes("b", "bar"); - assertEquals(6, backingStorage.attrCallCount); + assertEquals(5, backingStorage.attrCallCount); cache.getAttributes("b", "face"); assertEquals(6, backingStorage.attrCallCount); @@ -91,6 +93,8 @@ protected static class DummyBackingStorage implements N5JsonCacheableContainer { int existsCallCount = 0; int isGroupCallCount = 0; int isDatasetCallCount = 0; + int isGroupFromAttrsCallCount = 0; + int isDatasetFromAttrsCallCount = 0; int listCallCount = 0; public DummyBackingStorage() { @@ -98,10 +102,12 @@ public DummyBackingStorage() { public JsonElement getAttributesFromContainer(final String key, final String cacheKey) { attrCallCount++; - return null; + final JsonObject obj = new JsonObject(); + obj.addProperty("key", "value"); + return obj; } - public boolean existsFromContainer(final String key) { + public boolean existsFromContainer(final String path, final String cacheKey) { existsCallCount++; return true; } @@ -120,6 +126,18 @@ public String[] listFromContainer(final String key) { listCallCount++; return new String[] { "list" }; } + + @Override + public boolean isGroupFromAttributes(final String cacheKey, final JsonElement attributes) { + isGroupFromAttrsCallCount++; + return true; + } + + @Override + public boolean isDatasetFromAttributes(final String cacheKey, final JsonElement attributes) { + isDatasetFromAttrsCallCount++; + return true; + } } } From 15ec1d9761f359f5f11af0a216d9e31e6af4480d Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Tue, 23 May 2023 09:33:46 -0400 Subject: [PATCH 180/243] feat!: KeyValueWriter remove initializeGroup * implementation no longer used by zarr --- .../saalfeldlab/n5/CachedGsonKeyValueWriter.java | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueWriter.java b/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueWriter.java index 2d29fe3d..d52668d0 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueWriter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueWriter.java @@ -53,18 +53,6 @@ default void setVersion(final String path) throws IOException { setAttribute("/", VERSION_KEY, VERSION.toString());; } - /** - * Performs any necessary initialization to ensure the key given by the - * argument {@code normalPath} is a valid group after creation. Called by - * {@link #createGroup(String)}. - * - * @param normalPath the group path. - */ - default void initializeGroup(final String normalPath) { - // Nothing to do here, but other implementations (e.g. zarr) use this. - // TODO remove if not used by zarr - } - @Override default void createGroup(final String path) throws IOException { @@ -86,7 +74,6 @@ default void createGroup(final String path) throws IOException { parent = childPath; } } - initializeGroup(normalPath); } /** From 6d69039d338b610d932953d875b5fca626e8425c Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Tue, 23 May 2023 09:35:36 -0400 Subject: [PATCH 181/243] feat: make N5JsonCache more extensible * add and use methods for modifying N5CacheInfo * zarr extends this class and uses these methods --- .../saalfeldlab/n5/cache/N5JsonCache.java | 136 +++++++++++------- 1 file changed, 87 insertions(+), 49 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/cache/N5JsonCache.java b/src/main/java/org/janelia/saalfeldlab/n5/cache/N5JsonCache.java index 30c4ea96..5defc231 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/cache/N5JsonCache.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/cache/N5JsonCache.java @@ -11,27 +11,27 @@ public class N5JsonCache { public static final N5CacheInfo emptyCacheInfo = new N5CacheInfo(); public static final String jsonFile = "attributes.json"; - private final N5JsonCacheableContainer container; + protected final N5JsonCacheableContainer container; /** * Data object for caching meta data. Elements that are null are not yet * cached. */ - private static class N5CacheInfo { + protected static class N5CacheInfo { - private final HashMap attributesCache = new HashMap<>(); - private HashSet children = null; - private Boolean isDataset = false; - private Boolean isGroup = false; + protected final HashMap attributesCache = new HashMap<>(); + protected HashSet children = null; + protected boolean isDataset = false; + protected boolean isGroup = false; - private JsonElement getCache(final String normalCacheKey) { + protected JsonElement getCache(final String normalCacheKey) { synchronized (attributesCache) { return attributesCache.get(normalCacheKey); } } - private boolean containsKey(final String normalCacheKey) { + protected boolean containsKey(final String normalCacheKey) { synchronized (attributesCache) { return attributesCache.containsKey(normalCacheKey); @@ -69,7 +69,8 @@ public JsonElement getAttributes(final String normalPathKey, final String normal } } - return getCacheInfo(normalPathKey).getCache(normalCacheKey); + final JsonElement output = getCacheInfo(normalPathKey).getCache(normalCacheKey); + return output == null ? null : output.deepCopy(); } private void cacheAttributes(final String normalPathKey, final String normalCacheKey) { @@ -137,7 +138,7 @@ public N5CacheInfo addNewCacheInfo(String normalPathKey, String normalCacheKey, final N5CacheInfo cacheInfo; if ( container.existsFromContainer(normalPathKey, null)) { - cacheInfo = new N5CacheInfo(); + cacheInfo = newCacheInfo(); } else { cacheInfo = emptyCacheInfo; } @@ -147,36 +148,28 @@ public N5CacheInfo addNewCacheInfo(String normalPathKey, String normalCacheKey, ? container.getAttributesFromContainer(normalPathKey, normalCacheKey) : uncachedAttributes; - synchronized (cacheInfo.attributesCache) { - cacheInfo.attributesCache.put(normalCacheKey, attributes); - } - cacheInfo.isGroup = container.isGroupFromAttributes(normalCacheKey, attributes); - cacheInfo.isDataset = container.isDatasetFromAttributes(normalCacheKey, attributes); + updateCacheAttributes(cacheInfo, normalCacheKey, attributes); + updateCacheIsGroup(cacheInfo, container.isGroupFromAttributes(normalCacheKey, attributes)); + updateCacheIsDataset(cacheInfo, container.isDatasetFromAttributes(normalCacheKey, attributes)); } else { - cacheInfo.isGroup = container.isGroupFromContainer(normalPathKey); - cacheInfo.isDataset = container.isDatasetFromContainer(normalPathKey); - } - - synchronized (containerPathToCache) { - containerPathToCache.put(normalPathKey, cacheInfo); + updateCacheIsGroup(cacheInfo, container.isGroupFromContainer(normalPathKey)); + updateCacheIsGroup(cacheInfo, container.isDatasetFromContainer(normalPathKey)); } + updateCache(normalPathKey, cacheInfo); return cacheInfo; } public N5CacheInfo forceAddNewCacheInfo(String normalPathKey, String normalCacheKey, JsonElement uncachedAttributes, boolean isGroup, boolean isDataset) { - final N5CacheInfo cacheInfo = new N5CacheInfo(); + final N5CacheInfo cacheInfo = newCacheInfo(); if (normalCacheKey != null) { synchronized (cacheInfo.attributesCache) { cacheInfo.attributesCache.put(normalCacheKey, uncachedAttributes); } } - cacheInfo.isGroup = isGroup; - cacheInfo.isDataset = isDataset; - addChild(cacheInfo, normalPathKey); - synchronized (containerPathToCache) { - containerPathToCache.put(normalPathKey, cacheInfo); - } + updateCacheIsGroup(cacheInfo, isGroup); + updateCacheIsDataset(cacheInfo, isDataset); + updateCache(normalPathKey, cacheInfo); return cacheInfo; } @@ -204,27 +197,23 @@ public void updateCacheInfo(final String normalPathKey, final String normalCache cacheInfo = emptyCacheInfo; } else { if( cacheInfo == emptyCacheInfo ) - cacheInfo = new N5CacheInfo(); + cacheInfo = newCacheInfo(); if (normalCacheKey != null) { final JsonElement attributesToCache = uncachedAttributes == null ? container.getAttributesFromContainer(normalPathKey, normalCacheKey) : uncachedAttributes; - synchronized (cacheInfo.attributesCache) { - cacheInfo.attributesCache.put(normalCacheKey, attributesToCache); - } - cacheInfo.isGroup = container.isGroupFromAttributes(normalCacheKey, attributesToCache); - cacheInfo.isDataset = container.isDatasetFromAttributes(normalCacheKey, attributesToCache); + + updateCacheAttributes(cacheInfo, normalCacheKey, attributesToCache); + updateCacheIsGroup(cacheInfo, container.isGroupFromAttributes(normalCacheKey, attributesToCache)); + updateCacheIsDataset(cacheInfo, container.isDatasetFromAttributes(normalCacheKey, attributesToCache)); } else { - cacheInfo.isGroup = container.isGroupFromContainer(normalPathKey); - cacheInfo.isDataset = container.isDatasetFromContainer(normalPathKey); + updateCacheIsGroup(cacheInfo, container.isGroupFromContainer(normalPathKey)); + updateCacheIsGroup(cacheInfo, container.isDatasetFromContainer(normalPathKey)); } } - - synchronized (containerPathToCache) { - containerPathToCache.put(normalPathKey, cacheInfo); - } + updateCache( normalPathKey, cacheInfo); } public void setAttributes(final String normalPathKey, final String normalCacheKey, final JsonElement attributes ) { @@ -235,27 +224,51 @@ public void setAttributes(final String normalPathKey, final String normalCacheKe } if( cacheInfo == emptyCacheInfo ) { - cacheInfo = new N5CacheInfo(); + cacheInfo = newCacheInfo(); update = true; } - synchronized (cacheInfo.attributesCache) { - cacheInfo.attributesCache.put(normalCacheKey, attributes); - } + updateCacheAttributes(cacheInfo, normalCacheKey, attributes); if( update ) - synchronized (containerPathToCache) { - containerPathToCache.put(normalPathKey, cacheInfo); - } + updateCache(normalPathKey, cacheInfo); } - public void addChild(final String parent, final String child) { + /** + * Adds child to the parent's children list, only if the parent has been + * cached, and its children list already exists. + * + * @param parent parent path + * @param child child path + */ + public void addChildIfPresent(final String parent, final String child) { final N5CacheInfo cacheInfo = getCacheInfo(parent); if (cacheInfo == null) return; + + if( cacheInfo.children != null ) + cacheInfo.children.add(child); + } + + /** + * Adds child to the parent's children list, only if the parent has been + * cached, creating a children list if it does not already exist. + * + * @param parent parent path + * @param child child path + */ + public void addChild(final String parent, final String child ) { + + final N5CacheInfo cacheInfo = getCacheInfo(parent); + if (cacheInfo == null) return; + + if( cacheInfo.children == null ) + cacheInfo.children = new HashSet<>(); + cacheInfo.children.add(child); } + public void removeCache(final String normalParentPathKey, final String normalPathKey) { synchronized (containerPathToCache) { containerPathToCache.put(normalPathKey, emptyCacheInfo); @@ -275,11 +288,36 @@ public void clearCache(final String normalPathKey, final String normalCacheKey) } } - private N5CacheInfo getCacheInfo(String pathKey) { + protected N5CacheInfo getCacheInfo(String pathKey) { synchronized (containerPathToCache) { return containerPathToCache.get(pathKey); } } + protected N5CacheInfo newCacheInfo() { + return new N5CacheInfo(); + } + + protected void updateCache(final String normalPathKey, N5CacheInfo cacheInfo) { + synchronized (containerPathToCache) { + containerPathToCache.put(normalPathKey, cacheInfo); + } + } + + protected void updateCacheAttributes(final N5CacheInfo cacheInfo, final String normalCacheKey, + final JsonElement attributes) { + synchronized (cacheInfo.attributesCache) { + cacheInfo.attributesCache.put(normalCacheKey, attributes); + } + } + + protected void updateCacheIsGroup(final N5CacheInfo cacheInfo, final boolean isGroup) { + cacheInfo.isGroup = isGroup; + } + + protected void updateCacheIsDataset(final N5CacheInfo cacheInfo, final boolean isDataset) { + cacheInfo.isDataset = isDataset; + } + } From 94b4971b05407e9e0ab086b081178b3ae45e4107 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Tue, 23 May 2023 09:36:20 -0400 Subject: [PATCH 182/243] fix: createGroup updates parent children only when necessary --- .../janelia/saalfeldlab/n5/CachedGsonKeyValueWriter.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueWriter.java b/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueWriter.java index d52668d0..9054a194 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueWriter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueWriter.java @@ -59,6 +59,9 @@ default void createGroup(final String path) throws IOException { final String normalPath = N5URL.normalizeGroupPath(path); getKeyValueAccess().createDirectories(groupPath(normalPath)); if (cacheMeta()) { + getCache().forceAddNewCacheInfo(normalPath, N5JsonCache.jsonFile, null, true, false ); + + // check all nodes that are parents of the added node, if they have a children set, add the new child to it String[] pathParts = getKeyValueAccess().components(normalPath); String parent = N5URL.normalizeGroupPath("/"); if (pathParts.length == 0) { @@ -66,10 +69,9 @@ default void createGroup(final String path) throws IOException { } for (String child : pathParts) { final String childPath = getKeyValueAccess().compose(parent, child); - // add the group to the cache - getCache().forceAddNewCacheInfo(childPath, N5JsonCache.jsonFile, null, true, false ); + // only add if the parent exists and has children cached alreay if( parent != null && !child.isEmpty()) - getCache().addChild(parent, child); + getCache().addChildIfPresent(parent, child); parent = childPath; } From a150ca17df40f069356bcc9d560d3986c1336961 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Tue, 23 May 2023 17:12:49 -0400 Subject: [PATCH 183/243] fix: remove unused methods --- .../saalfeldlab/n5/cache/N5JsonCache.java | 22 +------------------ 1 file changed, 1 insertion(+), 21 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/cache/N5JsonCache.java b/src/main/java/org/janelia/saalfeldlab/n5/cache/N5JsonCache.java index 5defc231..5c453122 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/cache/N5JsonCache.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/cache/N5JsonCache.java @@ -45,22 +45,10 @@ public N5JsonCache( final N5JsonCacheableContainer container ) { this.container = container; } - /** - * Get cached attributes for the group identified by a normalPath - * - * @param cacheInfo - * @param normalCacheKey normalized cache key - * @return cached attributes - * null if the group exists but no attributes are set, the group does not exist, or if attributes have not been cached - */ - private JsonElement getCachedAttributes(final N5CacheInfo cacheInfo, final String normalCacheKey) { - return cacheInfo == null ? null : cacheInfo.getCache(normalCacheKey); - } - public JsonElement getAttributes(final String normalPathKey, final String normalCacheKey) { N5CacheInfo cacheInfo = getCacheInfo(normalPathKey); if (cacheInfo == null) { - cacheAttributes(normalPathKey, normalCacheKey); + addNewCacheInfo(normalPathKey, normalCacheKey, null); } cacheInfo = getCacheInfo(normalPathKey); synchronized (cacheInfo) { @@ -73,14 +61,6 @@ public JsonElement getAttributes(final String normalPathKey, final String normal return output == null ? null : output.deepCopy(); } - private void cacheAttributes(final String normalPathKey, final String normalCacheKey) { - - final JsonElement uncachedValue = container.getAttributesFromContainer(normalPathKey, normalCacheKey); - if (uncachedValue != null || container.existsFromContainer(normalPathKey, normalCacheKey)) { - addNewCacheInfo(normalPathKey, normalCacheKey, uncachedValue); - } - } - public boolean isDataset(final String normalPathKey, final String normalCacheKey ) { final N5CacheInfo cacheInfo = getCacheInfo(normalPathKey); From 953ec4ea0a61207535ff7e06c917b6a0ddd3ca29 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Wed, 24 May 2023 08:50:07 -0400 Subject: [PATCH 184/243] feat: move n5 cache key variable to GsonKVR --- .../saalfeldlab/n5/CachedGsonKeyValueReader.java | 11 ++++++----- .../saalfeldlab/n5/CachedGsonKeyValueWriter.java | 4 ++-- .../janelia/saalfeldlab/n5/GsonKeyValueReader.java | 5 +++-- .../org/janelia/saalfeldlab/n5/cache/N5JsonCache.java | 1 - 4 files changed, 11 insertions(+), 10 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueReader.java b/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueReader.java index ef619526..daa05493 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueReader.java @@ -44,6 +44,7 @@ */ public interface CachedGsonKeyValueReader extends GsonKeyValueReader, N5JsonCacheableContainer { + default N5JsonCache newCache() { return new N5JsonCache( this ); } @@ -69,7 +70,7 @@ default DatasetAttributes getDatasetAttributes(final String pathName) { return null; if (cacheMeta()) { - attributes = getCache().getAttributes(normalPath, N5JsonCache.jsonFile); + attributes = getCache().getAttributes(normalPath, jsonFile); } else { attributes = GsonKeyValueReader.super.getAttributes(normalPath); } @@ -95,7 +96,7 @@ default T getAttribute( final JsonElement attributes; if (cacheMeta()) { - attributes = getCache().getAttributes(normalPathName, N5JsonCache.jsonFile); + attributes = getCache().getAttributes(normalPathName, jsonFile); } else { attributes = GsonKeyValueReader.super.getAttributes(normalPathName); } @@ -112,7 +113,7 @@ default T getAttribute( final String normalizedAttributePath = N5URL.normalizeAttributePath(key); JsonElement attributes; if (cacheMeta()) { - attributes = getCache().getAttributes(normalPathName, N5JsonCache.jsonFile); + attributes = getCache().getAttributes(normalPathName, jsonFile); } else { attributes = GsonKeyValueReader.super.getAttributes(normalPathName); } @@ -124,7 +125,7 @@ default boolean exists(final String pathName) { final String normalPathName = N5URL.normalizeGroupPath(pathName); if (cacheMeta()) - return getCache().isGroup(normalPathName, N5JsonCache.jsonFile); + return getCache().isGroup(normalPathName, jsonFile); else { return existsFromContainer(normalPathName, null); } @@ -189,7 +190,7 @@ default JsonElement getAttributes(final String pathName) throws N5Exception.N5IO /* If cached, return the cache*/ if (cacheMeta()) { - return getCache().getAttributes(groupPath, N5JsonCache.jsonFile); + return getCache().getAttributes(groupPath, jsonFile); } else { return GsonKeyValueReader.super.getAttributes(groupPath); } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueWriter.java b/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueWriter.java index 9054a194..a9ae636c 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueWriter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueWriter.java @@ -59,7 +59,7 @@ default void createGroup(final String path) throws IOException { final String normalPath = N5URL.normalizeGroupPath(path); getKeyValueAccess().createDirectories(groupPath(normalPath)); if (cacheMeta()) { - getCache().forceAddNewCacheInfo(normalPath, N5JsonCache.jsonFile, null, true, false ); + getCache().forceAddNewCacheInfo(normalPath, jsonFile, null, true, false ); // check all nodes that are parents of the added node, if they have a children set, add the new child to it String[] pathParts = getKeyValueAccess().components(normalPath); @@ -159,7 +159,7 @@ default void writeAndCacheAttributes(final String normalGroupPath, final JsonEle nullRespectingAttributes = getGson().toJsonTree(attributes); } /* Update the cache, and write to the writer */ - getCache().updateCacheInfo(normalGroupPath, N5JsonCache.jsonFile, nullRespectingAttributes); + getCache().updateCacheInfo(normalGroupPath, jsonFile, nullRespectingAttributes); } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueReader.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueReader.java index 8231617b..cbcdfeec 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueReader.java @@ -28,7 +28,6 @@ import com.google.gson.Gson; import com.google.gson.JsonElement; import com.google.gson.JsonObject; -import org.janelia.saalfeldlab.n5.cache.N5JsonCache; import java.io.IOException; import java.lang.reflect.Type; @@ -45,6 +44,8 @@ */ public interface GsonKeyValueReader extends N5Reader { + public static final String jsonFile = "attributes.json"; + Gson getGson(); KeyValueAccess getKeyValueAccess(); @@ -220,7 +221,7 @@ default String groupPath(final String normalGroupPath) { */ default String attributesPath(final String normalPath) { - return getKeyValueAccess().compose(getBasePath(), normalPath, N5JsonCache.jsonFile); + return getKeyValueAccess().compose(getBasePath(), normalPath, jsonFile); } static DatasetAttributes createDatasetAttributes( diff --git a/src/main/java/org/janelia/saalfeldlab/n5/cache/N5JsonCache.java b/src/main/java/org/janelia/saalfeldlab/n5/cache/N5JsonCache.java index 5c453122..af000435 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/cache/N5JsonCache.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/cache/N5JsonCache.java @@ -9,7 +9,6 @@ public class N5JsonCache { public static final N5CacheInfo emptyCacheInfo = new N5CacheInfo(); - public static final String jsonFile = "attributes.json"; protected final N5JsonCacheableContainer container; From e22ce118c2f33a5ad5b2dc13cb2c0eb9edd7cbb7 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Wed, 24 May 2023 09:21:48 -0400 Subject: [PATCH 185/243] feat: move and rename ATTRIBUTES_JSON to N5KVR --- .../saalfeldlab/n5/CachedGsonKeyValueReader.java | 10 +++++----- .../saalfeldlab/n5/CachedGsonKeyValueWriter.java | 6 ++---- .../org/janelia/saalfeldlab/n5/GsonKeyValueReader.java | 4 +--- .../org/janelia/saalfeldlab/n5/N5KeyValueReader.java | 2 ++ 4 files changed, 10 insertions(+), 12 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueReader.java b/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueReader.java index daa05493..aadad98e 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueReader.java @@ -70,7 +70,7 @@ default DatasetAttributes getDatasetAttributes(final String pathName) { return null; if (cacheMeta()) { - attributes = getCache().getAttributes(normalPath, jsonFile); + attributes = getCache().getAttributes(normalPath, N5KeyValueReader.ATTRIBUTES_JSON); } else { attributes = GsonKeyValueReader.super.getAttributes(normalPath); } @@ -96,7 +96,7 @@ default T getAttribute( final JsonElement attributes; if (cacheMeta()) { - attributes = getCache().getAttributes(normalPathName, jsonFile); + attributes = getCache().getAttributes(normalPathName, N5KeyValueReader.ATTRIBUTES_JSON); } else { attributes = GsonKeyValueReader.super.getAttributes(normalPathName); } @@ -113,7 +113,7 @@ default T getAttribute( final String normalizedAttributePath = N5URL.normalizeAttributePath(key); JsonElement attributes; if (cacheMeta()) { - attributes = getCache().getAttributes(normalPathName, jsonFile); + attributes = getCache().getAttributes(normalPathName, N5KeyValueReader.ATTRIBUTES_JSON); } else { attributes = GsonKeyValueReader.super.getAttributes(normalPathName); } @@ -125,7 +125,7 @@ default boolean exists(final String pathName) { final String normalPathName = N5URL.normalizeGroupPath(pathName); if (cacheMeta()) - return getCache().isGroup(normalPathName, jsonFile); + return getCache().isGroup(normalPathName, N5KeyValueReader.ATTRIBUTES_JSON); else { return existsFromContainer(normalPathName, null); } @@ -190,7 +190,7 @@ default JsonElement getAttributes(final String pathName) throws N5Exception.N5IO /* If cached, return the cache*/ if (cacheMeta()) { - return getCache().getAttributes(groupPath, jsonFile); + return getCache().getAttributes(groupPath, N5KeyValueReader.ATTRIBUTES_JSON); } else { return GsonKeyValueReader.super.getAttributes(groupPath); } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueWriter.java b/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueWriter.java index a9ae636c..021ff65f 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueWriter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueWriter.java @@ -29,8 +29,6 @@ import com.google.gson.JsonNull; import com.google.gson.JsonObject; -import org.janelia.saalfeldlab.n5.cache.N5JsonCache; - import java.io.IOException; import java.util.Arrays; import java.util.List; @@ -59,7 +57,7 @@ default void createGroup(final String path) throws IOException { final String normalPath = N5URL.normalizeGroupPath(path); getKeyValueAccess().createDirectories(groupPath(normalPath)); if (cacheMeta()) { - getCache().forceAddNewCacheInfo(normalPath, jsonFile, null, true, false ); + getCache().forceAddNewCacheInfo(normalPath, N5KeyValueReader.ATTRIBUTES_JSON, null, true, false ); // check all nodes that are parents of the added node, if they have a children set, add the new child to it String[] pathParts = getKeyValueAccess().components(normalPath); @@ -159,7 +157,7 @@ default void writeAndCacheAttributes(final String normalGroupPath, final JsonEle nullRespectingAttributes = getGson().toJsonTree(attributes); } /* Update the cache, and write to the writer */ - getCache().updateCacheInfo(normalGroupPath, jsonFile, nullRespectingAttributes); + getCache().updateCacheInfo(normalGroupPath, N5KeyValueReader.ATTRIBUTES_JSON, nullRespectingAttributes); } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueReader.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueReader.java index cbcdfeec..fff01ecd 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueReader.java @@ -44,8 +44,6 @@ */ public interface GsonKeyValueReader extends N5Reader { - public static final String jsonFile = "attributes.json"; - Gson getGson(); KeyValueAccess getKeyValueAccess(); @@ -221,7 +219,7 @@ default String groupPath(final String normalGroupPath) { */ default String attributesPath(final String normalPath) { - return getKeyValueAccess().compose(getBasePath(), normalPath, jsonFile); + return getKeyValueAccess().compose(getBasePath(), normalPath, N5KeyValueReader.ATTRIBUTES_JSON); } static DatasetAttributes createDatasetAttributes( diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java index d8a713a4..24452017 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java @@ -44,6 +44,8 @@ */ public class N5KeyValueReader implements CachedGsonKeyValueReader { + public static final String ATTRIBUTES_JSON = "attributes.json"; + protected final KeyValueAccess keyValueAccess; protected final Gson gson; From 9168830efe111476dc79cc9c4af67b86a0824037 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Wed, 24 May 2023 12:14:46 -0400 Subject: [PATCH 186/243] perf: N5JsonCache avoid calling synchronized method twice --- .../saalfeldlab/n5/cache/N5JsonCache.java | 22 ++++++++++++------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/cache/N5JsonCache.java b/src/main/java/org/janelia/saalfeldlab/n5/cache/N5JsonCache.java index af000435..47b4f497 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/cache/N5JsonCache.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/cache/N5JsonCache.java @@ -48,8 +48,11 @@ public JsonElement getAttributes(final String normalPathKey, final String normal N5CacheInfo cacheInfo = getCacheInfo(normalPathKey); if (cacheInfo == null) { addNewCacheInfo(normalPathKey, normalCacheKey, null); + cacheInfo = getCacheInfo(normalPathKey); + } + if (cacheInfo == emptyCacheInfo) { + return null; } - cacheInfo = getCacheInfo(normalPathKey); synchronized (cacheInfo) { if (!cacheInfo.containsKey(normalCacheKey)) { updateCacheInfo(normalPathKey, normalCacheKey, null); @@ -62,20 +65,22 @@ public JsonElement getAttributes(final String normalPathKey, final String normal public boolean isDataset(final String normalPathKey, final String normalCacheKey ) { - final N5CacheInfo cacheInfo = getCacheInfo(normalPathKey); + N5CacheInfo cacheInfo = getCacheInfo(normalPathKey); if (cacheInfo == null) { addNewCacheInfo(normalPathKey, normalCacheKey, null); + cacheInfo = getCacheInfo(normalPathKey); } - return getCacheInfo(normalPathKey).isDataset; + return cacheInfo.isDataset; } public boolean isGroup(final String normalPathKey, String cacheKey) { - final N5CacheInfo cacheInfo = getCacheInfo(normalPathKey); + N5CacheInfo cacheInfo = getCacheInfo(normalPathKey); if (cacheInfo == null) { addNewCacheInfo(normalPathKey, cacheKey, null); + cacheInfo = getCacheInfo(normalPathKey); } - return getCacheInfo(normalPathKey).isGroup; + return cacheInfo.isGroup; } /** @@ -87,19 +92,20 @@ public boolean isGroup(final String normalPathKey, String cacheKey) { */ public boolean exists(final String normalPathKey, final String normalCacheKey) { - final N5CacheInfo cacheInfo = getCacheInfo(normalPathKey); + N5CacheInfo cacheInfo = getCacheInfo(normalPathKey); if (cacheInfo == null) { addNewCacheInfo( normalPathKey, normalCacheKey, null); + cacheInfo = getCacheInfo(normalPathKey); } - return getCacheInfo(normalPathKey) != emptyCacheInfo; + return cacheInfo != emptyCacheInfo; } public String[] list(String normalPathKey) { N5CacheInfo cacheInfo = getCacheInfo(normalPathKey); if (cacheInfo == null) { addNewCacheInfo(normalPathKey); + cacheInfo = getCacheInfo(normalPathKey); } - cacheInfo = getCacheInfo(normalPathKey); if (cacheInfo == emptyCacheInfo) return null; if( cacheInfo.children == null ) From e65d1a049ecac01fb6ec4e44622838ea188de427 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Wed, 24 May 2023 12:16:55 -0400 Subject: [PATCH 187/243] perf/test: fewer backend calls when caching * add tests for number of backend calls * modify addNewCacheInfo, updateCacheInfo to min number of calls * to match expected behavior defined by the new test --- .../saalfeldlab/n5/cache/N5JsonCache.java | 67 ++++--- .../saalfeldlab/n5/N5CachedFSTest.java | 184 ++++++++++++++++++ 2 files changed, 222 insertions(+), 29 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/cache/N5JsonCache.java b/src/main/java/org/janelia/saalfeldlab/n5/cache/N5JsonCache.java index 47b4f497..a7fbfcc8 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/cache/N5JsonCache.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/cache/N5JsonCache.java @@ -122,23 +122,25 @@ public String[] list(String normalPathKey) { public N5CacheInfo addNewCacheInfo(String normalPathKey, String normalCacheKey, JsonElement uncachedAttributes) { final N5CacheInfo cacheInfo; - if ( container.existsFromContainer(normalPathKey, null)) { + if (container.existsFromContainer(normalPathKey, null)) { cacheInfo = newCacheInfo(); } else { cacheInfo = emptyCacheInfo; } - if (cacheInfo != emptyCacheInfo && normalCacheKey != null) { - final JsonElement attributes = (uncachedAttributes == null) - ? container.getAttributesFromContainer(normalPathKey, normalCacheKey) - : uncachedAttributes; + if (cacheInfo != emptyCacheInfo) { + if (normalCacheKey != null) { + final JsonElement attributes = (uncachedAttributes == null) + ? container.getAttributesFromContainer(normalPathKey, normalCacheKey) + : uncachedAttributes; - updateCacheAttributes(cacheInfo, normalCacheKey, attributes); - updateCacheIsGroup(cacheInfo, container.isGroupFromAttributes(normalCacheKey, attributes)); - updateCacheIsDataset(cacheInfo, container.isDatasetFromAttributes(normalCacheKey, attributes)); - } else { - updateCacheIsGroup(cacheInfo, container.isGroupFromContainer(normalPathKey)); - updateCacheIsGroup(cacheInfo, container.isDatasetFromContainer(normalPathKey)); + updateCacheAttributes(cacheInfo, normalCacheKey, attributes); + updateCacheIsGroup(cacheInfo, container.isGroupFromAttributes(normalCacheKey, attributes)); + updateCacheIsDataset(cacheInfo, container.isDatasetFromAttributes(normalCacheKey, attributes)); + } else { + updateCacheIsGroup(cacheInfo, container.isGroupFromContainer(normalPathKey)); + updateCacheIsGroup(cacheInfo, container.isDatasetFromContainer(normalPathKey)); + } } updateCache(normalPathKey, cacheInfo); return cacheInfo; @@ -172,33 +174,40 @@ private void addChild(N5CacheInfo cacheInfo, String normalPathKey) { Collections.addAll(cacheInfo.children, children); } + /** + * Updates the cache attributes for the given normalPathKey and normalCacheKey, + * adding the appropriate node to the cache if necessary. + * + * @param normalPathKey the normalized path key + * @param normalCacheKey the normalized cache key + * @param uncachedAttributes attributes to be cached + */ public void updateCacheInfo(final String normalPathKey, final String normalCacheKey, final JsonElement uncachedAttributes) { N5CacheInfo cacheInfo = getCacheInfo(normalPathKey); if (cacheInfo == null ){ addNewCacheInfo(normalPathKey, normalCacheKey, uncachedAttributes ); return; - } else if (!container.existsFromContainer(normalPathKey, normalCacheKey)) { - cacheInfo = emptyCacheInfo; - } else { - if( cacheInfo == emptyCacheInfo ) - cacheInfo = newCacheInfo(); + } - if (normalCacheKey != null) { - final JsonElement attributesToCache = uncachedAttributes == null - ? container.getAttributesFromContainer(normalPathKey, normalCacheKey) - : uncachedAttributes; + // TODO go through this again with Caleb + if (cacheInfo == emptyCacheInfo) + cacheInfo = newCacheInfo(); - updateCacheAttributes(cacheInfo, normalCacheKey, attributesToCache); - updateCacheIsGroup(cacheInfo, container.isGroupFromAttributes(normalCacheKey, attributesToCache)); - updateCacheIsDataset(cacheInfo, container.isDatasetFromAttributes(normalCacheKey, attributesToCache)); - } - else { - updateCacheIsGroup(cacheInfo, container.isGroupFromContainer(normalPathKey)); - updateCacheIsGroup(cacheInfo, container.isDatasetFromContainer(normalPathKey)); - } + if (normalCacheKey != null) { + final JsonElement attributesToCache = uncachedAttributes == null + ? container.getAttributesFromContainer(normalPathKey, normalCacheKey) + : uncachedAttributes; + + updateCacheAttributes(cacheInfo, normalCacheKey, attributesToCache); + updateCacheIsGroup(cacheInfo, container.isGroupFromAttributes(normalCacheKey, attributesToCache)); + updateCacheIsDataset(cacheInfo, container.isDatasetFromAttributes(normalCacheKey, attributesToCache)); } - updateCache( normalPathKey, cacheInfo); + else { + updateCacheIsGroup(cacheInfo, container.isGroupFromContainer(normalPathKey)); + updateCacheIsGroup(cacheInfo, container.isDatasetFromContainer(normalPathKey)); + } + updateCache(normalPathKey, cacheInfo); } public void setAttributes(final String normalPathKey, final String normalCacheKey, final JsonElement attributes ) { diff --git a/src/test/java/org/janelia/saalfeldlab/n5/N5CachedFSTest.java b/src/test/java/org/janelia/saalfeldlab/n5/N5CachedFSTest.java index dece78db..647d0541 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/N5CachedFSTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/N5CachedFSTest.java @@ -121,4 +121,188 @@ public void cacheGroupDatasetTest() throws IOException, URISyntaxException { } } + @Test + public void emptyCacheInfoBehavior() throws IOException, URISyntaxException { + + final String loc = tempN5Location(); + // make an uncached n5 writer + try (final TrackingStorage n5 = new TrackingStorage(new FileSystemKeyValueAccess(FileSystems.getDefault()), loc, + new GsonBuilder(), true)) { + + final String group = "mygroup"; + boolean groupExists = n5.exists(group); + assertFalse(groupExists); // group does not exist + assertEquals(1, n5.existsCallCount); + assertEquals(0, n5.isGroupCallCount); + assertEquals(0, n5.isDatasetCallCount); + assertEquals(0, n5.attrCallCount); + + n5.createGroup(group); + groupExists = n5.exists(group); + assertTrue(groupExists); // group now exists + assertEquals(1, n5.existsCallCount); + assertEquals(0, n5.isGroupCallCount); + assertEquals(0, n5.isDatasetCallCount); + assertEquals(0, n5.attrCallCount); + } + } + + @Test + public void attributeListingSeparationTest() throws IOException, URISyntaxException { + + final String loc = tempN5Location(); + // make an uncached n5 writer + try (final TrackingStorage n5 = new TrackingStorage( + new FileSystemKeyValueAccess(FileSystems.getDefault()), loc, new GsonBuilder(), true)) { + + final String cachedGroup = "cachedGroup"; + // should not check existence when creating a group + n5.createGroup(cachedGroup); + n5.createGroup(cachedGroup); // be annoying + assertEquals(0, n5.existsCallCount); + assertEquals(0, n5.isGroupCallCount); + assertEquals(0, n5.isDatasetCallCount); + assertEquals(0, n5.attrCallCount); + assertEquals(0, n5.listCallCount); + + // should not check existence when this instance created a group + n5.exists(cachedGroup); + n5.groupExists(cachedGroup); + n5.datasetExists(cachedGroup); + assertEquals(0, n5.existsCallCount); + assertEquals(0, n5.isGroupCallCount); + assertEquals(0, n5.isDatasetCallCount); + assertEquals(0, n5.attrCallCount); + assertEquals(0, n5.listCallCount); + + // should not read attributes from container when setting them + n5.setAttribute(cachedGroup, "one", 1); + assertEquals(0, n5.existsCallCount); + assertEquals(0, n5.isGroupCallCount); + assertEquals(0, n5.isDatasetCallCount); + assertEquals(0, n5.attrCallCount); + assertEquals(0, n5.listCallCount); + + n5.setAttribute(cachedGroup, "two", 2); + assertEquals(0, n5.existsCallCount); + assertEquals(0, n5.isGroupCallCount); + assertEquals(0, n5.isDatasetCallCount); + assertEquals(0, n5.attrCallCount); + assertEquals(0, n5.listCallCount); + + n5.list("/"); + assertEquals(0, n5.existsCallCount); + assertEquals(0, n5.isGroupCallCount); + assertEquals(0, n5.isDatasetCallCount); + assertEquals(0, n5.attrCallCount); + assertEquals(1, n5.listCallCount); + + n5.list(cachedGroup); + assertEquals(0, n5.existsCallCount); + assertEquals(0, n5.isGroupCallCount); + assertEquals(0, n5.isDatasetCallCount); + assertEquals(0, n5.attrCallCount); + assertEquals(2, n5.listCallCount); + + /* + * Check existence for groups that have not been made by this reader but isGroup + * and isDatatset must be false if it does not exists so then should not be + * called. + * + * Similarly, attributes can not exist for a non-existent group, so should not + * attempt to get attributes from the container. + * + * Finally,listing on a non-existent group is pointless, so don't call the + * backend storage + */ + final String nonExistentGroup = "doesNotExist"; + n5.exists(nonExistentGroup); + assertEquals(1, n5.existsCallCount); + assertEquals(0, n5.isGroupCallCount); + assertEquals(0, n5.isDatasetCallCount); + assertEquals(0, n5.attrCallCount); + assertEquals(2, n5.listCallCount); + + n5.groupExists(nonExistentGroup); + assertEquals(1, n5.existsCallCount); + assertEquals(0, n5.isGroupCallCount); + assertEquals(0, n5.isDatasetCallCount); + assertEquals(0, n5.attrCallCount); + assertEquals(2, n5.listCallCount); + + n5.datasetExists(nonExistentGroup); + assertEquals(1, n5.existsCallCount); + assertEquals(0, n5.isGroupCallCount); + assertEquals(0, n5.isDatasetCallCount); + assertEquals(0, n5.attrCallCount); + assertEquals(2, n5.listCallCount); + + n5.getAttributes(nonExistentGroup); + assertEquals(1, n5.existsCallCount); + assertEquals(0, n5.isGroupCallCount); + assertEquals(0, n5.isDatasetCallCount); + assertEquals(0, n5.attrCallCount); + assertEquals(2, n5.listCallCount); + + n5.list(nonExistentGroup); + assertEquals(1, n5.existsCallCount); + assertEquals(0, n5.isGroupCallCount); + assertEquals(0, n5.isDatasetCallCount); + assertEquals(0, n5.attrCallCount); + assertEquals(2, n5.listCallCount); + } + } + + protected static class TrackingStorage extends N5KeyValueWriter { + + int attrCallCount = 0; + int existsCallCount = 0; + int isGroupCallCount = 0; + int isGroupAttrCallCount = 0; + int isDatasetCallCount = 0; + int isDatasetAttrCallCount = 0; + int listCallCount = 0; + + public TrackingStorage(final KeyValueAccess keyValueAccess, final String basePath, + final GsonBuilder gsonBuilder, final boolean cacheAttributes) throws IOException { + + super(keyValueAccess, basePath, gsonBuilder, cacheAttributes); + } + + public JsonElement getAttributesFromContainer(final String key, final String cacheKey) { + attrCallCount++; + return super.getAttributesFromContainer(key, cacheKey); + } + + public boolean existsFromContainer(final String path, final String cacheKey) { + existsCallCount++; + return super.existsFromContainer(path, cacheKey); + } + + public boolean isGroupFromContainer(final String key) { + isGroupCallCount++; + return super.isGroupFromContainer(key); + } + + public boolean isGroupFromAttributes(final String normalCacheKey, final JsonElement attributes) { + isGroupAttrCallCount++; + return super.isGroupFromAttributes(normalCacheKey, attributes); + } + + public boolean isDatasetFromContainer(final String key) { + isDatasetCallCount++; + return super.isDatasetFromContainer(key); + } + + public boolean isDatasetFromAttributes(final String normalCacheKey, final JsonElement attributes) { + isDatasetAttrCallCount++; + return super.isDatasetFromAttributes(normalCacheKey, attributes); + } + + public String[] listFromContainer(final String key) { + listCallCount++; + return super.listFromContainer(key); + } + } + } From 65f2e8a87f2b852c2dce9eaf7b3e8447ae6beca8 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Wed, 24 May 2023 17:38:59 -0400 Subject: [PATCH 188/243] fix/test: correct cache behavior for nested groups * existence of children correct in cache when rm parent * no backend check needed * add a test --- .../n5/CachedGsonKeyValueWriter.java | 8 +- .../saalfeldlab/n5/cache/N5JsonCache.java | 29 +- .../saalfeldlab/n5/N5CachedFSTest.java | 382 ++++++++++++------ 3 files changed, 271 insertions(+), 148 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueWriter.java b/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueWriter.java index 021ff65f..6fecd722 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueWriter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueWriter.java @@ -57,8 +57,6 @@ default void createGroup(final String path) throws IOException { final String normalPath = N5URL.normalizeGroupPath(path); getKeyValueAccess().createDirectories(groupPath(normalPath)); if (cacheMeta()) { - getCache().forceAddNewCacheInfo(normalPath, N5KeyValueReader.ATTRIBUTES_JSON, null, true, false ); - // check all nodes that are parents of the added node, if they have a children set, add the new child to it String[] pathParts = getKeyValueAccess().components(normalPath); String parent = N5URL.normalizeGroupPath("/"); @@ -66,8 +64,11 @@ default void createGroup(final String path) throws IOException { pathParts = new String[]{""}; } for (String child : pathParts) { + final String childPath = getKeyValueAccess().compose(parent, child); - // only add if the parent exists and has children cached alreay + getCache().forceAddNewCacheInfo(childPath, N5KeyValueReader.ATTRIBUTES_JSON, null, true, false ); + + // only add if the parent exists and has children cached already if( parent != null && !child.isEmpty()) getCache().addChildIfPresent(parent, child); @@ -242,6 +243,7 @@ default boolean remove(final String path) throws IOException { final String groupPath = groupPath(normalPath); if (getKeyValueAccess().exists(groupPath)) getKeyValueAccess().delete(groupPath); + if (cacheMeta()) { final String[] pathParts = getKeyValueAccess().components(normalPath); final String parent; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/cache/N5JsonCache.java b/src/main/java/org/janelia/saalfeldlab/n5/cache/N5JsonCache.java index a7fbfcc8..95f49f4d 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/cache/N5JsonCache.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/cache/N5JsonCache.java @@ -59,7 +59,7 @@ public JsonElement getAttributes(final String normalPathKey, final String normal } } - final JsonElement output = getCacheInfo(normalPathKey).getCache(normalCacheKey); + final JsonElement output = cacheInfo.getCache(normalCacheKey); return output == null ? null : output.deepCopy(); } @@ -148,7 +148,11 @@ public N5CacheInfo addNewCacheInfo(String normalPathKey, String normalCacheKey, public N5CacheInfo forceAddNewCacheInfo(String normalPathKey, String normalCacheKey, JsonElement uncachedAttributes, boolean isGroup, boolean isDataset) { - final N5CacheInfo cacheInfo = newCacheInfo(); + // getting the current cache info is useful if it already has had its children listed + N5CacheInfo cacheInfo = getCacheInfo(normalPathKey); + if( cacheInfo == null || cacheInfo == emptyCacheInfo ) + cacheInfo = newCacheInfo(); + if (normalCacheKey != null) { synchronized (cacheInfo.attributesCache) { cacheInfo.attributesCache.put(normalCacheKey, uncachedAttributes); @@ -264,21 +268,20 @@ public void addChild(final String parent, final String child ) { public void removeCache(final String normalParentPathKey, final String normalPathKey) { + // this path and all children should be removed = set to emptyCacheInfo synchronized (containerPathToCache) { containerPathToCache.put(normalPathKey, emptyCacheInfo); - final N5CacheInfo parentCache = containerPathToCache.get(normalParentPathKey); - if (parentCache != null && parentCache.children != null ) { - parentCache.children.remove(normalPathKey); - } + containerPathToCache.keySet().stream().filter(x -> { + return x.startsWith(normalPathKey + "/"); + }).forEach(x -> { + containerPathToCache.put(x, emptyCacheInfo); + }); } - } - public void clearCache(final String normalPathKey, final String normalCacheKey) { - final N5CacheInfo cacheInfo = getCacheInfo(normalPathKey); - if (cacheInfo != null && cacheInfo != emptyCacheInfo) { - synchronized (cacheInfo.attributesCache) { - cacheInfo.attributesCache.remove(normalCacheKey); - } + // update the parent's children, if present (remove the normalPathKey) + final N5CacheInfo parentCache = containerPathToCache.get(normalParentPathKey); + if (parentCache != null && parentCache.children != null ) { + parentCache.children.remove(normalPathKey); } } diff --git a/src/test/java/org/janelia/saalfeldlab/n5/N5CachedFSTest.java b/src/test/java/org/janelia/saalfeldlab/n5/N5CachedFSTest.java index 647d0541..a4facb3f 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/N5CachedFSTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/N5CachedFSTest.java @@ -1,10 +1,14 @@ package org.janelia.saalfeldlab.n5; import com.google.gson.GsonBuilder; +import com.google.gson.JsonElement; + import java.net.URI; import java.net.URISyntaxException; + import org.junit.Test; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; @@ -13,6 +17,7 @@ import java.io.File; import java.io.IOException; +import java.nio.file.FileSystems; import java.nio.file.Files; import java.nio.file.Paths; import java.util.ArrayList; @@ -122,148 +127,233 @@ public void cacheGroupDatasetTest() throws IOException, URISyntaxException { } @Test - public void emptyCacheInfoBehavior() throws IOException, URISyntaxException { + public void cacheBehaviorTest() throws IOException, URISyntaxException { final String loc = tempN5Location(); // make an uncached n5 writer - try (final TrackingStorage n5 = new TrackingStorage(new FileSystemKeyValueAccess(FileSystems.getDefault()), loc, + try (final N5TrackingStorage n5 = new N5TrackingStorage(new FileSystemKeyValueAccess(FileSystems.getDefault()), loc, new GsonBuilder(), true)) { - final String group = "mygroup"; - boolean groupExists = n5.exists(group); - assertFalse(groupExists); // group does not exist - assertEquals(1, n5.existsCallCount); - assertEquals(0, n5.isGroupCallCount); - assertEquals(0, n5.isDatasetCallCount); - assertEquals(0, n5.attrCallCount); - - n5.createGroup(group); - groupExists = n5.exists(group); - assertTrue(groupExists); // group now exists - assertEquals(1, n5.existsCallCount); - assertEquals(0, n5.isGroupCallCount); - assertEquals(0, n5.isDatasetCallCount); - assertEquals(0, n5.attrCallCount); + cacheBehaviorHelper(n5); } } - @Test - public void attributeListingSeparationTest() throws IOException, URISyntaxException { + public static void cacheBehaviorHelper(final TrackingStorage n5) throws IOException, URISyntaxException { + + // non existant group + final String groupA = "groupA"; + final String groupB = "groupB"; + + boolean exists = n5.exists(groupA); + boolean groupExists = n5.groupExists(groupA); + boolean datasetExists = n5.datasetExists(groupA); + assertFalse(exists); // group does not exist + assertFalse(groupExists); // group does not exist + assertFalse(datasetExists); // dataset does not exist + assertEquals(1, n5.getExistCallCount()); + assertEquals(0, n5.getGroupCallCount()); + assertEquals(0, n5.getDatasetCallCount()); + assertEquals(0, n5.getAttrCallCount()); + + n5.createGroup(groupA); + + // group B + exists = n5.exists(groupB); + groupExists = n5.groupExists(groupB); + datasetExists = n5.datasetExists(groupB); + assertFalse(exists); // group now exists + assertFalse(groupExists); // group now exists + assertFalse(datasetExists); // dataset does not exist + assertEquals(2, n5.getExistCallCount()); + assertEquals(0, n5.getGroupCallCount()); + assertEquals(0, n5.getDatasetCallCount()); + assertEquals(0, n5.getAttrCallCount()); + + exists = n5.exists(groupA); + groupExists = n5.groupExists(groupA); + datasetExists = n5.datasetExists(groupA); + assertTrue(exists); // group now exists + assertTrue(groupExists); // group now exists + assertFalse(datasetExists); // dataset does not exist + assertEquals(2, n5.getExistCallCount()); + assertEquals(0, n5.getGroupCallCount()); + assertEquals(0, n5.getDatasetCallCount()); + assertEquals(0, n5.getAttrCallCount()); + + final String cachedGroup = "cachedGroup"; + // should not check existence when creating a group + n5.createGroup(cachedGroup); + n5.createGroup(cachedGroup); // be annoying + assertEquals(2, n5.getExistCallCount()); + assertEquals(0, n5.getGroupCallCount()); + assertEquals(0, n5.getDatasetCallCount()); + assertEquals(0, n5.getAttrCallCount()); + assertEquals(0, n5.getListCallCount()); + + // should not check existence when this instance created a group + n5.exists(cachedGroup); + n5.groupExists(cachedGroup); + n5.datasetExists(cachedGroup); + assertEquals(2, n5.getExistCallCount()); + assertEquals(0, n5.getGroupCallCount()); + assertEquals(0, n5.getDatasetCallCount()); + assertEquals(0, n5.getAttrCallCount()); + assertEquals(0, n5.getListCallCount()); + + // should not read attributes from container when setting them + n5.setAttribute(cachedGroup, "one", 1); + assertEquals(2, n5.getExistCallCount()); + assertEquals(0, n5.getGroupCallCount()); + assertEquals(0, n5.getDatasetCallCount()); + assertEquals(0, n5.getAttrCallCount()); + assertEquals(0, n5.getListCallCount()); + + n5.setAttribute(cachedGroup, "two", 2); + assertEquals(2, n5.getExistCallCount()); + assertEquals(0, n5.getGroupCallCount()); + assertEquals(0, n5.getDatasetCallCount()); + assertEquals(0, n5.getAttrCallCount()); + assertEquals(0, n5.getListCallCount()); + + n5.list(""); + assertEquals(2, n5.getExistCallCount()); + assertEquals(0, n5.getGroupCallCount()); + assertEquals(0, n5.getDatasetCallCount()); + assertEquals(0, n5.getAttrCallCount()); + assertEquals(1, n5.getListCallCount()); + + n5.list(cachedGroup); + assertEquals(2, n5.getExistCallCount()); + assertEquals(0, n5.getGroupCallCount()); + assertEquals(0, n5.getDatasetCallCount()); + assertEquals(0, n5.getAttrCallCount()); + assertEquals(2, n5.getListCallCount()); + + /* + * Check existence for groups that have not been made by this reader but isGroup + * and isDatatset must be false if it does not exists so then should not be + * called. + * + * Similarly, attributes can not exist for a non-existent group, so should not + * attempt to get attributes from the container. + * + * Finally,listing on a non-existent group is pointless, so don't call the + * backend storage + */ + final String nonExistentGroup = "doesNotExist"; + n5.exists(nonExistentGroup); + assertEquals(3, n5.getExistCallCount()); + assertEquals(0, n5.getGroupCallCount()); + assertEquals(0, n5.getDatasetCallCount()); + assertEquals(0, n5.getAttrCallCount()); + assertEquals(2, n5.getListCallCount()); + + n5.groupExists(nonExistentGroup); + assertEquals(3, n5.getExistCallCount()); + assertEquals(0, n5.getGroupCallCount()); + assertEquals(0, n5.getDatasetCallCount()); + assertEquals(0, n5.getAttrCallCount()); + assertEquals(2, n5.getListCallCount()); + + n5.datasetExists(nonExistentGroup); + assertEquals(3, n5.getExistCallCount()); + assertEquals(0, n5.getGroupCallCount()); + assertEquals(0, n5.getDatasetCallCount()); + assertEquals(0, n5.getAttrCallCount()); + assertEquals(2, n5.getListCallCount()); + + n5.getAttributes(nonExistentGroup); + assertEquals(3, n5.getExistCallCount()); + assertEquals(0, n5.getGroupCallCount()); + assertEquals(0, n5.getDatasetCallCount()); + assertEquals(0, n5.getAttrCallCount()); + assertEquals(2, n5.getListCallCount()); + + n5.list(nonExistentGroup); + assertEquals(3, n5.getExistCallCount()); + assertEquals(0, n5.getGroupCallCount()); + assertEquals(0, n5.getDatasetCallCount()); + assertEquals(0, n5.getAttrCallCount()); + assertEquals(2, n5.getListCallCount()); + + String a = "a"; + String ab = "a/b"; + String abc = "a/b/c"; + // create "a/b/c" + n5.createGroup(abc); + assertTrue(n5.exists(abc)); + assertTrue(n5.groupExists(abc)); + assertFalse(n5.datasetExists(abc)); + assertEquals(3, n5.getExistCallCount()); + assertEquals(0, n5.getGroupCallCount()); + assertEquals(0, n5.getDatasetCallCount()); + assertEquals(0, n5.getAttrCallCount()); + assertEquals(2, n5.getListCallCount()); + + // ensure that backend need not be checked when testing existence of "a/b" + assertTrue(n5.exists(ab)); + assertTrue(n5.groupExists(ab)); + assertFalse(n5.datasetExists(ab)); + assertEquals(3, n5.getExistCallCount()); + assertEquals(0, n5.getGroupCallCount()); + assertEquals(0, n5.getDatasetCallCount()); + assertEquals(0, n5.getAttrCallCount()); + assertEquals(2, n5.getListCallCount()); + + // remove a nested group + // checks for all children should not require a backend check + n5.remove(a); + assertFalse(n5.exists(a)); + assertFalse(n5.groupExists(a)); + assertFalse(n5.datasetExists(a)); + assertEquals(3, n5.getExistCallCount()); + assertEquals(0, n5.getGroupCallCount()); + assertEquals(0, n5.getDatasetCallCount()); + assertEquals(0, n5.getAttrCallCount()); + assertEquals(2, n5.getListCallCount()); + + assertFalse(n5.exists(ab)); + assertFalse(n5.groupExists(ab)); + assertFalse(n5.datasetExists(ab)); + assertEquals(3, n5.getExistCallCount()); + assertEquals(0, n5.getGroupCallCount()); + assertEquals(0, n5.getDatasetCallCount()); + assertEquals(0, n5.getAttrCallCount()); + assertEquals(2, n5.getListCallCount()); + + assertFalse(n5.exists(abc)); + assertFalse(n5.groupExists(abc)); + assertFalse(n5.datasetExists(abc)); + assertEquals(3, n5.getExistCallCount()); + assertEquals(0, n5.getGroupCallCount()); + assertEquals(0, n5.getDatasetCallCount()); + assertEquals(0, n5.getAttrCallCount()); + assertEquals(2, n5.getListCallCount()); - final String loc = tempN5Location(); - // make an uncached n5 writer - try (final TrackingStorage n5 = new TrackingStorage( - new FileSystemKeyValueAccess(FileSystems.getDefault()), loc, new GsonBuilder(), true)) { + } - final String cachedGroup = "cachedGroup"; - // should not check existence when creating a group - n5.createGroup(cachedGroup); - n5.createGroup(cachedGroup); // be annoying - assertEquals(0, n5.existsCallCount); - assertEquals(0, n5.isGroupCallCount); - assertEquals(0, n5.isDatasetCallCount); - assertEquals(0, n5.attrCallCount); - assertEquals(0, n5.listCallCount); - - // should not check existence when this instance created a group - n5.exists(cachedGroup); - n5.groupExists(cachedGroup); - n5.datasetExists(cachedGroup); - assertEquals(0, n5.existsCallCount); - assertEquals(0, n5.isGroupCallCount); - assertEquals(0, n5.isDatasetCallCount); - assertEquals(0, n5.attrCallCount); - assertEquals(0, n5.listCallCount); - - // should not read attributes from container when setting them - n5.setAttribute(cachedGroup, "one", 1); - assertEquals(0, n5.existsCallCount); - assertEquals(0, n5.isGroupCallCount); - assertEquals(0, n5.isDatasetCallCount); - assertEquals(0, n5.attrCallCount); - assertEquals(0, n5.listCallCount); - - n5.setAttribute(cachedGroup, "two", 2); - assertEquals(0, n5.existsCallCount); - assertEquals(0, n5.isGroupCallCount); - assertEquals(0, n5.isDatasetCallCount); - assertEquals(0, n5.attrCallCount); - assertEquals(0, n5.listCallCount); - - n5.list("/"); - assertEquals(0, n5.existsCallCount); - assertEquals(0, n5.isGroupCallCount); - assertEquals(0, n5.isDatasetCallCount); - assertEquals(0, n5.attrCallCount); - assertEquals(1, n5.listCallCount); - - n5.list(cachedGroup); - assertEquals(0, n5.existsCallCount); - assertEquals(0, n5.isGroupCallCount); - assertEquals(0, n5.isDatasetCallCount); - assertEquals(0, n5.attrCallCount); - assertEquals(2, n5.listCallCount); - - /* - * Check existence for groups that have not been made by this reader but isGroup - * and isDatatset must be false if it does not exists so then should not be - * called. - * - * Similarly, attributes can not exist for a non-existent group, so should not - * attempt to get attributes from the container. - * - * Finally,listing on a non-existent group is pointless, so don't call the - * backend storage - */ - final String nonExistentGroup = "doesNotExist"; - n5.exists(nonExistentGroup); - assertEquals(1, n5.existsCallCount); - assertEquals(0, n5.isGroupCallCount); - assertEquals(0, n5.isDatasetCallCount); - assertEquals(0, n5.attrCallCount); - assertEquals(2, n5.listCallCount); - - n5.groupExists(nonExistentGroup); - assertEquals(1, n5.existsCallCount); - assertEquals(0, n5.isGroupCallCount); - assertEquals(0, n5.isDatasetCallCount); - assertEquals(0, n5.attrCallCount); - assertEquals(2, n5.listCallCount); - - n5.datasetExists(nonExistentGroup); - assertEquals(1, n5.existsCallCount); - assertEquals(0, n5.isGroupCallCount); - assertEquals(0, n5.isDatasetCallCount); - assertEquals(0, n5.attrCallCount); - assertEquals(2, n5.listCallCount); - - n5.getAttributes(nonExistentGroup); - assertEquals(1, n5.existsCallCount); - assertEquals(0, n5.isGroupCallCount); - assertEquals(0, n5.isDatasetCallCount); - assertEquals(0, n5.attrCallCount); - assertEquals(2, n5.listCallCount); - - n5.list(nonExistentGroup); - assertEquals(1, n5.existsCallCount); - assertEquals(0, n5.isGroupCallCount); - assertEquals(0, n5.isDatasetCallCount); - assertEquals(0, n5.attrCallCount); - assertEquals(2, n5.listCallCount); - } + public static interface TrackingStorage extends CachedGsonKeyValueWriter { + + public int getAttrCallCount(); + public int getExistCallCount(); + public int getGroupCallCount(); + public int getGroupAttrCallCount(); + public int getDatasetCallCount(); + public int getDatasetAttrCallCount(); + public int getListCallCount(); } - protected static class TrackingStorage extends N5KeyValueWriter { + public static class N5TrackingStorage extends N5KeyValueWriter implements TrackingStorage { - int attrCallCount = 0; - int existsCallCount = 0; - int isGroupCallCount = 0; - int isGroupAttrCallCount = 0; - int isDatasetCallCount = 0; - int isDatasetAttrCallCount = 0; - int listCallCount = 0; + public int attrCallCount = 0; + public int existsCallCount = 0; + public int groupCallCount = 0; + public int groupAttrCallCount = 0; + public int datasetCallCount = 0; + public int datasetAttrCallCount = 0; + public int listCallCount = 0; - public TrackingStorage(final KeyValueAccess keyValueAccess, final String basePath, + public N5TrackingStorage(final KeyValueAccess keyValueAccess, final String basePath, final GsonBuilder gsonBuilder, final boolean cacheAttributes) throws IOException { super(keyValueAccess, basePath, gsonBuilder, cacheAttributes); @@ -280,22 +370,22 @@ public boolean existsFromContainer(final String path, final String cacheKey) { } public boolean isGroupFromContainer(final String key) { - isGroupCallCount++; + groupCallCount++; return super.isGroupFromContainer(key); } public boolean isGroupFromAttributes(final String normalCacheKey, final JsonElement attributes) { - isGroupAttrCallCount++; + groupAttrCallCount++; return super.isGroupFromAttributes(normalCacheKey, attributes); } public boolean isDatasetFromContainer(final String key) { - isDatasetCallCount++; + datasetCallCount++; return super.isDatasetFromContainer(key); } public boolean isDatasetFromAttributes(final String normalCacheKey, final JsonElement attributes) { - isDatasetAttrCallCount++; + datasetAttrCallCount++; return super.isDatasetFromAttributes(normalCacheKey, attributes); } @@ -303,6 +393,34 @@ public String[] listFromContainer(final String key) { listCallCount++; return super.listFromContainer(key); } + + public int getAttrCallCount() { + return attrCallCount; + } + + public int getExistCallCount() { + return existsCallCount; + } + + public int getGroupCallCount() { + return groupCallCount; + } + + public int getGroupAttrCallCount() { + return groupAttrCallCount; + } + + public int getDatasetCallCount() { + return datasetCallCount; + } + + public int getDatasetAttrCallCount() { + return datasetAttrCallCount; + } + + public int getListCallCount() { + return listCallCount; + } } } From e877063d78f1eea1c488245cf497bcc2fe0086ae Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Wed, 24 May 2023 17:53:42 -0400 Subject: [PATCH 189/243] fix/test: removeCache correctly updates children for listing * test the desired behavior --- .../saalfeldlab/n5/cache/N5JsonCache.java | 2 +- .../saalfeldlab/n5/N5CachedFSTest.java | 20 +++++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/cache/N5JsonCache.java b/src/main/java/org/janelia/saalfeldlab/n5/cache/N5JsonCache.java index 95f49f4d..0bbc16b8 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/cache/N5JsonCache.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/cache/N5JsonCache.java @@ -281,7 +281,7 @@ public void removeCache(final String normalParentPathKey, final String normalPat // update the parent's children, if present (remove the normalPathKey) final N5CacheInfo parentCache = containerPathToCache.get(normalParentPathKey); if (parentCache != null && parentCache.children != null ) { - parentCache.children.remove(normalPathKey); + parentCache.children.remove(normalPathKey.replaceFirst(normalParentPathKey + "/", "")); } } diff --git a/src/test/java/org/janelia/saalfeldlab/n5/N5CachedFSTest.java b/src/test/java/org/janelia/saalfeldlab/n5/N5CachedFSTest.java index a4facb3f..e9f8a1c8 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/N5CachedFSTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/N5CachedFSTest.java @@ -330,6 +330,26 @@ public static void cacheBehaviorHelper(final TrackingStorage n5) throws IOExcept assertEquals(0, n5.getAttrCallCount()); assertEquals(2, n5.getListCallCount()); + n5.createGroup("a"); + n5.createGroup("a/a"); + n5.createGroup("a/b"); + n5.createGroup("a/c"); + assertArrayEquals(new String[] {"a", "b", "c"}, n5.list("a")); // call list + assertEquals(3, n5.getExistCallCount()); + assertEquals(0, n5.getGroupCallCount()); + assertEquals(0, n5.getDatasetCallCount()); + assertEquals(0, n5.getAttrCallCount()); + assertEquals(3, n5.getListCallCount()); // list incremented + + // remove a + n5.remove("a/a"); + assertArrayEquals(new String[] {"b", "c"}, n5.list("a")); // call list + assertEquals(3, n5.getExistCallCount()); + assertEquals(0, n5.getGroupCallCount()); + assertEquals(0, n5.getDatasetCallCount()); + assertEquals(0, n5.getAttrCallCount()); + assertEquals(3, n5.getListCallCount()); // list NOT incremented + } public static interface TrackingStorage extends CachedGsonKeyValueWriter { From f3a4bbcba6d41daa37315e8b8fe44e76a294a9bf Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Thu, 25 May 2023 11:28:38 -0400 Subject: [PATCH 190/243] style: remove outdated commented line --- src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java b/src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java index e53db44c..97ef57b9 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java @@ -35,7 +35,6 @@ * * @author Stephan Saalfeld */ -//public class N5FSWriter extends N5KeyValueWriter implements N5Writer { public class N5FSWriter extends N5KeyValueWriter { /** From f9ec10fe0cafcc9cd0d5d90b9bbc2901a7086e12 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Thu, 25 May 2023 11:30:07 -0400 Subject: [PATCH 191/243] fix: missing import --- src/test/java/org/janelia/saalfeldlab/n5/N5CachedFSTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/N5CachedFSTest.java b/src/test/java/org/janelia/saalfeldlab/n5/N5CachedFSTest.java index e9f8a1c8..89ac93f7 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/N5CachedFSTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/N5CachedFSTest.java @@ -8,6 +8,7 @@ import org.junit.Test; +import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; @@ -57,7 +58,6 @@ protected N5Reader createN5Reader(final String location, final GsonBuilder gson, public void cacheTest() throws IOException, URISyntaxException { /* Test the cache by setting many attributes, then manually deleting the underlying file. * The only possible way for the test to succeed is if it never again attempts to read the file, and relies on the cache. */ - try (N5KeyValueWriter n5 = (N5KeyValueWriter) createN5Writer()) { final String cachedGroup = "cachedGroup"; final String attributesPath = n5.attributesPath(cachedGroup); From 19367844346c73babacae866e4230a91214a9506 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Thu, 25 May 2023 11:54:24 -0400 Subject: [PATCH 192/243] test: more readable cache behavior helper --- .../saalfeldlab/n5/N5CachedFSTest.java | 211 +++++++++--------- 1 file changed, 109 insertions(+), 102 deletions(-) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/N5CachedFSTest.java b/src/test/java/org/janelia/saalfeldlab/n5/N5CachedFSTest.java index 89ac93f7..954b54b0 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/N5CachedFSTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/N5CachedFSTest.java @@ -144,16 +144,23 @@ public static void cacheBehaviorHelper(final TrackingStorage n5) throws IOExcept final String groupA = "groupA"; final String groupB = "groupB"; + // expected backend method call counts + int expectedExistCount = 0; + int expectedGroupCount = 0; + int expectedDatasetCount = 0; + int expectedAttributeCount = 0; + int expectedListCount = 0; + boolean exists = n5.exists(groupA); boolean groupExists = n5.groupExists(groupA); boolean datasetExists = n5.datasetExists(groupA); assertFalse(exists); // group does not exist assertFalse(groupExists); // group does not exist assertFalse(datasetExists); // dataset does not exist - assertEquals(1, n5.getExistCallCount()); - assertEquals(0, n5.getGroupCallCount()); - assertEquals(0, n5.getDatasetCallCount()); - assertEquals(0, n5.getAttrCallCount()); + assertEquals(++expectedExistCount, n5.getExistCallCount()); + assertEquals(expectedGroupCount, n5.getGroupCallCount()); + assertEquals(expectedDatasetCount, n5.getDatasetCallCount()); + assertEquals(expectedAttributeCount, n5.getAttrCallCount()); n5.createGroup(groupA); @@ -164,10 +171,10 @@ public static void cacheBehaviorHelper(final TrackingStorage n5) throws IOExcept assertFalse(exists); // group now exists assertFalse(groupExists); // group now exists assertFalse(datasetExists); // dataset does not exist - assertEquals(2, n5.getExistCallCount()); - assertEquals(0, n5.getGroupCallCount()); - assertEquals(0, n5.getDatasetCallCount()); - assertEquals(0, n5.getAttrCallCount()); + assertEquals(++expectedExistCount, n5.getExistCallCount()); + assertEquals(expectedGroupCount, n5.getGroupCallCount()); + assertEquals(expectedDatasetCount, n5.getDatasetCallCount()); + assertEquals(expectedAttributeCount, n5.getAttrCallCount()); exists = n5.exists(groupA); groupExists = n5.groupExists(groupA); @@ -175,59 +182,59 @@ public static void cacheBehaviorHelper(final TrackingStorage n5) throws IOExcept assertTrue(exists); // group now exists assertTrue(groupExists); // group now exists assertFalse(datasetExists); // dataset does not exist - assertEquals(2, n5.getExistCallCount()); - assertEquals(0, n5.getGroupCallCount()); - assertEquals(0, n5.getDatasetCallCount()); - assertEquals(0, n5.getAttrCallCount()); + assertEquals(expectedExistCount, n5.getExistCallCount()); + assertEquals(expectedGroupCount, n5.getGroupCallCount()); + assertEquals(expectedDatasetCount, n5.getDatasetCallCount()); + assertEquals(expectedAttributeCount, n5.getAttrCallCount()); final String cachedGroup = "cachedGroup"; // should not check existence when creating a group n5.createGroup(cachedGroup); n5.createGroup(cachedGroup); // be annoying - assertEquals(2, n5.getExistCallCount()); - assertEquals(0, n5.getGroupCallCount()); - assertEquals(0, n5.getDatasetCallCount()); - assertEquals(0, n5.getAttrCallCount()); - assertEquals(0, n5.getListCallCount()); + assertEquals(expectedExistCount, n5.getExistCallCount()); + assertEquals(expectedGroupCount, n5.getGroupCallCount()); + assertEquals(expectedDatasetCount, n5.getDatasetCallCount()); + assertEquals(expectedAttributeCount, n5.getAttrCallCount()); + assertEquals(expectedListCount, n5.getListCallCount()); // should not check existence when this instance created a group n5.exists(cachedGroup); n5.groupExists(cachedGroup); n5.datasetExists(cachedGroup); - assertEquals(2, n5.getExistCallCount()); - assertEquals(0, n5.getGroupCallCount()); - assertEquals(0, n5.getDatasetCallCount()); - assertEquals(0, n5.getAttrCallCount()); - assertEquals(0, n5.getListCallCount()); + assertEquals(expectedExistCount, n5.getExistCallCount()); + assertEquals(expectedGroupCount, n5.getGroupCallCount()); + assertEquals(expectedDatasetCount, n5.getDatasetCallCount()); + assertEquals(expectedAttributeCount, n5.getAttrCallCount()); + assertEquals(expectedListCount, n5.getListCallCount()); // should not read attributes from container when setting them n5.setAttribute(cachedGroup, "one", 1); - assertEquals(2, n5.getExistCallCount()); - assertEquals(0, n5.getGroupCallCount()); - assertEquals(0, n5.getDatasetCallCount()); - assertEquals(0, n5.getAttrCallCount()); - assertEquals(0, n5.getListCallCount()); + assertEquals(expectedExistCount, n5.getExistCallCount()); + assertEquals(expectedGroupCount, n5.getGroupCallCount()); + assertEquals(expectedDatasetCount, n5.getDatasetCallCount()); + assertEquals(expectedAttributeCount, n5.getAttrCallCount()); + assertEquals(expectedListCount, n5.getListCallCount()); n5.setAttribute(cachedGroup, "two", 2); - assertEquals(2, n5.getExistCallCount()); - assertEquals(0, n5.getGroupCallCount()); - assertEquals(0, n5.getDatasetCallCount()); - assertEquals(0, n5.getAttrCallCount()); - assertEquals(0, n5.getListCallCount()); + assertEquals(expectedExistCount, n5.getExistCallCount()); + assertEquals(expectedGroupCount, n5.getGroupCallCount()); + assertEquals(expectedDatasetCount, n5.getDatasetCallCount()); + assertEquals(expectedAttributeCount, n5.getAttrCallCount()); + assertEquals(expectedListCount, n5.getListCallCount()); n5.list(""); - assertEquals(2, n5.getExistCallCount()); - assertEquals(0, n5.getGroupCallCount()); - assertEquals(0, n5.getDatasetCallCount()); - assertEquals(0, n5.getAttrCallCount()); - assertEquals(1, n5.getListCallCount()); + assertEquals(expectedExistCount, n5.getExistCallCount()); + assertEquals(expectedGroupCount, n5.getGroupCallCount()); + assertEquals(expectedDatasetCount, n5.getDatasetCallCount()); + assertEquals(expectedAttributeCount, n5.getAttrCallCount()); + assertEquals(++expectedListCount, n5.getListCallCount()); n5.list(cachedGroup); - assertEquals(2, n5.getExistCallCount()); - assertEquals(0, n5.getGroupCallCount()); - assertEquals(0, n5.getDatasetCallCount()); - assertEquals(0, n5.getAttrCallCount()); - assertEquals(2, n5.getListCallCount()); + assertEquals(expectedExistCount, n5.getExistCallCount()); + assertEquals(expectedGroupCount, n5.getGroupCallCount()); + assertEquals(expectedDatasetCount, n5.getDatasetCallCount()); + assertEquals(expectedAttributeCount, n5.getAttrCallCount()); + assertEquals(++expectedListCount, n5.getListCallCount()); /* * Check existence for groups that have not been made by this reader but isGroup @@ -242,39 +249,39 @@ public static void cacheBehaviorHelper(final TrackingStorage n5) throws IOExcept */ final String nonExistentGroup = "doesNotExist"; n5.exists(nonExistentGroup); - assertEquals(3, n5.getExistCallCount()); - assertEquals(0, n5.getGroupCallCount()); - assertEquals(0, n5.getDatasetCallCount()); - assertEquals(0, n5.getAttrCallCount()); - assertEquals(2, n5.getListCallCount()); + assertEquals(++expectedExistCount, n5.getExistCallCount()); + assertEquals(expectedGroupCount, n5.getGroupCallCount()); + assertEquals(expectedDatasetCount, n5.getDatasetCallCount()); + assertEquals(expectedAttributeCount, n5.getAttrCallCount()); + assertEquals(expectedListCount, n5.getListCallCount()); n5.groupExists(nonExistentGroup); - assertEquals(3, n5.getExistCallCount()); - assertEquals(0, n5.getGroupCallCount()); - assertEquals(0, n5.getDatasetCallCount()); - assertEquals(0, n5.getAttrCallCount()); - assertEquals(2, n5.getListCallCount()); + assertEquals(expectedExistCount, n5.getExistCallCount()); + assertEquals(expectedGroupCount, n5.getGroupCallCount()); + assertEquals(expectedDatasetCount, n5.getDatasetCallCount()); + assertEquals(expectedAttributeCount, n5.getAttrCallCount()); + assertEquals(expectedListCount, n5.getListCallCount()); n5.datasetExists(nonExistentGroup); - assertEquals(3, n5.getExistCallCount()); - assertEquals(0, n5.getGroupCallCount()); - assertEquals(0, n5.getDatasetCallCount()); - assertEquals(0, n5.getAttrCallCount()); - assertEquals(2, n5.getListCallCount()); + assertEquals(expectedExistCount, n5.getExistCallCount()); + assertEquals(expectedGroupCount, n5.getGroupCallCount()); + assertEquals(expectedDatasetCount, n5.getDatasetCallCount()); + assertEquals(expectedAttributeCount, n5.getAttrCallCount()); + assertEquals(expectedListCount, n5.getListCallCount()); n5.getAttributes(nonExistentGroup); - assertEquals(3, n5.getExistCallCount()); - assertEquals(0, n5.getGroupCallCount()); - assertEquals(0, n5.getDatasetCallCount()); - assertEquals(0, n5.getAttrCallCount()); - assertEquals(2, n5.getListCallCount()); + assertEquals(expectedExistCount, n5.getExistCallCount()); + assertEquals(expectedGroupCount, n5.getGroupCallCount()); + assertEquals(expectedDatasetCount, n5.getDatasetCallCount()); + assertEquals(expectedAttributeCount, n5.getAttrCallCount()); + assertEquals(expectedListCount, n5.getListCallCount()); n5.list(nonExistentGroup); - assertEquals(3, n5.getExistCallCount()); - assertEquals(0, n5.getGroupCallCount()); - assertEquals(0, n5.getDatasetCallCount()); - assertEquals(0, n5.getAttrCallCount()); - assertEquals(2, n5.getListCallCount()); + assertEquals(expectedExistCount, n5.getExistCallCount()); + assertEquals(expectedGroupCount, n5.getGroupCallCount()); + assertEquals(expectedDatasetCount, n5.getDatasetCallCount()); + assertEquals(expectedAttributeCount, n5.getAttrCallCount()); + assertEquals(expectedListCount, n5.getListCallCount()); String a = "a"; String ab = "a/b"; @@ -284,21 +291,21 @@ public static void cacheBehaviorHelper(final TrackingStorage n5) throws IOExcept assertTrue(n5.exists(abc)); assertTrue(n5.groupExists(abc)); assertFalse(n5.datasetExists(abc)); - assertEquals(3, n5.getExistCallCount()); - assertEquals(0, n5.getGroupCallCount()); - assertEquals(0, n5.getDatasetCallCount()); - assertEquals(0, n5.getAttrCallCount()); - assertEquals(2, n5.getListCallCount()); + assertEquals(expectedExistCount, n5.getExistCallCount()); + assertEquals(expectedGroupCount, n5.getGroupCallCount()); + assertEquals(expectedDatasetCount, n5.getDatasetCallCount()); + assertEquals(expectedAttributeCount, n5.getAttrCallCount()); + assertEquals(expectedListCount, n5.getListCallCount()); // ensure that backend need not be checked when testing existence of "a/b" assertTrue(n5.exists(ab)); assertTrue(n5.groupExists(ab)); assertFalse(n5.datasetExists(ab)); - assertEquals(3, n5.getExistCallCount()); - assertEquals(0, n5.getGroupCallCount()); - assertEquals(0, n5.getDatasetCallCount()); - assertEquals(0, n5.getAttrCallCount()); - assertEquals(2, n5.getListCallCount()); + assertEquals(expectedExistCount, n5.getExistCallCount()); + assertEquals(expectedGroupCount, n5.getGroupCallCount()); + assertEquals(expectedDatasetCount, n5.getDatasetCallCount()); + assertEquals(expectedAttributeCount, n5.getAttrCallCount()); + assertEquals(expectedListCount, n5.getListCallCount()); // remove a nested group // checks for all children should not require a backend check @@ -306,49 +313,49 @@ public static void cacheBehaviorHelper(final TrackingStorage n5) throws IOExcept assertFalse(n5.exists(a)); assertFalse(n5.groupExists(a)); assertFalse(n5.datasetExists(a)); - assertEquals(3, n5.getExistCallCount()); - assertEquals(0, n5.getGroupCallCount()); - assertEquals(0, n5.getDatasetCallCount()); - assertEquals(0, n5.getAttrCallCount()); - assertEquals(2, n5.getListCallCount()); + assertEquals(expectedExistCount, n5.getExistCallCount()); + assertEquals(expectedGroupCount, n5.getGroupCallCount()); + assertEquals(expectedDatasetCount, n5.getDatasetCallCount()); + assertEquals(expectedAttributeCount, n5.getAttrCallCount()); + assertEquals(expectedListCount, n5.getListCallCount()); assertFalse(n5.exists(ab)); assertFalse(n5.groupExists(ab)); assertFalse(n5.datasetExists(ab)); - assertEquals(3, n5.getExistCallCount()); - assertEquals(0, n5.getGroupCallCount()); - assertEquals(0, n5.getDatasetCallCount()); - assertEquals(0, n5.getAttrCallCount()); - assertEquals(2, n5.getListCallCount()); + assertEquals(expectedExistCount, n5.getExistCallCount()); + assertEquals(expectedGroupCount, n5.getGroupCallCount()); + assertEquals(expectedDatasetCount, n5.getDatasetCallCount()); + assertEquals(expectedAttributeCount, n5.getAttrCallCount()); + assertEquals(expectedListCount, n5.getListCallCount()); assertFalse(n5.exists(abc)); assertFalse(n5.groupExists(abc)); assertFalse(n5.datasetExists(abc)); - assertEquals(3, n5.getExistCallCount()); - assertEquals(0, n5.getGroupCallCount()); - assertEquals(0, n5.getDatasetCallCount()); - assertEquals(0, n5.getAttrCallCount()); - assertEquals(2, n5.getListCallCount()); + assertEquals(expectedExistCount, n5.getExistCallCount()); + assertEquals(expectedGroupCount, n5.getGroupCallCount()); + assertEquals(expectedDatasetCount, n5.getDatasetCallCount()); + assertEquals(expectedAttributeCount, n5.getAttrCallCount()); + assertEquals(expectedListCount, n5.getListCallCount()); n5.createGroup("a"); n5.createGroup("a/a"); n5.createGroup("a/b"); n5.createGroup("a/c"); assertArrayEquals(new String[] {"a", "b", "c"}, n5.list("a")); // call list - assertEquals(3, n5.getExistCallCount()); - assertEquals(0, n5.getGroupCallCount()); - assertEquals(0, n5.getDatasetCallCount()); - assertEquals(0, n5.getAttrCallCount()); - assertEquals(3, n5.getListCallCount()); // list incremented + assertEquals(expectedExistCount, n5.getExistCallCount()); + assertEquals(expectedGroupCount, n5.getGroupCallCount()); + assertEquals(expectedDatasetCount, n5.getDatasetCallCount()); + assertEquals(expectedAttributeCount, n5.getAttrCallCount()); + assertEquals(++expectedListCount, n5.getListCallCount()); // list incremented // remove a n5.remove("a/a"); assertArrayEquals(new String[] {"b", "c"}, n5.list("a")); // call list - assertEquals(3, n5.getExistCallCount()); - assertEquals(0, n5.getGroupCallCount()); - assertEquals(0, n5.getDatasetCallCount()); - assertEquals(0, n5.getAttrCallCount()); - assertEquals(3, n5.getListCallCount()); // list NOT incremented + assertEquals(expectedExistCount, n5.getExistCallCount()); + assertEquals(expectedGroupCount, n5.getGroupCallCount()); + assertEquals(expectedDatasetCount, n5.getDatasetCallCount()); + assertEquals(expectedAttributeCount, n5.getAttrCallCount()); + assertEquals(expectedListCount, n5.getListCallCount()); // list NOT incremented } From 095bda3f47d8d4915454ff32cbdc553bb659260f Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Fri, 26 May 2023 10:14:13 -0400 Subject: [PATCH 193/243] fix: cache now correctly sets isGroup, isDataset on add and update --- .../java/org/janelia/saalfeldlab/n5/cache/N5JsonCache.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/cache/N5JsonCache.java b/src/main/java/org/janelia/saalfeldlab/n5/cache/N5JsonCache.java index 0bbc16b8..ea9e3161 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/cache/N5JsonCache.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/cache/N5JsonCache.java @@ -139,7 +139,7 @@ public N5CacheInfo addNewCacheInfo(String normalPathKey, String normalCacheKey, updateCacheIsDataset(cacheInfo, container.isDatasetFromAttributes(normalCacheKey, attributes)); } else { updateCacheIsGroup(cacheInfo, container.isGroupFromContainer(normalPathKey)); - updateCacheIsGroup(cacheInfo, container.isDatasetFromContainer(normalPathKey)); + updateCacheIsDataset(cacheInfo, container.isDatasetFromContainer(normalPathKey)); } } updateCache(normalPathKey, cacheInfo); @@ -209,7 +209,7 @@ public void updateCacheInfo(final String normalPathKey, final String normalCache } else { updateCacheIsGroup(cacheInfo, container.isGroupFromContainer(normalPathKey)); - updateCacheIsGroup(cacheInfo, container.isDatasetFromContainer(normalPathKey)); + updateCacheIsDataset(cacheInfo, container.isDatasetFromContainer(normalPathKey)); } updateCache(normalPathKey, cacheInfo); } From 7afb2b8314d6e6175637bac549a9ce7f6c470180 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Fri, 26 May 2023 10:29:18 -0400 Subject: [PATCH 194/243] perf: use KVA isFile and isDirectory, not exists * these methods perform better on cloud storage * minor changes to exists implementations --- .../n5/CachedGsonKeyValueReader.java | 10 ++++++---- .../n5/CachedGsonKeyValueWriter.java | 6 +++--- .../saalfeldlab/n5/GsonKeyValueReader.java | 13 ++++++------ .../saalfeldlab/n5/GsonKeyValueWriter.java | 20 ++++++++++--------- .../saalfeldlab/n5/N5KeyValueReader.java | 7 +++++++ 5 files changed, 33 insertions(+), 23 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueReader.java b/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueReader.java index aadad98e..05de923f 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueReader.java @@ -134,9 +134,9 @@ default boolean exists(final String pathName) { default boolean existsFromContainer(final String normalPathName, final String normalCacheKey) { if( normalCacheKey == null ) - return getKeyValueAccess().exists(groupPath(normalPathName)); + return getKeyValueAccess().isDirectory(groupPath(normalPathName)); else - return getKeyValueAccess().exists(getKeyValueAccess().compose(groupPath(normalPathName), normalCacheKey)); + return getKeyValueAccess().isFile(getKeyValueAccess().compose(groupPath(normalPathName), normalCacheKey)); } default boolean groupExists(final String pathName) { @@ -151,7 +151,8 @@ default boolean groupExists(final String pathName) { default boolean isGroupFromContainer(final String pathName) { - return GsonKeyValueReader.super.groupExists(pathName); + final String normalPathName = N5URL.normalizeGroupPath(pathName); + return GsonKeyValueReader.super.groupExists(groupPath(normalPathName)); } default boolean isGroupFromAttributes(final String normalCacheKey, final JsonElement attributes) { @@ -170,10 +171,11 @@ default boolean datasetExists(final String pathName) throws N5Exception.N5IOExce default boolean isDatasetFromContainer(final String pathName) throws N5Exception.N5IOException { - return isGroupFromContainer(groupPath(pathName)) && normalGetDatasetAttributes(pathName) != null; + return normalGetDatasetAttributes(pathName) != null; } default boolean isDatasetFromAttributes(final String normalCacheKey, final JsonElement attributes) { + return isGroupFromAttributes(normalCacheKey, attributes) && createDatasetAttributes(attributes) != null; } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueWriter.java b/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueWriter.java index 6fecd722..ae93af41 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueWriter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueWriter.java @@ -181,7 +181,7 @@ default boolean removeAttribute(final String pathName, final String key) throws final String absoluteNormalPath = getKeyValueAccess().compose(getBasePath(), normalPath); final String normalKey = N5URL.normalizeAttributePath(key); - if (!getKeyValueAccess().exists(absoluteNormalPath)) + if (!getKeyValueAccess().isDirectory(absoluteNormalPath)) return false; if (key.equals("/")) { @@ -241,7 +241,7 @@ default boolean remove(final String path) throws IOException { final String normalPath = N5URL.normalizeGroupPath(path); final String groupPath = groupPath(normalPath); - if (getKeyValueAccess().exists(groupPath)) + if (getKeyValueAccess().isDirectory(groupPath)) getKeyValueAccess().delete(groupPath); if (cacheMeta()) { @@ -266,7 +266,7 @@ default boolean deleteBlock( final long... gridPosition) throws IOException { final String blockPath = getDataBlockPath(N5URL.normalizeGroupPath(path), gridPosition); - if (getKeyValueAccess().exists(blockPath)) + if (getKeyValueAccess().isFile(blockPath)) getKeyValueAccess().delete(blockPath); /* an IOException should have occurred if anything had failed midway */ diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueReader.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueReader.java index fff01ecd..293ddc48 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueReader.java @@ -108,21 +108,20 @@ default T getAttribute(final String pathName, final String key, final Type t default boolean groupExists(final String absoluteNormalPath) { - return getKeyValueAccess().exists(absoluteNormalPath) && getKeyValueAccess().isDirectory(absoluteNormalPath); + return getKeyValueAccess().isDirectory(absoluteNormalPath); } @Override default boolean exists(final String pathName) { - final String normalPathName = N5URL.normalizeGroupPath(pathName); - return getKeyValueAccess().exists(groupPath(normalPathName)) || datasetExists(normalPathName); + final String normalPath = N5URL.normalizeGroupPath(pathName); + return groupExists(normalPath) || datasetExists(normalPath); } @Override default boolean datasetExists(final String pathName) throws N5Exception.N5IOException { - // for n5, every dataset must be a group - return groupExists(groupPath(pathName)) && getDatasetAttributes(pathName) != null; + return getDatasetAttributes(pathName) != null; } /** @@ -137,7 +136,7 @@ default JsonElement getAttributes(final String pathName) throws N5Exception.N5IO final String groupPath = N5URL.normalizeGroupPath(pathName); final String attributesPath = attributesPath(groupPath); - if (!getKeyValueAccess().exists(attributesPath)) + if (!getKeyValueAccess().isFile(attributesPath)) return null; try (final LockedChannel lockedChannel = getKeyValueAccess().lockForReading(attributesPath)) { @@ -152,7 +151,7 @@ default JsonElement getAttributes(final String pathName) throws N5Exception.N5IO default DataBlock readBlock( final String pathName, final DatasetAttributes datasetAttributes, final long... gridPosition) throws IOException { final String path = getDataBlockPath(N5URL.normalizeGroupPath(pathName), gridPosition); - if (!getKeyValueAccess().exists(path)) + if (!getKeyValueAccess().isFile(path)) return null; try (final LockedChannel lockedChannel = getKeyValueAccess().lockForReading(path)) { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueWriter.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueWriter.java index a9b6f9a8..d90c5b40 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueWriter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueWriter.java @@ -143,7 +143,7 @@ default boolean removeAttribute(final String pathName, final String key) throws final String absoluteNormalPath = getKeyValueAccess().compose(getBasePath(), normalPath); final String normalKey = N5URL.normalizeAttributePath(key); - if (!getKeyValueAccess().exists(absoluteNormalPath)) + if (!getKeyValueAccess().isDirectory(absoluteNormalPath)) return false; if (key.equals("/")) { @@ -203,11 +203,12 @@ default boolean remove(final String path) throws IOException { final String normalPath = N5URL.normalizeGroupPath(path); final String groupPath = groupPath(normalPath); - if (getKeyValueAccess().exists(groupPath)) + if (getKeyValueAccess().isDirectory(groupPath)) { getKeyValueAccess().delete(groupPath); - - /* an IOException should have occurred if anything had failed midway */ - return true; + /* an IOException should have occurred if anything had failed midway */ + return true; + } else + return false; } @Override @@ -216,10 +217,11 @@ default boolean deleteBlock( final long... gridPosition) throws IOException { final String blockPath = getDataBlockPath(N5URL.normalizeGroupPath(path), gridPosition); - if (getKeyValueAccess().exists(blockPath)) + if (getKeyValueAccess().isFile(blockPath)) { getKeyValueAccess().delete(blockPath); - - /* an IOException should have occurred if anything had failed midway */ - return true; + /* an IOException should have occurred if anything had failed midway */ + return true; + } else + return false; } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java index 24452017..db765d7f 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java @@ -171,4 +171,11 @@ public N5JsonCache getCache() { public String groupPath(String... nodes) { return keyValueAccess.compose(Stream.concat(Stream.of(basePath), Arrays.stream(nodes)).toArray(String[]::new)); } + + @Override + public boolean exists(final String pathName) { + + final String normalPath = N5URL.normalizeGroupPath(pathName); + return groupExists(normalPath); + } } From e9dc3a6700a92cb983b4038c6ea9b0d7cc1da9a4 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Fri, 26 May 2023 13:45:44 -0400 Subject: [PATCH 195/243] fix: GsonKeyValueWriter remove default initializeGroup --- .../java/org/janelia/saalfeldlab/n5/GsonKeyValueWriter.java | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueWriter.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueWriter.java index d90c5b40..3137e378 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueWriter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueWriter.java @@ -55,16 +55,11 @@ static String initializeContainer( return normBasePath; } - default void initializeGroup(String normalPath) { - - } - @Override default void createGroup(final String path) throws IOException { final String normalPath = N5URL.normalizeGroupPath(path); getKeyValueAccess().createDirectories(groupPath(normalPath)); - initializeGroup(normalPath); } @Override From 992872422e25c19519de0ee4a9712fdbbc7f3802 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Fri, 26 May 2023 15:41:22 -0400 Subject: [PATCH 196/243] fix: tmp test container locations * were appearing in a "file:" folder --- .../java/org/janelia/saalfeldlab/n5/N5CachedFSTest.java | 8 ++------ src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java | 9 +++++---- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/N5CachedFSTest.java b/src/test/java/org/janelia/saalfeldlab/n5/N5CachedFSTest.java index 954b54b0..520e2723 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/N5CachedFSTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/N5CachedFSTest.java @@ -3,7 +3,6 @@ import com.google.gson.GsonBuilder; import com.google.gson.JsonElement; -import java.net.URI; import java.net.URISyntaxException; import org.junit.Test; @@ -16,7 +15,6 @@ import static org.junit.Assert.assertThrows; import static org.junit.Assert.assertTrue; -import java.io.File; import java.io.IOException; import java.nio.file.FileSystems; import java.nio.file.Files; @@ -39,8 +37,7 @@ protected N5Reader createN5Reader(final String location, final GsonBuilder gson) protected N5Writer createN5Writer(final String location, final GsonBuilder gson, final boolean cache) throws IOException, URISyntaxException { - final String basePath = new File(new URI(location)).getCanonicalPath(); - return new N5FSWriter(basePath, gson, cache); + return new N5FSWriter(location, gson, cache); } protected N5Writer createN5Writer(final String location, final boolean cache) throws IOException, URISyntaxException { @@ -50,8 +47,7 @@ protected N5Writer createN5Writer(final String location, final boolean cache) th protected N5Reader createN5Reader(final String location, final GsonBuilder gson, final boolean cache) throws IOException, URISyntaxException { - final String basePath = new File(new URI(location)).getCanonicalPath(); - return new N5FSReader(basePath, gson, cache); + return new N5FSReader(location, gson, cache); } @Test diff --git a/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java b/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java index abb9f713..ad979c2b 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java @@ -25,6 +25,7 @@ */ package org.janelia.saalfeldlab.n5; +import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import java.io.File; @@ -48,6 +49,8 @@ import org.junit.AfterClass; import org.junit.Test; import com.google.gson.GsonBuilder; +import com.google.gson.JsonObject; + import org.janelia.saalfeldlab.n5.url.UrlAttributeTest; import java.util.ArrayList; @@ -86,14 +89,12 @@ protected String tempN5Location() throws URISyntaxException { @Override protected N5Writer createN5Writer(final String location, final GsonBuilder gson) throws IOException, URISyntaxException { - final String basePath = new File(new URI(location)).getCanonicalPath(); - return new N5FSWriter(basePath, gson); + return new N5FSWriter(location, gson); } @Override protected N5Reader createN5Reader(final String location, final GsonBuilder gson) throws IOException, URISyntaxException { - final String basePath = new File(new URI(location)).getCanonicalPath(); - return new N5FSReader(basePath, gson); + return new N5FSReader(location, gson); } @AfterClass From 310ea0644762fc303da9aa330925af77ee460b93 Mon Sep 17 00:00:00 2001 From: Stephan Saalfeld Date: Thu, 1 Jun 2023 21:07:10 -0400 Subject: [PATCH 197/243] breaking: update and clean up interface hierarchy, fix bugs, introduce new bugs TODO almost all tests failing for FS, some for cached FS (weird) I should mention that these changes compiled and passed ALL tests before my attempt to merge with John's recent changes from last week, so I expect something very simple yet core being broken and everything else failing on top, time to commit and let everybody look at it. --- .../saalfeldlab/n5/AbstractGsonReader.java | 148 ------- ...r.java => CachedGsonKeyValueN5Reader.java} | 90 +++-- .../n5/CachedGsonKeyValueN5Writer.java | 160 ++++++++ .../n5/CachedGsonKeyValueWriter.java | 275 ------------- .../saalfeldlab/n5/DatasetAttributes.java | 36 ++ .../saalfeldlab/n5/GsonAttributesParser.java | 261 ------------ ...eReader.java => GsonKeyValueN5Reader.java} | 153 ++----- ...eWriter.java => GsonKeyValueN5Writer.java} | 183 ++++++--- .../janelia/saalfeldlab/n5/GsonN5Reader.java | 113 ++++++ .../janelia/saalfeldlab/n5/GsonN5Writer.java | 50 +++ .../org/janelia/saalfeldlab/n5/GsonUtils.java | 5 +- .../n5/LinkedAttributePathToken.java | 4 +- .../saalfeldlab/n5/N5KeyValueReader.java | 2 +- .../saalfeldlab/n5/N5KeyValueWriter.java | 2 +- .../org/janelia/saalfeldlab/n5/N5Reader.java | 381 +++++++++++------- .../org/janelia/saalfeldlab/n5/N5Writer.java | 269 ++++++++----- .../saalfeldlab/n5/cache/N5JsonCache.java | 32 +- .../n5/cache/N5JsonCacheableContainer.java | 4 +- .../saalfeldlab/n5/AbstractN5Test.java | 81 ++-- .../janelia/saalfeldlab/n5/N5Benchmark.java | 8 +- .../saalfeldlab/n5/N5CachedFSTest.java | 43 +- .../org/janelia/saalfeldlab/n5/N5FSTest.java | 51 ++- 22 files changed, 1100 insertions(+), 1251 deletions(-) delete mode 100644 src/main/java/org/janelia/saalfeldlab/n5/AbstractGsonReader.java rename src/main/java/org/janelia/saalfeldlab/n5/{CachedGsonKeyValueReader.java => CachedGsonKeyValueN5Reader.java} (79%) create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Writer.java delete mode 100644 src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueWriter.java delete mode 100644 src/main/java/org/janelia/saalfeldlab/n5/GsonAttributesParser.java rename src/main/java/org/janelia/saalfeldlab/n5/{GsonKeyValueReader.java => GsonKeyValueN5Reader.java} (52%) rename src/main/java/org/janelia/saalfeldlab/n5/{GsonKeyValueWriter.java => GsonKeyValueN5Writer.java} (51%) create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/GsonN5Reader.java create mode 100644 src/main/java/org/janelia/saalfeldlab/n5/GsonN5Writer.java diff --git a/src/main/java/org/janelia/saalfeldlab/n5/AbstractGsonReader.java b/src/main/java/org/janelia/saalfeldlab/n5/AbstractGsonReader.java deleted file mode 100644 index d1029bb1..00000000 --- a/src/main/java/org/janelia/saalfeldlab/n5/AbstractGsonReader.java +++ /dev/null @@ -1,148 +0,0 @@ -/** - * Copyright (c) 2017, Stephan Saalfeld - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ -package org.janelia.saalfeldlab.n5; - -import java.io.IOException; -import java.lang.reflect.Type; -import java.util.Arrays; -import java.util.HashMap; - -import com.google.gson.Gson; -import com.google.gson.GsonBuilder; -import com.google.gson.JsonElement; - -/** - * Abstract base class implementing {@link N5Reader} with JSON attributes - * parsed with {@link Gson}. - * - * @author Stephan Saalfeld - * @author Igor Pisarev - * @author Philipp Hanslovsky - */ -@Deprecated -public abstract class AbstractGsonReader implements GsonAttributesParser, N5Reader { - - protected final Gson gson; - - /** - * Constructs an {@link AbstractGsonReader} with a custom - * {@link GsonBuilder} to support custom attributes. - * - * @param gsonBuilder the gson builder - */ - @Deprecated - public AbstractGsonReader(final GsonBuilder gsonBuilder) { - - gsonBuilder.registerTypeAdapter(DataType.class, new DataType.JsonAdapter()); - gsonBuilder.registerTypeHierarchyAdapter(Compression.class, CompressionAdapter.getJsonAdapter()); - gsonBuilder.disableHtmlEscaping(); - this.gson = gsonBuilder.create(); - } - - /** - * Constructs an {@link AbstractGsonReader} with a default - * {@link GsonBuilder}. - */ - @Deprecated - public AbstractGsonReader() { - - this(new GsonBuilder()); - } - - @Override - @Deprecated - public Gson getGson() { - - return gson; - } - - @Override - @Deprecated - public DatasetAttributes getDatasetAttributes(final String pathName) throws IOException { - - final HashMap map = getAttributes(pathName); - final Gson gson = getGson(); - - final long[] dimensions = GsonAttributesParser.parseAttribute(map, DatasetAttributes.DIMENSIONS_KEY, long[].class, gson); - if (dimensions == null) - return null; - - final DataType dataType = GsonAttributesParser.parseAttribute(map, DatasetAttributes.DATA_TYPE_KEY, DataType.class, gson); - if (dataType == null) - return null; - - int[] blockSize = GsonAttributesParser.parseAttribute(map, DatasetAttributes.BLOCK_SIZE_KEY, int[].class, gson); - if (blockSize == null) - blockSize = Arrays.stream(dimensions).mapToInt(a -> (int)a).toArray(); - - Compression compression = GsonAttributesParser.parseAttribute(map, DatasetAttributes.COMPRESSION_KEY, Compression.class, gson); - - /* version 0 */ - if (compression == null) { - switch (GsonAttributesParser.parseAttribute(map, DatasetAttributes.compressionTypeKey, String.class, gson)) { - case "raw": - compression = new RawCompression(); - break; - case "gzip": - compression = new GzipCompression(); - break; - case "bzip2": - compression = new Bzip2Compression(); - break; - case "lz4": - compression = new Lz4Compression(); - break; - case "xz": - compression = new XzCompression(); - break; - } - } - - return new DatasetAttributes(dimensions, blockSize, dataType, compression); - } - - @Override - @Deprecated - public T getAttribute( - final String pathName, - final String key, - final Class clazz) throws IOException { - - final HashMap map = getAttributes(pathName); - return GsonAttributesParser.parseAttribute(map, key, clazz, getGson()); - } - - @Override - @Deprecated - public T getAttribute( - final String pathName, - final String key, - final Type type) throws IOException { - - final HashMap map = getAttributes(pathName); - return GsonAttributesParser.parseAttribute(map, key, type, getGson()); - } -} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueReader.java b/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Reader.java similarity index 79% rename from src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueReader.java rename to src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Reader.java index 05de923f..a38cb90d 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Reader.java @@ -25,39 +25,38 @@ */ package org.janelia.saalfeldlab.n5; -import com.google.gson.Gson; -import com.google.gson.JsonElement; -import com.google.gson.JsonObject; +import java.io.IOException; +import java.lang.reflect.Type; + import org.janelia.saalfeldlab.n5.cache.N5JsonCache; import org.janelia.saalfeldlab.n5.cache.N5JsonCacheableContainer; -import java.io.IOException; -import java.lang.reflect.Type; +import com.google.gson.Gson; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; /** * {@link N5Reader} implementation through {@link KeyValueAccess} with JSON * attributes parsed with {@link Gson}. * - * @author Stephan Saalfeld - * @author Igor Pisarev - * @author Philipp Hanslovsky */ - -public interface CachedGsonKeyValueReader extends GsonKeyValueReader, N5JsonCacheableContainer { +public interface CachedGsonKeyValueN5Reader extends GsonKeyValueN5Reader, N5JsonCacheableContainer { default N5JsonCache newCache() { - return new N5JsonCache( this ); + + return new N5JsonCache(this); } boolean cacheMeta(); N5JsonCache getCache(); + @Override default JsonElement getAttributesFromContainer(final String normalPathName, final String normalCacheKey) { // this implementation doesn't use cache key, but rather depends on // attributesPath being implemented - return GsonKeyValueReader.super.getAttributes(normalPathName); + return GsonKeyValueN5Reader.super.getAttributes(normalPathName); } @Override @@ -72,7 +71,7 @@ default DatasetAttributes getDatasetAttributes(final String pathName) { if (cacheMeta()) { attributes = getCache().getAttributes(normalPath, N5KeyValueReader.ATTRIBUTES_JSON); } else { - attributes = GsonKeyValueReader.super.getAttributes(normalPath); + attributes = GsonKeyValueN5Reader.super.getAttributes(normalPath); } return createDatasetAttributes(attributes); @@ -81,7 +80,7 @@ default DatasetAttributes getDatasetAttributes(final String pathName) { default DatasetAttributes normalGetDatasetAttributes(final String pathName) throws N5Exception.N5IOException { final String normalPath = N5URL.normalizeGroupPath(pathName); - final JsonElement attributes = GsonKeyValueReader.super.getAttributes(normalPath); + final JsonElement attributes = GsonKeyValueN5Reader.super.getAttributes(normalPath); return createDatasetAttributes(attributes); } @@ -89,7 +88,7 @@ default DatasetAttributes normalGetDatasetAttributes(final String pathName) thro default T getAttribute( final String pathName, final String key, - final Class clazz) throws IOException { + final Class clazz) throws N5Exception { final String normalPathName = N5URL.normalizeGroupPath(pathName); final String normalizedAttributePath = N5URL.normalizeAttributePath(key); @@ -98,7 +97,7 @@ default T getAttribute( if (cacheMeta()) { attributes = getCache().getAttributes(normalPathName, N5KeyValueReader.ATTRIBUTES_JSON); } else { - attributes = GsonKeyValueReader.super.getAttributes(normalPathName); + attributes = GsonKeyValueN5Reader.super.getAttributes(normalPathName); } return GsonUtils.readAttribute(attributes, normalizedAttributePath, clazz, getGson()); } @@ -107,7 +106,7 @@ default T getAttribute( default T getAttribute( final String pathName, final String key, - final Type type) throws IOException { + final Type type) throws N5Exception { final String normalPathName = N5URL.normalizeGroupPath(pathName); final String normalizedAttributePath = N5URL.normalizeAttributePath(key); @@ -115,7 +114,7 @@ default T getAttribute( if (cacheMeta()) { attributes = getCache().getAttributes(normalPathName, N5KeyValueReader.ATTRIBUTES_JSON); } else { - attributes = GsonKeyValueReader.super.getAttributes(normalPathName); + attributes = GsonKeyValueN5Reader.super.getAttributes(normalPathName); } return GsonUtils.readAttribute(attributes, normalizedAttributePath, type, getGson()); } @@ -131,14 +130,16 @@ default boolean exists(final String pathName) { } } + @Override default boolean existsFromContainer(final String normalPathName, final String normalCacheKey) { - if( normalCacheKey == null ) + if (normalCacheKey == null) return getKeyValueAccess().isDirectory(groupPath(normalPathName)); else return getKeyValueAccess().isFile(getKeyValueAccess().compose(groupPath(normalPathName), normalCacheKey)); } + @Override default boolean groupExists(final String pathName) { final String normalPathName = N5URL.normalizeGroupPath(pathName); @@ -149,31 +150,35 @@ default boolean groupExists(final String pathName) { } } - default boolean isGroupFromContainer(final String pathName) { + @Override + default boolean isGroupFromContainer(final String normalPathName) { - final String normalPathName = N5URL.normalizeGroupPath(pathName); - return GsonKeyValueReader.super.groupExists(groupPath(normalPathName)); + return GsonKeyValueN5Reader.super.groupExists(normalPathName); } + @Override default boolean isGroupFromAttributes(final String normalCacheKey, final JsonElement attributes) { + return true; } @Override default boolean datasetExists(final String pathName) throws N5Exception.N5IOException { + final String normalPathName = N5URL.normalizeGroupPath(pathName); if (cacheMeta()) { - final String normalPathName = N5URL.normalizeGroupPath(pathName); - return getCache().isDataset(normalPathName, null); + return getCache().isDataset(normalPathName, N5KeyValueReader.ATTRIBUTES_JSON); } - return isDatasetFromContainer(pathName); + return isDatasetFromContainer(normalPathName); } - default boolean isDatasetFromContainer(final String pathName) throws N5Exception.N5IOException { + @Override + default boolean isDatasetFromContainer(final String normalPathName) throws N5Exception.N5IOException { - return normalGetDatasetAttributes(pathName) != null; + return normalGetDatasetAttributes(normalPathName) != null; } + @Override default boolean isDatasetFromAttributes(final String normalCacheKey, final JsonElement attributes) { return isGroupFromAttributes(normalCacheKey, attributes) && createDatasetAttributes(attributes) != null; @@ -182,21 +187,22 @@ default boolean isDatasetFromAttributes(final String normalCacheKey, final JsonE /** * Reads or creates the attributes map of a group or dataset. * - * @param pathName group path + * @param pathName + * group path * @return * @throws IOException */ + @Override default JsonElement getAttributes(final String pathName) throws N5Exception.N5IOException { final String groupPath = N5URL.normalizeGroupPath(pathName); - /* If cached, return the cache*/ + /* If cached, return the cache */ if (cacheMeta()) { return getCache().getAttributes(groupPath, N5KeyValueReader.ATTRIBUTES_JSON); } else { - return GsonKeyValueReader.super.getAttributes(groupPath); + return GsonKeyValueN5Reader.super.getAttributes(groupPath); } - } @Override @@ -206,30 +212,35 @@ default String[] list(final String pathName) throws N5Exception.N5IOException { if (cacheMeta()) { return getCache().list(normalPath); } else { - return GsonKeyValueReader.super.list(normalPath); + return GsonKeyValueN5Reader.super.list(normalPath); } } + @Override default String[] listFromContainer(final String normalPathName) { // this implementation doesn't use cache key, but rather depends on - return GsonKeyValueReader.super.list(normalPathName); + return GsonKeyValueN5Reader.super.list(normalPathName); } /** - * Constructs the path for a data block in a dataset at a given grid position. + * Constructs the path for a data block in a dataset at a given grid + * position. *

    * The returned path is + * * 

     	 * $basePath/datasetPathName/$gridPosition[0]/$gridPosition[1]/.../$gridPosition[n]
 	 *
    *

    * This is the file into which the data block will be stored. * - * @param normalPath normalized dataset path + * @param normalPath + * normalized dataset path * @param gridPosition * @return */ + @Override default String getDataBlockPath( final String normalPath, final long... gridPosition) { @@ -247,8 +258,10 @@ default String getDataBlockPath( /** * Check for attributes that are required for a group to be a dataset. * - * @param attributes to check for dataset attributes - * @return if {@link DatasetAttributes#DIMENSIONS_KEY} and {@link DatasetAttributes#DATA_TYPE_KEY} are present + * @param attributes + * to check for dataset attributes + * @return if {@link DatasetAttributes#DIMENSIONS_KEY} and + * {@link DatasetAttributes#DATA_TYPE_KEY} are present */ static boolean hasDatasetAttributes(final JsonElement attributes) { @@ -257,6 +270,7 @@ static boolean hasDatasetAttributes(final JsonElement attributes) { } final JsonObject metadataCache = attributes.getAsJsonObject(); - return metadataCache.has(DatasetAttributes.DIMENSIONS_KEY) && metadataCache.has(DatasetAttributes.DATA_TYPE_KEY); + return metadataCache.has(DatasetAttributes.DIMENSIONS_KEY) + && metadataCache.has(DatasetAttributes.DATA_TYPE_KEY); } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Writer.java new file mode 100644 index 00000000..2aaf5884 --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Writer.java @@ -0,0 +1,160 @@ +/** + * Copyright (c) 2017--2021, Stephan Saalfeld + * All rights reserved. + *

    + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + *

    + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + *

    + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +package org.janelia.saalfeldlab.n5; + +import java.io.IOException; +import java.util.Arrays; + +import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; + +import com.google.gson.Gson; +import com.google.gson.JsonElement; + +/** + * Cached default implementation of {@link N5Writer} with JSON attributes parsed + * with {@link Gson}. + */ +public interface CachedGsonKeyValueN5Writer extends CachedGsonKeyValueN5Reader, GsonKeyValueN5Writer { + + @Override + default void setVersion(final String path) throws N5Exception { + + final Version version = getVersion(); + if (!VERSION.isCompatible(version)) + throw new N5IOException("Incompatible version " + version + " (this is " + VERSION + ")."); + + if (!VERSION.equals(version)) + setAttribute("/", VERSION_KEY, VERSION.toString());; + } + + @Override + default void createGroup(final String path) throws N5Exception { + + // N5Writer.super.createGroup(path); + /* + * the 6 lines below duplicate the single line above but would have to + * call + * normalizeGroupPath again the below duplicates code, but avoids extra + * work + */ + final String normalPath = N5URL.normalizeGroupPath(path); + try { + getKeyValueAccess().createDirectories(groupPath(normalPath)); + } catch (final IOException e) { + throw new N5Exception.N5IOException("Failed to create group " + path, e); + } + + if (cacheMeta()) { + // check all nodes that are parents of the added node, if they have + // a children set, add the new child to it + String[] pathParts = getKeyValueAccess().components(normalPath); + String parent = N5URL.normalizeGroupPath("/"); + if (pathParts.length == 0) { + pathParts = new String[]{""}; + } + for (final String child : pathParts) { + + final String childPath = getKeyValueAccess().compose(parent, child); + getCache().forceAddNewCacheInfo(childPath, N5KeyValueReader.ATTRIBUTES_JSON, null, true, false); + + // only add if the parent exists and has children cached already + if (parent != null && !child.isEmpty()) + getCache().addChildIfPresent(parent, child); + + parent = childPath; + } + } + } + + @Override + default void writeAttributes( + final String normalGroupPath, + final JsonElement attributes) throws N5Exception { + + writeAndCacheAttributes(normalGroupPath, attributes); + } + + default void writeAndCacheAttributes( + final String normalGroupPath, + final JsonElement attributes) throws N5Exception { + + GsonKeyValueN5Writer.super.writeAttributes(normalGroupPath, attributes); + + if (cacheMeta()) { + JsonElement nullRespectingAttributes = attributes; + /* + * Gson only filters out nulls when you write the JsonElement. This + * means it doesn't filter them out when caching. + * To handle this, we explicitly writer the existing JsonElement to + * a new JsonElement. + * The output is identical to the input if: + * - serializeNulls is true + * - no null values are present + * - cacheing is turned off + */ + if (!getGson().serializeNulls()) { + nullRespectingAttributes = getGson().toJsonTree(attributes); + } + /* Update the cache, and write to the writer */ + getCache().updateCacheInfo(normalGroupPath, N5KeyValueReader.ATTRIBUTES_JSON, nullRespectingAttributes); + } + } + + @Override + default boolean remove(final String path) throws N5Exception { + + // GsonKeyValueN5Writer.super.remove(path) + /* + * the 8 lines below duplicate the single line above but would have to + * call + * normalizeGroupPath again the below duplicates code, but avoids extra + * work + */ + final String normalPath = N5URL.normalizeGroupPath(path); + final String groupPath = groupPath(normalPath); + try { + if (getKeyValueAccess().isDirectory(groupPath)) + getKeyValueAccess().delete(groupPath); + } catch (final IOException e) { + throw new N5IOException("Failed to remove " + path, e); + } + + if (cacheMeta()) { + final String[] pathParts = getKeyValueAccess().components(normalPath); + final String parent; + if (pathParts.length <= 1) { + parent = N5URL.normalizeGroupPath("/"); + } else { + final int parentPathLength = pathParts.length - 1; + parent = getKeyValueAccess().compose(Arrays.copyOf(pathParts, parentPathLength)); + } + getCache().removeCache(parent, normalPath); + } + + /* an IOException should have occurred if anything had failed midway */ + return true; + } +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueWriter.java b/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueWriter.java deleted file mode 100644 index ae93af41..00000000 --- a/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueWriter.java +++ /dev/null @@ -1,275 +0,0 @@ -/** - * Copyright (c) 2017--2021, Stephan Saalfeld - * All rights reserved. - *

    - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - *

    - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - *

    - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ -package org.janelia.saalfeldlab.n5; - -import com.google.gson.JsonElement; -import com.google.gson.JsonNull; -import com.google.gson.JsonObject; - -import java.io.IOException; -import java.util.Arrays; -import java.util.List; -import java.util.Map; - -/** - * Filesystem {@link N5Writer} implementation with version compatibility check. - * - * @author Stephan Saalfeld - */ -public interface CachedGsonKeyValueWriter extends CachedGsonKeyValueReader, N5Writer { - - default void setVersion(final String path) throws IOException { - - final Version version = getVersion(); - if (!VERSION.isCompatible(version)) - throw new N5Exception.N5IOException("Incompatible version " + version + " (this is " + VERSION + ")."); - - if (!VERSION.equals(version)) - setAttribute("/", VERSION_KEY, VERSION.toString());; - } - - @Override - default void createGroup(final String path) throws IOException { - - final String normalPath = N5URL.normalizeGroupPath(path); - getKeyValueAccess().createDirectories(groupPath(normalPath)); - if (cacheMeta()) { - // check all nodes that are parents of the added node, if they have a children set, add the new child to it - String[] pathParts = getKeyValueAccess().components(normalPath); - String parent = N5URL.normalizeGroupPath("/"); - if (pathParts.length == 0) { - pathParts = new String[]{""}; - } - for (String child : pathParts) { - - final String childPath = getKeyValueAccess().compose(parent, child); - getCache().forceAddNewCacheInfo(childPath, N5KeyValueReader.ATTRIBUTES_JSON, null, true, false ); - - // only add if the parent exists and has children cached already - if( parent != null && !child.isEmpty()) - getCache().addChildIfPresent(parent, child); - - parent = childPath; - } - } - } - - /** - * Creates a dataset. This does not create any data but the path and - * mandatory attributes only. - * - * @param pathName dataset path - * @param datasetAttributes the dataset attributes - * @throws IOException the exception - */ - @Override - default void createDataset( - final String pathName, - final DatasetAttributes datasetAttributes) throws IOException { - - // N5Writer.super.createDataset(pathName, datasetAttributes); - /* the three lines below duplicate the single line above but would have to call - * normalizeGroupPath again the below duplicates code, but avoids extra work */ - final String normalPath = N5URL.normalizeGroupPath(pathName); - createGroup(normalPath); - setDatasetAttributes(normalPath, datasetAttributes); - } - - /** - * Helper method that reads an existing JsonElement representing the root - * attributes for {@code normalGroupPath}, inserts and overrides the - * provided attributes, and writes them back into the attributes store. - * - * @param normalGroupPath to write the attributes to - * @param attributes to write - * @throws IOException if unable to read the attributes at {@code normalGroupPath} - *

    - * TODO consider cache (or you read the attributes twice?) - */ - default void writeAttributes( - final String normalGroupPath, - final JsonElement attributes) throws IOException { - - JsonElement root = getAttributes(normalGroupPath); - root = GsonUtils.insertAttribute(root, "/", attributes, getGson()); - writeAndCacheAttributes(normalGroupPath, root); - } - - /** - * Helper method that reads the existing map of attributes, JSON encodes, - * inserts and overrides the provided attributes, and writes them back into - * the attributes store. - * - * @param normalGroupPath to write the attributes to - * @param attributes to write - * @throws IOException if unable to read the attributes at {@code normalGroupPath} - */ - default void writeAttributes( - final String normalGroupPath, - final Map attributes) throws IOException { - - if (attributes != null && !attributes.isEmpty()) { - JsonElement existingAttributes = getAttributes(normalGroupPath); - existingAttributes = existingAttributes != null && existingAttributes.isJsonObject() - ? existingAttributes.getAsJsonObject() - : new JsonObject(); - JsonElement newAttributes = GsonUtils.insertAttributes(existingAttributes, attributes, getGson()); - writeAndCacheAttributes(normalGroupPath, newAttributes); - } - } - - default void writeAndCacheAttributes(final String normalGroupPath, final JsonElement attributes) throws IOException { - - try (final LockedChannel lock = getKeyValueAccess().lockForWriting(attributesPath(normalGroupPath))) { - GsonUtils.writeAttributes(lock.newWriter(), attributes, getGson()); - } - if (cacheMeta()) { - JsonElement nullRespectingAttributes = attributes; - /* Gson only filters out nulls when you write the JsonElement. This means it doesn't filter them out when caching. - * To handle this, we explicitly writer the existing JsonElement to a new JsonElement. - * The output is identical to the input if: - * - serializeNulls is true - * - no null values are present - * - cacheing is turned off */ - if (!getGson().serializeNulls()) { - nullRespectingAttributes = getGson().toJsonTree(attributes); - } - /* Update the cache, and write to the writer */ - getCache().updateCacheInfo(normalGroupPath, N5KeyValueReader.ATTRIBUTES_JSON, nullRespectingAttributes); - } - } - - @Override - default void setAttributes( - final String path, - final Map attributes) throws IOException { - - final String normalPath = N5URL.normalizeGroupPath(path); - if (!exists(normalPath)) - throw new N5Exception.N5IOException("" + normalPath + " is not a group or dataset."); - - writeAttributes(normalPath, attributes); - } - - @Override - default boolean removeAttribute(final String pathName, final String key) throws IOException { - - final String normalPath = N5URL.normalizeGroupPath(pathName); - final String absoluteNormalPath = getKeyValueAccess().compose(getBasePath(), normalPath); - final String normalKey = N5URL.normalizeAttributePath(key); - - if (!getKeyValueAccess().isDirectory(absoluteNormalPath)) - return false; - - if (key.equals("/")) { - writeAttributes(normalPath, JsonNull.INSTANCE); - return true; - } - - final JsonElement attributes = getAttributes(normalPath); - if (GsonUtils.removeAttribute(attributes, normalKey) != null) { - writeAttributes(normalPath, attributes); - return true; - } - return false; - } - - @Override - default T removeAttribute(final String pathName, final String key, final Class cls) throws IOException { - - final String normalPath = N5URL.normalizeGroupPath(pathName); - final String normalKey = N5URL.normalizeAttributePath(key); - - final JsonElement attributes = getAttributes(normalPath); - final T obj = GsonUtils.removeAttribute(attributes, normalKey, cls, getGson()); - if (obj != null) { - writeAttributes(normalPath, attributes); - } - return obj; - } - - @Override - default boolean removeAttributes(final String pathName, final List attributes) throws IOException { - - final String normalPath = N5URL.normalizeGroupPath(pathName); - boolean removed = false; - for (final String attribute : attributes) { - final String normalKey = N5URL.normalizeAttributePath(attribute); - removed |= removeAttribute(normalPath, attribute); - } - return removed; - } - - @Override - default void writeBlock( - final String path, - final DatasetAttributes datasetAttributes, - final DataBlock dataBlock) throws IOException { - - final String blockPath = getDataBlockPath(N5URL.normalizeGroupPath(path), dataBlock.getGridPosition()); - try (final LockedChannel lock = getKeyValueAccess().lockForWriting(blockPath)) { - - DefaultBlockWriter.writeBlock(lock.newOutputStream(), datasetAttributes, dataBlock); - } - } - - @Override - default boolean remove(final String path) throws IOException { - - final String normalPath = N5URL.normalizeGroupPath(path); - final String groupPath = groupPath(normalPath); - if (getKeyValueAccess().isDirectory(groupPath)) - getKeyValueAccess().delete(groupPath); - - if (cacheMeta()) { - final String[] pathParts = getKeyValueAccess().components(normalPath); - final String parent; - if (pathParts.length <= 1) { - parent = N5URL.normalizeGroupPath("/"); - } else { - final int parentPathLength = pathParts.length - 1; - parent = getKeyValueAccess().compose(Arrays.copyOf(pathParts, parentPathLength)); - } - getCache().removeCache(parent, normalPath); - } - - /* an IOException should have occurred if anything had failed midway */ - return true; - } - - @Override - default boolean deleteBlock( - final String path, - final long... gridPosition) throws IOException { - - final String blockPath = getDataBlockPath(N5URL.normalizeGroupPath(path), gridPosition); - if (getKeyValueAccess().isFile(blockPath)) - getKeyValueAccess().delete(blockPath); - - /* an IOException should have occurred if anything had failed midway */ - return true; - } -} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java b/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java index 17f098b8..f56db529 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java @@ -26,6 +26,7 @@ package org.janelia.saalfeldlab.n5; import java.io.Serializable; +import java.util.Arrays; import java.util.HashMap; /** @@ -104,4 +105,39 @@ public HashMap asMap() { map.put(COMPRESSION_KEY, compression); return map; } + + static DatasetAttributes from( + final long[] dimensions, + final DataType dataType, + int[] blockSize, + Compression compression, + final String compressionVersion0Name + ) { + + if (blockSize == null) + blockSize = Arrays.stream(dimensions).mapToInt(a -> (int)a).toArray(); + + /* version 0 */ + if (compression == null) { + switch (compressionVersion0Name) { + case "raw": + compression = new RawCompression(); + break; + case "gzip": + compression = new GzipCompression(); + break; + case "bzip2": + compression = new Bzip2Compression(); + break; + case "lz4": + compression = new Lz4Compression(); + break; + case "xz": + compression = new XzCompression(); + break; + } + } + + return new DatasetAttributes(dimensions, blockSize, dataType, compression); + } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonAttributesParser.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonAttributesParser.java deleted file mode 100644 index 876426f2..00000000 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonAttributesParser.java +++ /dev/null @@ -1,261 +0,0 @@ -/** - * Copyright (c) 2017, Stephan Saalfeld - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ -package org.janelia.saalfeldlab.n5; - -import java.io.IOException; -import java.io.Reader; -import java.io.Writer; -import java.lang.reflect.Array; -import java.lang.reflect.Type; -import java.util.HashMap; -import java.util.Map; -import java.util.Map.Entry; - -import com.google.gson.Gson; -import com.google.gson.JsonArray; -import com.google.gson.JsonElement; -import com.google.gson.JsonPrimitive; -import com.google.gson.reflect.TypeToken; - -/** - * {@link N5Reader} for JSON attributes parsed by {@link Gson}. - * - * @author Stephan Saalfeld - */ -@Deprecated -public interface GsonAttributesParser extends N5Reader { - - @Deprecated - public Gson getGson(); - - /** - * Reads or creates the attributes map of a group or dataset. - * - * @param pathName group path - * @return the attributes - * @throws IOException the exception - */ - @Deprecated - public HashMap getAttributes(final String pathName) throws IOException; - - /** - * Parses an attribute from the given attributes map. - * - * @param map the attributes map - * @param key the key - * @param clazz the class - * @param gson the gson - * @param the attribute type - * @return the attribute - * @throws IOException the exception - */ - @Deprecated - public static T parseAttribute( - final HashMap map, - final String key, - final Class clazz, - final Gson gson) throws IOException { - - final JsonElement attribute = map.get(key); - if (attribute != null) - return gson.fromJson(attribute, clazz); - else - return null; - } - - /** - * Parses an attribute from the given attributes map. - * - * @param map the attributes map - * @param key the key - * @param type the type - * @param gson the gson - * @param the attribute type - * @return the attribute - * @throws IOException the exception - */ - @Deprecated - public static T parseAttribute( - final HashMap map, - final String key, - final Type type, - final Gson gson) throws IOException { - - final JsonElement attribute = map.get(key); - if (attribute != null) - return gson.fromJson(attribute, type); - else - return null; - } - - /** - * Reads the attributes map from a given {@link Reader}. - * - * @param reader the reader - * @param gson the gson - * @return the attributes map - * @throws IOException the exception - */ - @Deprecated - public static HashMap readAttributes(final Reader reader, final Gson gson) throws IOException { - - final Type mapType = new TypeToken>(){}.getType(); - final HashMap map = gson.fromJson(reader, mapType); - return map == null ? new HashMap<>() : map; - } - - /** - * Inserts new the JSON export of attributes into the given attributes map. - * - * @param map the target attributes map to be edited - * @param attributes the source attributes map - * @param gson the gson - * @throws IOException the exception - */ - @Deprecated - public static void insertAttributes( - final HashMap map, - final Map attributes, - final Gson gson) throws IOException { - - for (final Entry entry : attributes.entrySet()) - map.put(entry.getKey(), gson.toJsonTree(entry.getValue())); - } - - /** - * Writes the attributes map to a given {@link Writer}. - * - * @param writer the writer - * @param map the attributes map - * @param gson the gson - * @throws IOException the exception - */ - @Deprecated - public static void writeAttributes( - final Writer writer, - final HashMap map, - final Gson gson) throws IOException { - - final Type mapType = new TypeToken>(){}.getType(); - gson.toJson(map, mapType, writer); - writer.flush(); - } - - /** - * Return a reasonable class for a {@link JsonPrimitive}. Possible return - * types are - *

      - *
    • boolean
      • - *
    • double
      • - *
    • String
      • - *
    • Object
      • - *
    - * - * @param jsonPrimitive the json - * @return the class - */ - @Deprecated - public static Class classForJsonPrimitive(final JsonPrimitive jsonPrimitive) { - - if (jsonPrimitive.isBoolean()) - return boolean.class; - else if (jsonPrimitive.isNumber()) { - final Number number = jsonPrimitive.getAsNumber(); - if (number.longValue() == number.doubleValue()) - return long.class; - else - return double.class; - } else if (jsonPrimitive.isString()) - return String.class; - else return Object.class; - } - - /** - * Best effort implementation of {@link N5Reader#listAttributes(String)} - * with limited type resolution. Possible return types are - *
      - *
    • null
      • - *
    • boolean
      • - *
    • double
      • - *
    • String
      • - *
    • Object
      • - *
    • boolean[]
      • - *
    • double[]
      • - *
    • String[]
      • - *
    • Object[]
      • - *
    - * - * @param pathName the path - * @return the attributes map - */ - @Override - @Deprecated - public default Map> listAttributes(final String pathName) throws IOException { - - final HashMap jsonElementMap = getAttributes(pathName); - final HashMap> attributes = new HashMap<>(); - jsonElementMap.forEach( - (key, jsonElement) -> { - final Class clazz; - if (jsonElement.isJsonNull()) - clazz = null; - else if (jsonElement.isJsonPrimitive()) - clazz = classForJsonPrimitive((JsonPrimitive)jsonElement); - else if (jsonElement.isJsonArray()) { - final JsonArray jsonArray = (JsonArray)jsonElement; - Class arrayElementClass = Object.class; - if (jsonArray.size() > 0) { - final JsonElement firstElement = jsonArray.get(0); - if (firstElement.isJsonPrimitive()) { - arrayElementClass = classForJsonPrimitive(firstElement.getAsJsonPrimitive()); - for (int i = 1; i < jsonArray.size() && arrayElementClass != Object.class; ++i) { - final JsonElement element = jsonArray.get(i); - if (element.isJsonPrimitive()) { - final Class nextArrayElementClass = classForJsonPrimitive(element.getAsJsonPrimitive()); - if (nextArrayElementClass != arrayElementClass) - if (nextArrayElementClass == double.class && arrayElementClass == long.class) - arrayElementClass = double.class; - else { - arrayElementClass = Object.class; - break; - } - } else { - arrayElementClass = Object.class; - break; - } - } - } - clazz = Array.newInstance(arrayElementClass, 0).getClass(); - } else - clazz = Object[].class; - } - else - clazz = Object.class; - attributes.put(key, clazz); - }); - return attributes; - } -} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueReader.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java similarity index 52% rename from src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueReader.java rename to src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java index 293ddc48..fc600693 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java @@ -25,87 +25,20 @@ */ package org.janelia.saalfeldlab.n5; +import java.io.IOException; + import com.google.gson.Gson; import com.google.gson.JsonElement; -import com.google.gson.JsonObject; - -import java.io.IOException; -import java.lang.reflect.Type; -import java.util.Arrays; -import java.util.Map; /** * {@link N5Reader} implementation through {@link KeyValueAccess} with JSON * attributes parsed with {@link Gson}. * - * @author Stephan Saalfeld - * @author Igor Pisarev - * @author Philipp Hanslovsky */ -public interface GsonKeyValueReader extends N5Reader { - - Gson getGson(); +public interface GsonKeyValueN5Reader extends GsonN5Reader { KeyValueAccess getKeyValueAccess(); - @Override - default Map> listAttributes(final String pathName) throws IOException { - - return GsonUtils.listAttributes(getAttributes(pathName)); - } - - @Override - default DatasetAttributes getDatasetAttributes(final String pathName) throws N5Exception.N5IOException { - - final String normalPath = N5URL.normalizeGroupPath(pathName); - final JsonElement attributes = getAttributes(normalPath); - return createDatasetAttributes(attributes); - } - - default DatasetAttributes createDatasetAttributes(JsonElement attributes) { - - final long[] dimensions = GsonUtils.readAttribute(attributes, DatasetAttributes.DIMENSIONS_KEY, long[].class, getGson()); - if (dimensions == null) { - return null; - } - - final DataType dataType = GsonUtils.readAttribute(attributes, DatasetAttributes.DATA_TYPE_KEY, DataType.class, getGson()); - if (dataType == null) { - return null; - } - - final int[] blockSize = GsonUtils.readAttribute(attributes, DatasetAttributes.BLOCK_SIZE_KEY, int[].class, getGson()); - - final Compression compression = GsonUtils.readAttribute(attributes, DatasetAttributes.COMPRESSION_KEY, Compression.class, getGson()); - - /* version 0 */ - final String compressionVersion0Name = compression - == null - ? GsonUtils.readAttribute(attributes, DatasetAttributes.compressionTypeKey, String.class, getGson()) - : null; - - return createDatasetAttributes(dimensions, dataType, blockSize, compression, compressionVersion0Name); - } - - @Override - default T getAttribute(final String pathName, final String key, final Class clazz) throws IOException { - - final String normalPathName = N5URL.normalizeGroupPath(pathName); - final String normalizedAttributePath = N5URL.normalizeAttributePath(key); - - final JsonElement attributes = getAttributes(normalPathName); - return GsonUtils.readAttribute(attributes, normalizedAttributePath, clazz, getGson()); - } - - @Override - default T getAttribute(final String pathName, final String key, final Type type) throws IOException { - - final String normalPathName = N5URL.normalizeGroupPath(pathName); - final String normalizedAttributePath = N5URL.normalizeAttributePath(key); - JsonElement attributes = getAttributes(normalPathName); - return GsonUtils.readAttribute(attributes, normalizedAttributePath, type, getGson()); - } - default boolean groupExists(final String absoluteNormalPath) { return getKeyValueAccess().isDirectory(absoluteNormalPath); @@ -121,16 +54,19 @@ default boolean exists(final String pathName) { @Override default boolean datasetExists(final String pathName) throws N5Exception.N5IOException { + // for n5, every dataset must be a group return getDatasetAttributes(pathName) != null; } /** * Reads or creates the attributes map of a group or dataset. * - * @param pathName group path + * @param pathName + * group path * @return * @throws IOException */ + @Override default JsonElement getAttributes(final String pathName) throws N5Exception.N5IOException { final String groupPath = N5URL.normalizeGroupPath(pathName); @@ -141,14 +77,17 @@ default JsonElement getAttributes(final String pathName) throws N5Exception.N5IO try (final LockedChannel lockedChannel = getKeyValueAccess().lockForReading(attributesPath)) { return GsonUtils.readAttributes(lockedChannel.newReader(), getGson()); - } catch (IOException e) { + } catch (final IOException e) { throw new N5Exception.N5IOException("Cannot open lock for Reading", e); } } @Override - default DataBlock readBlock( final String pathName, final DatasetAttributes datasetAttributes, final long... gridPosition) throws IOException { + default DataBlock readBlock( + final String pathName, + final DatasetAttributes datasetAttributes, + final long... gridPosition) throws N5Exception.N5IOException { final String path = getDataBlockPath(N5URL.normalizeGroupPath(pathName), gridPosition); if (!getKeyValueAccess().isFile(path)) @@ -156,6 +95,8 @@ default DataBlock readBlock( final String pathName, final DatasetAttributes d try (final LockedChannel lockedChannel = getKeyValueAccess().lockForReading(path)) { return DefaultBlockReader.readBlock(lockedChannel.newInputStream(), datasetAttributes, gridPosition); + } catch (final IOException e) { + throw new N5Exception.N5IOException("Cannot open lock for Reading", e); } } @@ -164,22 +105,25 @@ default String[] list(final String pathName) throws N5Exception.N5IOException { try { return getKeyValueAccess().listDirectories(groupPath(pathName)); - } catch (IOException e) { + } catch (final IOException e) { throw new N5Exception.N5IOException("Cannot list directories for group " + pathName, e); } } /** - * Constructs the path for a data block in a dataset at a given grid position. + * Constructs the path for a data block in a dataset at a given grid + * position. *

    * The returned path is + * * 

     	 * $basePath/datasetPathName/$gridPosition[0]/$gridPosition[1]/.../$gridPosition[n]
 	 *
    *

    * This is the file into which the data block will be stored. * - * @param normalPath normalized dataset path + * @param normalPath + * normalized dataset path * @param gridPosition * @return */ @@ -201,7 +145,8 @@ default String getDataBlockPath( * Constructs the absolute path (in terms of this store) for the group or * dataset. * - * @param normalGroupPath normalized group path without leading slash + * @param normalGroupPath + * normalized group path without leading slash * @return the absolute path to the group */ default String groupPath(final String normalGroupPath) { @@ -213,62 +158,12 @@ default String groupPath(final String normalGroupPath) { * Constructs the absolute path (in terms of this store) for the attributes * file of a group or dataset. * - * @param normalPath normalized group path without leading slash + * @param normalPath + * normalized group path without leading slash * @return the absolute path to the attributes */ default String attributesPath(final String normalPath) { return getKeyValueAccess().compose(getBasePath(), normalPath, N5KeyValueReader.ATTRIBUTES_JSON); } - - static DatasetAttributes createDatasetAttributes( - final long[] dimensions, - final DataType dataType, - int[] blockSize, - Compression compression, - final String compressionVersion0Name - ) { - - if (blockSize == null) - blockSize = Arrays.stream(dimensions).mapToInt(a -> (int)a).toArray(); - - /* version 0 */ - if (compression == null) { - switch (compressionVersion0Name) { - case "raw": - compression = new RawCompression(); - break; - case "gzip": - compression = new GzipCompression(); - break; - case "bzip2": - compression = new Bzip2Compression(); - break; - case "lz4": - compression = new Lz4Compression(); - break; - case "xz": - compression = new XzCompression(); - break; - } - } - - return new DatasetAttributes(dimensions, blockSize, dataType, compression); - } - - /** - * Check for attributes that are required for a group to be a dataset. - * - * @param attributes to check for dataset attributes - * @return if {@link DatasetAttributes#DIMENSIONS_KEY} and {@link DatasetAttributes#DATA_TYPE_KEY} are present - */ - static boolean hasDatasetAttributes(final JsonElement attributes) { - - if (attributes == null || !attributes.isJsonObject()) { - return false; - } - - final JsonObject metadataCache = attributes.getAsJsonObject(); - return metadataCache.has(DatasetAttributes.DIMENSIONS_KEY) && metadataCache.has(DatasetAttributes.DATA_TYPE_KEY); - } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueWriter.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java similarity index 51% rename from src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueWriter.java rename to src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java index 3137e378..003eeeda 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueWriter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java @@ -25,21 +25,32 @@ */ package org.janelia.saalfeldlab.n5; -import com.google.gson.JsonElement; -import com.google.gson.JsonNull; -import com.google.gson.JsonObject; - import java.io.IOException; +import java.util.Arrays; import java.util.List; import java.util.Map; +import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; + +import com.google.gson.Gson; +import com.google.gson.JsonElement; +import com.google.gson.JsonNull; +import com.google.gson.JsonObject; + /** - * Filesystem {@link N5Writer} implementation with version compatibility check. - * - * @author Stephan Saalfeld + * Default implementation of {@link N5Writer} with JSON attributes parsed with + * {@link Gson}. */ -public interface GsonKeyValueWriter extends GsonKeyValueReader, N5Writer { +public interface GsonKeyValueN5Writer extends GsonN5Writer, GsonKeyValueN5Reader { + /** + * TODO This overrides the version even if incompatible, check + * if this is the desired behavior or if it is always overridden, e.g. as by + * the caching version. If this is true, delete this implementation. + * + * @param path + * @throws IOException + */ default void setVersion(final String path) throws IOException { if (!VERSION.equals(getVersion())) @@ -55,107 +66,134 @@ static String initializeContainer( return normBasePath; } - @Override - default void createGroup(final String path) throws IOException { + /** + * Performs any necessary initialization to ensure the key given by the + * argument {@code normalPath} is a valid group after creation. Called by + * {@link #createGroup(String)}. + * + * @param normalPath the group path. + */ + default void initializeGroup(final String normalPath) { - final String normalPath = N5URL.normalizeGroupPath(path); - getKeyValueAccess().createDirectories(groupPath(normalPath)); + // Nothing to do here, but other implementations (e.g. zarr) use this. + // TODO remove if not used by zarr } @Override - default void createDataset( - final String path, - final DatasetAttributes datasetAttributes) throws IOException { + default void createGroup(final String path) throws N5Exception { final String normalPath = N5URL.normalizeGroupPath(path); - createGroup(path); - setDatasetAttributes(normalPath, datasetAttributes); + try { + getKeyValueAccess().createDirectories(groupPath(normalPath)); + } catch (final IOException e) { + throw new N5Exception.N5IOException("Failed to create group " + path, e); + } + initializeGroup(normalPath); } /** - * Helper method that reads an existing JsonElement representing the root - * attributes for {@code normalGroupPath}, inserts and overrides the - * provided attributes, and writes them back into the attributes store. + * Helper method that writes an attributes tree into the store + * + * TODO This method is not part of the public API and should be protected + * in Java >8 * - * @param normalGroupPath to write the attributes to - * @param attributes to write - * @throws IOException if unable to read the attributes at {@code normalGroupPath} - *

    - * TODO consider cache (or you read the attributes twice?) + * @param normalGroupPath + * to write the attributes to + * @param attributes + * to write + * @throws N5Exception + * if unable to write the attributes at {@code normalGroupPath} */ default void writeAttributes( final String normalGroupPath, - final JsonElement attributes) throws IOException { + final JsonElement attributes) throws N5Exception { - JsonElement root = getAttributes(normalGroupPath); - root = GsonUtils.insertAttribute(root, "/", attributes, getGson()); try (final LockedChannel lock = getKeyValueAccess().lockForWriting(attributesPath(normalGroupPath))) { - GsonUtils.writeAttributes(lock.newWriter(), root, getGson()); + GsonUtils.writeAttributes(lock.newWriter(), attributes, getGson()); + } catch (final IOException e) { + throw new N5Exception.N5IOException("Failed to write attributes into " + normalGroupPath, e); } } + @Override + default void setAttributes( + final String path, + final JsonElement attributes) throws N5Exception { + + final String normalPath = N5URL.normalizeGroupPath(path); + if (!exists(normalPath)) + throw new N5IOException("" + normalPath + " is not a group or dataset."); + + writeAttributes(normalPath, attributes); + } + /** * Helper method that reads the existing map of attributes, JSON encodes, * inserts and overrides the provided attributes, and writes them back into * the attributes store. * - * @param normalGroupPath to write the attributes to - * @param attributes to write - * @throws IOException if unable to read the attributes at {@code normalGroupPath} + * TODO This method is not part of the public API and should be protected + * in Java >8 + * + * @param normalGroupPath + * to write the attributes to + * @param attributes + * to write + * @throws N5Exception + * if unable to read or write the attributes at + * {@code normalGroupPath} */ default void writeAttributes( final String normalGroupPath, - final Map attributes) throws IOException { + final Map attributes) throws N5Exception { if (attributes != null && !attributes.isEmpty()) { - JsonElement existingAttributes = getAttributes(normalGroupPath); - existingAttributes = existingAttributes != null && existingAttributes.isJsonObject() - ? existingAttributes.getAsJsonObject() + JsonElement root = getAttributes(normalGroupPath); + root = root != null && root.isJsonObject() + ? root.getAsJsonObject() : new JsonObject(); - JsonElement newAttributes = GsonUtils.insertAttributes(existingAttributes, attributes, getGson()); - try (final LockedChannel lock = getKeyValueAccess().lockForWriting(attributesPath(normalGroupPath))) { - GsonUtils.writeAttributes(lock.newWriter(), newAttributes, getGson()); - } + root = GsonUtils.insertAttributes(root, attributes, getGson()); + writeAttributes(normalGroupPath, root); } } @Override default void setAttributes( final String path, - final Map attributes) throws IOException { + final Map attributes) throws N5Exception { final String normalPath = N5URL.normalizeGroupPath(path); if (!exists(normalPath)) - throw new N5Exception.N5IOException("" + normalPath + " is not a group or dataset."); + throw new N5IOException("" + normalPath + " is not a group or dataset."); writeAttributes(normalPath, attributes); } @Override - default boolean removeAttribute(final String pathName, final String key) throws IOException { + default boolean removeAttribute(final String groupPath, final String attributePath) throws N5Exception { - final String normalPath = N5URL.normalizeGroupPath(pathName); + final String normalPath = N5URL.normalizeGroupPath(groupPath); final String absoluteNormalPath = getKeyValueAccess().compose(getBasePath(), normalPath); - final String normalKey = N5URL.normalizeAttributePath(key); + final String normalKey = N5URL.normalizeAttributePath(attributePath); if (!getKeyValueAccess().isDirectory(absoluteNormalPath)) return false; - if (key.equals("/")) { - writeAttributes(normalPath, JsonNull.INSTANCE); + if (attributePath.equals("/")) { + setAttributes(normalPath, JsonNull.INSTANCE); return true; } final JsonElement attributes = getAttributes(normalPath); if (GsonUtils.removeAttribute(attributes, normalKey) != null) { - writeAttributes(normalPath, attributes); + setAttributes(normalPath, attributes); return true; } return false; } @Override - default T removeAttribute(final String pathName, final String key, final Class cls) throws IOException { + default T removeAttribute(final String pathName, final String key, final Class cls) throws N5Exception { final String normalPath = N5URL.normalizeGroupPath(pathName); final String normalKey = N5URL.normalizeAttributePath(key); @@ -169,13 +207,13 @@ default T removeAttribute(final String pathName, final String key, final Cla } @Override - default boolean removeAttributes(final String pathName, final List attributes) throws IOException { + default boolean removeAttributes(final String pathName, final List attributes) throws N5Exception { final String normalPath = N5URL.normalizeGroupPath(pathName); boolean removed = false; for (final String attribute : attributes) { final String normalKey = N5URL.normalizeAttributePath(attribute); - removed |= removeAttribute(normalPath, attribute); + removed |= removeAttribute(normalPath, normalKey); } return removed; } @@ -184,39 +222,50 @@ default boolean removeAttributes(final String pathName, final List attri default void writeBlock( final String path, final DatasetAttributes datasetAttributes, - final DataBlock dataBlock) throws IOException { + final DataBlock dataBlock) throws N5Exception { final String blockPath = getDataBlockPath(N5URL.normalizeGroupPath(path), dataBlock.getGridPosition()); try (final LockedChannel lock = getKeyValueAccess().lockForWriting(blockPath)) { - DefaultBlockWriter.writeBlock(lock.newOutputStream(), datasetAttributes, dataBlock); + } catch (final IOException e) { + throw new N5IOException( + "Failed to write block " + Arrays.toString(dataBlock.getGridPosition()) + " into dataset " + path, + e); } } @Override - default boolean remove(final String path) throws IOException { + default boolean remove(final String path) throws N5Exception { final String normalPath = N5URL.normalizeGroupPath(path); final String groupPath = groupPath(normalPath); - if (getKeyValueAccess().isDirectory(groupPath)) { - getKeyValueAccess().delete(groupPath); - /* an IOException should have occurred if anything had failed midway */ - return true; - } else - return false; + try { + if (getKeyValueAccess().isDirectory(groupPath)) + getKeyValueAccess().delete(groupPath); + } catch (final IOException e) { + throw new N5IOException("Failed to remove " + path, e); + } + + /* an IOException should have occurred if anything had failed midway */ + return true; } @Override default boolean deleteBlock( final String path, - final long... gridPosition) throws IOException { + final long... gridPosition) throws N5Exception { final String blockPath = getDataBlockPath(N5URL.normalizeGroupPath(path), gridPosition); - if (getKeyValueAccess().isFile(blockPath)) { - getKeyValueAccess().delete(blockPath); - /* an IOException should have occurred if anything had failed midway */ - return true; - } else - return false; + try { + if (getKeyValueAccess().isFile(blockPath)) + getKeyValueAccess().delete(blockPath); + } catch (final IOException e) { + throw new N5IOException( + "Failed to delete block " + Arrays.toString(gridPosition) + " from dataset " + path, + e); + } + + /* an IOException should have occurred if anything had failed midway */ + return true; } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonN5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonN5Reader.java new file mode 100644 index 00000000..b3295778 --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonN5Reader.java @@ -0,0 +1,113 @@ +/** + * Copyright (c) 2017--2021, Stephan Saalfeld + * All rights reserved. + *

    + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + *

    + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + *

    + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +package org.janelia.saalfeldlab.n5; + +import java.io.IOException; +import java.lang.reflect.Type; +import java.util.Map; + +import com.google.gson.Gson; +import com.google.gson.JsonElement; + +/** + * {@link N5Reader} with JSON attributes parsed with {@link Gson}. + * + */ +public interface GsonN5Reader extends N5Reader { + + Gson getGson(); + + @Override + default Map> listAttributes(final String pathName) throws N5Exception { + + return GsonUtils.listAttributes(getAttributes(pathName)); + } + + @Override + default DatasetAttributes getDatasetAttributes(final String pathName) throws N5Exception { + + final String normalPath = N5URL.normalizeGroupPath(pathName); + final JsonElement attributes = getAttributes(normalPath); + return createDatasetAttributes(attributes); + } + + default DatasetAttributes createDatasetAttributes(final JsonElement attributes) { + + final long[] dimensions = GsonUtils + .readAttribute(attributes, DatasetAttributes.DIMENSIONS_KEY, long[].class, getGson()); + if (dimensions == null) { + return null; + } + + final DataType dataType = GsonUtils + .readAttribute(attributes, DatasetAttributes.DATA_TYPE_KEY, DataType.class, getGson()); + if (dataType == null) { + return null; + } + + final int[] blockSize = GsonUtils + .readAttribute(attributes, DatasetAttributes.BLOCK_SIZE_KEY, int[].class, getGson()); + + final Compression compression = GsonUtils + .readAttribute(attributes, DatasetAttributes.COMPRESSION_KEY, Compression.class, getGson()); + + /* version 0 */ + final String compressionVersion0Name = compression == null + ? GsonUtils.readAttribute(attributes, DatasetAttributes.compressionTypeKey, String.class, getGson()) + : null; + + return DatasetAttributes.from(dimensions, dataType, blockSize, compression, compressionVersion0Name); + } + + @Override + default T getAttribute(final String pathName, final String key, final Class clazz) throws N5Exception { + + final String normalPathName = N5URL.normalizeGroupPath(pathName); + final String normalizedAttributePath = N5URL.normalizeAttributePath(key); + + final JsonElement attributes = getAttributes(normalPathName); + return GsonUtils.readAttribute(attributes, normalizedAttributePath, clazz, getGson()); + } + + @Override + default T getAttribute(final String pathName, final String key, final Type type) throws N5Exception { + + final String normalPathName = N5URL.normalizeGroupPath(pathName); + final String normalizedAttributePath = N5URL.normalizeAttributePath(key); + final JsonElement attributes = getAttributes(normalPathName); + return GsonUtils.readAttribute(attributes, normalizedAttributePath, type, getGson()); + } + + /** + * Reads or creates the attributes map of a group or dataset. + * + * @param pathName + * group path + * @return + * @throws IOException + */ + JsonElement getAttributes(final String pathName) throws N5Exception; +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonN5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonN5Writer.java new file mode 100644 index 00000000..4eff1184 --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonN5Writer.java @@ -0,0 +1,50 @@ +/** + * Copyright (c) 2017--2021, Stephan Saalfeld + * All rights reserved. + *

    + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + *

    + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + *

    + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +package org.janelia.saalfeldlab.n5; + +import com.google.gson.Gson; +import com.google.gson.JsonElement; + +/** + * {@link N5Writer} with JSON attributes parsed with {@link Gson}. + * + */ +public interface GsonN5Writer extends GsonN5Reader, N5Writer { + + /** + * Set the attributes of a group. This result of this method is equivalent + * with {@link #setAttribute(groupPath, "/", attributes)}. + * + * @param groupPath + * to write the attributes to + * @param attributes + * to write + * @throws N5Exception + */ + void setAttributes( + final String groupPath, + final JsonElement attributes) throws N5Exception; +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonUtils.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonUtils.java index af055778..03988f77 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonUtils.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonUtils.java @@ -183,7 +183,7 @@ static JsonElement getAttribute(JsonElement root, final String normalizedAttribu *

  • String[]
    • *
  • Object[]
    • * - * + * * @param root the json element * @return the attribute map */ @@ -516,7 +516,8 @@ static JsonElement insertAttributes(JsonElement root, final Map attri static JsonElement insertAttribute(JsonElement root, final String normalizedAttributePath, final T attribute, final Gson gson) { LinkedAttributePathToken pathToken = N5URL.getAttributePathTokens(normalizedAttributePath); - /* No path to traverse or build; just write the value */ + + /* No path to traverse or build; just return the value */ if (pathToken == null) return gson.toJsonTree(attribute); diff --git a/src/main/java/org/janelia/saalfeldlab/n5/LinkedAttributePathToken.java b/src/main/java/org/janelia/saalfeldlab/n5/LinkedAttributePathToken.java index d1df5f40..ca7a9504 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/LinkedAttributePathToken.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/LinkedAttributePathToken.java @@ -27,7 +27,7 @@ public abstract class LinkedAttributePathToken implements public abstract T getJsonType(); /** - * This method will retgurn the child element, if the {@link #parentJson} + * This method will return the child element, if the {@link #parentJson} * contains a child that * is valid for this token. If the {@link #parentJson} does not have a * child, this method will @@ -77,7 +77,7 @@ public JsonElement writeChild(final Gson gson, final Object value) { /** * Check if the provided {@code json} is compatible with this token. *
    - * Compatibility means thath the provided {@code JsonElement} does not + * Compatibility means that the provided {@code JsonElement} does not * explicitly conflict with this token. That means that {@code null} is * always compatible. The only real incompatibility is when the * {@code JsonElement} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java index db765d7f..78837ef9 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java @@ -42,7 +42,7 @@ * @author Igor Pisarev * @author Philipp Hanslovsky */ -public class N5KeyValueReader implements CachedGsonKeyValueReader { +public class N5KeyValueReader implements CachedGsonKeyValueN5Reader { public static final String ATTRIBUTES_JSON = "attributes.json"; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriter.java b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriter.java index 8763a567..910e7128 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriter.java @@ -36,7 +36,7 @@ * * @author Stephan Saalfeld */ -public class N5KeyValueWriter extends N5KeyValueReader implements CachedGsonKeyValueWriter { +public class N5KeyValueWriter extends N5KeyValueReader implements CachedGsonKeyValueN5Writer { /** * Opens an {@link N5KeyValueWriter} at a given base path with a custom diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java index 89b7d0ef..98b17521 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java @@ -83,7 +83,9 @@ public Version( * If the version string is null or not a SemVer version, this * version will be "0.0.0" *

    - * @param versionString the string representation of the version + * + * @param versionString + * the string representation of the version */ public Version(final String versionString) { @@ -157,8 +159,7 @@ public boolean equals(final Object other) { if (other instanceof Version) { final Version otherVersion = (Version)other; - return - (major == otherVersion.major) & + return (major == otherVersion.major) & (minor == otherVersion.minor) & (patch == otherVersion.patch) & (suffix.equals(otherVersion.suffix)); @@ -173,7 +174,8 @@ public boolean equals(final Object other) { * Currently, this means that the version is less than or equal to * 1.X.X. * - * @param version the version + * @param version + * the version * @return true if this version is compatible */ public boolean isCompatible(final Version version) { @@ -185,7 +187,7 @@ public boolean isCompatible(final Version version) { /** * SemVer version of this N5 spec. */ - public static final Version VERSION = new Version(2, 6, 1); + public static final Version VERSION = new Version(3, 0, 0); /** * Version attribute key. @@ -202,9 +204,10 @@ public boolean isCompatible(final Version version) { * case. * * @return the version - * @throws IOException the exception + * @throws N5Exception + * the exception */ - default Version getVersion() throws IOException { + default Version getVersion() throws N5Exception { return new Version(getAttribute("/", VERSION_KEY, String.class)); } @@ -228,76 +231,97 @@ default Version getVersion() throws IOException { /** * Reads an attribute. * - * @param pathName group path - * @param key the key - * @param clazz attribute class - * @param the attribute type + * @param pathName + * group path + * @param key + * the key + * @param clazz + * attribute class + * @param + * the attribute type * @return the attribute - * @throws IOException the exception + * @throws N5Exception + * the exception */ T getAttribute( final String pathName, final String key, - final Class clazz) throws IOException; + final Class clazz) throws N5Exception; /** * Reads an attribute. * * @param pathName * group path - * @param key the key + * @param key + * the key * @param type * attribute Type (use this for specifying generic types) - * @param the attribute type - * @return the attribute - * @throws IOException the exception + * @param + * the attribute type + * @return the attribute + * @throws N5Exception + * the exception */ T getAttribute( final String pathName, final String key, - final Type type) throws IOException; + final Type type) throws N5Exception; /** * Get mandatory dataset attributes. * - * @param pathName dataset path - * @return dataset attributes or null if either dimensions or dataType are not - * set - * @throws IOException the exception + * @param pathName + * dataset path + * @return dataset attributes or null if either dimensions or dataType are + * not set + * @throws N5Exception + * the exception */ - DatasetAttributes getDatasetAttributes(final String pathName) throws IOException; + DatasetAttributes getDatasetAttributes(final String pathName) throws N5Exception; /** * Reads a {@link DataBlock}. * - * @param pathName dataset path - * @param datasetAttributes the dataset attributes - * @param gridPosition the grid position + * @param pathName + * dataset path + * @param datasetAttributes + * the dataset attributes + * @param gridPosition + * the grid position * @return the data block - * @throws IOException the exception + * @throws N5Exception + * the exception */ DataBlock readBlock( final String pathName, final DatasetAttributes datasetAttributes, - final long... gridPosition) throws IOException; + final long... gridPosition) throws N5Exception; /** - * Load a {@link DataBlock} as a {@link Serializable}. The offset is given in + * Load a {@link DataBlock} as a {@link Serializable}. The offset is given + * in * {@link DataBlock} grid coordinates. * - * @param dataset the dataset path - * @param attributes the dataset attributes - * @param the data block type - * @param gridPosition the grid position + * @param dataset + * the dataset path + * @param attributes + * the dataset attributes + * @param + * the data block type + * @param gridPosition + * the grid position * @return the data block - * @throws IOException the io exception - * @throws ClassNotFoundException the class not found exception + * @throws N5Exception + * the exception + * @throws ClassNotFoundException + * the class not found exception */ @SuppressWarnings("unchecked") default T readSerializedBlock( final String dataset, final DatasetAttributes attributes, - final long... gridPosition) throws IOException, ClassNotFoundException { + final long... gridPosition) throws N5Exception, ClassNotFoundException { final DataBlock block = readBlock(dataset, attributes, gridPosition); if (block == null) @@ -306,13 +330,16 @@ default T readSerializedBlock( final ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(block.toByteBuffer().array()); try (ObjectInputStream in = new ObjectInputStream(byteArrayInputStream)) { return (T)in.readObject(); + } catch (final IOException e) { + throw new N5Exception.N5IOException(e); } } /** * Test whether a group or dataset exists at a given path. * - * @param pathName group path + * @param pathName + * group path * @return true if the path exists */ boolean exists(final String pathName); @@ -320,11 +347,13 @@ default T readSerializedBlock( /** * Test whether a dataset exists. * - * @param pathName dataset path + * @param pathName + * dataset path * @return true if a dataset exists - * @throws IOException the exception + * @throws N5Exception + * the exception */ - default boolean datasetExists(final String pathName) throws IOException { + default boolean datasetExists(final String pathName) throws N5Exception { return exists(pathName) && getDatasetAttributes(pathName) != null; } @@ -332,12 +361,13 @@ default boolean datasetExists(final String pathName) throws IOException { /** * List all groups (including datasets) in a group. * - * @param pathName group path + * @param pathName + * group path * @return list of children - * @throws IOException the exception + * @throws N5Exception + * the exception */ - String[] list(final String pathName) throws IOException; - + String[] list(final String pathName) throws N5Exception; /** * Recursively list all groups (including datasets) in the given group. @@ -350,78 +380,93 @@ default boolean datasetExists(final String pathName) throws IOException { * @param filter * filter for children to be included * @return list of groups - * @throws IOException the exception + * @throws N5Exception + * the exception */ default String[] deepList( final String pathName, - final Predicate filter) throws IOException { + final Predicate filter) throws N5Exception { final String groupSeparator = getGroupSeparator(); - final String normalPathName = - pathName.replaceAll( + final String normalPathName = pathName + .replaceAll( "(^" + groupSeparator + "*)|(" + groupSeparator + "*$)", ""); final List absolutePaths = deepList(this, normalPathName, false, filter); - return absolutePaths.stream() + return absolutePaths + .stream() .map(a -> a.replaceFirst(normalPathName + "(" + groupSeparator + "?)", "")) - .filter(a -> !a.isEmpty()).toArray(String[]::new); + .filter(a -> !a.isEmpty()) + .toArray(String[]::new); } /** * Recursively list all groups (including datasets) in the given group. * - * @param pathName base group path + * @param pathName + * base group path * @return list of groups - * @throws IOException the exception + * @throws N5Exception + * the exception */ - default String[] deepList(final String pathName) throws IOException { + default String[] deepList(final String pathName) throws N5Exception { return deepList(pathName, a -> true); } /** - * Recursively list all datasets in the given group. Only paths that satisfy the + * Recursively list all datasets in the given group. Only paths that satisfy + * the * provided filter will be included, but the children of paths that were * excluded may be included (filter does not apply to the subtree). * *

    * This method delivers the same results as *

    - * - * 
    {@code
+	 *
+	 * 
    +	 * {@code
 	 * n5.deepList(prefix, a -> {
 	 * 	try {
 	 * 		return n5.datasetExists(a) && filter.test(a);
-	 * 	} catch (final IOException e) {
+	 * 	} catch (final N5Exception e) {
 	 * 		return false;
 	 * 	}
 	 * });
-	 * }
    
+	 * }
+	 *
    *

    - * but will execute {@link #datasetExists(String)} only once per node. This can - * be relevant for performance on high latency backends such as cloud stores. + * but will execute {@link #datasetExists(String)} only once per node. This + * can + * be relevant for performance on high latency backends such as cloud + * stores. *

    * - * @param pathName base group path - * @param filter filter for datasets to be included + * @param pathName + * base group path + * @param filter + * filter for datasets to be included * @return list of groups - * @throws IOException the exception + * @throws N5Exception + * the exception */ default String[] deepListDatasets( final String pathName, - final Predicate filter) throws IOException { + final Predicate filter) throws N5Exception { final String groupSeparator = getGroupSeparator(); - final String normalPathName = - pathName.replaceAll( + final String normalPathName = pathName + .replaceAll( "(^" + groupSeparator + "*)|(" + groupSeparator + "*$)", ""); final List absolutePaths = deepList(this, normalPathName, true, filter); - return absolutePaths.stream() + return absolutePaths + .stream() .map(a -> a.replaceFirst(normalPathName + "(" + groupSeparator + "?)", "")) - .filter(a -> !a.isEmpty()).toArray(String[]::new); + .filter(a -> !a.isEmpty()) + .toArray(String[]::new); } /** @@ -430,26 +475,32 @@ default String[] deepListDatasets( *

    * This method delivers the same results as *

    - * - * 
    {@code
+	 *
+	 * 
    +	 * {@code
 	 * n5.deepList(prefix, a -> {
 	 * 	try {
 	 * 		return n5.datasetExists(a);
-	 * 	} catch (final IOException e) {
+	 * 	} catch (final N5Exception e) {
 	 * 		return false;
 	 * 	}
 	 * });
-	 * }
    
+	 * }
+	 *
    *

    - * but will execute {@link #datasetExists(String)} only once per node. This can - * be relevant for performance on high latency backends such as cloud stores. + * but will execute {@link #datasetExists(String)} only once per node. This + * can + * be relevant for performance on high latency backends such as cloud + * stores. *

    * - * @param pathName base group path + * @param pathName + * base group path * @return list of groups - * @throws IOException the exception + * @throws N5Exception + * the exception */ - default String[] deepListDatasets(final String pathName) throws IOException { + default String[] deepListDatasets(final String pathName) throws N5Exception { return deepListDatasets(pathName, a -> true); } @@ -460,25 +511,31 @@ default String[] deepListDatasets(final String pathName) throws IOException { * private interface methods yet. * * TODO make private when committing to Java versions newer than 8 - * - * @param n5 the n5 reader - * @param pathName the base group path - * @param datasetsOnly true if only dataset paths should be returned - * @param filter a dataset filter + * + * @param n5 + * the n5 reader + * @param pathName + * the base group path + * @param datasetsOnly + * true if only dataset paths should be returned + * @param filter + * a dataset filter * @return the list of all children - * @throws IOException the exception + * @throws N5Exception + * the exception */ static ArrayList deepList( final N5Reader n5, final String pathName, final boolean datasetsOnly, - final Predicate filter ) throws IOException { + final Predicate filter) throws N5Exception { final ArrayList children = new ArrayList<>(); final boolean isDataset = n5.datasetExists(pathName); final boolean passDatasetTest = datasetsOnly && !isDataset; - if (!passDatasetTest && filter.test(pathName)) children.add(pathName); + if (!passDatasetTest && filter.test(pathName)) + children.add(pathName); if (!isDataset) { final String groupSeparator = n5.getGroupSeparator(); @@ -492,22 +549,29 @@ static ArrayList deepList( /** * Recursively list all groups (including datasets) in the given group, in - * parallel, using the given {@link ExecutorService}. Only paths that satisfy + * parallel, using the given {@link ExecutorService}. Only paths that + * satisfy * the provided filter will be included, but the children of paths that were * excluded may be included (filter does not apply to the subtree). * - * @param pathName base group path - * @param filter filter for children to be included - * @param executor executor service + * @param pathName + * base group path + * @param filter + * filter for children to be included + * @param executor + * executor service * @return list of datasets - * @throws IOException the io exception - * @throws ExecutionException the execution exception - * @throws InterruptedException the interrupted exception + * @throws N5Exception + * the exception + * @throws ExecutionException + * the execution exception + * @throws InterruptedException + * the interrupted exception */ default String[] deepList( final String pathName, final Predicate filter, - final ExecutorService executor) throws IOException, InterruptedException, ExecutionException { + final ExecutorService executor) throws N5Exception, InterruptedException, ExecutionException { final String groupSeparator = getGroupSeparator(); final String normalPathName = pathName.replaceAll("(^" + groupSeparator + "*)|(" + groupSeparator + "*$)", ""); @@ -529,56 +593,72 @@ default String[] deepList( * Recursively list all groups (including datasets) in the given group, in * parallel, using the given {@link ExecutorService}. * - * @param pathName base group path - * @param executor executor service + * @param pathName + * base group path + * @param executor + * executor service * @return list of groups - * @throws IOException the io exception - * @throws ExecutionException the execution exception - * @throws InterruptedException the interrupted exception + * @throws N5Exception + * the exception + * @throws ExecutionException + * the execution exception + * @throws InterruptedException + * the interrupted exception */ default String[] deepList( final String pathName, - final ExecutorService executor) throws IOException, InterruptedException, ExecutionException { + final ExecutorService executor) throws N5Exception, InterruptedException, ExecutionException { - return deepList( pathName, a -> true, executor ); + return deepList(pathName, a -> true, executor); } /** * Recursively list all datasets in the given group, in parallel, using the - * given {@link ExecutorService}. Only paths that satisfy the provided filter + * given {@link ExecutorService}. Only paths that satisfy the provided + * filter * will be included, but the children of paths that were excluded may be * included (filter does not apply to the subtree). * *

    * This method delivers the same results as *

    - * - * 
    {@code
+	 *
+	 * 
    +	 * {@code
 	 * n5.deepList(prefix, a -> {
 	 * 	try {
 	 * 		return n5.datasetExists(a) && filter.test(a);
-	 * 	} catch (final IOException e) {
+	 * 	} catch (final N5Exception e) {
 	 * 		return false;
 	 * 	}
 	 * }, exec);
-	 * }
    
+	 * }
+	 *
    *

    - * but will execute {@link #datasetExists(String)} only once per node. This can - * be relevant for performance on high latency backends such as cloud stores. + * but will execute {@link #datasetExists(String)} only once per node. This + * can + * be relevant for performance on high latency backends such as cloud + * stores. *

    * - * @param pathName base group path - * @param filter filter for datasets to be included - * @param executor executor service + * @param pathName + * base group path + * @param filter + * filter for datasets to be included + * @param executor + * executor service * @return list of datasets - * @throws IOException the io exception - * @throws ExecutionException the execution exception - * @throws InterruptedException the interrupted exception + * @throws N5Exception + * the exception + * @throws ExecutionException + * the execution exception + * @throws InterruptedException + * the interrupted exception */ default String[] deepListDatasets( final String pathName, final Predicate filter, - final ExecutorService executor) throws IOException, InterruptedException, ExecutionException { + final ExecutorService executor) throws N5Exception, InterruptedException, ExecutionException { final String groupSeparator = getGroupSeparator(); final String normalPathName = pathName.replaceAll("(^" + groupSeparator + "*)|(" + groupSeparator + "*$)", ""); @@ -603,33 +683,42 @@ default String[] deepListDatasets( *

    * This method delivers the same results as *

    - * - * 
    {@code
+	 *
+	 * 
    +	 * {@code
 	 * n5.deepList(prefix, a -> {
 	 * 	try {
 	 * 		return n5.datasetExists(a);
-	 * 	} catch (final IOException e) {
+	 * 	} catch (final N5Exception e) {
 	 * 		return false;
 	 * 	}
 	 * }, exec);
-	 * }
    
+	 * }
+	 *
    *

    - * but will execute {@link #datasetExists(String)} only once per node. This can - * be relevant for performance on high latency backends such as cloud stores. + * but will execute {@link #datasetExists(String)} only once per node. This + * can + * be relevant for performance on high latency backends such as cloud + * stores. *

    * - * @param pathName base group path - * @param executor executor service + * @param pathName + * base group path + * @param executor + * executor service * @return list of groups - * @throws IOException the io exception - * @throws ExecutionException the execution exception - * @throws InterruptedException the interrupted exception + * @throws N5Exception + * the exception + * @throws ExecutionException + * the execution exception + * @throws InterruptedException + * the interrupted exception */ default String[] deepListDatasets( final String pathName, - final ExecutorService executor) throws IOException, InterruptedException, ExecutionException { + final ExecutorService executor) throws N5Exception, InterruptedException, ExecutionException { - return deepListDatasets( pathName, a -> true, executor ); + return deepListDatasets(pathName, a -> true, executor); } /** @@ -639,12 +728,18 @@ default String[] deepListDatasets( * * TODO make private when committing to Java versions newer than 8 * - * @param n5 the n5 reader - * @param path the base path - * @param datasetsOnly true if only dataset paths should be returned - * @param filter filter for datasets to be included - * @param executor the executor service - * @param datasetFutures result futures + * @param n5 + * the n5 reader + * @param path + * the base path + * @param datasetsOnly + * true if only dataset paths should be returned + * @param filter + * filter for datasets to be included + * @param executor + * the executor service + * @param datasetFutures + * result futures */ static void deepListHelper( final N5Reader n5, @@ -661,7 +756,7 @@ static void deepListHelper( boolean isDataset = false; try { isDataset = n5.datasetExists(path); - } catch (final IOException e) {} + } catch (final N5Exception e) {} if (!isDataset) { String[] children = null; @@ -671,7 +766,7 @@ static void deepListHelper( final String fullChildPath = path + groupSeparator + child; deepListHelper(n5, fullChildPath, datasetsOnly, filter, executor, datasetFutures); } - } catch (final IOException e) {} + } catch (final N5Exception e) {} } final boolean passDatasetTest = datasetsOnly && !isDataset; return !passDatasetTest && filter.test(path) ? path : null; @@ -681,11 +776,13 @@ static void deepListHelper( /** * List all attributes and their class of a group. * - * @param pathName group path + * @param pathName + * group path * @return the attribute map - * @throws IOException the exception + * @throws N5Exception + * the exception */ - Map> listAttributes(final String pathName) throws IOException; + Map> listAttributes(final String pathName) throws N5Exception; /** * Returns the symbol that is used to separate nodes in a group path. @@ -698,14 +795,16 @@ default String getGroupSeparator() { } /** - * Returns an absolute path to a group by concatenating all nodes with the node separator + * Returns an absolute path to a group by concatenating all nodes with the + * node separator * defined by {@link #getGroupSeparator()}. - * @param nodes the group components + * + * @param nodes + * the group components * @return the absolute group path */ String groupPath(final String... nodes); - /** * Default implementation of {@link AutoCloseable#close()} for all * implementations that do not hold any closable resources. diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java index 5bf62555..c8ee52bb 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java @@ -45,211 +45,298 @@ public interface N5Writer extends N5Reader { /** * Sets an attribute. * - * @param pathName group path - * @param key the key - * @param attribute the attribute - * @param the attribute type type - * @throws IOException the exception + * @param groupPath + * group path + * @param attributePath + * the key + * @param attribute + * the attribute + * @param + * the attribute type type + * @throws N5Exception + * the exception */ default void setAttribute( - final String pathName, - final String key, - final T attribute) throws IOException { + final String groupPath, + final String attributePath, + final T attribute) throws N5Exception { - setAttributes(pathName, Collections.singletonMap(key, attribute)); + setAttributes(groupPath, Collections.singletonMap(attributePath, attribute)); } /** - * Sets a map of attributes. + * Sets a map of attributes. The passed attributes are inserted into the + * existing attribute tree. New attributes, including their parent + * objects will be added, existing attributes whose paths are not included + * will remain unchanged, those whose paths are included will be overridden. * - * @param pathName group path - * @param attributes the attribute map - * @throws IOException the exception + * @param groupPath + * group path + * @param attributes + * the attribute map of attribute paths and values + * @throws N5Exception + * the exception */ void setAttributes( - final String pathName, - final Map attributes) throws IOException; + final String groupPath, + final Map attributes) throws N5Exception; /** * Remove the attribute from group {@code pathName} with key {@code key}. * - * @param pathName group path - * @param key of attribute to remove + * @param groupPath + * group path + * @param attributePath + * of attribute to remove * @return true if attribute removed, else false - * @throws IOException the exception + * @throws N5Exception + * the exception */ - boolean removeAttribute(String pathName, String key) throws IOException; + boolean removeAttribute(String groupPath, String attributePath) throws N5Exception; /** - * Remove the attribute from group {@code pathName} with key {@code key} and type {@code T}. + * Remove the attribute from group {@code pathName} with key {@code key} and + * type {@code T}. *

    - * If an attribute at {@code pathName} and {@code key} exists, but is not of type {@code T}, it is not removed. - * - * @param pathName group path - * @param key of attribute to remove - * @param clazz of the attribute to remove - * @param of the attribute - * @return the removed attribute, as {@code T}, or {@code null} if no matching attribute - * @throws IOException the exception + * If an attribute at {@code pathName} and {@code key} exists, but is not of + * type {@code T}, it is not removed. + * + * @param groupPath + * group path + * @param attributePath + * of attribute to remove + * @param clazz + * of the attribute to remove + * @param + * of the attribute + * @return the removed attribute, as {@code T}, or {@code null} if no + * matching attribute + * @throws N5Exception + * if removing he attribute failed, parsing the attribute + * failed, or the attribute cannot be interpreted as T */ - T removeAttribute(String pathName, String key, Class clazz) throws IOException; + T removeAttribute(String groupPath, String attributePath, Class clazz) throws N5Exception; /** * Remove attributes as provided by {@code attributes}. *

    * If any element of {@code attributes} does not exist, it will be ignored. - * If at least one attribute from {@code attributes} is removed, this will return {@code true}. + * If at least one attribute from {@code attributes} is removed, this will + * return {@code true}. * * - * @param pathName group path - * @param attributes to remove + * @param groupPath + * group path + * @param attributePaths + * to remove * @return true if any of the listed attributes were removed - * @throws IOException the exception + * @throws N5Exception + * the exception */ - boolean removeAttributes( String pathName, List attributes) throws IOException; + default boolean removeAttributes(final String groupPath, final List attributePaths) throws N5Exception { + + final String normalPath = N5URL.normalizeGroupPath(groupPath); + boolean removed = false; + for (final String attribute : attributePaths) { + final String normalKey = N5URL.normalizeAttributePath(attribute); + removed |= removeAttribute(normalPath, attribute); + } + return removed; + } /** * Sets mandatory dataset attributes. * - * @param pathName dataset path - * @param datasetAttributes the dataset attributse - * @throws IOException the exception + * @param datasetPath + * dataset path + * @param datasetAttributes + * the dataset attributse + * @throws N5Exception + * the exception */ default void setDatasetAttributes( - final String pathName, - final DatasetAttributes datasetAttributes) throws IOException { + final String datasetPath, + final DatasetAttributes datasetAttributes) throws N5Exception { + + setAttributes(datasetPath, datasetAttributes.asMap()); + } - setAttributes(pathName, datasetAttributes.asMap()); + /** + * Set the SemVer version of this container as specified in the + * {@link N5Reader#VERSION_KEY} attribute of the root group. This default + * implementation writes the version only if the current version is not + * equal {@link N5Reader#VERSION}. + * + * @throws N5Exception + * the exception + */ + default void setVersion() throws N5Exception { + + if (!VERSION.equals(getVersion())) + setAttribute("/", VERSION_KEY, VERSION.toString()); } /** * Creates a group (directory) * - * @param pathName the path - * @throws IOException the exception + * @param groupPath + * the path + * @throws N5Exception + * the exception */ - void createGroup(final String pathName) throws IOException; + void createGroup(final String groupPath) throws N5Exception; /** * Removes a group or dataset (directory and all contained files). * - *

    {@link #remove(String) remove("/")} or + *

    + * {@link #remove(String) remove("")} or * {@link #remove(String) remove("")} will delete this N5 - * container. Please note that no checks for safety will be performed, + * container. Please note that no checks for safety will be performed, * e.g. {@link #remove(String) remove("..")} will try to * recursively delete the parent directory of this N5 container which * only fails because it attempts to delete the parent directory before it * is empty. * - * @param pathName group path + * @param groupPath + * group path * @return true if removal was successful, false otherwise - * @throws IOException the exception + * @throws N5Exception + * the exception */ - boolean remove(final String pathName) throws IOException; + boolean remove(final String groupPath) throws N5Exception; /** * Removes the N5 container. * * @return true if removal was successful, false otherwise - * @throws IOException the exception + * @throws N5Exception + * the exception */ - default boolean remove() throws IOException { + default boolean remove() throws N5Exception { return remove("/"); } /** - * Creates a dataset. This does not create any data but the path and + * Creates a dataset. This does not create any data but the path and * mandatory attributes only. * - * @param pathName dataset path - * @param datasetAttributes the dataset attributes - * @throws IOException the exception + * @param datasetPath + * dataset path + * @param datasetAttributes + * the dataset attributes + * @throws N5Exception + * the exception */ default void createDataset( - final String pathName, - final DatasetAttributes datasetAttributes) throws IOException { + final String datasetPath, + final DatasetAttributes datasetAttributes) throws N5Exception { - final String normalPath = N5URL.normalizeGroupPath(pathName); + final String normalPath = N5URL.normalizeGroupPath(datasetPath); createGroup(normalPath); setDatasetAttributes(normalPath, datasetAttributes); } /** - * Creates a dataset. This does not create any data but the path and mandatory + * Creates a dataset. This does not create any data but the path and + * mandatory * attributes only. * - * @param pathName dataset path - * @param dimensions the dataset dimensions - * @param blockSize the block size - * @param dataType the data type - * @param compression the compression - * @throws IOException the exception + * @param datasetPath + * dataset path + * @param dimensions + * the dataset dimensions + * @param blockSize + * the block size + * @param dataType + * the data type + * @param compression + * the compression + * @throws N5Exception + * the exception */ default void createDataset( - final String pathName, + final String datasetPath, final long[] dimensions, final int[] blockSize, final DataType dataType, - final Compression compression) throws IOException { + final Compression compression) throws N5Exception { - createDataset(pathName, new DatasetAttributes(dimensions, blockSize, dataType, compression)); + createDataset(datasetPath, new DatasetAttributes(dimensions, blockSize, dataType, compression)); } /** * Writes a {@link DataBlock}. * - * @param pathName dataset path - * @param datasetAttributes the dataset attributes - * @param dataBlock the data block - * @param the data block data type - * @throws IOException the exception + * @param datasetPath + * dataset path + * @param datasetAttributes + * the dataset attributes + * @param dataBlock + * the data block + * @param + * the data block data type + * @throws N5Exception + * the exception */ void writeBlock( - final String pathName, + final String datasetPath, final DatasetAttributes datasetAttributes, - final DataBlock dataBlock) throws IOException; - + final DataBlock dataBlock) throws N5Exception; /** * Deletes the block at {@code gridPosition} * - * @param pathName dataset path - * @param gridPosition position of block to be deleted - * @throws IOException the exception + * @param datasetPath + * dataset path + * @param gridPosition + * position of block to be deleted + * @throws N5Exception + * the exception * - * @return {@code true} if the block at {@code gridPosition} is "empty" after + * @return {@code true} if the block at {@code gridPosition} is "empty" + * after * deletion. The meaning of "empty" is implementation dependent. For - * example "empty" means that no file exists on the file system for the + * example "empty" means that no file exists on the file system for + * the * deleted block in case of the file system implementation. * */ boolean deleteBlock( - final String pathName, - final long... gridPosition) throws IOException; + final String datasetPath, + final long... gridPosition) throws N5Exception; /** - * Save a {@link Serializable} as an N5 {@link DataBlock} at a given offset. The + * Save a {@link Serializable} as an N5 {@link DataBlock} at a given offset. + * The * offset is given in {@link DataBlock} grid coordinates. * - * @param object the object to serialize - * @param dataset the dataset path - * @param datasetAttributes the dataset attributes - * @param gridPosition the grid position - * @throws IOException the exception + * @param object + * the object to serialize + * @param datasetPath + * the dataset path + * @param datasetAttributes + * the dataset attributes + * @param gridPosition + * the grid position + * @throws N5Exception + * the exception */ default void writeSerializedBlock( final Serializable object, - final String dataset, + final String datasetPath, final DatasetAttributes datasetAttributes, - final long... gridPosition) throws IOException { + final long... gridPosition) throws N5Exception { final ByteArrayOutputStream byteOutputStream = new ByteArrayOutputStream(); try (ObjectOutputStream out = new ObjectOutputStream(byteOutputStream)) { out.writeObject(object); + } catch (final IOException e) { + throw new N5Exception.N5IOException(e); } final byte[] bytes = byteOutputStream.toByteArray(); final DataBlock dataBlock = new ByteArrayDataBlock(null, gridPosition, bytes); - writeBlock(dataset, datasetAttributes, dataBlock); + writeBlock(datasetPath, datasetAttributes, dataBlock); } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/cache/N5JsonCache.java b/src/main/java/org/janelia/saalfeldlab/n5/cache/N5JsonCache.java index ea9e3161..0abe9e2b 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/cache/N5JsonCache.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/cache/N5JsonCache.java @@ -1,11 +1,11 @@ package org.janelia.saalfeldlab.n5.cache; -import com.google.gson.JsonElement; - import java.util.Collections; import java.util.HashMap; import java.util.HashSet; +import com.google.gson.JsonElement; + public class N5JsonCache { public static final N5CacheInfo emptyCacheInfo = new N5CacheInfo(); @@ -20,7 +20,7 @@ protected static class N5CacheInfo { protected final HashMap attributesCache = new HashMap<>(); protected HashSet children = null; - protected boolean isDataset = false; + protected boolean isDataset = false; protected boolean isGroup = false; protected JsonElement getCache(final String normalCacheKey) { @@ -73,7 +73,7 @@ public boolean isDataset(final String normalPathKey, final String normalCacheKey return cacheInfo.isDataset; } - public boolean isGroup(final String normalPathKey, String cacheKey) { + public boolean isGroup(final String normalPathKey, final String cacheKey) { N5CacheInfo cacheInfo = getCacheInfo(normalPathKey); if (cacheInfo == null) { @@ -100,7 +100,7 @@ public boolean exists(final String normalPathKey, final String normalCacheKey) { return cacheInfo != emptyCacheInfo; } - public String[] list(String normalPathKey) { + public String[] list(final String normalPathKey) { N5CacheInfo cacheInfo = getCacheInfo(normalPathKey); if (cacheInfo == null) { addNewCacheInfo(normalPathKey); @@ -113,13 +113,13 @@ public String[] list(String normalPathKey) { final String[] children = new String[cacheInfo.children.size()]; int i = 0; - for (String child : cacheInfo.children) { + for (final String child : cacheInfo.children) { children[i++] = child; } return children; } - public N5CacheInfo addNewCacheInfo(String normalPathKey, String normalCacheKey, JsonElement uncachedAttributes) { + public N5CacheInfo addNewCacheInfo(final String normalPathKey, final String normalCacheKey, final JsonElement uncachedAttributes) { final N5CacheInfo cacheInfo; if (container.existsFromContainer(normalPathKey, null)) { @@ -146,9 +146,9 @@ public N5CacheInfo addNewCacheInfo(String normalPathKey, String normalCacheKey, return cacheInfo; } - public N5CacheInfo forceAddNewCacheInfo(String normalPathKey, String normalCacheKey, JsonElement uncachedAttributes, boolean isGroup, boolean isDataset) { + public N5CacheInfo forceAddNewCacheInfo(final String normalPathKey, final String normalCacheKey, final JsonElement uncachedAttributes, final boolean isGroup, final boolean isDataset) { - // getting the current cache info is useful if it already has had its children listed + // getting the current cache info is useful if it already has had its children listed N5CacheInfo cacheInfo = getCacheInfo(normalPathKey); if( cacheInfo == null || cacheInfo == emptyCacheInfo ) cacheInfo = newCacheInfo(); @@ -164,12 +164,12 @@ public N5CacheInfo forceAddNewCacheInfo(String normalPathKey, String normalCache return cacheInfo; } - private N5CacheInfo addNewCacheInfo(String normalPathKey) { + private N5CacheInfo addNewCacheInfo(final String normalPathKey) { return addNewCacheInfo(normalPathKey, null, null); } - private void addChild(N5CacheInfo cacheInfo, String normalPathKey) { + private void addChild(final N5CacheInfo cacheInfo, final String normalPathKey) { if( cacheInfo.children == null ) cacheInfo.children = new HashSet<>(); @@ -181,7 +181,7 @@ private void addChild(N5CacheInfo cacheInfo, String normalPathKey) { /** * Updates the cache attributes for the given normalPathKey and normalCacheKey, * adding the appropriate node to the cache if necessary. - * + * * @param normalPathKey the normalized path key * @param normalCacheKey the normalized cache key * @param uncachedAttributes attributes to be cached @@ -233,7 +233,7 @@ public void setAttributes(final String normalPathKey, final String normalCacheKe } /** - * Adds child to the parent's children list, only if the parent has been + * Adds child to the parent's children list, only if the parent has been * cached, and its children list already exists. * * @param parent parent path @@ -249,7 +249,7 @@ public void addChildIfPresent(final String parent, final String child) { } /** - * Adds child to the parent's children list, only if the parent has been + * Adds child to the parent's children list, only if the parent has been * cached, creating a children list if it does not already exist. * * @param parent parent path @@ -285,7 +285,7 @@ public void removeCache(final String normalParentPathKey, final String normalPat } } - protected N5CacheInfo getCacheInfo(String pathKey) { + protected N5CacheInfo getCacheInfo(final String pathKey) { synchronized (containerPathToCache) { return containerPathToCache.get(pathKey); @@ -296,7 +296,7 @@ protected N5CacheInfo newCacheInfo() { return new N5CacheInfo(); } - protected void updateCache(final String normalPathKey, N5CacheInfo cacheInfo) { + protected void updateCache(final String normalPathKey, final N5CacheInfo cacheInfo) { synchronized (containerPathToCache) { containerPathToCache.put(normalPathKey, cacheInfo); } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/cache/N5JsonCacheableContainer.java b/src/main/java/org/janelia/saalfeldlab/n5/cache/N5JsonCacheableContainer.java index 0a20ab60..1a35dc37 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/cache/N5JsonCacheableContainer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/cache/N5JsonCacheableContainer.java @@ -6,7 +6,7 @@ * An N5 container whose structure and attributes can be cached. *

    * Implementations of interface methods must explicitly query the backing - * storage. Cached implmentations (e.g {@link CachedGsonKeyValueReader}) call + * storage. Cached implmentations (e.g {@link CachedGsonKeyValueN5Reader}) call * these methods to update their {@link N5JsonCache}. Corresponding * {@link N5Reader} methods should use the cache, if present. */ @@ -19,7 +19,7 @@ public interface N5JsonCacheableContainer { * @param normalPathName the normalized path name * @param normalCacheKey the cache key * @return the attributes as a json element. - * @see GsonKeyValueReader#getAttributes + * @see GsonKeyValueN5Reader#getAttributes */ JsonElement getAttributesFromContainer(final String normalPathName, final String normalCacheKey); diff --git a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java index 81e235a1..e3014175 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java @@ -25,16 +25,14 @@ */ package org.janelia.saalfeldlab.n5; -import com.google.gson.GsonBuilder; -import com.google.gson.JsonArray; -import com.google.gson.JsonElement; -import com.google.gson.JsonNull; -import com.google.gson.JsonObject; -import com.google.gson.reflect.TypeToken; -import org.janelia.saalfeldlab.n5.N5Reader.Version; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.Test; +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; import java.io.IOException; import java.net.URISyntaxException; @@ -51,14 +49,17 @@ import java.util.concurrent.Executors; import java.util.function.Predicate; -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertThrows; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; +import org.janelia.saalfeldlab.n5.N5Reader.Version; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.Test; + +import com.google.gson.GsonBuilder; +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonNull; +import com.google.gson.JsonObject; +import com.google.gson.reflect.TypeToken; /** * Abstract base class for testing N5 functionality. @@ -93,13 +94,13 @@ protected N5Writer createN5Writer() throws IOException, URISyntaxException { return createN5Writer(tempN5Location()); } - protected N5Writer createN5Writer(String location) throws IOException, URISyntaxException { + protected N5Writer createN5Writer(final String location) throws IOException, URISyntaxException { return createN5Writer(location, new GsonBuilder()); } protected abstract N5Writer createN5Writer(String location, GsonBuilder gson) throws IOException, URISyntaxException; - protected N5Reader createN5Reader(String location) throws IOException, URISyntaxException { + protected N5Reader createN5Reader(final String location) throws IOException, URISyntaxException { return createN5Reader(location, new GsonBuilder()); } @@ -153,9 +154,9 @@ public void setUpOnce() throws IOException, URISyntaxException { public static void rampDownAfterClass() throws IOException { if (n5 != null) { - try { + try { assertTrue(n5.remove()); - } catch ( Exception e ) {} + } catch ( final Exception e ) {} n5 = null; } } @@ -165,7 +166,7 @@ public void testCreateGroup() { try { n5.createGroup(groupName); - } catch (final IOException e) { + } catch (final N5Exception e) { fail(e.getMessage()); } @@ -225,7 +226,7 @@ public void testWriteReadByteBlock() { assertTrue(n5.remove(datasetName)); - } catch (final IOException e) { + } catch (final N5Exception e) { e.printStackTrace(); fail("Block cannot be written."); } @@ -942,7 +943,7 @@ public void testDeepList() throws IOException, URISyntaxException, ExecutionExce a -> { try { return n5.datasetExists(a); - } catch (final IOException e) { + } catch (final N5Exception e) { return false; } })); @@ -956,7 +957,7 @@ public void testDeepList() throws IOException, URISyntaxException, ExecutionExce a -> { try { return n5.datasetExists(a) && isBorC.test(a); - } catch (final IOException e) { + } catch (final N5Exception e) { return false; } })); @@ -973,7 +974,7 @@ public void testDeepList() throws IOException, URISyntaxException, ExecutionExce a -> { try { return n5.datasetExists(a); - } catch (final IOException e) { + } catch (final N5Exception e) { return false; } }, @@ -989,7 +990,7 @@ public void testDeepList() throws IOException, URISyntaxException, ExecutionExce a -> { try { return n5.datasetExists(a) && isBorC.test(a); - } catch (final IOException e) { + } catch (final N5Exception e) { return false; } }, @@ -1063,7 +1064,7 @@ public void testListAttributes() { assertTrue(attributesMap.get("attr6") == long.class); assertTrue(attributesMap.get("attr7") == double[].class); assertTrue(attributesMap.get("attr8") == Object[].class); - } catch (final IOException e) { + } catch (final N5Exception e) { fail(e.getMessage()); } } @@ -1177,7 +1178,7 @@ public class TestData { public T attributeValue; public Class attributeClass; - public TestData(String groupPath, String key, T attributeValue) { + public TestData(final String groupPath, final String key, final T attributeValue) { this.groupPath = groupPath; this.attributePath = key; @@ -1186,7 +1187,7 @@ public TestData(String groupPath, String key, T attributeValue) { } } - protected static void addAndTest(N5Writer writer, ArrayList> existingTests, TestData testData) throws IOException { + protected static void addAndTest(final N5Writer writer, final ArrayList> existingTests, final TestData testData) throws IOException { /* test a new value on existing path */ writer.setAttribute(testData.groupPath, testData.attributePath, testData.attributeValue); assertEquals(testData.attributeValue, writer.getAttribute(testData.groupPath, testData.attributePath, testData.attributeClass)); @@ -1199,7 +1200,7 @@ protected static void addAndTest(N5Writer writer, ArrayList> existin final String normalizedTestKey = N5URL.from(null, "", test.attributePath).normalizeAttributePath().replaceAll("^/", ""); final String normalizedTestDataKey = N5URL.from(null, "", testData.attributePath).normalizeAttributePath().replaceAll("^/", ""); return normalizedTestKey.equals(normalizedTestDataKey); - } catch (URISyntaxException e) { + } catch (final URISyntaxException e) { throw new RuntimeException(e); } }); @@ -1207,9 +1208,9 @@ protected static void addAndTest(N5Writer writer, ArrayList> existin existingTests.add(testData); } - protected static void runTests(N5Writer writer, ArrayList> existingTests) throws IOException { + protected static void runTests(final N5Writer writer, final ArrayList> existingTests) throws IOException { - for (TestData test : existingTests) { + for (final TestData test : existingTests) { assertEquals(test.attributeValue, writer.getAttribute(test.groupPath, test.attributePath, test.attributeClass)); assertEquals(test.attributeValue, writer.getAttribute(test.groupPath, test.attributePath, TypeToken.get(test.attributeClass).getType())); } @@ -1220,7 +1221,7 @@ public void testAttributePaths() throws IOException, URISyntaxException { try (final N5Writer writer = createN5Writer()) { - String testGroup = "test"; + final String testGroup = "test"; writer.createGroup(testGroup); final ArrayList> existingTests = new ArrayList<>(); @@ -1401,7 +1402,7 @@ public void testAttributePathEscaping() throws IOException, URISyntaxException { /* * For readability above */ - private String jsonKeyVal(String key, String val) { + private String jsonKeyVal(final String key, final String val) { return String.format("\"%s\":\"%s\"", key, val); } @@ -1411,7 +1412,7 @@ private String readAttributesAsString(final String group) { final String basePath = ((N5FSWriter)n5).getBasePath(); try { return new String(Files.readAllBytes(Paths.get(basePath, group, "attributes.json"))); - } catch (IOException e) { + } catch (final IOException e) { } return null; } @@ -1455,7 +1456,7 @@ private String readAttributesAsString(final String group) { tests.add(new TestData<>(groupName, "/", "replace_empty_root")); tests.add(new TestData<>(groupName, "[0]", "array_root")); - for (TestData testData : tests) { + for (final TestData testData : tests) { try (final N5Writer writer = createN5Writer()) { writer.createGroup(testData.groupPath); writer.setAttribute(testData.groupPath, testData.attributePath, testData.attributeValue); @@ -1473,7 +1474,7 @@ private String readAttributesAsString(final String group) { try (final N5Writer writer = createN5Writer()) { writer.createGroup(groupName); - for (TestData testData : tests) { + for (final TestData testData : tests) { writer.setAttribute(testData.groupPath, testData.attributePath, testData.attributeValue); assertEquals(testData.attributeValue, writer.getAttribute(testData.groupPath, testData.attributePath, testData.attributeClass)); assertEquals(testData.attributeValue, @@ -1490,7 +1491,7 @@ private String readAttributesAsString(final String group) { tests.add(rootAsArray); try (final N5Writer writer = createN5Writer()) { writer.createGroup(groupName); - for (TestData test : tests) { + for (final TestData test : tests) { /* Set the root as Object*/ writer.setAttribute(rootAsObject.groupPath, rootAsObject.attributePath, rootAsObject.attributeValue); assertEquals(rootAsObject.attributeValue, diff --git a/src/test/java/org/janelia/saalfeldlab/n5/N5Benchmark.java b/src/test/java/org/janelia/saalfeldlab/n5/N5Benchmark.java index e60f85c4..8f72edaf 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/N5Benchmark.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/N5Benchmark.java @@ -68,7 +68,7 @@ public class N5Benchmark { static { try { testDirPath = Files.createTempDirectory("n5-benchmark-").toFile().getCanonicalPath(); - } catch (IOException e) { + } catch (final IOException e) { throw new RuntimeException(e); } } @@ -138,7 +138,7 @@ public void testDocExample() { final DatasetAttributes attributes = n5.getDatasetAttributes(compressedDatasetName); final ShortArrayDataBlock dataBlock = new ShortArrayDataBlock(new int[]{1, 2, 3}, new long[]{0, 0, 0}, dataBlockData); n5.writeBlock(compressedDatasetName, attributes, dataBlock); - } catch (final IOException e) { + } catch (final N5Exception e) { fail(e.getMessage()); } } @@ -163,7 +163,7 @@ public void benchmarkWritingSpeed() { final ShortArrayDataBlock dataBlock = new ShortArrayDataBlock(new int[]{64, 64, 64}, new long[]{x, y, z}, data); n5.writeBlock(compressedDatasetName, attributes, dataBlock); } - } catch (final IOException e) { + } catch (final N5Exception e) { fail(e.getMessage()); } System.out.println(String.format("%d : %s : %fs", i, compression.getType(), 0.001 * (System.currentTimeMillis() - t))); @@ -260,7 +260,7 @@ public void benchmarkParallelWritingSpeed() { f.get(); System.out.println(String.format("%d : %s : %fs", i, compression.getType(), 0.001 * (System.currentTimeMillis() - t))); - } catch (final IOException | InterruptedException | ExecutionException e) { + } catch (final N5Exception | InterruptedException | ExecutionException e) { fail(e.getMessage()); } } diff --git a/src/test/java/org/janelia/saalfeldlab/n5/N5CachedFSTest.java b/src/test/java/org/janelia/saalfeldlab/n5/N5CachedFSTest.java index 520e2723..82c667b2 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/N5CachedFSTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/N5CachedFSTest.java @@ -1,12 +1,5 @@ package org.janelia.saalfeldlab.n5; -import com.google.gson.GsonBuilder; -import com.google.gson.JsonElement; - -import java.net.URISyntaxException; - -import org.junit.Test; - import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -16,11 +9,17 @@ import static org.junit.Assert.assertTrue; import java.io.IOException; +import java.net.URISyntaxException; import java.nio.file.FileSystems; import java.nio.file.Files; import java.nio.file.Paths; import java.util.ArrayList; +import org.junit.Test; + +import com.google.gson.GsonBuilder; +import com.google.gson.JsonElement; + public class N5CachedFSTest extends N5FSTest { @Override @@ -142,9 +141,9 @@ public static void cacheBehaviorHelper(final TrackingStorage n5) throws IOExcept // expected backend method call counts int expectedExistCount = 0; - int expectedGroupCount = 0; - int expectedDatasetCount = 0; - int expectedAttributeCount = 0; + final int expectedGroupCount = 0; + final int expectedDatasetCount = 0; + final int expectedAttributeCount = 0; int expectedListCount = 0; boolean exists = n5.exists(groupA); @@ -239,7 +238,7 @@ public static void cacheBehaviorHelper(final TrackingStorage n5) throws IOExcept * * Similarly, attributes can not exist for a non-existent group, so should not * attempt to get attributes from the container. - * + * * Finally,listing on a non-existent group is pointless, so don't call the * backend storage */ @@ -279,9 +278,9 @@ public static void cacheBehaviorHelper(final TrackingStorage n5) throws IOExcept assertEquals(expectedAttributeCount, n5.getAttrCallCount()); assertEquals(expectedListCount, n5.getListCallCount()); - String a = "a"; - String ab = "a/b"; - String abc = "a/b/c"; + final String a = "a"; + final String ab = "a/b"; + final String abc = "a/b/c"; // create "a/b/c" n5.createGroup(abc); assertTrue(n5.exists(abc)); @@ -355,7 +354,7 @@ public static void cacheBehaviorHelper(final TrackingStorage n5) throws IOExcept } - public static interface TrackingStorage extends CachedGsonKeyValueWriter { + public static interface TrackingStorage extends CachedGsonKeyValueN5Writer { public int getAttrCallCount(); public int getExistCallCount(); @@ -382,65 +381,79 @@ public N5TrackingStorage(final KeyValueAccess keyValueAccess, final String baseP super(keyValueAccess, basePath, gsonBuilder, cacheAttributes); } + @Override public JsonElement getAttributesFromContainer(final String key, final String cacheKey) { attrCallCount++; return super.getAttributesFromContainer(key, cacheKey); } + @Override public boolean existsFromContainer(final String path, final String cacheKey) { existsCallCount++; return super.existsFromContainer(path, cacheKey); } + @Override public boolean isGroupFromContainer(final String key) { groupCallCount++; return super.isGroupFromContainer(key); } + @Override public boolean isGroupFromAttributes(final String normalCacheKey, final JsonElement attributes) { groupAttrCallCount++; return super.isGroupFromAttributes(normalCacheKey, attributes); } + @Override public boolean isDatasetFromContainer(final String key) { datasetCallCount++; return super.isDatasetFromContainer(key); } + @Override public boolean isDatasetFromAttributes(final String normalCacheKey, final JsonElement attributes) { datasetAttrCallCount++; return super.isDatasetFromAttributes(normalCacheKey, attributes); } + @Override public String[] listFromContainer(final String key) { listCallCount++; return super.listFromContainer(key); } + @Override public int getAttrCallCount() { return attrCallCount; } + @Override public int getExistCallCount() { return existsCallCount; } + @Override public int getGroupCallCount() { return groupCallCount; } + @Override public int getGroupAttrCallCount() { return groupAttrCallCount; } + @Override public int getDatasetCallCount() { return datasetCallCount; } + @Override public int getDatasetAttrCallCount() { return datasetAttrCallCount; } + @Override public int getListCallCount() { return listCallCount; } diff --git a/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java b/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java index ad979c2b..1ffe759d 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java @@ -25,7 +25,6 @@ */ package org.janelia.saalfeldlab.n5; -import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import java.io.File; @@ -36,6 +35,7 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.ArrayList; import java.util.HashSet; import java.util.Set; import java.util.concurrent.ExecutionException; @@ -46,14 +46,11 @@ import java.util.concurrent.TimeoutException; import org.apache.commons.io.FileUtils; +import org.janelia.saalfeldlab.n5.url.UrlAttributeTest; import org.junit.AfterClass; import org.junit.Test; -import com.google.gson.GsonBuilder; -import com.google.gson.JsonObject; -import org.janelia.saalfeldlab.n5.url.UrlAttributeTest; - -import java.util.ArrayList; +import com.google.gson.GsonBuilder; /** * Initiates testing of the filesystem-based N5 implementation. @@ -70,40 +67,48 @@ public class N5FSTest extends AbstractN5Test { private static String testDirPath = tempN5PathName(); private static String tempN5PathName() { + try { final File tmpFile = Files.createTempDirectory("n5-test-").toFile(); tmpFile.deleteOnExit(); final String tmpPath = tmpFile.getCanonicalPath(); tmpFiles.add(tmpPath); return tmpPath; - } catch (Exception e) { + } catch (final Exception e) { throw new RuntimeException(e); } } @Override protected String tempN5Location() throws URISyntaxException { + final String basePath = tempN5PathName(); return new URI("file", null, basePath, null).toString(); } @Override - protected N5Writer createN5Writer(final String location, final GsonBuilder gson) throws IOException, URISyntaxException { + protected N5Writer createN5Writer( + final String location, + final GsonBuilder gson) throws IOException, URISyntaxException { + return new N5FSWriter(location, gson); } @Override - protected N5Reader createN5Reader(final String location, final GsonBuilder gson) throws IOException, URISyntaxException { + protected N5Reader createN5Reader( + final String location, + final GsonBuilder gson) throws IOException, URISyntaxException { + return new N5FSReader(location, gson); } @AfterClass public static void cleanup() { - for (String tmpFile : tmpFiles) { - try{ + for (final String tmpFile : tmpFiles) { + try { FileUtils.deleteDirectory(new File(tmpFile)); - } catch (Exception e) { } + } catch (final Exception e) {} } } @@ -113,10 +118,22 @@ public void customObjectTest() throws IOException { final String testGroup = "test"; final ArrayList> existingTests = new ArrayList<>(); - final UrlAttributeTest.TestDoubles doubles1 = new UrlAttributeTest.TestDoubles("doubles", "doubles1", new double[]{5.7, 4.5, 3.4}); - final UrlAttributeTest.TestDoubles doubles2 = new UrlAttributeTest.TestDoubles("doubles", "doubles2", new double[]{5.8, 4.6, 3.5}); - final UrlAttributeTest.TestDoubles doubles3 = new UrlAttributeTest.TestDoubles("doubles", "doubles3", new double[]{5.9, 4.7, 3.6}); - final UrlAttributeTest.TestDoubles doubles4 = new UrlAttributeTest.TestDoubles("doubles", "doubles4", new double[]{5.10, 4.8, 3.7}); + final UrlAttributeTest.TestDoubles doubles1 = new UrlAttributeTest.TestDoubles( + "doubles", + "doubles1", + new double[]{5.7, 4.5, 3.4}); + final UrlAttributeTest.TestDoubles doubles2 = new UrlAttributeTest.TestDoubles( + "doubles", + "doubles2", + new double[]{5.8, 4.6, 3.5}); + final UrlAttributeTest.TestDoubles doubles3 = new UrlAttributeTest.TestDoubles( + "doubles", + "doubles3", + new double[]{5.9, 4.7, 3.6}); + final UrlAttributeTest.TestDoubles doubles4 = new UrlAttributeTest.TestDoubles( + "doubles", + "doubles4", + new double[]{5.10, 4.8, 3.7}); n5.createGroup(testGroup); addAndTest(n5, existingTests, new TestData<>(testGroup, "/doubles[1]", doubles1)); addAndTest(n5, existingTests, new TestData<>(testGroup, "/doubles[2]", doubles2)); @@ -127,7 +144,6 @@ public void customObjectTest() throws IOException { addAndTest(n5, existingTests, new TestData<>(testGroup, "/doubles[1]", doubles4)); } - // @Test public void testReadLock() throws IOException, InterruptedException { @@ -165,7 +181,6 @@ public void testReadLock() throws IOException, InterruptedException { exec.shutdownNow(); } - // @Test public void testWriteLock() throws IOException { From 7afb19559a46e44f62e593391eb5e9e4bbf1526e Mon Sep 17 00:00:00 2001 From: Stephan Saalfeld Date: Thu, 1 Jun 2023 22:51:02 -0400 Subject: [PATCH 198/243] fix: groupExists must use an absolute path, was that hidden by overrides? --- .../saalfeldlab/n5/GsonKeyValueN5Reader.java | 4 +- .../saalfeldlab/n5/N5KeyValueReader.java | 85 ++++++++++--------- 2 files changed, 47 insertions(+), 42 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java index fc600693..acf092b5 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java @@ -39,9 +39,9 @@ public interface GsonKeyValueN5Reader extends GsonN5Reader { KeyValueAccess getKeyValueAccess(); - default boolean groupExists(final String absoluteNormalPath) { + default boolean groupExists(final String normalPath) { - return getKeyValueAccess().isDirectory(absoluteNormalPath); + return getKeyValueAccess().isDirectory(groupPath(normalPath)); } @Override diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java index 78837ef9..489415ac 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java @@ -25,14 +25,15 @@ */ package org.janelia.saalfeldlab.n5; -import com.google.gson.Gson; -import com.google.gson.GsonBuilder; +import java.io.IOException; import java.net.URISyntaxException; +import java.util.Arrays; import java.util.stream.Stream; + import org.janelia.saalfeldlab.n5.cache.N5JsonCache; -import java.io.IOException; -import java.util.Arrays; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; /** * {@link N5Reader} implementation through {@link KeyValueAccess} with JSON @@ -59,55 +60,62 @@ public class N5KeyValueReader implements CachedGsonKeyValueN5Reader { * {@link GsonBuilder} to support custom attributes. * * @param keyValueAccess - * @param basePath N5 base path + * @param basePath + * N5 base path * @param gsonBuilder - * @param cacheMeta cache attributes and meta data - * Setting this to true avoids frequent reading and parsing of JSON - * encoded attributes and other meta data that requires accessing the - * store. This is most interesting for high latency backends. Changes - * of cached attributes and meta data by an independent writer will - * not be tracked. + * @param cacheMeta + * cache attributes and meta data + * Setting this to true avoids frequent reading and parsing of + * JSON encoded attributes and other meta data that requires + * accessing the store. This is most interesting for high latency + * backends. Changes of cached attributes and meta data by an + * independent writer will not be tracked. * * @throws IOException - * if the base path cannot be read or does not exist, - * if the N5 version of the container is not compatible with this - * implementation. + * if the base path cannot be read or does not exist, if the N5 + * version of the container is not compatible with this + * implementation. */ public N5KeyValueReader( final KeyValueAccess keyValueAccess, final String basePath, final GsonBuilder gsonBuilder, - final boolean cacheMeta) throws IOException { + final boolean cacheMeta) + throws IOException { - this( true, keyValueAccess, basePath, gsonBuilder, cacheMeta ); + this(true, keyValueAccess, basePath, gsonBuilder, cacheMeta); } /** * Opens an {@link N5KeyValueReader} at a given base path with a custom * {@link GsonBuilder} to support custom attributes. * - * @param checkVersion do the version check + * @param checkVersion + * do the version check * @param keyValueAccess - * @param basePath N5 base path + * @param basePath + * N5 base path * @param gsonBuilder - * @param cacheMeta cache attributes and meta data - * Setting this to true avoids frequent reading and parsing of JSON - * encoded attributes and other meta data that requires accessing the - * store. This is most interesting for high latency backends. Changes - * of cached attributes and meta data by an independent writer will - * not be tracked. + * @param cacheMeta + * cache attributes and meta data + * Setting this to true avoids frequent reading and parsing of + * JSON encoded attributes and other meta data that requires + * accessing the store. This is most interesting for high latency + * backends. Changes of cached attributes and meta data by an + * independent writer will not be tracked. * * @throws IOException - * if the base path cannot be read or does not exist, - * if the N5 version of the container is not compatible with this - * implementation. + * if the base path cannot be read or does not exist, if the N5 + * version of the container is not compatible with this + * implementation. */ protected N5KeyValueReader( final boolean checkVersion, final KeyValueAccess keyValueAccess, final String basePath, final GsonBuilder gsonBuilder, - final boolean cacheMeta) throws IOException { + final boolean cacheMeta) + throws IOException { this.keyValueAccess = keyValueAccess; this.basePath = keyValueAccess.normalize(basePath); @@ -121,21 +129,24 @@ protected N5KeyValueReader( try { final Version version = getVersion(); if (!VERSION.isCompatible(version)) - throw new N5Exception.N5IOException("Incompatible version " + version + " (this is " + VERSION + ")."); - } catch( NullPointerException e ) - { + throw new N5Exception.N5IOException( + "Incompatible version " + version + " (this is " + VERSION + ")."); + } catch (final NullPointerException e) { throw new N5Exception.N5IOException("Could not read version from " + basePath); } } } + @Override public Gson getGson() { return gson; } + @Override public KeyValueAccess getKeyValueAccess() { + return keyValueAccess; } @@ -150,7 +161,7 @@ public String getContainerURI() { try { return keyValueAccess.absoluteURI(basePath); - } catch (URISyntaxException e) { + } catch (final URISyntaxException e) { throw new RuntimeException(e); } } @@ -168,14 +179,8 @@ public N5JsonCache getCache() { } @Override - public String groupPath(String... nodes) { - return keyValueAccess.compose(Stream.concat(Stream.of(basePath), Arrays.stream(nodes)).toArray(String[]::new)); - } - - @Override - public boolean exists(final String pathName) { + public String groupPath(final String... nodes) { - final String normalPath = N5URL.normalizeGroupPath(pathName); - return groupExists(normalPath); + return keyValueAccess.compose(Stream.concat(Stream.of(basePath), Arrays.stream(nodes)).toArray(String[]::new)); } } From 34850f30431566d84bd53070cf095ed879292fdc Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Fri, 2 Jun 2023 10:11:35 -0400 Subject: [PATCH 199/243] merge wip/basePath2Uri --- .../n5/CachedGsonKeyValueN5Reader.java | 2 +- .../n5/FileSystemKeyValueAccess.java | 7 +- .../saalfeldlab/n5/GsonKeyValueN5Reader.java | 6 +- .../saalfeldlab/n5/GsonKeyValueN5Writer.java | 2 +- .../saalfeldlab/n5/KeyValueAccess.java | 3 +- .../saalfeldlab/n5/N5KeyValueReader.java | 35 ++- .../org/janelia/saalfeldlab/n5/N5Reader.java | 14 +- .../org/janelia/saalfeldlab/n5/N5URL.java | 4 +- .../saalfeldlab/n5/AbstractN5Test.java | 7 +- .../org/janelia/saalfeldlab/n5/UriTest.java | 60 +++++ .../n5/demo/AttributePathDemo.java | 213 ++++++++++++++++++ 11 files changed, 310 insertions(+), 43 deletions(-) create mode 100644 src/test/java/org/janelia/saalfeldlab/n5/UriTest.java create mode 100644 src/test/java/org/janelia/saalfeldlab/n5/demo/AttributePathDemo.java diff --git a/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Reader.java index a38cb90d..29133446 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Reader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Reader.java @@ -246,7 +246,7 @@ default String getDataBlockPath( final long... gridPosition) { final String[] components = new String[gridPosition.length + 2]; - components[0] = getBasePath(); + components[0] = getURI().getPath(); components[1] = normalPath; int i = 1; for (final long p : gridPosition) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java index 15e1afd4..3aa951c4 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java @@ -25,6 +25,7 @@ */ package org.janelia.saalfeldlab.n5; +import java.io.File; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; @@ -273,8 +274,10 @@ public String normalize(final String path) { } @Override - public String absoluteURI(final String normalPath) throws URISyntaxException { - return new URI("file", null, normalPath, null).toString(); + public URI uri(final String normalPath) throws URISyntaxException { + // normalize make absolute the scheme specific part only + final URI uri = new URI( normalPath ); + return new URI( "file", normalize(new File( uri.getSchemeSpecificPart()).getAbsolutePath()), uri.getFragment() ); } @Override diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java index acf092b5..d2d348a2 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java @@ -132,7 +132,7 @@ default String getDataBlockPath( final long... gridPosition) { final String[] components = new String[gridPosition.length + 2]; - components[0] = getBasePath(); + components[0] = getURI().getPath(); components[1] = normalPath; int i = 1; for (final long p : gridPosition) @@ -151,7 +151,7 @@ default String getDataBlockPath( */ default String groupPath(final String normalGroupPath) { - return getKeyValueAccess().compose(getBasePath(), normalGroupPath); + return getKeyValueAccess().compose(getURI().getPath(), normalGroupPath); } /** @@ -164,6 +164,6 @@ default String groupPath(final String normalGroupPath) { */ default String attributesPath(final String normalPath) { - return getKeyValueAccess().compose(getBasePath(), normalPath, N5KeyValueReader.ATTRIBUTES_JSON); + return getKeyValueAccess().compose(getURI().getPath(), normalPath, N5KeyValueReader.ATTRIBUTES_JSON); } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java index 003eeeda..e5c43139 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java @@ -173,7 +173,7 @@ default void setAttributes( default boolean removeAttribute(final String groupPath, final String attributePath) throws N5Exception { final String normalPath = N5URL.normalizeGroupPath(groupPath); - final String absoluteNormalPath = getKeyValueAccess().compose(getBasePath(), normalPath); + final String absoluteNormalPath = getKeyValueAccess().compose(getURI().getPath(), normalPath); final String normalKey = N5URL.normalizeAttributePath(attributePath); if (!getKeyValueAccess().isDirectory(absoluteNormalPath)) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccess.java index e003a02c..20c14467 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccess.java @@ -26,6 +26,7 @@ package org.janelia.saalfeldlab.n5; import java.io.IOException; +import java.net.URI; import java.net.URISyntaxException; import java.nio.file.FileSystem; @@ -89,7 +90,7 @@ public interface KeyValueAccess { * efforts are made to normalize it. * @return absolute URI */ - public String absoluteURI(final String normalPath) throws URISyntaxException; + public URI uri(final String normalPath) throws URISyntaxException; /** * Test whether the path exists. diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java index 489415ac..54ecd10b 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java @@ -26,14 +26,16 @@ package org.janelia.saalfeldlab.n5; import java.io.IOException; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; + +import java.net.URI; import java.net.URISyntaxException; import java.util.Arrays; import java.util.stream.Stream; import org.janelia.saalfeldlab.n5.cache.N5JsonCache; -import com.google.gson.Gson; -import com.google.gson.GsonBuilder; /** * {@link N5Reader} implementation through {@link KeyValueAccess} with JSON @@ -51,8 +53,8 @@ public class N5KeyValueReader implements CachedGsonKeyValueN5Reader { protected final Gson gson; protected final boolean cacheMeta; + protected URI uri; - protected final String basePath; private final N5JsonCache cache; /** @@ -118,11 +120,16 @@ protected N5KeyValueReader( throws IOException { this.keyValueAccess = keyValueAccess; - this.basePath = keyValueAccess.normalize(basePath); this.gson = GsonUtils.registerGson(gsonBuilder); this.cacheMeta = cacheMeta; this.cache = newCache(); + try { + uri = keyValueAccess.uri(basePath); + } catch (URISyntaxException e) { + throw new N5Exception( e ); + } + if (checkVersion) { /* Existence checks, if any, go in subclasses */ /* Check that version (if there is one) is compatible. */ @@ -151,19 +158,8 @@ public KeyValueAccess getKeyValueAccess() { } @Override - public String getBasePath() { - - return this.basePath; - } - - @Override - public String getContainerURI() { - - try { - return keyValueAccess.absoluteURI(basePath); - } catch (final URISyntaxException e) { - throw new RuntimeException(e); - } + public URI getURI() { + return uri; } @Override @@ -179,8 +175,7 @@ public N5JsonCache getCache() { } @Override - public String groupPath(final String... nodes) { - - return keyValueAccess.compose(Stream.concat(Stream.of(basePath), Arrays.stream(nodes)).toArray(String[]::new)); + public String groupPath(String... nodes) { + return keyValueAccess.compose(Stream.concat(Stream.of(getURI().getPath()), Arrays.stream(nodes)).toArray(String[]::new)); } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java index 98b17521..de4e5884 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java @@ -30,6 +30,7 @@ import java.io.ObjectInputStream; import java.io.Serializable; import java.lang.reflect.Type; +import java.net.URI; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -213,20 +214,13 @@ default Version getVersion() throws N5Exception { } /** - * Returns a String representation of the path to the root of the container. + * Returns the URI of the container's root. * - * @return the base path - */ - String getBasePath(); - - /** - * Returns a String representation of the absolute URI of the container. - * - * @return the absolute URI of the container + * @return the base path URI */ // TODO: should this throw URISyntaxException or can we assume that this is // never possible if we were able to instantiate this N5Reader? - String getContainerURI(); + URI getURI(); /** * Reads an attribute. diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java b/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java index ffb393e4..23c98871 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java @@ -517,8 +517,8 @@ public static URI encodeAsUri(final String uri) throws URISyntaxException { public static N5URL from(final String container, final String group, final String attribute) throws URISyntaxException { final String containerPart = container != null ? container : ""; - final String groupPart = group != null ? "?" + group : "?"; - final String attributePart = attribute != null ? "#" + attribute : "#"; + final String groupPart = group != null ? "?" + group : ""; + final String attributePart = attribute != null ? "#" + attribute : ""; return new N5URL(containerPart + groupPart + attributePart); } diff --git a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java index e3014175..47f036af 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java @@ -596,6 +596,7 @@ public void testAttributes() throws IOException, URISyntaxException { } } + @Test public void testNullAttributes() throws IOException, URISyntaxException { @@ -799,7 +800,7 @@ public void testRemoveContainer() throws IOException, URISyntaxException { String location; try (final N5Writer n5 = createN5Writer()) { - location = n5.getContainerURI(); + location = n5.getURI().toString(); assertNotNull(createN5Reader(location)); n5.remove(); assertThrows(Exception.class, () -> createN5Reader(location)); @@ -1084,7 +1085,7 @@ public void testVersion() throws NumberFormatException, IOException, URISyntaxEx assertFalse(N5Reader.VERSION.isCompatible(version)); assertThrows(N5Exception.N5IOException.class, () -> { - final String containerPath = writer.getContainerURI(); + final String containerPath = writer.getURI().toString(); createN5Writer(containerPath); }); @@ -1409,7 +1410,7 @@ private String jsonKeyVal(final String key, final String val) { private String readAttributesAsString(final String group) { - final String basePath = ((N5FSWriter)n5).getBasePath(); + final String basePath = ((N5FSWriter)n5).getURI().getSchemeSpecificPart(); try { return new String(Files.readAllBytes(Paths.get(basePath, group, "attributes.json"))); } catch (final IOException e) { diff --git a/src/test/java/org/janelia/saalfeldlab/n5/UriTest.java b/src/test/java/org/janelia/saalfeldlab/n5/UriTest.java new file mode 100644 index 00000000..90084857 --- /dev/null +++ b/src/test/java/org/janelia/saalfeldlab/n5/UriTest.java @@ -0,0 +1,60 @@ +package org.janelia.saalfeldlab.n5; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.file.Paths; + +import org.junit.Before; +import org.junit.Test; + +public class UriTest { + + private N5FSWriter n5; + + private KeyValueAccess kva; + + private String relativePath = "src/test/resources/url/urlAttributes.n5"; + + private String relativeAbnormalPath = "src/test/resources/./url/urlAttributes.n5"; + + private String relativeAbnormalPath2 = "src/test/resources/../resources/url/urlAttributes.n5"; + + @Before + public void before() { + try { + n5 = new N5FSWriter(relativePath); + kva = n5.getKeyValueAccess(); + } catch (IOException e) { + e.printStackTrace(); + } + + } + + @Test + public void testUriParsing() { + + final URI uri = n5.getURI(); + try { + assertEquals("Container URI must contain scheme", "file", uri.getScheme()); + + assertEquals("Container URI must be absolute", + Paths.get(relativePath).toAbsolutePath().toString(), + uri.getPath()); + + assertEquals("Container URI must be normalized 1", uri, kva.uri(relativeAbnormalPath)); + assertEquals("Container URI must be normalized 2", uri, kva.uri(relativeAbnormalPath2)); + assertEquals("Container URI must be normalized 3", uri, kva.uri("file:" + relativePath)); + assertEquals("Container URI must be normalized 4", uri, kva.uri("file:" + relativeAbnormalPath)); + assertEquals("Container URI must be normalized 5", uri, kva.uri("file:" + relativeAbnormalPath2)); + + } catch (URISyntaxException e) { + fail(e.getMessage()); + } + + } + +} diff --git a/src/test/java/org/janelia/saalfeldlab/n5/demo/AttributePathDemo.java b/src/test/java/org/janelia/saalfeldlab/n5/demo/AttributePathDemo.java new file mode 100644 index 00000000..9db2a786 --- /dev/null +++ b/src/test/java/org/janelia/saalfeldlab/n5/demo/AttributePathDemo.java @@ -0,0 +1,213 @@ +package org.janelia.saalfeldlab.n5.demo; + +import java.io.IOException; +import java.nio.charset.Charset; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.Arrays; +import java.util.Collections; +import java.util.Map; + +import org.janelia.saalfeldlab.n5.N5FSWriter; +import org.janelia.saalfeldlab.n5.N5Reader; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.JsonObject; + +public class AttributePathDemo { + + final Gson gson; + + public AttributePathDemo() { + gson = new Gson(); + } + + public static void main(String[] args) throws IOException { + + new AttributePathDemo().demo(); + } + + public void demo() throws IOException { + + final String rootPath = "/home/john/projects/n5/demo.n5"; + final N5FSWriter n5 = new N5FSWriter( rootPath ); + final N5FSWriter n5WithNulls = new N5FSWriter( rootPath, new GsonBuilder().serializeNulls() ); + + final String group = ""; + final String specialGroup = "specialCharsGroup"; + final String rmAndNulls = "rmAndNulls"; + n5.createGroup(group); + n5.createGroup(specialGroup); + n5.createGroup(rmAndNulls); + + // clear all attributes + n5.setAttribute(group, "/", new JsonObject()); + n5.setAttribute(specialGroup, "/", new JsonObject()); + n5.setAttribute(rmAndNulls, "/", new JsonObject()); + + simple(n5, group); + arrays(n5, group); + objects(n5, group); + specialChars(n5, specialGroup); + removingAttributesAndNulls(n5, rmAndNulls); + removingAttributesAndNulls(n5WithNulls, rmAndNulls); + + n5.close(); + } + + public void simple(final N5FSWriter n5, final String group) throws IOException { + n5.setAttribute(group, "six", 6); + System.out.println(n5.getAttribute("/", "six", Integer.class)); // 6 + System.out.println(n5.getAttribute("/", "twelve", Integer.class)); // null + + final String longKey = "The Answer to the Ultimate Question"; + n5.setAttribute(group, longKey, 42); + System.out.println(n5.getAttribute(group, longKey, Integer.class)); // 42 + + n5.setAttribute(group, "name", "Marie Daly"); + System.out.println(n5.getAttribute(group, "name", String.class)); // returns "Marie Daly" + System.out.println(n5.getAttribute(group, "name", int.class)); // returns null + + n5.setAttribute(group, "year", "1921"); + System.out.println( "(String):" + n5.getAttribute(group, "year", String.class)); + System.out.println( "(int) :" + n5.getAttribute(group, "year", int.class)); + + n5.setAttribute(group, "animal", "aardvark"); + System.out.println( n5.getAttribute(group, "animal", String.class)); // "aardvark" + n5.setAttribute(group, "animal", new String[]{"bat", "cat", "dog"}); // overwrites "animal" + printAsJson( n5.getAttribute(group, "animal", String[].class)); // ["bat", "cat", "dog"] + + System.out.println(getRawJson(n5, group)); + // {"six":6,"The Answer to the Ultimate Question":42,"name":"Marie Daly","year":"1921","animal":["bat","cat","dog"]} + + n5.setAttribute(group, "/", new JsonObject()); // overwrites "animal" + System.out.println(getRawJson(n5, group)); + // {} + } + + public void arrays(final N5FSWriter n5, final String group) throws IOException { + + n5.setAttribute(group, "array", new double[] { 5, 6, 7, 8 }); + System.out.println( Arrays.toString(n5.getAttribute(group, "array", double[].class))); // [5.0, 6.0, 7.0, 8.0] + System.out.println( n5.getAttribute(group, "array[0]", double.class)); // 7.0 + System.out.println( n5.getAttribute(group, "array[2]", double.class)); // 7.0 + System.out.println( n5.getAttribute(group, "array[999]", double.class)); // null + System.out.println( n5.getAttribute(group, "array[-1]", double.class)); // null + + + n5.setAttribute(group, "array[1]", 0.6); + System.out.println( Arrays.toString(n5.getAttribute(group, "array", double[].class))); // [5.0, 0.6, 7.0, 8.0] + n5.setAttribute(group, "array[6]", 99.99 ); + System.out.println( Arrays.toString(n5.getAttribute(group, "array", double[].class))); // [5.0, 0.6, 7.0, 8.0, 0.0, 0.0, 99.99] + n5.setAttribute(group, "array[-5]", -5 ); + System.out.println( Arrays.toString(n5.getAttribute(group, "array", double[].class))); // [5.0, 0.6, 7.0, 8.0, 0.0, 0.0, 99.99] + + System.out.println( n5.getAttribute(group, "array", int.class)); // [5.0, 0.6, 7.0, 8.0, 0.0, 0.0, 99.99] + } + + @SuppressWarnings("rawtypes") + public void objects(final N5FSWriter n5, final String group) throws IOException { + Map a = Collections.singletonMap("a", "A"); + Map b = Collections.singletonMap("b", "B"); + Map c = Collections.singletonMap("c", "C"); + + n5.setAttribute(group, "obj", a ); + printAsJson(n5.getAttribute(group, "obj", Map.class)); // {"a":"A"} + System.out.println(""); + + n5.setAttribute(group, "obj/a", b); + printAsJson(n5.getAttribute(group, "obj", Map.class)); // {"a": {"b": "B"}} + printAsJson(n5.getAttribute(group, "obj/a", Map.class)); // {"b": "B"} + System.out.println(""); + + n5.setAttribute(group, "obj/a/b", c); + printAsJson(n5.getAttribute(group, "obj", Map.class)); // {"a": {"b": {"c": "C"}}} + printAsJson(n5.getAttribute(group, "obj/a", Map.class)); // {"b": {"c": "C"}} + printAsJson(n5.getAttribute(group, "obj/a/b", Map.class)); // {"c": "C"} + printAsJson(n5.getAttribute(group, "/", Map.class)); // returns {"obj": {"a": {"b": {"c": "C"}}}} + System.out.println(""); + + n5.setAttribute(group, "pet", new Pet("Pluto", 93)); + System.out.println(n5.getAttribute(group, "pet", Pet.class)); // Pet("Pluto", 93) + printAsJson(n5.getAttribute(group, "pet", Map.class)); // {"name": "Pluto", "age": 93} + + n5.setAttribute(group, "pet/likes", new String[]{"Micky"}); + printAsJson(n5.getAttribute(group, "pet", Map.class)); // {"name": "Pluto", "age": 93, "likes": ["Micky"]} + System.out.println(""); + + n5.removeAttribute(group, "/"); + System.out.println(getRawJson(n5, group)); // null + + n5.setAttribute(group, "one/[2]/three/[4]", 5); + System.out.println(getRawJson(n5, group)); // {"one":[null,null,{"three":[0,0,0,0,5]}]} + } + + public void specialChars(final N5FSWriter n5, final String group) throws IOException { + n5.setAttribute(group, "\\/", "fwdSlash"); + printAsJson(n5.getAttribute(group, "\\/", String.class )); // "fwdSlash" + + n5.setAttribute(group, "\\\\", "bckSlash"); + printAsJson(n5.getAttribute(group, "\\\\", String.class )); // "bckSlash" + + // print out the contents of attributes.json + System.out.println("\n" + getRawJson(n5, group)); // {"/":"fwdSlash","\\\\":"bckSlash"} + } + + public void removingAttributesAndNulls(final N5FSWriter n5, final String group) throws IOException { + + n5.setAttribute(group, "cow", "moo"); + n5.setAttribute(group, "dog", "woof"); + n5.setAttribute(group, "sheep", "baa"); + System.out.println(getRawJson(n5, group)); // {"sheep":"baa","cow":"moo","dog":"woof"} + + n5.removeAttribute(group, "cow"); // void method + System.out.println(getRawJson(n5, group)); // {"sheep":"baa","dog":"woof"} + + String theDogSays = n5.removeAttribute(group, "dog", String.class); // returns type + System.out.println(theDogSays); // woof + System.out.println(getRawJson(n5, group)); // {"sheep":"baa"} + + n5.removeAttribute(group, "sheep", int.class); // returns type + System.out.println(getRawJson(n5, group)); // {"sheep":"baa"} + + System.out.println( n5.removeAttribute(group, "sheep", String.class)); // "baa" + System.out.println(getRawJson(n5, group)); // {} + + n5.setAttribute(group, "attr", "value"); + System.out.println(getRawJson(n5, group)); // {"attr":"value"} + n5.setAttribute(group, "attr", null); + System.out.println(getRawJson(n5, group)); // if serializeNulls {"attr":null} + + n5.setAttribute(group, "foo", 12); + System.out.println(getRawJson(n5, group)); // {"foo":12} + n5.removeAttribute(group, "foo"); + System.out.println(getRawJson(n5, group)); // {} + } + + public String getRawJson(final N5Reader n5, final String group) throws IOException { + return new String( + Files.readAllBytes( + Paths.get(Paths.get(n5.getURI()).toAbsolutePath().toString(), group, "attributes.json")), + Charset.defaultCharset()); + } + + public void printAsJson(final Object obj) { + System.out.println(gson.toJson(obj)); + } + + class Pet { + String name; + int age; + + public Pet(String name, int age) { + this.name = name; + this.age = age; + } + + public String toString() { + return String.format("pet %s is %d", name, age); + } + } + +} From b717b8fc01bd0440eb6e0508f1a99cedf2b935bd Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Fri, 2 Jun 2023 10:15:06 -0400 Subject: [PATCH 200/243] test: fix url version assertion --- .../java/org/janelia/saalfeldlab/n5/url/UrlAttributeTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/url/UrlAttributeTest.java b/src/test/java/org/janelia/saalfeldlab/n5/url/UrlAttributeTest.java index 16d5e93b..8547b6a8 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/url/UrlAttributeTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/url/UrlAttributeTest.java @@ -68,7 +68,7 @@ public void testRootAttributes() throws URISyntaxException, IOException { // get Map< String, Object > everything = getAttribute( n5, new N5URL( "" ), Map.class ); - final String version = N5Reader.VERSION.toString(); + final String version = "2.6.1"; // the n5 version at the time the test data were created assertEquals( "empty url", version, everything.get( "n5" ) ); Map< String, Object > everything2 = getAttribute( n5, new N5URL( "#/" ), Map.class ); From 3f03eeb5c842f0c41f2b3018c82191e251a6c928 Mon Sep 17 00:00:00 2001 From: Stephan Saalfeld Date: Fri, 2 Jun 2023 17:32:14 -0400 Subject: [PATCH 201/243] BREAKING: remove unused initializeGroup --- .../saalfeldlab/n5/GsonKeyValueN5Reader.java | 19 +++++++----- .../saalfeldlab/n5/GsonKeyValueN5Writer.java | 14 --------- .../janelia/saalfeldlab/n5/N5Exception.java | 29 +++++++++++-------- 3 files changed, 29 insertions(+), 33 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java index d2d348a2..3b0d4b41 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java @@ -26,6 +26,9 @@ package org.janelia.saalfeldlab.n5; import java.io.IOException; +import java.util.Arrays; + +import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; import com.google.gson.Gson; import com.google.gson.JsonElement; @@ -52,7 +55,7 @@ default boolean exists(final String pathName) { } @Override - default boolean datasetExists(final String pathName) throws N5Exception.N5IOException { + default boolean datasetExists(final String pathName) throws N5Exception { // for n5, every dataset must be a group return getDatasetAttributes(pathName) != null; @@ -67,7 +70,7 @@ default boolean datasetExists(final String pathName) throws N5Exception.N5IOExce * @throws IOException */ @Override - default JsonElement getAttributes(final String pathName) throws N5Exception.N5IOException { + default JsonElement getAttributes(final String pathName) throws N5Exception { final String groupPath = N5URL.normalizeGroupPath(pathName); final String attributesPath = attributesPath(groupPath); @@ -78,7 +81,7 @@ default JsonElement getAttributes(final String pathName) throws N5Exception.N5IO try (final LockedChannel lockedChannel = getKeyValueAccess().lockForReading(attributesPath)) { return GsonUtils.readAttributes(lockedChannel.newReader(), getGson()); } catch (final IOException e) { - throw new N5Exception.N5IOException("Cannot open lock for Reading", e); + throw new N5IOException("Failed to read attributes from dataset " + pathName, e); } } @@ -87,7 +90,7 @@ default JsonElement getAttributes(final String pathName) throws N5Exception.N5IO default DataBlock readBlock( final String pathName, final DatasetAttributes datasetAttributes, - final long... gridPosition) throws N5Exception.N5IOException { + final long... gridPosition) throws N5Exception { final String path = getDataBlockPath(N5URL.normalizeGroupPath(pathName), gridPosition); if (!getKeyValueAccess().isFile(path)) @@ -96,17 +99,19 @@ default DataBlock readBlock( try (final LockedChannel lockedChannel = getKeyValueAccess().lockForReading(path)) { return DefaultBlockReader.readBlock(lockedChannel.newInputStream(), datasetAttributes, gridPosition); } catch (final IOException e) { - throw new N5Exception.N5IOException("Cannot open lock for Reading", e); + throw new N5IOException( + "Failed to read block " + Arrays.toString(gridPosition) + " from dataset " + path, + e); } } @Override - default String[] list(final String pathName) throws N5Exception.N5IOException { + default String[] list(final String pathName) throws N5Exception { try { return getKeyValueAccess().listDirectories(groupPath(pathName)); } catch (final IOException e) { - throw new N5Exception.N5IOException("Cannot list directories for group " + pathName, e); + throw new N5IOException("Cannot list directories for group " + pathName, e); } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java index e5c43139..895cdcd3 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java @@ -66,19 +66,6 @@ static String initializeContainer( return normBasePath; } - /** - * Performs any necessary initialization to ensure the key given by the - * argument {@code normalPath} is a valid group after creation. Called by - * {@link #createGroup(String)}. - * - * @param normalPath the group path. - */ - default void initializeGroup(final String normalPath) { - - // Nothing to do here, but other implementations (e.g. zarr) use this. - // TODO remove if not used by zarr - } - @Override default void createGroup(final String path) throws N5Exception { @@ -88,7 +75,6 @@ default void createGroup(final String path) throws N5Exception { } catch (final IOException e) { throw new N5Exception.N5IOException("Failed to create group " + path, e); } - initializeGroup(normalPath); } /** diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5Exception.java b/src/main/java/org/janelia/saalfeldlab/n5/N5Exception.java index 93e87fc3..a2dea42b 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5Exception.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5Exception.java @@ -2,56 +2,61 @@ import java.io.IOException; -public class N5Exception extends RuntimeException{ +public class N5Exception extends RuntimeException { public N5Exception() { super(); } - public N5Exception(String message) { + public N5Exception(final String message) { super(message); } - public N5Exception(String message, Throwable cause) { + public N5Exception(final String message, final Throwable cause) { super(message, cause); } - public N5Exception(Throwable cause) { + public N5Exception(final Throwable cause) { super(cause); } - protected N5Exception(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) { + protected N5Exception( + final String message, + final Throwable cause, + final boolean enableSuppression, + final boolean writableStackTrace) { super(message, cause, enableSuppression, writableStackTrace); } public static class N5IOException extends N5Exception { - public N5IOException(String message) { + public N5IOException(final String message) { super(message); } - public N5IOException(String message, IOException cause) { + public N5IOException(final String message, final IOException cause) { super(message, cause); } - public N5IOException(IOException cause) { + public N5IOException(final IOException cause) { super(cause); } - protected N5IOException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) { + protected N5IOException( + final String message, + final Throwable cause, + final boolean enableSuppression, + final boolean writableStackTrace) { super(message, cause, enableSuppression, writableStackTrace); } - - } - } From bb2037080d4dc3988374b2b4b753f43cbb85ca7c Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Wed, 7 Jun 2023 11:46:02 -0400 Subject: [PATCH 202/243] fix: implement equals for compression implementations * useful for tests --- .../org/janelia/saalfeldlab/n5/Bzip2Compression.java | 9 +++++++++ .../org/janelia/saalfeldlab/n5/GzipCompression.java | 10 ++++++++++ .../org/janelia/saalfeldlab/n5/Lz4Compression.java | 8 ++++++++ .../org/janelia/saalfeldlab/n5/RawCompression.java | 8 ++++++++ .../java/org/janelia/saalfeldlab/n5/XzCompression.java | 7 +++++++ 5 files changed, 42 insertions(+) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/Bzip2Compression.java b/src/main/java/org/janelia/saalfeldlab/n5/Bzip2Compression.java index 45236a18..0ac07d04 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/Bzip2Compression.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/Bzip2Compression.java @@ -74,4 +74,13 @@ public Bzip2Compression getWriter() { return this; } + + @Override + public boolean equals(Object other) { + if (other == null || other.getClass() != Bzip2Compression.class) + return false; + else + return blockSize == ((Bzip2Compression) other).blockSize; + } + } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GzipCompression.java b/src/main/java/org/janelia/saalfeldlab/n5/GzipCompression.java index e57b1af0..5646aadf 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GzipCompression.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GzipCompression.java @@ -105,4 +105,14 @@ private void readObject(final ObjectInputStream in) throws Exception { in.defaultReadObject(); ReflectionUtils.setFieldValue(this, "parameters", new GzipParameters()); } + + @Override + public boolean equals(Object other) { + if (other == null || other.getClass() != GzipCompression.class) + return false; + else { + GzipCompression gz = ((GzipCompression) other); + return useZlib == gz.useZlib && level == gz.level; + } + } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/Lz4Compression.java b/src/main/java/org/janelia/saalfeldlab/n5/Lz4Compression.java index b20b95a8..682f0bc2 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/Lz4Compression.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/Lz4Compression.java @@ -75,4 +75,12 @@ public Lz4Compression getWriter() { return this; } + + @Override + public boolean equals(Object other) { + if (other == null || other.getClass() != Lz4Compression.class) + return false; + else + return blockSize == ((Lz4Compression) other).blockSize; + } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/RawCompression.java b/src/main/java/org/janelia/saalfeldlab/n5/RawCompression.java index 1a3127d7..e7a2a8f4 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/RawCompression.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/RawCompression.java @@ -59,4 +59,12 @@ public RawCompression getWriter() { return this; } + + @Override + public boolean equals(Object other) { + if (other == null || other.getClass() != RawCompression.class) + return false; + else + return true; + } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/XzCompression.java b/src/main/java/org/janelia/saalfeldlab/n5/XzCompression.java index 6226abb5..0de4db57 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/XzCompression.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/XzCompression.java @@ -76,5 +76,12 @@ public XzCompression getWriter() { return this; } + @Override + public boolean equals(Object other) { + if (other == null || other.getClass() != XzCompression.class) + return false; + else + return preset == ((XzCompression) other).preset; + } } From 88db247d5c892ab14ba8a079ffb262d2cda4ad93 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Wed, 7 Jun 2023 14:35:42 -0400 Subject: [PATCH 203/243] fix/test: correct testWriteReadByteBlock * WriteReadBlocktests are quiet --- .../saalfeldlab/n5/AbstractN5Test.java | 19 ++++--------------- 1 file changed, 4 insertions(+), 15 deletions(-) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java index 47f036af..6b7f234f 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java @@ -206,27 +206,23 @@ public void testCreateDataset() throws IOException, URISyntaxException { } @Test - public void testWriteReadByteBlock() { + public void testWriteReadByteBlock() throws URISyntaxException { for (final Compression compression : getCompressions()) { for (final DataType dataType : new DataType[]{ - DataType.UINT8, - DataType.INT8}) { + DataType.UINT8, DataType.INT8}) { - System.out.println("Testing " + compression.getType() + " " + dataType); - try { + try (final N5Writer n5 = createN5Writer()) { n5.createDataset(datasetName, dimensions, blockSize, dataType, compression); final DatasetAttributes attributes = n5.getDatasetAttributes(datasetName); final ByteArrayDataBlock dataBlock = new ByteArrayDataBlock(blockSize, new long[]{0, 0, 0}, byteBlock); n5.writeBlock(datasetName, attributes, dataBlock); final DataBlock loadedDataBlock = n5.readBlock(datasetName, attributes, new long[]{0, 0, 0}); - assertArrayEquals(byteBlock, (byte[])loadedDataBlock.getData()); - assertTrue(n5.remove(datasetName)); - } catch (final N5Exception e) { + } catch (final IOException e) { e.printStackTrace(); fail("Block cannot be written."); } @@ -242,7 +238,6 @@ public void testWriteReadShortBlock() throws URISyntaxException { DataType.UINT16, DataType.INT16}) { - System.out.println("Testing " + compression.getType() + " " + dataType); try (final N5Writer n5 = createN5Writer()) { n5.createDataset(datasetName, dimensions, blockSize, dataType, compression); final DatasetAttributes attributes = n5.getDatasetAttributes(datasetName); @@ -271,7 +266,6 @@ public void testWriteReadIntBlock() throws URISyntaxException { DataType.UINT32, DataType.INT32}) { - System.out.println("Testing " + compression.getType() + " " + dataType); try (final N5Writer n5 = createN5Writer()) { n5.createDataset(datasetName, dimensions, blockSize, dataType, compression); final DatasetAttributes attributes = n5.getDatasetAttributes(datasetName); @@ -300,7 +294,6 @@ public void testWriteReadLongBlock() throws URISyntaxException { DataType.UINT64, DataType.INT64}) { - System.out.println("Testing " + compression.getType() + " " + dataType); try (final N5Writer n5 = createN5Writer()) { n5.createDataset(datasetName, dimensions, blockSize, dataType, compression); final DatasetAttributes attributes = n5.getDatasetAttributes(datasetName); @@ -325,7 +318,6 @@ public void testWriteReadLongBlock() throws URISyntaxException { public void testWriteReadFloatBlock() throws URISyntaxException { for (final Compression compression : getCompressions()) { - System.out.println("Testing " + compression.getType() + " float32"); try (final N5Writer n5 = createN5Writer()) { n5.createDataset(datasetName, dimensions, blockSize, DataType.FLOAT32, compression); final DatasetAttributes attributes = n5.getDatasetAttributes(datasetName); @@ -349,7 +341,6 @@ public void testWriteReadFloatBlock() throws URISyntaxException { public void testWriteReadDoubleBlock() throws URISyntaxException { for (final Compression compression : getCompressions()) { - System.out.println("Testing " + compression.getType() + " float64"); try (final N5Writer n5 = createN5Writer()) { n5.createDataset(datasetName, dimensions, blockSize, DataType.FLOAT64, compression); final DatasetAttributes attributes = n5.getDatasetAttributes(datasetName); @@ -379,7 +370,6 @@ public void testMode1WriteReadByteBlock() throws URISyntaxException { DataType.UINT8, DataType.INT8}) { - System.out.println("Testing " + compression.getType() + " " + dataType + " (mode=1)"); try (final N5Writer n5 = createN5Writer()) { n5.createDataset(datasetName, dimensions, differentBlockSize, dataType, compression); final DatasetAttributes attributes = n5.getDatasetAttributes(datasetName); @@ -406,7 +396,6 @@ public void testWriteReadSerializableBlock() throws ClassNotFoundException, URIS for (final Compression compression : getCompressions()) { final DataType dataType = DataType.OBJECT; - System.out.println("Testing " + compression.getType() + " " + dataType + " (mode=2)"); try (final N5Writer n5 = createN5Writer()) { n5.createDataset(datasetName, dimensions, blockSize, dataType, compression); final DatasetAttributes attributes = n5.getDatasetAttributes(datasetName); From e7b1e736aa5dfdbeeb7667c1fefaad0da81d0385 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Wed, 7 Jun 2023 14:38:19 -0400 Subject: [PATCH 204/243] BREAKING: cache remove forceAddNewCacheInfo * add initializeNonemptyCache * add two argument updateCacheInfo * implement createGroup using the above * causes test changes --- .../n5/CachedGsonKeyValueN5Writer.java | 16 +++- .../saalfeldlab/n5/cache/N5JsonCache.java | 85 ++++++++++++------- .../saalfeldlab/n5/N5CachedFSTest.java | 15 +++- 3 files changed, 80 insertions(+), 36 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Writer.java index 2aaf5884..1cb93074 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Writer.java @@ -53,6 +53,18 @@ default void setVersion(final String path) throws N5Exception { @Override default void createGroup(final String path) throws N5Exception { + final String normalPath = N5URL.normalizeGroupPath(path); + // TODO: John document this! + // if you are a group, avoid hitting the backend + // if something exists, be safe + if (cacheMeta()) { + if( getCache().isGroup(normalPath, N5KeyValueReader.ATTRIBUTES_JSON)) + return; + else if ( getCache().exists(normalPath, N5KeyValueReader.ATTRIBUTES_JSON)){ + throw new N5Exception("Can't make a group on existing path."); + } + } + // N5Writer.super.createGroup(path); /* * the 6 lines below duplicate the single line above but would have to @@ -60,7 +72,6 @@ default void createGroup(final String path) throws N5Exception { * normalizeGroupPath again the below duplicates code, but avoids extra * work */ - final String normalPath = N5URL.normalizeGroupPath(path); try { getKeyValueAccess().createDirectories(groupPath(normalPath)); } catch (final IOException e) { @@ -78,7 +89,8 @@ default void createGroup(final String path) throws N5Exception { for (final String child : pathParts) { final String childPath = getKeyValueAccess().compose(parent, child); - getCache().forceAddNewCacheInfo(childPath, N5KeyValueReader.ATTRIBUTES_JSON, null, true, false); + getCache().initializeNonemptyCache(childPath, N5KeyValueReader.ATTRIBUTES_JSON); + getCache().updateCacheInfo(childPath, N5KeyValueReader.ATTRIBUTES_JSON); // only add if the parent exists and has children cached already if (parent != null && !child.isEmpty()) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/cache/N5JsonCache.java b/src/main/java/org/janelia/saalfeldlab/n5/cache/N5JsonCache.java index 0abe9e2b..eae2684a 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/cache/N5JsonCache.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/cache/N5JsonCache.java @@ -4,12 +4,16 @@ import java.util.HashMap; import java.util.HashSet; +import org.janelia.saalfeldlab.n5.N5Exception; + import com.google.gson.JsonElement; public class N5JsonCache { public static final N5CacheInfo emptyCacheInfo = new N5CacheInfo(); + public static final EmptyJson emptyJson = new EmptyJson(); + protected final N5JsonCacheableContainer container; /** @@ -38,6 +42,15 @@ protected boolean containsKey(final String normalCacheKey) { } } + protected static class EmptyJson extends JsonElement { + + @Override + public JsonElement deepCopy() { + throw new N5Exception("Do not copy EmptyJson, you naughty person"); + } + + } + private final HashMap containerPathToCache = new HashMap<>(); public N5JsonCache( final N5JsonCacheableContainer container ) { @@ -50,7 +63,7 @@ public JsonElement getAttributes(final String normalPathKey, final String normal addNewCacheInfo(normalPathKey, normalCacheKey, null); cacheInfo = getCacheInfo(normalPathKey); } - if (cacheInfo == emptyCacheInfo) { + if (cacheInfo == emptyCacheInfo || cacheInfo.getCache(normalCacheKey) == emptyJson) { return null; } synchronized (cacheInfo) { @@ -146,24 +159,6 @@ public N5CacheInfo addNewCacheInfo(final String normalPathKey, final String norm return cacheInfo; } - public N5CacheInfo forceAddNewCacheInfo(final String normalPathKey, final String normalCacheKey, final JsonElement uncachedAttributes, final boolean isGroup, final boolean isDataset) { - - // getting the current cache info is useful if it already has had its children listed - N5CacheInfo cacheInfo = getCacheInfo(normalPathKey); - if( cacheInfo == null || cacheInfo == emptyCacheInfo ) - cacheInfo = newCacheInfo(); - - if (normalCacheKey != null) { - synchronized (cacheInfo.attributesCache) { - cacheInfo.attributesCache.put(normalCacheKey, uncachedAttributes); - } - } - updateCacheIsGroup(cacheInfo, isGroup); - updateCacheIsDataset(cacheInfo, isDataset); - updateCache(normalPathKey, cacheInfo); - return cacheInfo; - } - private N5CacheInfo addNewCacheInfo(final String normalPathKey) { return addNewCacheInfo(normalPathKey, null, null); @@ -178,6 +173,32 @@ private void addChild(final N5CacheInfo cacheInfo, final String normalPathKey) { Collections.addAll(cacheInfo.children, children); } + protected N5CacheInfo getOrMakeCacheInfo(final String normalPathKey) { + + N5CacheInfo cacheInfo = getCacheInfo(normalPathKey); + if (cacheInfo == null) { + return addNewCacheInfo(normalPathKey, null, null); + } + + if (cacheInfo == emptyCacheInfo) + cacheInfo = newCacheInfo(); + + return cacheInfo; + } + + /** + * Updates the cache attributes for the given normalPathKey + * adding the appropriate node to the cache if necessary. + * + * @param normalPathKey the normalized path key + */ + public void updateCacheInfo(final String normalPathKey, final String normalCacheKey) { + + final N5CacheInfo cacheInfo = getOrMakeCacheInfo(normalPathKey); + final JsonElement attrs = cacheInfo.attributesCache.get(normalCacheKey); + updateCacheInfo(normalPathKey, normalCacheKey, attrs); + } + /** * Updates the cache attributes for the given normalPathKey and normalCacheKey, * adding the appropriate node to the cache if necessary. @@ -188,16 +209,7 @@ private void addChild(final N5CacheInfo cacheInfo, final String normalPathKey) { */ public void updateCacheInfo(final String normalPathKey, final String normalCacheKey, final JsonElement uncachedAttributes) { - N5CacheInfo cacheInfo = getCacheInfo(normalPathKey); - if (cacheInfo == null ){ - addNewCacheInfo(normalPathKey, normalCacheKey, uncachedAttributes ); - return; - } - - // TODO go through this again with Caleb - if (cacheInfo == emptyCacheInfo) - cacheInfo = newCacheInfo(); - + final N5CacheInfo cacheInfo = getOrMakeCacheInfo(normalPathKey); if (normalCacheKey != null) { final JsonElement attributesToCache = uncachedAttributes == null ? container.getAttributesFromContainer(normalPathKey, normalCacheKey) @@ -206,14 +218,25 @@ public void updateCacheInfo(final String normalPathKey, final String normalCache updateCacheAttributes(cacheInfo, normalCacheKey, attributesToCache); updateCacheIsGroup(cacheInfo, container.isGroupFromAttributes(normalCacheKey, attributesToCache)); updateCacheIsDataset(cacheInfo, container.isDatasetFromAttributes(normalCacheKey, attributesToCache)); - } - else { + } else { updateCacheIsGroup(cacheInfo, container.isGroupFromContainer(normalPathKey)); updateCacheIsDataset(cacheInfo, container.isDatasetFromContainer(normalPathKey)); } updateCache(normalPathKey, cacheInfo); } + public void initializeNonemptyCache(final String normalPathKey, final String normalCacheKey) { + + final N5CacheInfo cacheInfo = getCacheInfo(normalPathKey); + if (cacheInfo == null || cacheInfo == emptyCacheInfo ) { + final N5CacheInfo info = newCacheInfo(); + if( normalCacheKey != null ) + info.attributesCache.put(normalCacheKey, emptyJson); + + updateCache(normalPathKey, info); + } + } + public void setAttributes(final String normalPathKey, final String normalCacheKey, final JsonElement attributes ) { N5CacheInfo cacheInfo = getCacheInfo(normalPathKey); boolean update = false; diff --git a/src/test/java/org/janelia/saalfeldlab/n5/N5CachedFSTest.java b/src/test/java/org/janelia/saalfeldlab/n5/N5CachedFSTest.java index 82c667b2..dcf4b795 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/N5CachedFSTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/N5CachedFSTest.java @@ -158,6 +158,7 @@ public static void cacheBehaviorHelper(final TrackingStorage n5) throws IOExcept assertEquals(expectedAttributeCount, n5.getAttrCallCount()); n5.createGroup(groupA); + assertEquals(expectedAttributeCount, n5.getAttrCallCount()); // group B exists = n5.exists(groupB); @@ -186,7 +187,7 @@ public static void cacheBehaviorHelper(final TrackingStorage n5) throws IOExcept // should not check existence when creating a group n5.createGroup(cachedGroup); n5.createGroup(cachedGroup); // be annoying - assertEquals(expectedExistCount, n5.getExistCallCount()); + assertEquals(++expectedExistCount, n5.getExistCallCount()); assertEquals(expectedGroupCount, n5.getGroupCallCount()); assertEquals(expectedDatasetCount, n5.getDatasetCallCount()); assertEquals(expectedAttributeCount, n5.getAttrCallCount()); @@ -203,7 +204,9 @@ public static void cacheBehaviorHelper(final TrackingStorage n5) throws IOExcept assertEquals(expectedListCount, n5.getListCallCount()); // should not read attributes from container when setting them + System.out.println(n5.getAttrCallCount()); n5.setAttribute(cachedGroup, "one", 1); + System.out.println(n5.getAttrCallCount()); assertEquals(expectedExistCount, n5.getExistCallCount()); assertEquals(expectedGroupCount, n5.getGroupCallCount()); assertEquals(expectedDatasetCount, n5.getDatasetCallCount()); @@ -286,13 +289,14 @@ public static void cacheBehaviorHelper(final TrackingStorage n5) throws IOExcept assertTrue(n5.exists(abc)); assertTrue(n5.groupExists(abc)); assertFalse(n5.datasetExists(abc)); - assertEquals(expectedExistCount, n5.getExistCallCount()); + assertEquals(++expectedExistCount, n5.getExistCallCount()); assertEquals(expectedGroupCount, n5.getGroupCallCount()); assertEquals(expectedDatasetCount, n5.getDatasetCallCount()); assertEquals(expectedAttributeCount, n5.getAttrCallCount()); assertEquals(expectedListCount, n5.getListCallCount()); // ensure that backend need not be checked when testing existence of "a/b" + // TODO how does this work assertTrue(n5.exists(ab)); assertTrue(n5.groupExists(ab)); assertFalse(n5.datasetExists(ab)); @@ -333,11 +337,15 @@ public static void cacheBehaviorHelper(final TrackingStorage n5) throws IOExcept assertEquals(expectedListCount, n5.getListCallCount()); n5.createGroup("a"); + assertEquals(expectedExistCount, n5.getExistCallCount()); n5.createGroup("a/a"); + assertEquals(++expectedExistCount, n5.getExistCallCount()); n5.createGroup("a/b"); + assertEquals(expectedExistCount, n5.getExistCallCount()); n5.createGroup("a/c"); + assertEquals(++expectedExistCount, n5.getExistCallCount()); + assertArrayEquals(new String[] {"a", "b", "c"}, n5.list("a")); // call list - assertEquals(expectedExistCount, n5.getExistCallCount()); assertEquals(expectedGroupCount, n5.getGroupCallCount()); assertEquals(expectedDatasetCount, n5.getDatasetCallCount()); assertEquals(expectedAttributeCount, n5.getAttrCallCount()); @@ -352,6 +360,7 @@ public static void cacheBehaviorHelper(final TrackingStorage n5) throws IOExcept assertEquals(expectedAttributeCount, n5.getAttrCallCount()); assertEquals(expectedListCount, n5.getListCallCount()); // list NOT incremented + // TODO repeat the above exercise when creating dataset } public static interface TrackingStorage extends CachedGsonKeyValueN5Writer { From 97cc420126e56565f129465a7f94b18c7b2c29ec Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Fri, 9 Jun 2023 10:49:49 -0400 Subject: [PATCH 205/243] feat: remove unused method it is not claer what to do to normalize container paths, when multiple are supported, and different filesystems handle them differently. For now, just don't expose this explicitly --- src/main/java/org/janelia/saalfeldlab/n5/N5URL.java | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java b/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java index 23c98871..37aa512d 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java @@ -62,14 +62,6 @@ public String getGroupPath() { return group != null ? group : "/"; } - /** - * @return the normalized container path - */ - public String normalizeContainerPath() { - - return normalizePath(getContainerPath()); - } - /** * @return the normalized group path */ From 1a0a5f592c4951a32702e264ba246438c1cb0edb Mon Sep 17 00:00:00 2001 From: Stephan Saalfeld Date: Fri, 9 Jun 2023 13:21:01 -0400 Subject: [PATCH 206/243] fix, style: N5Exceptions in N5KeyValue/FS/Reader/Writer --- .../saalfeldlab/n5/AbstractDataBlock.java | 3 +- .../janelia/saalfeldlab/n5/BlockReader.java | 15 +- .../janelia/saalfeldlab/n5/BlockWriter.java | 12 +- .../saalfeldlab/n5/ByteArrayDataBlock.java | 4 +- .../saalfeldlab/n5/Bzip2Compression.java | 5 +- .../n5/CachedGsonKeyValueN5Writer.java | 4 +- .../saalfeldlab/n5/CompressionAdapter.java | 17 +- .../org/janelia/saalfeldlab/n5/DataBlock.java | 19 +- .../org/janelia/saalfeldlab/n5/DataType.java | 93 +++++-- .../saalfeldlab/n5/DatasetAttributes.java | 3 +- .../saalfeldlab/n5/DefaultBlockReader.java | 12 +- .../saalfeldlab/n5/DefaultBlockWriter.java | 14 +- .../saalfeldlab/n5/DoubleArrayDataBlock.java | 4 +- .../n5/FileSystemKeyValueAccess.java | 215 +++++++++------- .../saalfeldlab/n5/FloatArrayDataBlock.java | 5 +- .../saalfeldlab/n5/GsonKeyValueN5Writer.java | 4 +- .../janelia/saalfeldlab/n5/GsonN5Writer.java | 2 +- .../org/janelia/saalfeldlab/n5/GsonUtils.java | 183 ++++++++----- .../saalfeldlab/n5/GzipCompression.java | 5 +- .../saalfeldlab/n5/IntArrayDataBlock.java | 4 +- .../saalfeldlab/n5/KeyValueAccess.java | 111 ++++---- .../janelia/saalfeldlab/n5/LockedChannel.java | 8 +- .../saalfeldlab/n5/LongArrayDataBlock.java | 4 +- .../saalfeldlab/n5/Lz4Compression.java | 5 +- .../janelia/saalfeldlab/n5/N5FSReader.java | 90 ++++--- .../janelia/saalfeldlab/n5/N5FSWriter.java | 86 ++++--- .../saalfeldlab/n5/N5KeyValueReader.java | 57 +++-- .../saalfeldlab/n5/N5KeyValueWriter.java | 29 +-- .../org/janelia/saalfeldlab/n5/N5Reader.java | 2 +- .../org/janelia/saalfeldlab/n5/N5URL.java | 241 +++++++++++------- .../org/janelia/saalfeldlab/n5/N5Writer.java | 151 ++++------- .../saalfeldlab/n5/RawCompression.java | 3 +- .../saalfeldlab/n5/ShortArrayDataBlock.java | 4 +- .../janelia/saalfeldlab/n5/XzCompression.java | 6 +- .../saalfeldlab/n5/cache/N5JsonCache.java | 102 +++++--- .../n5/cache/N5JsonCacheableContainer.java | 47 ++-- 36 files changed, 921 insertions(+), 648 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/AbstractDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/AbstractDataBlock.java index ca85eb7b..f1cbc352 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/AbstractDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/AbstractDataBlock.java @@ -28,7 +28,8 @@ /** * Abstract base class for {@link DataBlock} implementations. * - * @param the block data type + * @param + * the block data type * * @author Stephan Saalfeld */ diff --git a/src/main/java/org/janelia/saalfeldlab/n5/BlockReader.java b/src/main/java/org/janelia/saalfeldlab/n5/BlockReader.java index 8f63ddd2..b788a3f4 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/BlockReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/BlockReader.java @@ -39,11 +39,16 @@ public interface BlockReader { /** * Reads a {@link DataBlock} from an {@link InputStream}. * - * @param dataBlock the data block - * @param in the input stream - * @param the block data type - * @param the block type - * @throws IOException the exception + * @param dataBlock + * the data block + * @param in + * the input stream + * @param + * the block data type + * @param + * the block type + * @throws IOException + * the exception */ public > void read(final B dataBlock, final InputStream in) throws IOException; } \ No newline at end of file diff --git a/src/main/java/org/janelia/saalfeldlab/n5/BlockWriter.java b/src/main/java/org/janelia/saalfeldlab/n5/BlockWriter.java index 8bb384d3..5120e745 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/BlockWriter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/BlockWriter.java @@ -39,10 +39,14 @@ public interface BlockWriter { /** * Writes a {@link DataBlock} into an {@link OutputStream}. * - * @param dataBlock the data block - * @param out the output stream - * @param the block data type - * @throws IOException the exception + * @param dataBlock + * the data block + * @param out + * the output stream + * @param + * the block data type + * @throws IOException + * the exception */ public void write(final DataBlock dataBlock, final OutputStream out) throws IOException; } \ No newline at end of file diff --git a/src/main/java/org/janelia/saalfeldlab/n5/ByteArrayDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/ByteArrayDataBlock.java index ac427da8..5717ad2e 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/ByteArrayDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/ByteArrayDataBlock.java @@ -46,10 +46,10 @@ public void readData(final ByteBuffer buffer) { if (buffer.array() != getData()) buffer.get(getData()); } - + @Override public int getNumElements() { - + return data.length; } } \ No newline at end of file diff --git a/src/main/java/org/janelia/saalfeldlab/n5/Bzip2Compression.java b/src/main/java/org/janelia/saalfeldlab/n5/Bzip2Compression.java index 0ac07d04..5d3d6161 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/Bzip2Compression.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/Bzip2Compression.java @@ -76,11 +76,12 @@ public Bzip2Compression getWriter() { } @Override - public boolean equals(Object other) { + public boolean equals(final Object other) { + if (other == null || other.getClass() != Bzip2Compression.class) return false; else - return blockSize == ((Bzip2Compression) other).blockSize; + return blockSize == ((Bzip2Compression)other).blockSize; } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Writer.java index 1cb93074..ae7ee209 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Writer.java @@ -58,9 +58,9 @@ default void createGroup(final String path) throws N5Exception { // if you are a group, avoid hitting the backend // if something exists, be safe if (cacheMeta()) { - if( getCache().isGroup(normalPath, N5KeyValueReader.ATTRIBUTES_JSON)) + if (getCache().isGroup(normalPath, N5KeyValueReader.ATTRIBUTES_JSON)) return; - else if ( getCache().exists(normalPath, N5KeyValueReader.ATTRIBUTES_JSON)){ + else if (getCache().exists(normalPath, N5KeyValueReader.ATTRIBUTES_JSON)) { throw new N5Exception("Can't make a group on existing path."); } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/CompressionAdapter.java b/src/main/java/org/janelia/saalfeldlab/n5/CompressionAdapter.java index 9845855e..7ab8aa87 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/CompressionAdapter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/CompressionAdapter.java @@ -96,7 +96,8 @@ public static synchronized void update(final boolean override) { newInstance.compressionConstructors.put(type, constructor); newInstance.compressionParameters.put(type, parameters); - } catch (final ClassNotFoundException | NoSuchMethodException | ClassCastException | UnsatisfiedLinkError e) { + } catch (final ClassNotFoundException | NoSuchMethodException | ClassCastException + | UnsatisfiedLinkError e) { System.err.println("Compression '" + item.className() + "' could not be registered because:"); e.printStackTrace(System.err); } @@ -112,7 +113,10 @@ public static void update() { } @Override - public JsonElement serialize(final Compression compression, final Type typeOfSrc, final JsonSerializationContext context) { + public JsonElement serialize( + final Compression compression, + final Type typeOfSrc, + final JsonSerializationContext context) { final String type = compression.getType(); final Class clazz = compression.getClass(); @@ -140,8 +144,10 @@ public JsonElement serialize(final Compression compression, final Type typeOfSrc } @Override - public Compression deserialize(final JsonElement json, final Type typeOfT, final JsonDeserializationContext context) - throws JsonParseException { + public Compression deserialize( + final JsonElement json, + final Type typeOfT, + final JsonDeserializationContext context) throws JsonParseException { final JsonObject jsonObject = json.getAsJsonObject(); final JsonElement jsonType = jsonObject.get("type"); @@ -161,7 +167,8 @@ public Compression deserialize(final JsonElement json, final Type typeOfT, final ReflectionUtils.setFieldValue(compression, name, parameter); } } - } catch (InstantiationException | IllegalAccessException | IllegalArgumentException | InvocationTargetException | SecurityException | NoSuchFieldException e) { + } catch (InstantiationException | IllegalAccessException | IllegalArgumentException | InvocationTargetException + | SecurityException | NoSuchFieldException e) { e.printStackTrace(System.err); return null; } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/DataBlock.java index dc6366f4..c79dcf26 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DataBlock.java @@ -28,7 +28,7 @@ import java.nio.ByteBuffer; /** - * Interface for data blocks. A data block has data, a position on the block + * Interface for data blocks. A data block has data, a position on the block * grid, a size, and can read itself from and write itself into a * {@link ByteBuffer}. * @@ -42,8 +42,8 @@ public interface DataBlock { * Returns the size of this data block. * * The size of a data block is expected to be smaller than or equal to the - * spacing of the block grid. The dimensionality of size is expected to be - * equal to the dimensionality of the dataset. Consistency is not enforced. + * spacing of the block grid. The dimensionality of size is expected to be + * equal to the dimensionality of the dataset. Consistency is not enforced. * * @return size of the data block */ @@ -53,7 +53,7 @@ public interface DataBlock { * Returns the position of this data block on the block grid. * * The dimensionality of the grid position is expected to be equal to the - * dimensionality of the dataset. Consistency is not enforced. + * dimensionality of the dataset. Consistency is not enforced. * * @return position on the block grid */ @@ -71,7 +71,7 @@ public interface DataBlock { * block. * * The {@link ByteBuffer} may or may not map directly to the data - * object of this data block. I.e. modifying the {@link ByteBuffer} after + * object of this data block. I.e. modifying the {@link ByteBuffer} after * calling this method may or may not change the data of this data block. * modifying the data object of this data block after calling this method * may or may not change the content of the {@link ByteBuffer}. @@ -84,12 +84,13 @@ public interface DataBlock { * Reads the data object of this data block from a {@link ByteBuffer}. * * The {@link ByteBuffer} may or may not map directly to the data - * object of this data block. I.e. modifying the {@link ByteBuffer} after + * object of this data block. I.e. modifying the {@link ByteBuffer} after * calling this method may or may not change the data of this data block. * modifying the data object of this data block after calling this method * may or may not change the content of the {@link ByteBuffer}. * - * @param buffer the byte buffer + * @param buffer + * the byte buffer */ public void readData(final ByteBuffer buffer); @@ -105,10 +106,12 @@ public interface DataBlock { /** * Returns the number of elements in a box of given size. * - * @param size the size + * @param size + * the size * @return the number of elements */ public static int getNumElements(final int[] size) { + int n = size[0]; for (int i = 1; i < size.length; ++i) n *= size[i]; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DataType.java b/src/main/java/org/janelia/saalfeldlab/n5/DataType.java index 76c906fd..7b4bac50 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DataType.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DataType.java @@ -42,17 +42,72 @@ */ public enum DataType { - UINT8("uint8", (blockSize, gridPosition, numElements) -> new ByteArrayDataBlock(blockSize, gridPosition, new byte[numElements])), - UINT16("uint16", (blockSize, gridPosition, numElements) -> new ShortArrayDataBlock(blockSize, gridPosition, new short[numElements])), - UINT32("uint32", (blockSize, gridPosition, numElements) -> new IntArrayDataBlock(blockSize, gridPosition, new int[numElements])), - UINT64("uint64", (blockSize, gridPosition, numElements) -> new LongArrayDataBlock(blockSize, gridPosition, new long[numElements])), - INT8("int8", (blockSize, gridPosition, numElements) -> new ByteArrayDataBlock(blockSize, gridPosition, new byte[numElements])), - INT16("int16", (blockSize, gridPosition, numElements) -> new ShortArrayDataBlock(blockSize, gridPosition, new short[numElements])), - INT32("int32", (blockSize, gridPosition, numElements) -> new IntArrayDataBlock(blockSize, gridPosition, new int[numElements])), - INT64("int64", (blockSize, gridPosition, numElements) -> new LongArrayDataBlock(blockSize, gridPosition, new long[numElements])), - FLOAT32("float32", (blockSize, gridPosition, numElements) -> new FloatArrayDataBlock(blockSize, gridPosition, new float[numElements])), - FLOAT64("float64", (blockSize, gridPosition, numElements) -> new DoubleArrayDataBlock(blockSize, gridPosition, new double[numElements])), - OBJECT("object", (blockSize, gridPosition, numElements) -> new ByteArrayDataBlock(blockSize, gridPosition, new byte[numElements])); + UINT8( + "uint8", + (blockSize, gridPosition, numElements) -> new ByteArrayDataBlock( + blockSize, + gridPosition, + new byte[numElements])), + UINT16( + "uint16", + (blockSize, gridPosition, numElements) -> new ShortArrayDataBlock( + blockSize, + gridPosition, + new short[numElements])), + UINT32( + "uint32", + (blockSize, gridPosition, numElements) -> new IntArrayDataBlock( + blockSize, + gridPosition, + new int[numElements])), + UINT64( + "uint64", + (blockSize, gridPosition, numElements) -> new LongArrayDataBlock( + blockSize, + gridPosition, + new long[numElements])), + INT8( + "int8", + (blockSize, gridPosition, numElements) -> new ByteArrayDataBlock( + blockSize, + gridPosition, + new byte[numElements])), + INT16( + "int16", + (blockSize, gridPosition, numElements) -> new ShortArrayDataBlock( + blockSize, + gridPosition, + new short[numElements])), + INT32( + "int32", + (blockSize, gridPosition, numElements) -> new IntArrayDataBlock( + blockSize, + gridPosition, + new int[numElements])), + INT64( + "int64", + (blockSize, gridPosition, numElements) -> new LongArrayDataBlock( + blockSize, + gridPosition, + new long[numElements])), + FLOAT32( + "float32", + (blockSize, gridPosition, numElements) -> new FloatArrayDataBlock( + blockSize, + gridPosition, + new float[numElements])), + FLOAT64( + "float64", + (blockSize, gridPosition, numElements) -> new DoubleArrayDataBlock( + blockSize, + gridPosition, + new double[numElements])), + OBJECT( + "object", + (blockSize, gridPosition, numElements) -> new ByteArrayDataBlock( + blockSize, + gridPosition, + new byte[numElements])); private final String label; @@ -81,9 +136,13 @@ public static DataType fromString(final String string) { /** * Factory for {@link DataBlock DataBlocks}. * - * @param blockSize the block size - * @param gridPosition the grid position - * @param numElements the number of elements (not necessarily one element per block element) + * @param blockSize + * the block size + * @param gridPosition + * the grid position + * @param numElements + * the number of elements (not necessarily one element per block + * element) * @return the data block */ public DataBlock createDataBlock(final int[] blockSize, final long[] gridPosition, final int numElements) { @@ -95,8 +154,10 @@ public DataBlock createDataBlock(final int[] blockSize, final long[] gridPosi * Factory for {@link DataBlock DataBlocks} with one data element for each * block element (e.g. pixel image). * - * @param blockSize the block size - * @param gridPosition the grid position + * @param blockSize + * the block size + * @param gridPosition + * the grid position * @return the data block */ public DataBlock createDataBlock(final int[] blockSize, final long[] gridPosition) { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java b/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java index f56db529..f4aea9fe 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DatasetAttributes.java @@ -111,8 +111,7 @@ static DatasetAttributes from( final DataType dataType, int[] blockSize, Compression compression, - final String compressionVersion0Name - ) { + final String compressionVersion0Name) { if (blockSize == null) blockSize = Arrays.stream(dimensions).mapToInt(a -> (int)a).toArray(); diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockReader.java b/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockReader.java index d103ddf2..58c59780 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockReader.java @@ -56,11 +56,15 @@ public default > void read( /** * Reads a {@link DataBlock} from an {@link InputStream}. * - * @param in the input stream - * @param datasetAttributes the dataset attributes - * @param gridPosition the grid position + * @param in + * the input stream + * @param datasetAttributes + * the dataset attributes + * @param gridPosition + * the grid position * @return the block - * @throws IOException the exception + * @throws IOException + * the exception */ public static DataBlock readBlock( final InputStream in, diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockWriter.java b/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockWriter.java index aa9c5fc4..1f1aad39 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockWriter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockWriter.java @@ -55,11 +55,15 @@ public default void write( /** * Writes a {@link DataBlock} into an {@link OutputStream}. * - * @param out the output stream - * @param datasetAttributes the dataset attributes - * @param dataBlock the data block - * @param the block data type - * @throws IOException the exception + * + * @param out + * the output stream + * @param datasetAttributes + * the dataset attributes + * @param dataBlock + * the data block the block data type + * @throws IOException + * the exception */ public static void writeBlock( final OutputStream out, diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DoubleArrayDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/DoubleArrayDataBlock.java index a3fec497..426c7944 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DoubleArrayDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DoubleArrayDataBlock.java @@ -47,10 +47,10 @@ public void readData(final ByteBuffer buffer) { buffer.asDoubleBuffer().get(data); } - + @Override public int getNumElements() { - + return data.length; } } \ No newline at end of file diff --git a/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java index 3aa951c4..cd165823 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java @@ -64,7 +64,7 @@ public class FileSystemKeyValueAccess implements KeyValueAccess { /** * A {@link FileChannel} wrapper that attempts to acquire a lock and waits * for existing locks to be lifted before returning if the - * {@link FileSystem} supports that. If the {@link FileSystem} does not + * {@link FileSystem} supports that. If the {@link FileSystem} does not * support locking, it returns immediately. */ protected class LockedFileChannel implements LockedChannel { @@ -83,7 +83,8 @@ protected LockedFileChannel(final Path path, final boolean readOnly) throws IOEx options = new OpenOption[]{StandardOpenOption.READ}; channel = FileChannel.open(path, options); } else { - options = new OpenOption[]{StandardOpenOption.READ, StandardOpenOption.WRITE, StandardOpenOption.CREATE}; + options = new OpenOption[]{StandardOpenOption.READ, StandardOpenOption.WRITE, + StandardOpenOption.CREATE}; FileChannel tryChannel = null; try { tryChannel = FileChannel.open(path, options); @@ -232,8 +233,7 @@ public String[] components(final String path) { if (root == null) { components = new String[fsPath.getNameCount()]; o = 0; - } - else { + } else { components = new String[fsPath.getNameCount() + 1]; components[0] = root.toString(); o = 1; @@ -248,8 +248,10 @@ public String[] components(final String path) { public String parent(final String path) { final Path parent = fileSystem.getPath(path).getParent(); - if (parent == null) return null; - else return parent.toString(); + if (parent == null) + return null; + else + return parent.toString(); } @Override @@ -260,7 +262,8 @@ public String relativize(final String path, final String base) { } /** - * Returns a normalized path. It ensures correctness on both Unix and Windows, + * Returns a normalized path. It ensures correctness on both Unix and + * Windows, * otherwise {@code pathName} is treated as UNC path on Windows, and * {@code Paths.get(pathName, ...)} fails with {@code InvalidPathException}. * @@ -275,9 +278,10 @@ public String normalize(final String path) { @Override public URI uri(final String normalPath) throws URISyntaxException { + // normalize make absolute the scheme specific part only - final URI uri = new URI( normalPath ); - return new URI( "file", normalize(new File( uri.getSchemeSpecificPart()).getAbsolutePath()), uri.getFragment() ); + final URI uri = new URI(normalPath); + return new URI("file", normalize(new File(uri.getSchemeSpecificPart()).getAbsolutePath()), uri.getFragment()); } @Override @@ -325,7 +329,8 @@ protected static void tryDelete(final Path path) throws IOException { try { Files.delete(path); } catch (final DirectoryNotEmptyException e) { - /* Even though path is expected to be an empty directory, sometimes + /* + * Even though path is expected to be an empty directory, sometimes * deletion fails on network filesystems when lock files are not * cleared immediately after the leaves have been removed. */ @@ -346,97 +351,109 @@ protected static void tryDelete(final Path path) throws IOException { * * Workaround for https://bugs.openjdk.java.net/browse/JDK-8130464 * - * Creates a directory by creating all nonexistent parent directories first. - * Unlike the {@link Files#createDirectories} method, an exception - * is not thrown if the directory could not be created because it already - * exists. - * - *

    The {@code attrs} parameter is optional {@link FileAttribute - * file-attributes} to set atomically when creating the nonexistent - * directories. Each file attribute is identified by its {@link - * FileAttribute#name name}. If more than one attribute of the same name is - * included in the array then all but the last occurrence is ignored. - * - *

    If this method fails, then it may do so after creating some, but not - * all, of the parent directories. - * - * @param dir - * the directory to create - * - * @param attrs - * an optional list of file attributes to set atomically when - * creating the directory - * - * @return the directory - * - * @throws UnsupportedOperationException - * if the array contains an attribute that cannot be set atomically - * when creating the directory - * @throws FileAlreadyExistsException - * if {@code dir} exists but is not a directory (optional specific - * exception) - * @throws IOException - * if an I/O error occurs - * @throws SecurityException - * in the case of the default provider, and a security manager is - * installed, the {@link SecurityManager#checkWrite(String) checkWrite} - * method is invoked prior to attempting to create a directory and - * its {@link SecurityManager#checkRead(String) checkRead} is - * invoked for each parent directory that is checked. If {@code - * dir} is not an absolute path then its {@link Path#toAbsolutePath - * toAbsolutePath} may need to be invoked to get its absolute path. - * This may invoke the security manager's {@link - * SecurityManager#checkPropertyAccess(String) checkPropertyAccess} - * method to check access to the system property {@code user.dir} - */ + * Creates a directory by creating all nonexistent parent directories first. + * Unlike the {@link Files#createDirectories} method, an exception + * is not thrown if the directory could not be created because it already + * exists. + * + *

    + * The {@code attrs} parameter is optional {@link FileAttribute + * file-attributes} to set atomically when creating the nonexistent + * directories. Each file attribute is identified by its {@link + * FileAttribute#name name}. If more than one attribute of the same name is + * included in the array then all but the last occurrence is ignored. + * + *

    + * If this method fails, then it may do so after creating some, but not + * all, of the parent directories. + * + * @param dir + * the directory to create + * + * @param attrs + * an optional list of file attributes to set atomically when + * creating the directory + * + * @return the directory + * + * @throws UnsupportedOperationException + * if the array contains an attribute that cannot be set + * atomically + * when creating the directory + * @throws FileAlreadyExistsException + * if {@code dir} exists but is not a directory (optional + * specific + * exception) + * @throws IOException + * if an I/O error occurs + * @throws SecurityException + * in the case of the default provider, and a security manager + * is + * installed, the {@link SecurityManager#checkWrite(String) + * checkWrite} + * method is invoked prior to attempting to create a directory + * and + * its {@link SecurityManager#checkRead(String) checkRead} is + * invoked for each parent directory that is checked. If {@code + * dir} is not an absolute path then its {@link Path#toAbsolutePath + * toAbsolutePath} may need to be invoked to get its absolute + * path. + * This may invoke the security manager's {@link + * SecurityManager#checkPropertyAccess(String) + * checkPropertyAccess} + * method to check access to the system property + * {@code user.dir} + */ protected static Path createDirectories(Path dir, final FileAttribute... attrs) throws IOException { - // attempt to create the directory - try { - createAndCheckIsDirectory(dir, attrs); - return dir; - } catch (final FileAlreadyExistsException x) { - // file exists and is not a directory - throw x; - } catch (final IOException x) { - // parent may not exist or other reason - } - SecurityException se = null; - try { - dir = dir.toAbsolutePath(); - } catch (final SecurityException x) { - // don't have permission to get absolute path - se = x; - } - // find a decendent that exists - Path parent = dir.getParent(); - while (parent != null) { - try { - parent.getFileSystem().provider().checkAccess(parent); - break; - } catch (final NoSuchFileException x) { - // does not exist - } - parent = parent.getParent(); - } - if (parent == null) { - // unable to find existing parent - if (se == null) { - throw new FileSystemException(dir.toString(), null, - "Unable to determine if root directory exists"); - } else { - throw se; - } - } - - // create directories - Path child = parent; - for (final Path name: parent.relativize(dir)) { - child = child.resolve(name); - createAndCheckIsDirectory(child, attrs); - } - return dir; - } + // attempt to create the directory + try { + createAndCheckIsDirectory(dir, attrs); + return dir; + } catch (final FileAlreadyExistsException x) { + // file exists and is not a directory + throw x; + } catch (final IOException x) { + // parent may not exist or other reason + } + SecurityException se = null; + try { + dir = dir.toAbsolutePath(); + } catch (final SecurityException x) { + // don't have permission to get absolute path + se = x; + } + // find a decendent that exists + Path parent = dir.getParent(); + while (parent != null) { + try { + parent.getFileSystem().provider().checkAccess(parent); + break; + } catch (final NoSuchFileException x) { + // does not exist + } + parent = parent.getParent(); + } + if (parent == null) { + // unable to find existing parent + if (se == null) { + throw new FileSystemException( + dir.toString(), + null, + "Unable to determine if root directory exists"); + } else { + throw se; + } + } + + // create directories + Path child = parent; + for (final Path name : parent.relativize(dir)) { + child = child.resolve(name); + createAndCheckIsDirectory(child, attrs); + } + return dir; + } /** * This is a copy of a previous diff --git a/src/main/java/org/janelia/saalfeldlab/n5/FloatArrayDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/FloatArrayDataBlock.java index 284b0ab9..b8d30999 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/FloatArrayDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/FloatArrayDataBlock.java @@ -30,6 +30,7 @@ public class FloatArrayDataBlock extends AbstractDataBlock { public FloatArrayDataBlock(final int[] size, final long[] gridPosition, final float[] data) { + super(size, gridPosition, data); } @@ -46,10 +47,10 @@ public void readData(final ByteBuffer buffer) { buffer.asFloatBuffer().get(data); } - + @Override public int getNumElements() { - + return data.length; } } \ No newline at end of file diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java index 895cdcd3..3bdbf067 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java @@ -45,8 +45,8 @@ public interface GsonKeyValueN5Writer extends GsonN5Writer, GsonKeyValueN5Reader /** * TODO This overrides the version even if incompatible, check - * if this is the desired behavior or if it is always overridden, e.g. as by - * the caching version. If this is true, delete this implementation. + * if this is the desired behavior or if it is always overridden, e.g. as by + * the caching version. If this is true, delete this implementation. * * @param path * @throws IOException diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonN5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonN5Writer.java index 4eff1184..74291514 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonN5Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonN5Writer.java @@ -35,7 +35,7 @@ public interface GsonN5Writer extends GsonN5Reader, N5Writer { /** - * Set the attributes of a group. This result of this method is equivalent + * Set the attributes of a group. This result of this method is equivalent * with {@link #setAttribute(groupPath, "/", attributes)}. * * @param groupPath diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonUtils.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonUtils.java index 03988f77..5ba8e596 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonUtils.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonUtils.java @@ -61,9 +61,11 @@ static Gson registerGson(final GsonBuilder gsonBuilder) { /** * Reads the attributes json from a given {@link Reader}. * - * @param reader the reader + * @param reader + * the reader * @return the root {@link JsonObject} of the attributes - * @throws IOException the exception + * @throws IOException + * the exception */ static JsonElement readAttributes(final Reader reader, final Gson gson) throws IOException { @@ -71,12 +73,20 @@ static JsonElement readAttributes(final Reader reader, final Gson gson) throws I return json; } - static T readAttribute(final JsonElement root, final String normalizedAttributePath, final Class cls, final Gson gson) { + static T readAttribute( + final JsonElement root, + final String normalizedAttributePath, + final Class cls, + final Gson gson) { return readAttribute(root, normalizedAttributePath, TypeToken.get(cls).getType(), gson); } - static T readAttribute(final JsonElement root, final String normalizedAttributePath, final Type type, final Gson gson) { + static T readAttribute( + final JsonElement root, + final String normalizedAttributePath, + final Type type, + final Gson gson) { final JsonElement attribute = getAttribute(root, normalizedAttributePath); return parseAttributeElement(attribute, gson, type); @@ -85,11 +95,16 @@ static T readAttribute(final JsonElement root, final String normalizedAttrib /** * Deserialize the {@code attribute} as {@link Type type} {@code T}. * - * @param attribute to deserialize as {@link Type type} - * @param gson used to deserialize {@code attribute} - * @param type to desrialize {@code attribute} as - * @param return type represented by {@link Type type} - * @return the deserialized attribute object, or {@code null} if {@code attribute} cannot deserialize to {@code T} + * @param attribute + * to deserialize as {@link Type type} + * @param gson + * used to deserialize {@code attribute} + * @param type + * to desrialize {@code attribute} as + * @param + * return type represented by {@link Type type} + * @return the deserialized attribute object, or {@code null} if + * {@code attribute} cannot deserialize to {@code T} */ static T parseAttributeElement(final JsonElement attribute, final Gson gson, final Type type) { @@ -102,7 +117,7 @@ static T parseAttributeElement(final JsonElement attribute, final Gson gson, }.getType(); final Map retMap = gson.fromJson(attribute, mapType); - //noinspection unchecked + // noinspection unchecked return (T)retMap; } if (attribute instanceof JsonArray) { @@ -115,7 +130,7 @@ static T parseAttributeElement(final JsonElement attribute, final Gson gson, if (type == String.class) return (T)gson.toJson(attribute); return null; - } catch ( final NumberFormatException nfe ) { + } catch (final NumberFormatException nfe) { return null; } } @@ -125,7 +140,7 @@ static T parseAttributeElement(final JsonElement attribute, final Gson gson, if (type == String.class) return (T)gson.toJson(attribute); return null; - } catch ( final NumberFormatException nfe ) { + } catch (final NumberFormatException nfe) { return null; } } @@ -133,10 +148,11 @@ static T parseAttributeElement(final JsonElement attribute, final Gson gson, /** * Return the attribute at {@code normalizedAttributePath} as a * {@link JsonElement}. Does not attempt to parse the attribute. + * to search for the {@link JsonElement} at location + * {@code normalizedAttributePath} * - * @param root to search for the {@link JsonElement} at - * location {@code normalizedAttributePath} - * @param normalizedAttributePath to the attribute + * @param normalizedAttributePath + * to the attribute * @return the attribute as a {@link JsonElement}. */ static JsonElement getAttribute(JsonElement root, final String normalizedAttributePath) { @@ -171,7 +187,7 @@ static JsonElement getAttribute(JsonElement root, final String normalizedAttribu /** * Best effort implementation of {@link N5Reader#listAttributes(String)} - * with limited type resolution. Possible return types are + * with limited type resolution. Possible return types are *

      *
    • null
      • *
    • boolean
      • @@ -184,7 +200,8 @@ static JsonElement getAttribute(JsonElement root, final String normalizedAttribu *
    • Object[]
      • *
    * - * @param root the json element + * @param root + * the json element * @return the attribute map */ static Map> listAttributes(final JsonElement root) throws N5Exception.N5IOException { @@ -212,7 +229,8 @@ else if (jsonElement.isJsonArray()) { for (int i = 1; i < jsonArray.size() && arrayElementClass != Object.class; ++i) { final JsonElement element = jsonArray.get(i); if (element.isJsonPrimitive()) { - final Class nextArrayElementClass = classForJsonPrimitive(element.getAsJsonPrimitive()); + final Class nextArrayElementClass = classForJsonPrimitive( + element.getAsJsonPrimitive()); if (nextArrayElementClass != arrayElementClass) if (nextArrayElementClass == double.class && arrayElementClass == long.class) arrayElementClass = double.class; @@ -307,14 +325,14 @@ static T getJsonAsArray(final Gson gson, final JsonArray array, final Type t for (int i = 0; i < array.size(); i++) { clsArray[i] = gson.fromJson(array.get(i), componentCls); } - //noinspection unchecked + // noinspection unchecked return (T)clsArray; } return null; } /** - * Return a reasonable class for a {@link JsonPrimitive}. Possible return + * Return a reasonable class for a {@link JsonPrimitive}. Possible return * types are *
      *
    • boolean
      • @@ -323,7 +341,8 @@ static T getJsonAsArray(final Gson gson, final JsonArray array, final Type t *
    • Object
      • *
    * - * @param jsonPrimitive the json primitive + * @param jsonPrimitive + * the json primitive * @return the class */ static Class classForJsonPrimitive(final JsonPrimitive jsonPrimitive) { @@ -343,21 +362,26 @@ else if (jsonPrimitive.isNumber()) { } /** - * If there is an attribute in {@code root} such that it can be parsed and deserialized as {@code T}, - * then remove it from {@code root}, write {@code root} to the {@code writer}, and return the removed attribute. + * If there is an attribute in {@code root} such that it can be parsed and + * deserialized as {@code T}, + * then remove it from {@code root}, write {@code root} to the + * {@code writer}, and return the removed attribute. *

    - * If there is an attribute at the location specified by {@code normalizedAttributePath} but it cannot be deserialized to {@code T}, then it is not removed. + * If there is an attribute at the location specified by + * {@code normalizedAttributePath} but it cannot be deserialized to + * {@code T}, then it is not removed. *

    - * If nothing is removed, then {@code root} is not written to the {@code writer}. + * If nothing is removed, then {@code root} is not written to the + * {@code writer}. + * to write the modified {@code root} to after removal of the attribute to + * remove the attribute from * - * @param writer to write the modified {@code root} to after removal of the attribute - * @param root to remove the attribute from - * @param normalizedAttributePath to the attribute location - * @param cls of the attribute to remove - * @param gson to deserialize the attribute with - * @param of the removed attribute + * @param normalizedAttributePath + * to the attribute location of the attribute to remove to + * deserialize the attribute with of the removed attribute * @return the removed attribute, or null if nothing removed - * @throws IOException the exception + * @throws IOException + * the exception */ static T removeAttribute( final Writer writer, @@ -374,12 +398,13 @@ static T removeAttribute( } /** - * If there is an attribute in {@code root} at location {@code normalizedAttributePath} then remove it from {@code root}.. + * If there is an attribute in {@code root} at location + * {@code normalizedAttributePath} then remove it from {@code root}.. + * to write the modified {@code root} to after removal of the attribute to + * remove the attribute from * - * @param writer to write the modified {@code root} to after removal of the attribute - * @param root to remove the attribute from - * @param normalizedAttributePath to the attribute location - * @param gson to deserialize the attribute with + * @param normalizedAttributePath + * to the attribute location to deserialize the attribute with * @return if the attribute was removed or not */ static boolean removeAttribute( @@ -397,19 +422,25 @@ static boolean removeAttribute( } /** - * If there is an attribute in {@code root} such that it can be parsed and desrialized as {@code T}, + * If there is an attribute in {@code root} such that it can be parsed and + * desrialized as {@code T}, * then remove it from {@code root} and return the removed attribute. *

    - * If there is an attribute at the location specified by {@code normalizedAttributePath} but it cannot be deserialized to {@code T}, then it is not removed. + * If there is an attribute at the location specified by + * {@code normalizedAttributePath} but it cannot be deserialized to + * {@code T}, then it is not removed. + * to remove the attribute from * - * @param root to remove the attribute from - * @param normalizedAttributePath to the attribute location - * @param cls of the attribute to remove - * @param gson to deserialize the attribute with - * @param of the removed attribute + * @param normalizedAttributePath + * to the attribute location of the attribute to remove to + * deserialize the attribute with of the removed attribute * @return the removed attribute, or null if nothing removed */ - static T removeAttribute(final JsonElement root, final String normalizedAttributePath, final Class cls, final Gson gson) { + static T removeAttribute( + final JsonElement root, + final String normalizedAttributePath, + final Class cls, + final Gson gson) { final T attribute = GsonUtils.readAttribute(root, normalizedAttributePath, cls, gson); if (attribute != null) { @@ -419,11 +450,14 @@ static T removeAttribute(final JsonElement root, final String normalizedAttr } /** - * Remove and return the attribute at {@code normalizedAttributePath} as a {@link JsonElement}. + * Remove and return the attribute at {@code normalizedAttributePath} as a + * {@link JsonElement}. * Does not attempt to parse the attribute. + * to search for the {@link JsonElement} at location + * {@code normalizedAttributePath} * - * @param root to search for the {@link JsonElement} at location {@code normalizedAttributePath} - * @param normalizedAttributePath to the attribute + * @param normalizedAttributePath + * to the attribute * @return the attribute as a {@link JsonElement}. */ static JsonElement removeAttribute(JsonElement root, final String normalizedAttributePath) { @@ -463,17 +497,26 @@ static JsonElement removeAttribute(JsonElement root, final String normalizedAttr } /** - * Inserts {@code attribute} into {@code root} at location {@code normalizedAttributePath} and write the resulting {@code root}. + * Inserts {@code attribute} into {@code root} at location + * {@code normalizedAttributePath} and write the resulting {@code root}. *

    - * If {@code root} is not a {@link JsonObject}, then it is overwritten with an object containing {@code "normalizedAttributePath": attribute } + * If {@code root} is not a {@link JsonObject}, then it is overwritten with + * an object containing {@code "normalizedAttributePath": attribute } * - * @param writer the writer - * @param root the root json element - * @param normalizedAttributePath the attribute path - * @param attribute the attribute - * @param gson the gson - * @param the attribute type - * @throws IOException the exception + * @param writer + * the writer + * @param root + * the root json element + * @param normalizedAttributePath + * the attribute path + * @param attribute + * the attribute + * @param gson + * the gson + * @param + * the attribute type + * @throws IOException + * the exception */ static void writeAttribute( final Writer writer, @@ -490,11 +533,16 @@ static void writeAttribute( * Writes the attributes JsonElemnt to a given {@link Writer}. * This will overwrite any existing attributes. * - * @param writer the writer - * @param root the root json element - * @param gson the gson - * @param the attribute type - * @throws IOException the exception + * @param writer + * the writer + * @param root + * the root json element + * @param gson + * the gson + * @param + * the attribute type + * @throws IOException + * the exception */ static void writeAttributes( final Writer writer, @@ -513,7 +561,11 @@ static JsonElement insertAttributes(JsonElement root, final Map attri return root; } - static JsonElement insertAttribute(JsonElement root, final String normalizedAttributePath, final T attribute, final Gson gson) { + static JsonElement insertAttribute( + JsonElement root, + final String normalizedAttributePath, + final T attribute, + final Gson gson) { LinkedAttributePathToken pathToken = N5URL.getAttributePathTokens(normalizedAttributePath); @@ -526,7 +578,10 @@ static JsonElement insertAttribute(JsonElement root, final String normalized final JsonElement parent = pathToken.setAndCreateParentElement(json); - /* We may need to create or override the existing root if it is non-existent or incompatible. */ + /* + * We may need to create or override the existing root if it is + * non-existent or incompatible. + */ final boolean rootOverriden = json == root && parent != json; if (root == null || rootOverriden) { root = parent; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GzipCompression.java b/src/main/java/org/janelia/saalfeldlab/n5/GzipCompression.java index 5646aadf..b691a6d3 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GzipCompression.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GzipCompression.java @@ -107,11 +107,12 @@ private void readObject(final ObjectInputStream in) throws Exception { } @Override - public boolean equals(Object other) { + public boolean equals(final Object other) { + if (other == null || other.getClass() != GzipCompression.class) return false; else { - GzipCompression gz = ((GzipCompression) other); + final GzipCompression gz = ((GzipCompression)other); return useZlib == gz.useZlib && level == gz.level; } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/IntArrayDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/IntArrayDataBlock.java index 7ec2c022..98c5577d 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/IntArrayDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/IntArrayDataBlock.java @@ -47,10 +47,10 @@ public void readData(final ByteBuffer buffer) { buffer.asIntBuffer().get(data); } - + @Override public int getNumElements() { - + return data.length; } } \ No newline at end of file diff --git a/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccess.java index 20c14467..6d3b8a5f 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccess.java @@ -32,8 +32,9 @@ /** * Key value read primitives used by {@link N5KeyValueReader} - * implementations. This interface implements a subset of access primitives - * provided by {@link FileSystem} to reduce the implementation burden for backends + * implementations. This interface implements a subset of access primitives + * provided by {@link FileSystem} to reduce the implementation burden for + * backends * lacking a {@link FileSystem} implementation (such as AWS-S3). * * @author Stephan Saalfeld @@ -43,7 +44,8 @@ public interface KeyValueAccess { /** * Split a path string into its components. * - * @param path the path + * @param path + * the path * @return the path components */ public String[] components(final String path); @@ -51,7 +53,8 @@ public interface KeyValueAccess { /** * Compose a path from components. * - * @param components the path components + * @param components + * the path components * @return the path */ public String compose(final String... components); @@ -59,7 +62,8 @@ public interface KeyValueAccess { /** * Get the parent of a path string. * - * @param path the path + * @param path + * the path * @return the parent path or null if the path has no parent */ public String parent(final String path); @@ -67,18 +71,21 @@ public interface KeyValueAccess { /** * Relativize path relative to base. * - * @param path the path - * @param base the base path + * @param path + * the path + * @param base + * the base path * @return the result or null if the path has no parent */ public String relativize(final String path, final String base); /** - * Normalize a path to canonical form. All paths pointing to the same - * location return the same output. This is most important for cached + * Normalize a path to canonical form. All paths pointing to the same + * location return the same output. This is most important for cached * data pointing at the same location getting the same key. * - * @param path the path + * @param path + * the path * @return the normalized path */ public String normalize(final String path); @@ -86,8 +93,9 @@ public interface KeyValueAccess { /** * Get the absolute (including scheme) URI of the given path * - * @param normalPath is expected to be in normalized form, no further - * efforts are made to normalize it. + * @param normalPath + * is expected to be in normalized form, no further + * efforts are made to normalize it. * @return absolute URI */ public URI uri(final String normalPath) throws URISyntaxException; @@ -95,8 +103,9 @@ public interface KeyValueAccess { /** * Test whether the path exists. * - * @param normalPath is expected to be in normalized form, no further - * efforts are made to normalize it. + * @param normalPath + * is expected to be in normalized form, no further + * efforts are made to normalize it. * @return true if the path exists */ public boolean exists(final String normalPath); @@ -104,8 +113,9 @@ public interface KeyValueAccess { /** * Test whether the path is a directory. * - * @param normalPath is expected to be in normalized form, no further - * efforts are made to normalize it. + * @param normalPath + * is expected to be in normalized form, no further + * efforts are made to normalize it. * @return true if the path is a directory */ public boolean isDirectory(String normalPath); @@ -113,82 +123,97 @@ public interface KeyValueAccess { /** * Test whether the path is a file. * - * @param normalPath is expected to be in normalized form, no further - * efforts are made to normalize it. + * @param normalPath + * is expected to be in normalized form, no further + * efforts are made to normalize it. * @return true if the path is a file */ public boolean isFile(String normalPath); // TODO: Looks un-used. Remove? /** - * Create a lock on a path for reading. This isn't meant to be kept - * around. Create, use, [auto]close, e.g. + * Create a lock on a path for reading. This isn't meant to be kept + * around. Create, use, [auto]close, e.g. * * try (final lock = store.lockForReading()) { * ... * } * * - * @param normalPath is expected to be in normalized form, no further - * efforts are made to normalize it. + * @param normalPath + * is expected to be in normalized form, no further + * efforts are made to normalize it. * @return the locked channel - * @throws IOException the exception + * @throws IOException + * the exception */ public LockedChannel lockForReading(final String normalPath) throws IOException; /** - * Create an exclusive lock on a path for writing. If the file doesn't + * Create an exclusive lock on a path for writing. If the file doesn't * exist yet, it will be created, including all directories leading up to - * it. This lock isn't meant to be kept around. Create, use, [auto]close, e.g. + * it. This lock isn't meant to be kept around. Create, use, [auto]close, + * e.g. * * try (final lock = store.lockForWriting()) { * ... * } * * - * @param normalPath is expected to be in normalized form, no further - * efforts are made to normalize it. + * @param normalPath + * is expected to be in normalized form, no further + * efforts are made to normalize it. * @return the locked channel - * @throws IOException the exception + * @throws IOException + * the exception */ public LockedChannel lockForWriting(final String normalPath) throws IOException; /** * List all 'directory'-like children of a path. * - * @param normalPath is expected to be in normalized form, no further - * efforts are made to normalize it. + * @param normalPath + * is expected to be in normalized form, no further + * efforts are made to normalize it. * @return the directories - * @throws IOException the exception + * @throws IOException + * the exception */ public String[] listDirectories(final String normalPath) throws IOException; /** * List all children of a path. * - * @param normalPath is expected to be in normalized form, no further - * efforts are made to normalize it. + * @param normalPath + * is expected to be in normalized form, no further + * efforts are made to normalize it. * @return the the child paths - * @throws IOException the exception + * @throws IOException + * the exception + * + * */ - public String[] list(final String normalPath) throws IOException; // TODO: Looks un-used. Remove? + public String[] list(final String normalPath) throws IOException; /** - * Create a directory and all parent paths along the way. The directory - * and parent paths are discoverable. On a filesystem, this usually means + * Create a directory and all parent paths along the way. The directory + * and parent paths are discoverable. On a filesystem, this usually means * that the directories exist, on a key value store that is unaware of * directories, this may be implemented as creating an object for each path. * - * @param normalPath is expected to be in normalized form, no further - * efforts are made to normalize it. - * @throws IOException the exception + * @param normalPath + * is expected to be in normalized form, no further + * efforts are made to normalize it. + * @throws IOException + * the exception */ public void createDirectories(final String normalPath) throws IOException; /** - * Delete a path. If the path is a directory, delete it recursively. + * Delete a path. If the path is a directory, delete it recursively. * - * @param normalPath is expected to be in normalized form, no further - * efforts are made to normalize it. + * @param normalPath + * is expected to be in normalized form, no further + * efforts are made to normalize it. */ public void delete(final String normalPath) throws IOException; } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/LockedChannel.java b/src/main/java/org/janelia/saalfeldlab/n5/LockedChannel.java index a2ae6f4c..3e931eb9 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/LockedChannel.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/LockedChannel.java @@ -44,7 +44,8 @@ public interface LockedChannel extends Closeable { * Create a UTF-8 {@link Reader}. * * @return the reader - * @throws IOException the exception + * @throws IOException + * the exception */ public Reader newReader() throws IOException; @@ -52,18 +53,21 @@ public interface LockedChannel extends Closeable { * Create a new {@link InputStream}. * * @return the reader - * @throws IOException the exception + * @throws IOException + * the exception */ public InputStream newInputStream() throws IOException; /** * Create a new UTF-8 {@link Writer}. + * * @return the writer */ public Writer newWriter() throws IOException; /** * Create a new {@link OutputStream}. + * * @return the output stream */ public OutputStream newOutputStream() throws IOException; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/LongArrayDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/LongArrayDataBlock.java index 8873f67b..d3f3fc9c 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/LongArrayDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/LongArrayDataBlock.java @@ -47,10 +47,10 @@ public void readData(final ByteBuffer buffer) { buffer.asLongBuffer().get(data); } - + @Override public int getNumElements() { - + return data.length; } } \ No newline at end of file diff --git a/src/main/java/org/janelia/saalfeldlab/n5/Lz4Compression.java b/src/main/java/org/janelia/saalfeldlab/n5/Lz4Compression.java index 682f0bc2..d76e4fe5 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/Lz4Compression.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/Lz4Compression.java @@ -77,10 +77,11 @@ public Lz4Compression getWriter() { } @Override - public boolean equals(Object other) { + public boolean equals(final Object other) { + if (other == null || other.getClass() != Lz4Compression.class) return false; else - return blockSize == ((Lz4Compression) other).blockSize; + return blockSize == ((Lz4Compression)other).blockSize; } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5FSReader.java b/src/main/java/org/janelia/saalfeldlab/n5/N5FSReader.java index 93311358..aa0505ee 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5FSReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5FSReader.java @@ -25,7 +25,6 @@ */ package org.janelia.saalfeldlab.n5; -import java.io.IOException; import java.nio.file.FileSystems; import com.google.gson.Gson; @@ -46,21 +45,25 @@ public class N5FSReader extends N5KeyValueReader { * Opens an {@link N5FSReader} at a given base path with a custom * {@link GsonBuilder} to support custom attributes. * - * @param basePath N5 base path - * @param gsonBuilder the gson builder - * @param cacheMeta cache attributes and meta data - * Setting this to true avoids frequent reading and parsing of JSON - * encoded attributes and other meta data that requires accessing the - * store. This is most interesting for high latency backends. Changes - * of cached attributes and meta data by an independent writer on the - * same container will not be tracked. + * @param basePath + * N5 base path + * @param gsonBuilder + * the gson builder + * @param cacheMeta + * cache attributes and meta data + * Setting this to true avoids frequent reading and parsing of + * JSON encoded attributes and other meta data that requires + * accessing the store. This is most interesting for high latency + * backends. Changes of cached attributes and meta data by an + * independent writer on the same container will not be tracked. * - * @throws IOException - * if the base path cannot be read or does not exist, - * if the N5 version of the container is not compatible with this - * implementation. + * @throws N5Exception + * if the base path cannot be read or does not exist, if the N5 + * version of the container is not compatible with this + * implementation. */ - public N5FSReader(final String basePath, final GsonBuilder gsonBuilder, final boolean cacheMeta) throws IOException { + public N5FSReader(final String basePath, final GsonBuilder gsonBuilder, final boolean cacheMeta) + throws N5Exception { super( new FileSystemKeyValueAccess(FileSystems.getDefault()), @@ -68,27 +71,29 @@ public N5FSReader(final String basePath, final GsonBuilder gsonBuilder, final bo gsonBuilder, cacheMeta); - if( !exists("/")) - throw new N5Exception.N5IOException("No container exists at " + basePath ); + if (!exists("/")) + throw new N5Exception.N5IOException("No container exists at " + basePath); } /** * Opens an {@link N5FSReader} at a given base path. * - * @param basePath N5 base path - * @param cacheMeta cache attributes and meta data - * Setting this to true avoids frequent reading and parsing of JSON - * encoded attributes and other meta data that requires accessing the - * store. This is most interesting for high latency backends. Changes - * of cached attributes and meta data by an independent writer on the - * same container will not be tracked. + * @param basePath + * N5 base path + * @param cacheMeta + * cache attributes and meta data + * Setting this to true avoids frequent reading and parsing of + * JSON encoded attributes and other meta data that requires + * accessing the store. This is most interesting for high latency + * backends. Changes of cached attributes and meta data by an + * independent writer on the same container will not be tracked. * - * @throws IOException - * if the base path cannot be read or does not exist, - * if the N5 version of the container is not compatible with this - * implementation. + * @throws N5Exception + * if the base path cannot be read or does not exist, if the N5 + * version of the container is not compatible with this + * implementation. */ - public N5FSReader(final String basePath, final boolean cacheMeta) throws IOException { + public N5FSReader(final String basePath, final boolean cacheMeta) throws N5Exception { this(basePath, new GsonBuilder(), cacheMeta); } @@ -97,14 +102,16 @@ public N5FSReader(final String basePath, final boolean cacheMeta) throws IOExcep * Opens an {@link N5FSReader} at a given base path with a custom * {@link GsonBuilder} to support custom attributes. * - * @param basePath N5 base path - * @param gsonBuilder the gson builder - * @throws IOException - * if the base path cannot be read or does not exist, - * if the N5 version of the container is not compatible with this - * implementation. + * @param basePath + * N5 base path + * @param gsonBuilder + * the gson builder + * @throws N5Exception + * if the base path cannot be read or does not exist, if the N5 + * version of the container is not compatible with this + * implementation. */ - public N5FSReader(final String basePath, final GsonBuilder gsonBuilder) throws IOException { + public N5FSReader(final String basePath, final GsonBuilder gsonBuilder) throws N5Exception { this(basePath, gsonBuilder, false); } @@ -112,13 +119,14 @@ public N5FSReader(final String basePath, final GsonBuilder gsonBuilder) throws I /** * Opens an {@link N5FSReader} at a given base path. * - * @param basePath N5 base path - * @throws IOException - * if the base path cannot be read or does not exist, - * if the N5 version of the container is not compatible with this - * implementation. + * @param basePath + * N5 base path + * @throws N5Exception + * if the base path cannot be read or does not exist, if the N5 + * version of the container is not compatible with this + * implementation. */ - public N5FSReader(final String basePath) throws IOException { + public N5FSReader(final String basePath) throws N5Exception { this(basePath, new GsonBuilder(), false); } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java b/src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java index 97ef57b9..d69740f9 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java @@ -25,7 +25,6 @@ */ package org.janelia.saalfeldlab.n5; -import java.io.IOException; import java.nio.file.FileSystems; import com.google.gson.GsonBuilder; @@ -47,21 +46,25 @@ public class N5FSWriter extends N5KeyValueWriter { * compatible with this implementation, the N5 version of this container * will be set to the current N5 version of this implementation. * - * @param basePath n5 base path - * @param gsonBuilder the gson builder - * @param cacheAttributes cache attributes and meta data - * Setting this to true avoids frequent reading and parsing of JSON - * encoded attributes and other meta data that requires accessing the - * store. This is most interesting for high latency backends. Changes - * of cached attributes and meta data by an independent writer on the - * same container will not be tracked. + * @param basePath + * n5 base path + * @param gsonBuilder + * the gson builder + * @param cacheAttributes + * cache attributes and meta data + * Setting this to true avoidsfrequent reading and parsing of + * JSON encoded attributes andother meta data that requires + * accessing the store. This ismost interesting for high latency + * backends. Changes of cachedattributes and meta data by an + * independent writer on the samecontainer will not be tracked. * - * @throws IOException - * if the base path cannot be written to or cannot be created, - * if the N5 version of the container is not compatible with this - * implementation. + * @throws N5Exception + * if the base path cannot be written to or cannot be created, + * if the N5 version of the container is not compatible with + * this implementation. */ - public N5FSWriter(final String basePath, final GsonBuilder gsonBuilder, final boolean cacheAttributes) throws IOException { + public N5FSWriter(final String basePath, final GsonBuilder gsonBuilder, final boolean cacheAttributes) + throws N5Exception { super( new FileSystemKeyValueAccess(FileSystems.getDefault()), @@ -79,20 +82,22 @@ public N5FSWriter(final String basePath, final GsonBuilder gsonBuilder, final bo * compatible with this implementation, the N5 version of this container * will be set to the current N5 version of this implementation. * - * @param basePath n5 base path - * @param cacheAttributes cache attributes and meta data - * Setting this to true avoids frequent reading and parsing of JSON - * encoded attributes and other meta data that requires accessing the - * store. This is most interesting for high latency backends. Changes - * of cached attributes and meta data by an independent writer on the - * same container will not be tracked. + * @param basePathn5 + * base path + * @param cacheAttributescache + * attributes and meta data + * Setting this to true avoidsfrequent reading and parsing of + * JSON encoded attributes andother meta data that requires + * accessing the store. This ismost interesting for high latency + * backends. Changes of cachedattributes and meta data by an + * independent writer on the samecontainer will not be tracked. * - * @throws IOException - * if the base path cannot be written to or cannot be created, - * if the N5 version of the container is not compatible with this - * implementation. + * @throws N5Exception + * if the base path cannot be written to or cannot be created, + * if the N5 version of the container is not compatible with + * this implementation. */ - public N5FSWriter(final String basePath, final boolean cacheAttributes) throws IOException { + public N5FSWriter(final String basePath, final boolean cacheAttributes) throws N5Exception { this(basePath, new GsonBuilder(), cacheAttributes); } @@ -109,15 +114,17 @@ public N5FSWriter(final String basePath, final boolean cacheAttributes) throws I * will be set to the current N5 version of this implementation. *

    * - * @param basePath n5 base path - * @param gsonBuilder the gson builder + * @param basePathn5 + * base path + * @param gsonBuilderthe + * gson builder * - * @throws IOException - * if the base path cannot be written to or cannot be created, - * if the N5 version of the container is not compatible with this - * implementation. + * @throws N5Exception + * if the base path cannot be written to or cannot be created, + * if the N5 version of the container is not compatible with + * this implementation. */ - public N5FSWriter(final String basePath, final GsonBuilder gsonBuilder) throws IOException { + public N5FSWriter(final String basePath, final GsonBuilder gsonBuilder) throws N5Exception { this(basePath, gsonBuilder, false); } @@ -133,14 +140,15 @@ public N5FSWriter(final String basePath, final GsonBuilder gsonBuilder) throws I * will be set to the current N5 version of this implementation. *

    * - * @param basePath n5 base path + * @param basePath + * n5 base path * - * @throws IOException - * if the base path cannot be written to or cannot be created, - * if the N5 version of the container is not compatible with this - * implementation. + * @throws N5Exception + * if the base path cannot be written to or cannot be created, + * if the N5 version of the container is not compatible with + * this implementation. */ - public N5FSWriter(final String basePath) throws IOException { + public N5FSWriter(final String basePath) throws N5Exception { this(basePath, new GsonBuilder()); } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java index 54ecd10b..17b50bdb 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java @@ -25,10 +25,6 @@ */ package org.janelia.saalfeldlab.n5; -import java.io.IOException; -import com.google.gson.Gson; -import com.google.gson.GsonBuilder; - import java.net.URI; import java.net.URISyntaxException; import java.util.Arrays; @@ -36,6 +32,8 @@ import org.janelia.saalfeldlab.n5.cache.N5JsonCache; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; /** * {@link N5Reader} implementation through {@link KeyValueAccess} with JSON @@ -67,13 +65,13 @@ public class N5KeyValueReader implements CachedGsonKeyValueN5Reader { * @param gsonBuilder * @param cacheMeta * cache attributes and meta data - * Setting this to true avoids frequent reading and parsing of - * JSON encoded attributes and other meta data that requires - * accessing the store. This is most interesting for high latency - * backends. Changes of cached attributes and meta data by an - * independent writer will not be tracked. + * Setting this to true avoidsfrequent reading and parsing of + * JSON encoded attributes andother meta data that requires + * accessing the store. This ismost interesting for high latency + * backends. Changes of cachedattributes and meta data by an + * independent writer will not betracked. * - * @throws IOException + * @throws N5Exception * if the base path cannot be read or does not exist, if the N5 * version of the container is not compatible with this * implementation. @@ -83,7 +81,7 @@ public N5KeyValueReader( final String basePath, final GsonBuilder gsonBuilder, final boolean cacheMeta) - throws IOException { + throws N5Exception { this(true, keyValueAccess, basePath, gsonBuilder, cacheMeta); } @@ -92,21 +90,21 @@ public N5KeyValueReader( * Opens an {@link N5KeyValueReader} at a given base path with a custom * {@link GsonBuilder} to support custom attributes. * - * @param checkVersion - * do the version check + * @param checkVersiondo + * the version check * @param keyValueAccess - * @param basePath - * N5 base path + * @param basePathN5 + * base path * @param gsonBuilder - * @param cacheMeta - * cache attributes and meta data - * Setting this to true avoids frequent reading and parsing of - * JSON encoded attributes and other meta data that requires - * accessing the store. This is most interesting for high latency - * backends. Changes of cached attributes and meta data by an - * independent writer will not be tracked. + * @param cacheMetacache + * attributes and meta data + * Setting this to true avoidsfrequent reading and parsing of + * JSON encoded attributes andother meta data that requires + * accessing the store. This ismost interesting for high latency + * backends. Changes of cachedattributes and meta data by an + * independent writer will not betracked. * - * @throws IOException + * @throws N5Exception * if the base path cannot be read or does not exist, if the N5 * version of the container is not compatible with this * implementation. @@ -117,7 +115,7 @@ protected N5KeyValueReader( final String basePath, final GsonBuilder gsonBuilder, final boolean cacheMeta) - throws IOException { + throws N5Exception { this.keyValueAccess = keyValueAccess; this.gson = GsonUtils.registerGson(gsonBuilder); @@ -126,8 +124,8 @@ protected N5KeyValueReader( try { uri = keyValueAccess.uri(basePath); - } catch (URISyntaxException e) { - throw new N5Exception( e ); + } catch (final URISyntaxException e) { + throw new N5Exception(e); } if (checkVersion) { @@ -159,6 +157,7 @@ public KeyValueAccess getKeyValueAccess() { @Override public URI getURI() { + return uri; } @@ -175,7 +174,9 @@ public N5JsonCache getCache() { } @Override - public String groupPath(String... nodes) { - return keyValueAccess.compose(Stream.concat(Stream.of(getURI().getPath()), Arrays.stream(nodes)).toArray(String[]::new)); + public String groupPath(final String... nodes) { + + return keyValueAccess + .compose(Stream.concat(Stream.of(getURI().getPath()), Arrays.stream(nodes)).toArray(String[]::new)); } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriter.java b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriter.java index 910e7128..93e14c68 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriter.java @@ -27,10 +27,6 @@ import com.google.gson.GsonBuilder; -import java.io.IOException; - -import org.janelia.saalfeldlab.n5.N5Reader.Version; - /** * Filesystem {@link N5Writer} implementation with version compatibility check. * @@ -41,6 +37,7 @@ public class N5KeyValueWriter extends N5KeyValueReader implements CachedGsonKeyV /** * Opens an {@link N5KeyValueWriter} at a given base path with a custom * {@link GsonBuilder} to support custom attributes. + * *

    * If the base path does not exist, it will be created. *

    @@ -49,22 +46,25 @@ public class N5KeyValueWriter extends N5KeyValueReader implements CachedGsonKeyV * will be set to the current N5 version of this implementation. * * @param keyValueAccess - * @param basePath n5 base path + * @param basePath + * n5 base path * @param gsonBuilder - * @param cacheAttributes Setting this to true avoids frequent reading and parsing of - * JSON encoded attributes, this is most interesting for high - * latency file systems. Changes of attributes by an independent - * writer will not be tracked. - * @throws IOException if the base path cannot be written to or cannot be created, - * if the N5 version of the container is not compatible with - * this implementation. + * @param cacheAttributes + * Setting this to true avoids frequent reading and parsing of + * JSON encoded attributes, this is most interesting for high + * latency file systems. Changes of attributes by an independent + * writer will not be tracked. + * @throws N5Exception + * if the base path cannot be written to or cannot be created, + * if the N5 version of the container is not compatible with + * this implementation. */ public N5KeyValueWriter( final KeyValueAccess keyValueAccess, final String basePath, final GsonBuilder gsonBuilder, final boolean cacheAttributes) - throws IOException { + throws N5Exception { super(false, keyValueAccess, basePath, gsonBuilder, cacheAttributes); @@ -73,8 +73,7 @@ public N5KeyValueWriter( version = getVersion(); if (!VERSION.isCompatible(version)) throw new N5Exception.N5IOException("Incompatible version " + version + " (this is " + VERSION + ")."); - } catch (NullPointerException e) { - } + } catch (final NullPointerException e) {} if (version == null || version.equals(new Version(0, 0, 0, ""))) { createGroup("/"); diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java index de4e5884..13637985 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java @@ -219,7 +219,7 @@ default Version getVersion() throws N5Exception { * @return the base path URI */ // TODO: should this throw URISyntaxException or can we assume that this is - // never possible if we were able to instantiate this N5Reader? + // never possible if we were able to instantiate this N5Reader? URI getURI(); /** diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java b/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java index 37aa512d..90d991b0 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java @@ -98,11 +98,15 @@ public LinkedAttributePathToken getAttributePathTokens() { } /** - * Parses the {@link String normalizedAttributePath} to a list of {@link LinkedAttributePathToken}. - * This is useful for traversing or constructing a json representation of the provided {@link String normalizedAttributePath}. - * Note that {@link String normalizedAttributePath} should be normalized prior to generating this list + * Parses the {@link String normalizedAttributePath} to a list of + * {@link LinkedAttributePathToken}. + * This is useful for traversing or constructing a json representation of + * the provided {@link String normalizedAttributePath}. + * Note that {@link String normalizedAttributePath} should be normalized + * prior to generating this list * - * @param normalizedAttributePath to parse into {@link LinkedAttributePathToken}s + * @param normalizedAttributePath + * to parse into {@link LinkedAttributePathToken}s * @return the head of the {@link LinkedAttributePathToken}s */ public static LinkedAttributePathToken getAttributePathTokens(final String normalizedAttributePath) { @@ -161,20 +165,22 @@ private String getAttributePart() { return attribute == null ? "" : "#" + attribute; } - @Override public String toString() { + @Override + public String toString() { return getContainerPart() + getGroupPart() + getAttributePart(); } private String getSchemeSpecificPartWithoutQuery() { - /* Why not substring "?"?*/ + /* Why not substring "?"? */ return uri.getSchemeSpecificPart().replace("?" + uri.getQuery(), ""); } /** * N5URL is always considered absolute if a scheme is provided. - * If no scheme is provided, the N5URL is absolute if it starts with either "/" or "[A-Z]:" + * If no scheme is provided, the N5URL is absolute if it starts with either + * "/" or "[A-Z]:" * * @return if the path for this N5URL is absolute */ @@ -191,12 +197,16 @@ public boolean isAbsolute() { } /** - * Generate a new N5URL which is the result of resolving {@link N5URL relativeN5Url} to this {@link N5URL}. - * If relativeN5Url is not relative to this N5URL, then the resulting N5URL is equivalent to relativeN5Url. + * Generate a new N5URL which is the result of resolving {@link N5URL + * relativeN5Url} to this {@link N5URL}. + * If relativeN5Url is not relative to this N5URL, then the resulting N5URL + * is equivalent to relativeN5Url. * - * @param relativeN5Url N5URL to resolve against ourselves + * @param relativeN5Url + * N5URL to resolve against ourselves * @return the result of the resolution. - * @throws URISyntaxException if the uri is malformed + * @throws URISyntaxException + * if the uri is malformed */ public N5URL resolve(final N5URL relativeN5Url) throws URISyntaxException { @@ -268,12 +278,16 @@ public N5URL resolve(final N5URL relativeN5Url) throws URISyntaxException { } /** - * Generate a new N5URL which is the result of resolving {@link URI relativeUri} to this {@link N5URL}. - * If relativeUri is not relative to this N5URL, then the resulting N5URL is equivalent to relativeUri. + * Generate a new N5URL which is the result of resolving {@link URI + * relativeUri} to this {@link N5URL}. + * If relativeUri is not relative to this N5URL, then the resulting N5URL is + * equivalent to relativeUri. * - * @param relativeUri URI to resolve against ourselves + * @param relativeUri + * URI to resolve against ourselves * @return the result of the resolution. - * @throws URISyntaxException if the uri is malformed + * @throws URISyntaxException + * if the uri is malformed */ public N5URL resolve(final URI relativeUri) throws URISyntaxException { @@ -281,12 +295,16 @@ public N5URL resolve(final URI relativeUri) throws URISyntaxException { } /** - * Generate a new N5URL which is the result of resolving {@link String relativeString} to this {@link N5URL} - * If relativeString is not relative to this N5URL, then the resulting N5URL is equivalent to relativeString. + * Generate a new N5URL which is the result of resolving {@link String + * relativeString} to this {@link N5URL} + * If relativeString is not relative to this N5URL, then the resulting N5URL + * is equivalent to relativeString. * - * @param relativeString String to resolve against ourselves + * @param relativeString + * String to resolve against ourselves * @return the result of the resolution. - * @throws URISyntaxException if the uri is malformed + * @throws URISyntaxException + * if the uri is malformed */ public N5URL resolve(final String relativeString) throws URISyntaxException { @@ -294,9 +312,11 @@ public N5URL resolve(final String relativeString) throws URISyntaxException { } /** - * Normalize a path, resulting in removal of redundant "/", "./", and resolution of relative "../". + * Normalize a path, resulting in removal of redundant "/", "./", and + * resolution of relative "../". * - * @param path to normalize + * @param path + * to normalize * @return the normalized path */ public static String normalizePath(String path) { @@ -308,7 +328,7 @@ public static String normalizePath(String path) { final StringBuilder curToken = new StringBuilder(); boolean escape = false; for (final char character : pathChars) { - /* Skip if we last saw escape*/ + /* Skip if we last saw escape */ if (escape) { escape = false; curToken.append(character); @@ -323,10 +343,16 @@ public static String normalizePath(String path) { curToken.append(character); } - /* The current token is complete, add it to the list, if it isn't empty */ + /* + * The current token is complete, add it to the list, if it + * isn't empty + */ final String newToken = curToken.toString(); if (!newToken.isEmpty()) { - /* If our token is '..' then remove the last token instead of adding a new one */ + /* + * If our token is '..' then remove the last token instead + * of adding a new one + */ if (newToken.equals("..")) { tokens.remove(tokens.size() - 1); } else { @@ -354,26 +380,34 @@ public static String normalizePath(String path) { tokens.remove(0); root = "/"; } - return root + tokens.stream() + return root + tokens + .stream() .filter(it -> !it.equals(".")) .filter(it -> !it.isEmpty()) - .reduce((l, r) -> l + "/" + r).orElse(""); + .reduce((l, r) -> l + "/" + r) + .orElse(""); } /** - * Normalize a group path relative to a container's root, resulting in removal of redundant "/", "./", resolution of relative "../", + * Normalize a group path relative to a container's root, resulting in + * removal of redundant "/", "./", resolution of relative "../", * and removal of leading slashes. * - * @param path to normalize + * @param path + * to normalize * @return the normalized path */ - public static String normalizeGroupPath( final String path ) { + public static String normalizeGroupPath(final String path) { - /* Alternatively, could do something like the below in every KeyValueReader implementation + /* + * Alternatively, could do something like the below in every + * KeyValueReader implementation * - * return keyValueAccess.relativize( N5URL.normalizeGroupPath(path), basePath); + * return keyValueAccess.relativize( N5URL.normalizeGroupPath(path), + * basePath); * - * has to be in the implementations, since KeyValueAccess doesn't have a basePath. + * has to be in the implementations, since KeyValueAccess doesn't have a + * basePath. */ return normalizePath(path.startsWith("/") || path.startsWith("\\") ? path.substring(1) : path); } @@ -383,66 +417,81 @@ public static String normalizeGroupPath( final String path ) { *

    * Attribute paths have a few of special characters: *

      - *
    • "." which represents the current element
      • - *
    • ".." which represent the previous elemnt
      • - *
    • "/" which is used to separate elements in the json tree
      • - *
    • [N] where N is an integer, refer to an index in the previous element in the tree; the previous element must be an array. - *

      - * Note: [N] also separates the previous and following elements, regardless of whether it is preceded by "/" or not. - *

      • - *
    • "\" which is an escape character, which indicates the subquent '/' or '[N]' should not be interpreted as a path delimeter, - * but as part of the current path name.
      • + *
    • "." which represents the current element
      • + *
    • ".." which represent the previous elemnt
      • + *
    • "/" which is used to separate elements in the json tree
      • + *
    • [N] where N is an integer, refer to an index in the previous element + * in the tree; the previous element must be an array. + *

      + * Note: [N] also separates the previous and following elements, regardless + * of whether it is preceded by "/" or not. + *

      • + *
    • "\" which is an escape character, which indicates the subquent '/' or + * '[N]' should not be interpreted as a path delimeter, + * but as part of the current path name.
      • * *
    - *

    + *

    * When normalizing: *

      - *
    • "/" are added before and after any indexing brackets [N]
      • - *
    • any redundant "/" are removed
      • - *
    • any relative ".." and "." are resolved
      • + *
    • "/" are added before and after any indexing brackets [N]
      • + *
    • any redundant "/" are removed
      • + *
    • any relative ".." and "." are resolved
      • *
    *

    * Examples of valid attribute paths, and their normalizations *

      - *
    • /a/b/c becomes /a/b/c
      • - *
    • /a/b/c/ becomes /a/b/c
      • - *
    • ///a///b///c becomes /a/b/c
      • - *
    • /a/././b/c becomes /a/b/c
      • - *
    • /a/b[1]c becomes /a/b/[1]/c
      • - *
    • /a/b/[1]c becomes /a/b/[1]/c
      • - *
    • /a/b[1]/c becomes /a/b/[1]/c
      • - *
    • /a/b[1]/c/.. becomes /a/b/[1]
      • + *
    • /a/b/c becomes /a/b/c
      • + *
    • /a/b/c/ becomes /a/b/c
      • + *
    • ///a///b///c becomes /a/b/c
      • + *
    • /a/././b/c becomes /a/b/c
      • + *
    • /a/b[1]c becomes /a/b/[1]/c
      • + *
    • /a/b/[1]c becomes /a/b/[1]/c
      • + *
    • /a/b[1]/c becomes /a/b/[1]/c
      • + *
    • /a/b[1]/c/.. becomes /a/b/[1]
      • *
    * - * @param attributePath to normalize + * @param attributePath + * to normalize * @return the normalized attribute path */ public static String normalizeAttributePath(final String attributePath) { - /* Short circuit if there are no non-escaped `/` or array indices (e.g. [N] where N is a non-negative integer) */ + /* + * Short circuit if there are no non-escaped `/` or array indices (e.g. + * [N] where N is a non-negative integer) + */ if (!attributePath.matches(".*((? `[10]/b`*/ + /* Add separator after arrays at the beginning `[10]b` -> `[10]/b` */ final String attrPathPlusFirstIndexSeparator = attributePath.replaceAll("^(?\\[[0-9]+])", "${array}/"); - /* Add separator before and after arrays not at the beginning `a[10]b` -> `a/[10]/b`*/ - final String attrPathPlusIndexSeparators = attrPathPlusFirstIndexSeparator.replaceAll("((?\\[[0-9]+]))", "/${array}/"); + /* + * Add separator before and after arrays not at the beginning `a[10]b` + * -> `a/[10]/b` + */ + final String attrPathPlusIndexSeparators = attrPathPlusFirstIndexSeparator + .replaceAll("((?\\[[0-9]+]))", "/${array}/"); /* - * The following has 4 possible matches, in each case it removes the match: - * The first 3 remove redundant separators of the form: - * 1.`a///b` -> `a/b` : (?<=/)/+ - * 2.`a/./b` -> `a/b` : (?<=(/|^))(\./)+ - * 3.`a/b/` -> `a/b` : ((/|(?<=/))\.)$ - * The last resolves relative paths: - * 4. `a/../b` -> `a/b` : (? `a/b` : (?<=/)/+ + * 2.`a/./b` -> `a/b` : (?<=(/|^))(\./)+ + * 3.`a/b/` -> `a/b` : ((/|(?<=/))\.)$ + * The last resolves relative paths: + * 4. `a/../b` -> `a/b` : + * (? - * As an example of the original implementation, a backslash inside a square brace would be encoded to "[%5C]", and - * when calling {@code decode("[%5C]")} it would not decode to "[\]" since the encode escape sequence is inside square braces. + * As an example of the original implementation, a backslash inside a square + * brace would be encoded to "[%5C]", and + * when calling {@code decode("[%5C]")} it would not decode to "[\]" since + * the encode escape sequence is inside square braces. *

    - * We keep all the decoding logic in this modified version, EXCEPT, that we don't check for and ignore encoded sequences inside square braces. + * We keep all the decoding logic in this modified version, EXCEPT, that we + * don't check for and ignore encoded sequences inside square braces. *

    * Thus, {@code decode("[%5C]")} -> "[\]". * @@ -568,15 +634,16 @@ private static String decodeFragment(final String rawFragment) { final ByteBuffer bb = ByteBuffer.allocate(n); final CharBuffer cb = CharBuffer.allocate(n); - final CharsetDecoder dec = UTF8.newDecoder() + final CharsetDecoder dec = UTF8 + .newDecoder() .onMalformedInput(CodingErrorAction.REPLACE) .onUnmappableCharacter(CodingErrorAction.REPLACE); // This is not horribly efficient, but it will do for now char c = rawFragment.charAt(0); - for (int i = 0; i < n; ) { - assert c == rawFragment.charAt(i); // Loop invariant + for (int i = 0; i < n;) { + assert c == rawFragment.charAt(i); // Loop invariant if (c != '%') { sb.append(c); if (++i >= n) @@ -585,7 +652,7 @@ private static String decodeFragment(final String rawFragment) { continue; } bb.clear(); - for (; ; ) { + for (;;) { assert (n - i >= 2); bb.put(decode(rawFragment.charAt(++i), rawFragment.charAt(++i))); if (++i >= n) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java index c8ee52bb..172580a9 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java @@ -45,16 +45,11 @@ public interface N5Writer extends N5Reader { /** * Sets an attribute. * - * @param groupPath - * group path - * @param attributePath - * the key - * @param attribute - * the attribute - * @param - * the attribute type type - * @throws N5Exception - * the exception + * @param groupPath group path + * @param attributePath the key + * @param attribute the attribute + * @param the attribute type type + * @throws N5Exception the exception */ default void setAttribute( final String groupPath, @@ -70,12 +65,9 @@ default void setAttribute( * objects will be added, existing attributes whose paths are not included * will remain unchanged, those whose paths are included will be overridden. * - * @param groupPath - * group path - * @param attributes - * the attribute map of attribute paths and values - * @throws N5Exception - * the exception + * @param groupPath group path + * @param attributes the attribute map of attribute paths and values + * @throws N5Exception the exception */ void setAttributes( final String groupPath, @@ -84,13 +76,10 @@ void setAttributes( /** * Remove the attribute from group {@code pathName} with key {@code key}. * - * @param groupPath - * group path - * @param attributePath - * of attribute to remove + * @param groupPath group path + * @param attributePath of attribute to remove * @return true if attribute removed, else false - * @throws N5Exception - * the exception + * @throws N5Exception the exception */ boolean removeAttribute(String groupPath, String attributePath) throws N5Exception; @@ -101,19 +90,13 @@ void setAttributes( * If an attribute at {@code pathName} and {@code key} exists, but is not of * type {@code T}, it is not removed. * - * @param groupPath - * group path - * @param attributePath - * of attribute to remove - * @param clazz - * of the attribute to remove - * @param - * of the attribute + * @param groupPath group path + * @param attributePath of attribute to remove + * @param clazz of the attribute to remove + * @param of the attribute * @return the removed attribute, as {@code T}, or {@code null} if no * matching attribute - * @throws N5Exception - * if removing he attribute failed, parsing the attribute - * failed, or the attribute cannot be interpreted as T + * @throws N5Exception if removing he attribute failed, parsing the attribute failed, or the attribute cannot be interpreted as T */ T removeAttribute(String groupPath, String attributePath, Class clazz) throws N5Exception; @@ -125,13 +108,10 @@ void setAttributes( * return {@code true}. * * - * @param groupPath - * group path - * @param attributePaths - * to remove + * @param groupPath group path + * @param attributePaths to remove * @return true if any of the listed attributes were removed - * @throws N5Exception - * the exception + * @throws N5Exception the exception */ default boolean removeAttributes(final String groupPath, final List attributePaths) throws N5Exception { @@ -147,12 +127,9 @@ default boolean removeAttributes(final String groupPath, final List attr /** * Sets mandatory dataset attributes. * - * @param datasetPath - * dataset path - * @param datasetAttributes - * the dataset attributse - * @throws N5Exception - * the exception + * @param datasetPath dataset path + * @param datasetAttributes the dataset attributse + * @throws N5Exception the exception */ default void setDatasetAttributes( final String datasetPath, @@ -167,8 +144,7 @@ default void setDatasetAttributes( * implementation writes the version only if the current version is not * equal {@link N5Reader#VERSION}. * - * @throws N5Exception - * the exception + * @throws N5Exception the exception */ default void setVersion() throws N5Exception { @@ -179,10 +155,8 @@ default void setVersion() throws N5Exception { /** * Creates a group (directory) * - * @param groupPath - * the path - * @throws N5Exception - * the exception + * @param groupPath the path + * @throws N5Exception the exception */ void createGroup(final String groupPath) throws N5Exception; @@ -198,11 +172,9 @@ default void setVersion() throws N5Exception { * only fails because it attempts to delete the parent directory before it * is empty. * - * @param groupPath - * group path + * @param groupPath group path * @return true if removal was successful, false otherwise - * @throws N5Exception - * the exception + * @throws N5Exception the exception */ boolean remove(final String groupPath) throws N5Exception; @@ -210,8 +182,7 @@ default void setVersion() throws N5Exception { * Removes the N5 container. * * @return true if removal was successful, false otherwise - * @throws N5Exception - * the exception + * @throws N5Exception the exception */ default boolean remove() throws N5Exception { @@ -222,12 +193,9 @@ default boolean remove() throws N5Exception { * Creates a dataset. This does not create any data but the path and * mandatory attributes only. * - * @param datasetPath - * dataset path - * @param datasetAttributes - * the dataset attributes - * @throws N5Exception - * the exception + * @param datasetPath dataset path + * @param datasetAttributes the dataset attributes + * @throws N5Exception the exception */ default void createDataset( final String datasetPath, @@ -243,18 +211,12 @@ default void createDataset( * mandatory * attributes only. * - * @param datasetPath - * dataset path - * @param dimensions - * the dataset dimensions - * @param blockSize - * the block size - * @param dataType - * the data type - * @param compression - * the compression - * @throws N5Exception - * the exception + * @param datasetPath dataset path + * @param dimensions the dataset dimensions + * @param blockSize the block size + * @param dataType the data type + * @param compression the compression + * @throws N5Exception the exception */ default void createDataset( final String datasetPath, @@ -269,16 +231,11 @@ default void createDataset( /** * Writes a {@link DataBlock}. * - * @param datasetPath - * dataset path - * @param datasetAttributes - * the dataset attributes - * @param dataBlock - * the data block - * @param - * the data block data type - * @throws N5Exception - * the exception + * @param datasetPath dataset path + * @param datasetAttributes the dataset attributes + * @param dataBlock the data block + * @param the data block data type + * @throws N5Exception the exception */ void writeBlock( final String datasetPath, @@ -288,12 +245,9 @@ void writeBlock( /** * Deletes the block at {@code gridPosition} * - * @param datasetPath - * dataset path - * @param gridPosition - * position of block to be deleted - * @throws N5Exception - * the exception + * @param datasetPath dataset path + * @param gridPosition position of block to be deleted + * @throws N5Exception the exception * * @return {@code true} if the block at {@code gridPosition} is "empty" * after @@ -312,16 +266,11 @@ boolean deleteBlock( * The * offset is given in {@link DataBlock} grid coordinates. * - * @param object - * the object to serialize - * @param datasetPath - * the dataset path - * @param datasetAttributes - * the dataset attributes - * @param gridPosition - * the grid position - * @throws N5Exception - * the exception + * @param object the object to serialize + * @param datasetPath the dataset path + * @param datasetAttributes the dataset attributes + * @param gridPosition the grid position + * @throws N5Exception the exception */ default void writeSerializedBlock( final Serializable object, diff --git a/src/main/java/org/janelia/saalfeldlab/n5/RawCompression.java b/src/main/java/org/janelia/saalfeldlab/n5/RawCompression.java index e7a2a8f4..ffa674fc 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/RawCompression.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/RawCompression.java @@ -61,7 +61,8 @@ public RawCompression getWriter() { } @Override - public boolean equals(Object other) { + public boolean equals(final Object other) { + if (other == null || other.getClass() != RawCompression.class) return false; else diff --git a/src/main/java/org/janelia/saalfeldlab/n5/ShortArrayDataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/ShortArrayDataBlock.java index 34f51a5b..2dbf6b17 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/ShortArrayDataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/ShortArrayDataBlock.java @@ -47,10 +47,10 @@ public void readData(final ByteBuffer buffer) { buffer.asShortBuffer().get(data); } - + @Override public int getNumElements() { - + return data.length; } } \ No newline at end of file diff --git a/src/main/java/org/janelia/saalfeldlab/n5/XzCompression.java b/src/main/java/org/janelia/saalfeldlab/n5/XzCompression.java index 0de4db57..5204e799 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/XzCompression.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/XzCompression.java @@ -51,7 +51,6 @@ public XzCompression() { this(6); } - @Override public InputStream getInputStream(final InputStream in) throws IOException { @@ -77,11 +76,12 @@ public XzCompression getWriter() { } @Override - public boolean equals(Object other) { + public boolean equals(final Object other) { + if (other == null || other.getClass() != XzCompression.class) return false; else - return preset == ((XzCompression) other).preset; + return preset == ((XzCompression)other).preset; } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/cache/N5JsonCache.java b/src/main/java/org/janelia/saalfeldlab/n5/cache/N5JsonCache.java index eae2684a..ffca797b 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/cache/N5JsonCache.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/cache/N5JsonCache.java @@ -17,7 +17,7 @@ public class N5JsonCache { protected final N5JsonCacheableContainer container; /** - * Data object for caching meta data. Elements that are null are not yet + * Data object for caching meta data. Elements that are null are not yet * cached. */ protected static class N5CacheInfo { @@ -46,6 +46,7 @@ protected static class EmptyJson extends JsonElement { @Override public JsonElement deepCopy() { + throw new N5Exception("Do not copy EmptyJson, you naughty person"); } @@ -53,11 +54,13 @@ public JsonElement deepCopy() { private final HashMap containerPathToCache = new HashMap<>(); - public N5JsonCache( final N5JsonCacheableContainer container ) { + public N5JsonCache(final N5JsonCacheableContainer container) { + this.container = container; } public JsonElement getAttributes(final String normalPathKey, final String normalCacheKey) { + N5CacheInfo cacheInfo = getCacheInfo(normalPathKey); if (cacheInfo == null) { addNewCacheInfo(normalPathKey, normalCacheKey, null); @@ -76,7 +79,7 @@ public JsonElement getAttributes(final String normalPathKey, final String normal return output == null ? null : output.deepCopy(); } - public boolean isDataset(final String normalPathKey, final String normalCacheKey ) { + public boolean isDataset(final String normalPathKey, final String normalCacheKey) { N5CacheInfo cacheInfo = getCacheInfo(normalPathKey); if (cacheInfo == null) { @@ -99,30 +102,34 @@ public boolean isGroup(final String normalPathKey, final String cacheKey) { /** * Returns true if a resource exists. * - * @param normalPathKey the container path - * @param normalCacheKey the cache key / resource (may be null) + * @param normalPathKey + * the container path + * @param normalCacheKey + * the cache key / resource (may be null) * @return true if exists */ public boolean exists(final String normalPathKey, final String normalCacheKey) { N5CacheInfo cacheInfo = getCacheInfo(normalPathKey); if (cacheInfo == null) { - addNewCacheInfo( normalPathKey, normalCacheKey, null); + addNewCacheInfo(normalPathKey, normalCacheKey, null); cacheInfo = getCacheInfo(normalPathKey); } return cacheInfo != emptyCacheInfo; } public String[] list(final String normalPathKey) { + N5CacheInfo cacheInfo = getCacheInfo(normalPathKey); if (cacheInfo == null) { addNewCacheInfo(normalPathKey); cacheInfo = getCacheInfo(normalPathKey); } - if (cacheInfo == emptyCacheInfo) return null; + if (cacheInfo == emptyCacheInfo) + return null; - if( cacheInfo.children == null ) - addChild( cacheInfo, normalPathKey ); + if (cacheInfo.children == null) + addChild(cacheInfo, normalPathKey); final String[] children = new String[cacheInfo.children.size()]; int i = 0; @@ -132,7 +139,10 @@ public String[] list(final String normalPathKey) { return children; } - public N5CacheInfo addNewCacheInfo(final String normalPathKey, final String normalCacheKey, final JsonElement uncachedAttributes) { + public N5CacheInfo addNewCacheInfo( + final String normalPathKey, + final String normalCacheKey, + final JsonElement uncachedAttributes) { final N5CacheInfo cacheInfo; if (container.existsFromContainer(normalPathKey, null)) { @@ -166,7 +176,7 @@ private N5CacheInfo addNewCacheInfo(final String normalPathKey) { private void addChild(final N5CacheInfo cacheInfo, final String normalPathKey) { - if( cacheInfo.children == null ) + if (cacheInfo.children == null) cacheInfo.children = new HashSet<>(); final String[] children = container.listFromContainer(normalPathKey); @@ -190,7 +200,8 @@ protected N5CacheInfo getOrMakeCacheInfo(final String normalPathKey) { * Updates the cache attributes for the given normalPathKey * adding the appropriate node to the cache if necessary. * - * @param normalPathKey the normalized path key + * @param normalPathKey + * the normalized path key */ public void updateCacheInfo(final String normalPathKey, final String normalCacheKey) { @@ -200,14 +211,21 @@ public void updateCacheInfo(final String normalPathKey, final String normalCache } /** - * Updates the cache attributes for the given normalPathKey and normalCacheKey, + * Updates the cache attributes for the given normalPathKey and + * normalCacheKey, * adding the appropriate node to the cache if necessary. * - * @param normalPathKey the normalized path key - * @param normalCacheKey the normalized cache key - * @param uncachedAttributes attributes to be cached + * @param normalPathKey + * the normalized path key + * @param normalCacheKey + * the normalized cache key + * @param uncachedAttributes + * attributes to be cached */ - public void updateCacheInfo(final String normalPathKey, final String normalCacheKey, final JsonElement uncachedAttributes) { + public void updateCacheInfo( + final String normalPathKey, + final String normalCacheKey, + final JsonElement uncachedAttributes) { final N5CacheInfo cacheInfo = getOrMakeCacheInfo(normalPathKey); if (normalCacheKey != null) { @@ -228,30 +246,31 @@ public void updateCacheInfo(final String normalPathKey, final String normalCache public void initializeNonemptyCache(final String normalPathKey, final String normalCacheKey) { final N5CacheInfo cacheInfo = getCacheInfo(normalPathKey); - if (cacheInfo == null || cacheInfo == emptyCacheInfo ) { + if (cacheInfo == null || cacheInfo == emptyCacheInfo) { final N5CacheInfo info = newCacheInfo(); - if( normalCacheKey != null ) + if (normalCacheKey != null) info.attributesCache.put(normalCacheKey, emptyJson); updateCache(normalPathKey, info); } } - public void setAttributes(final String normalPathKey, final String normalCacheKey, final JsonElement attributes ) { + public void setAttributes(final String normalPathKey, final String normalCacheKey, final JsonElement attributes) { + N5CacheInfo cacheInfo = getCacheInfo(normalPathKey); boolean update = false; - if (cacheInfo == null ){ + if (cacheInfo == null) { return; } - if( cacheInfo == emptyCacheInfo ) { + if (cacheInfo == emptyCacheInfo) { cacheInfo = newCacheInfo(); update = true; } updateCacheAttributes(cacheInfo, normalCacheKey, attributes); - if( update ) + if (update) updateCache(normalPathKey, cacheInfo); } @@ -259,15 +278,18 @@ public void setAttributes(final String normalPathKey, final String normalCacheKe * Adds child to the parent's children list, only if the parent has been * cached, and its children list already exists. * - * @param parent parent path - * @param child child path + * @param parent + * parent path + * @param child + * child path */ public void addChildIfPresent(final String parent, final String child) { final N5CacheInfo cacheInfo = getCacheInfo(parent); - if (cacheInfo == null) return; + if (cacheInfo == null) + return; - if( cacheInfo.children != null ) + if (cacheInfo.children != null) cacheInfo.children.add(child); } @@ -275,22 +297,25 @@ public void addChildIfPresent(final String parent, final String child) { * Adds child to the parent's children list, only if the parent has been * cached, creating a children list if it does not already exist. * - * @param parent parent path - * @param child child path + * @param parent + * parent path + * @param child + * child path */ - public void addChild(final String parent, final String child ) { + public void addChild(final String parent, final String child) { final N5CacheInfo cacheInfo = getCacheInfo(parent); - if (cacheInfo == null) return; + if (cacheInfo == null) + return; - if( cacheInfo.children == null ) + if (cacheInfo.children == null) cacheInfo.children = new HashSet<>(); cacheInfo.children.add(child); } - public void removeCache(final String normalParentPathKey, final String normalPathKey) { + // this path and all children should be removed = set to emptyCacheInfo synchronized (containerPathToCache) { containerPathToCache.put(normalPathKey, emptyCacheInfo); @@ -303,7 +328,7 @@ public void removeCache(final String normalParentPathKey, final String normalPat // update the parent's children, if present (remove the normalPathKey) final N5CacheInfo parentCache = containerPathToCache.get(normalParentPathKey); - if (parentCache != null && parentCache.children != null ) { + if (parentCache != null && parentCache.children != null) { parentCache.children.remove(normalPathKey.replaceFirst(normalParentPathKey + "/", "")); } } @@ -316,27 +341,34 @@ protected N5CacheInfo getCacheInfo(final String pathKey) { } protected N5CacheInfo newCacheInfo() { + return new N5CacheInfo(); } protected void updateCache(final String normalPathKey, final N5CacheInfo cacheInfo) { + synchronized (containerPathToCache) { containerPathToCache.put(normalPathKey, cacheInfo); } } - protected void updateCacheAttributes(final N5CacheInfo cacheInfo, final String normalCacheKey, + protected void updateCacheAttributes( + final N5CacheInfo cacheInfo, + final String normalCacheKey, final JsonElement attributes) { + synchronized (cacheInfo.attributesCache) { cacheInfo.attributesCache.put(normalCacheKey, attributes); } } protected void updateCacheIsGroup(final N5CacheInfo cacheInfo, final boolean isGroup) { + cacheInfo.isGroup = isGroup; } protected void updateCacheIsDataset(final N5CacheInfo cacheInfo, final boolean isDataset) { + cacheInfo.isDataset = isDataset; } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/cache/N5JsonCacheableContainer.java b/src/main/java/org/janelia/saalfeldlab/n5/cache/N5JsonCacheableContainer.java index 1a35dc37..e8d59301 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/cache/N5JsonCacheableContainer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/cache/N5JsonCacheableContainer.java @@ -1,5 +1,9 @@ package org.janelia.saalfeldlab.n5.cache; +import org.janelia.saalfeldlab.n5.CachedGsonKeyValueN5Reader; +import org.janelia.saalfeldlab.n5.GsonKeyValueN5Reader; +import org.janelia.saalfeldlab.n5.N5Reader; + import com.google.gson.JsonElement; /** @@ -13,37 +17,43 @@ public interface N5JsonCacheableContainer { /** - * Returns a {@link JsonElement} containing attributes at a given path, + * Returns a {@link JsonElement} containing attributes at a given path, * for a given cache key. * - * @param normalPathName the normalized path name - * @param normalCacheKey the cache key - * @return the attributes as a json element. + * @param normalPathName + * the normalized path name + * @param normalCacheKey + * the cache key + * @return the attributes as a json element. * @see GsonKeyValueN5Reader#getAttributes */ JsonElement getAttributesFromContainer(final String normalPathName, final String normalCacheKey); /** * Query whether a resource exists in this container. - * - * @param normalPathName the normalized path name - * @param normalCacheKey the normalized resource name (may be null). + * + * @param normalPathName + * the normalized path name + * @param normalCacheKey + * the normalized resource name (may be null). * @return true if the resouce exists */ boolean existsFromContainer(final String normalPathName, final String normalCacheKey); /** * Query whether a path in this container is a group. - * - * @param normalPathName the normalized path name + * + * @param normalPathName + * the normalized path name * @return true if the path is a group */ boolean isGroupFromContainer(final String normalPathName); /** * Query whether a path in this container is a dataset. - * - * @param normalPathName the normalized path name + * + * @param normalPathName + * the normalized path name * @return true if the path is a dataset * @see N5Reader#datasetExists */ @@ -53,8 +63,10 @@ public interface N5JsonCacheableContainer { * If this method is called, its parent path must exist, * and normalCacheKey must exist, with contents given by attributes. * - * @param normalCacheKey the cache key - * @param attributes the attributes + * @param normalCacheKey + * the cache key + * @param attributes + * the attributes * @return true if the path is a group */ boolean isGroupFromAttributes(final String normalCacheKey, final JsonElement attributes); @@ -63,8 +75,10 @@ public interface N5JsonCacheableContainer { * If this method is called, its parent path must exist, * and normalCacheKey must exist, with contents given by attributes. * - * @param normalCacheKey the cache key - * @param attributes the attributes + * @param normalCacheKey + * the cache key + * @param attributes + * the attributes * @return true if the path is a dataset */ boolean isDatasetFromAttributes(final String normalCacheKey, final JsonElement attributes); @@ -72,7 +86,8 @@ public interface N5JsonCacheableContainer { /** * List the children of a path for this container. * - * @param normalPathName the normalized path name + * @param normalPathName + * the normalized path name * @return list of children * @see N5Reader#list */ From 59a8897ba4a22e8d1c83fb71031f46736a8a432b Mon Sep 17 00:00:00 2001 From: Stephan Saalfeld Date: Fri, 9 Jun 2023 16:44:04 -0400 Subject: [PATCH 207/243] fix: N5FSReader do not check existence twice --- src/main/java/org/janelia/saalfeldlab/n5/N5FSReader.java | 3 --- src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java | 1 - 2 files changed, 4 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5FSReader.java b/src/main/java/org/janelia/saalfeldlab/n5/N5FSReader.java index aa0505ee..80d1f801 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5FSReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5FSReader.java @@ -70,9 +70,6 @@ public N5FSReader(final String basePath, final GsonBuilder gsonBuilder, final bo basePath, gsonBuilder, cacheMeta); - - if (!exists("/")) - throw new N5Exception.N5IOException("No container exists at " + basePath); } /** diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java index 17b50bdb..b9e9377b 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java @@ -139,7 +139,6 @@ protected N5KeyValueReader( } catch (final NullPointerException e) { throw new N5Exception.N5IOException("Could not read version from " + basePath); } - } } From c9e529c1ecae45d03c6a1cff86fdd9a02ba8d3e4 Mon Sep 17 00:00:00 2001 From: Stephan Saalfeld Date: Fri, 9 Jun 2023 17:03:44 -0400 Subject: [PATCH 208/243] Revert "fix: N5FSReader do not check existence twice" This reverts commit 59a8897ba4a22e8d1c83fb71031f46736a8a432b. --- src/main/java/org/janelia/saalfeldlab/n5/N5FSReader.java | 3 +++ src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java | 1 + 2 files changed, 4 insertions(+) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5FSReader.java b/src/main/java/org/janelia/saalfeldlab/n5/N5FSReader.java index 80d1f801..aa0505ee 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5FSReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5FSReader.java @@ -70,6 +70,9 @@ public N5FSReader(final String basePath, final GsonBuilder gsonBuilder, final bo basePath, gsonBuilder, cacheMeta); + + if (!exists("/")) + throw new N5Exception.N5IOException("No container exists at " + basePath); } /** diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java index b9e9377b..17b50bdb 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java @@ -139,6 +139,7 @@ protected N5KeyValueReader( } catch (final NullPointerException e) { throw new N5Exception.N5IOException("Could not read version from " + basePath); } + } } From 152a43142647eb65e2465950698915add3d53c2c Mon Sep 17 00:00:00 2001 From: Stephan Saalfeld Date: Fri, 9 Jun 2023 17:07:50 -0400 Subject: [PATCH 209/243] fix, style: remove unnecessary NullPointer exception catch, fix tests --- .../saalfeldlab/n5/N5KeyValueReader.java | 13 +- .../org/janelia/saalfeldlab/n5/UriTest.java | 14 +- .../saalfeldlab/n5/url/UrlAttributeTest.java | 366 +++++++++--------- 3 files changed, 195 insertions(+), 198 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java index 17b50bdb..2193797a 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java @@ -131,15 +131,10 @@ protected N5KeyValueReader( if (checkVersion) { /* Existence checks, if any, go in subclasses */ /* Check that version (if there is one) is compatible. */ - try { - final Version version = getVersion(); - if (!VERSION.isCompatible(version)) - throw new N5Exception.N5IOException( - "Incompatible version " + version + " (this is " + VERSION + ")."); - } catch (final NullPointerException e) { - throw new N5Exception.N5IOException("Could not read version from " + basePath); - } - + final Version version = getVersion(); + if (!VERSION.isCompatible(version)) + throw new N5Exception.N5IOException( + "Incompatible version " + version + " (this is " + VERSION + ")."); } } diff --git a/src/test/java/org/janelia/saalfeldlab/n5/UriTest.java b/src/test/java/org/janelia/saalfeldlab/n5/UriTest.java index 90084857..b94c0413 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/UriTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/UriTest.java @@ -3,7 +3,6 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; -import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; import java.nio.file.Paths; @@ -25,13 +24,9 @@ public class UriTest { @Before public void before() { - try { - n5 = new N5FSWriter(relativePath); - kva = n5.getKeyValueAccess(); - } catch (IOException e) { - e.printStackTrace(); - } + n5 = new N5FSWriter(relativePath); + kva = n5.getKeyValueAccess(); } @Test @@ -41,7 +36,8 @@ public void testUriParsing() { try { assertEquals("Container URI must contain scheme", "file", uri.getScheme()); - assertEquals("Container URI must be absolute", + assertEquals( + "Container URI must be absolute", Paths.get(relativePath).toAbsolutePath().toString(), uri.getPath()); @@ -51,7 +47,7 @@ public void testUriParsing() { assertEquals("Container URI must be normalized 4", uri, kva.uri("file:" + relativeAbnormalPath)); assertEquals("Container URI must be normalized 5", uri, kva.uri("file:" + relativeAbnormalPath2)); - } catch (URISyntaxException e) { + } catch (final URISyntaxException e) { fail(e.getMessage()); } diff --git a/src/test/java/org/janelia/saalfeldlab/n5/url/UrlAttributeTest.java b/src/test/java/org/janelia/saalfeldlab/n5/url/UrlAttributeTest.java index 8547b6a8..b748ca07 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/url/UrlAttributeTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/url/UrlAttributeTest.java @@ -1,5 +1,9 @@ package org.janelia.saalfeldlab.n5.url; +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + import java.io.IOException; import java.net.URISyntaxException; import java.util.Arrays; @@ -18,285 +22,287 @@ import org.junit.Before; import org.junit.Test; -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +public class UrlAttributeTest { -public class UrlAttributeTest -{ N5Reader n5; String rootContext = ""; int[] list; - HashMap< String, String > obj; + HashMap obj; - Set< String > rootKeys; + Set rootKeys; TestInts testObjInts; TestDoubles testObjDoubles; @Before - public void before() - { - try - { - n5 = new N5FSWriter( "src/test/resources/url/urlAttributes.n5" ); - rootContext = ""; - list = new int[] { 0, 1, 2, 3 }; - - obj = new HashMap<>(); - obj.put( "a", "aa" ); - obj.put( "b", "bb" ); - rootKeys = new HashSet<>(); - rootKeys.addAll( Stream.of( "n5", "foo", "list", "object" ).collect( Collectors.toList() ) ); - } - catch ( IOException e ) - { - e.printStackTrace(); - } + public void before() { + + n5 = new N5FSWriter("src/test/resources/url/urlAttributes.n5"); + rootContext = ""; + list = new int[]{0, 1, 2, 3}; + + obj = new HashMap<>(); + obj.put("a", "aa"); + obj.put("b", "bb"); + rootKeys = new HashSet<>(); + rootKeys.addAll(Stream.of("n5", "foo", "list", "object").collect(Collectors.toList())); - testObjInts = new TestInts( "ints", "intsName", new int[] { 5, 4, 3 } ); - testObjDoubles = new TestDoubles( "doubles", "doublesName", new double[] { 5.5, 4.4, 3.3 } ); + testObjInts = new TestInts("ints", "intsName", new int[]{5, 4, 3}); + testObjDoubles = new TestDoubles("doubles", "doublesName", new double[]{5.5, 4.4, 3.3}); } - @SuppressWarnings( "unchecked" ) + @SuppressWarnings("unchecked") @Test - public void testRootAttributes() throws URISyntaxException, IOException - { + public void testRootAttributes() throws URISyntaxException, IOException { + // get - Map< String, Object > everything = getAttribute( n5, new N5URL( "" ), Map.class ); - final String version = "2.6.1"; // the n5 version at the time the test data were created - assertEquals( "empty url", version, everything.get( "n5" ) ); + final Map everything = getAttribute(n5, new N5URL(""), Map.class); + final String version = "2.6.1"; // the n5 version at the time the test + // data were created + assertEquals("empty url", version, everything.get("n5")); - Map< String, Object > everything2 = getAttribute( n5, new N5URL( "#/" ), Map.class ); - assertEquals( "root attribute", version, everything2.get( "n5" ) ); + final Map everything2 = getAttribute(n5, new N5URL("#/"), Map.class); + assertEquals("root attribute", version, everything2.get("n5")); - assertEquals( "url to attribute", "bar", getAttribute( n5, new N5URL( "#foo" ), String.class ) ); - assertEquals( "url to attribute absolute", "bar", getAttribute( n5, new N5URL( "#/foo" ), String.class ) ); + assertEquals("url to attribute", "bar", getAttribute(n5, new N5URL("#foo"), String.class)); + assertEquals("url to attribute absolute", "bar", getAttribute(n5, new N5URL("#/foo"), String.class)); - assertEquals( "#foo", "bar", getAttribute( n5, new N5URL( "#foo" ), String.class ) ); - assertEquals( "#/foo", "bar", getAttribute( n5, new N5URL( "#/foo" ), String.class ) ); - assertEquals( "?#foo", "bar", getAttribute( n5, new N5URL( "?#foo" ), String.class ) ); - assertEquals( "?#/foo", "bar", getAttribute( n5, new N5URL( "?#/foo" ), String.class ) ); - assertEquals( "?/#/foo", "bar", getAttribute( n5, new N5URL( "?/#/foo" ), String.class ) ); - assertEquals( "?/.#/foo", "bar", getAttribute( n5, new N5URL( "?/.#/foo" ), String.class ) ); - assertEquals( "?./#/foo", "bar", getAttribute( n5, new N5URL( "?./#/foo" ), String.class ) ); - assertEquals( "?.#foo", "bar", getAttribute( n5, new N5URL( "?.#foo" ), String.class ) ); - assertEquals( "?/a/..#foo", "bar", getAttribute( n5, new N5URL( "?/a/..#foo" ), String.class ) ); - assertEquals( "?/a/../.#foo", "bar", getAttribute( n5, new N5URL( "?/a/../.#foo" ), String.class ) ); + assertEquals("#foo", "bar", getAttribute(n5, new N5URL("#foo"), String.class)); + assertEquals("#/foo", "bar", getAttribute(n5, new N5URL("#/foo"), String.class)); + assertEquals("?#foo", "bar", getAttribute(n5, new N5URL("?#foo"), String.class)); + assertEquals("?#/foo", "bar", getAttribute(n5, new N5URL("?#/foo"), String.class)); + assertEquals("?/#/foo", "bar", getAttribute(n5, new N5URL("?/#/foo"), String.class)); + assertEquals("?/.#/foo", "bar", getAttribute(n5, new N5URL("?/.#/foo"), String.class)); + assertEquals("?./#/foo", "bar", getAttribute(n5, new N5URL("?./#/foo"), String.class)); + assertEquals("?.#foo", "bar", getAttribute(n5, new N5URL("?.#foo"), String.class)); + assertEquals("?/a/..#foo", "bar", getAttribute(n5, new N5URL("?/a/..#foo"), String.class)); + assertEquals("?/a/../.#foo", "bar", getAttribute(n5, new N5URL("?/a/../.#foo"), String.class)); - /* whitespace-encoding-necesary, fragment-only test*/ - assertEquals( "#f o o", "b a r", getAttribute( n5, new N5URL( "#f o o" ), String.class ) ); + /* whitespace-encoding-necesary, fragment-only test */ + assertEquals("#f o o", "b a r", getAttribute(n5, new N5URL("#f o o"), String.class)); - Assert.assertArrayEquals( "url list", list, getAttribute( n5, new N5URL( "#list" ), int[].class ) ); + Assert.assertArrayEquals("url list", list, getAttribute(n5, new N5URL("#list"), int[].class)); // list - assertEquals( "url list[0]", list[ 0 ], ( int ) getAttribute( n5, new N5URL( "#list[0]" ), Integer.class ) ); - assertEquals( "url list[1]", list[ 1 ], ( int ) getAttribute( n5, new N5URL( "#list[1]" ), Integer.class ) ); - assertEquals( "url list[2]", list[ 2 ], ( int ) getAttribute( n5, new N5URL( "#list[2]" ), Integer.class ) ); + assertEquals("url list[0]", list[0], (int)getAttribute(n5, new N5URL("#list[0]"), Integer.class)); + assertEquals("url list[1]", list[1], (int)getAttribute(n5, new N5URL("#list[1]"), Integer.class)); + assertEquals("url list[2]", list[2], (int)getAttribute(n5, new N5URL("#list[2]"), Integer.class)); - assertEquals( "url list[3]", list[ 3 ], ( int ) getAttribute( n5, new N5URL( "#list[3]" ), Integer.class ) ); - assertEquals( "url list/[3]", list[ 3 ], ( int ) getAttribute( n5, new N5URL( "#list/[3]" ), Integer.class ) ); - assertEquals( "url list//[3]", list[ 3 ], ( int ) getAttribute( n5, new N5URL( "#list//[3]" ), Integer.class ) ); - assertEquals( "url //list//[3]", list[ 3 ], ( int ) getAttribute( n5, new N5URL( "#//list//[3]" ), Integer.class ) ); - assertEquals( "url //list//[3]//", list[ 3 ], ( int ) getAttribute( n5, new N5URL( "#//list////[3]//" ), Integer.class ) ); + assertEquals("url list[3]", list[3], (int)getAttribute(n5, new N5URL("#list[3]"), Integer.class)); + assertEquals("url list/[3]", list[3], (int)getAttribute(n5, new N5URL("#list/[3]"), Integer.class)); + assertEquals("url list//[3]", list[3], (int)getAttribute(n5, new N5URL("#list//[3]"), Integer.class)); + assertEquals("url //list//[3]", list[3], (int)getAttribute(n5, new N5URL("#//list//[3]"), Integer.class)); + assertEquals("url //list//[3]//", list[3], (int)getAttribute(n5, new N5URL("#//list////[3]//"), Integer.class)); // object - assertTrue( "url object", mapsEqual( obj, getAttribute( n5, new N5URL( "#object" ), Map.class ) ) ); - assertEquals( "url object/a", "aa", getAttribute( n5, new N5URL( "#object/a" ), String.class ) ); - assertEquals( "url object/b", "bb", getAttribute( n5, new N5URL( "#object/b" ), String.class ) ); + assertTrue("url object", mapsEqual(obj, getAttribute(n5, new N5URL("#object"), Map.class))); + assertEquals("url object/a", "aa", getAttribute(n5, new N5URL("#object/a"), String.class)); + assertEquals("url object/b", "bb", getAttribute(n5, new N5URL("#object/b"), String.class)); } @Test - public void testPathAttributes() throws URISyntaxException, IOException - { + public void testPathAttributes() throws URISyntaxException, IOException { final String a = "a"; final String aa = "aa"; final String aaa = "aaa"; - final N5URL aUrl = new N5URL( "?/a" ); - final N5URL aaUrl = new N5URL( "?/a/aa" ); - final N5URL aaaUrl = new N5URL( "?/a/aa/aaa" ); + final N5URL aUrl = new N5URL("?/a"); + final N5URL aaUrl = new N5URL("?/a/aa"); + final N5URL aaaUrl = new N5URL("?/a/aa/aaa"); // name of a - assertEquals( "name of a from root", a, getAttribute( n5, "?/a#name", String.class ) ); - assertEquals( "name of a from root", a, getAttribute( n5, new N5URL( "?a#name" ), String.class ) ); - assertEquals( "name of a from a", a, getAttribute( n5, aUrl.resolve( new N5URL( "?/a#name" ) ), String.class ) ); - assertEquals( "name of a from aa", a, getAttribute( n5, aaUrl.resolve( "?..#name" ), String.class ) ); - assertEquals( "name of a from aaa", a, getAttribute( n5, aaaUrl.resolve( new N5URL( "?../..#name" ) ), String.class ) ); + assertEquals("name of a from root", a, getAttribute(n5, "?/a#name", String.class)); + assertEquals("name of a from root", a, getAttribute(n5, new N5URL("?a#name"), String.class)); + assertEquals("name of a from a", a, getAttribute(n5, aUrl.resolve(new N5URL("?/a#name")), String.class)); + assertEquals("name of a from aa", a, getAttribute(n5, aaUrl.resolve("?..#name"), String.class)); + assertEquals("name of a from aaa", a, getAttribute(n5, aaaUrl.resolve(new N5URL("?../..#name")), String.class)); // name of aa - assertEquals( "name of aa from root", aa, getAttribute( n5, new N5URL( "?/a/aa#name" ), String.class ) ); - assertEquals( "name of aa from root", aa, getAttribute( n5, new N5URL( "?a/aa#name" ), String.class ) ); - assertEquals( "name of aa from a", aa, getAttribute( n5, aUrl.resolve( "?aa#name" ), String.class ) ); + assertEquals("name of aa from root", aa, getAttribute(n5, new N5URL("?/a/aa#name"), String.class)); + assertEquals("name of aa from root", aa, getAttribute(n5, new N5URL("?a/aa#name"), String.class)); + assertEquals("name of aa from a", aa, getAttribute(n5, aUrl.resolve("?aa#name"), String.class)); - assertEquals( "name of aa from aa", aa, getAttribute( n5, aaUrl.resolve( "?./#name" ), String.class ) ); + assertEquals("name of aa from aa", aa, getAttribute(n5, aaUrl.resolve("?./#name"), String.class)); - assertEquals( "name of aa from aa", aa, getAttribute( n5, aaUrl.resolve( "#name" ), String.class ) ); - assertEquals( "name of aa from aaa", aa, getAttribute( n5, aaaUrl.resolve( "?..#name" ), String.class ) ); + assertEquals("name of aa from aa", aa, getAttribute(n5, aaUrl.resolve("#name"), String.class)); + assertEquals("name of aa from aaa", aa, getAttribute(n5, aaaUrl.resolve("?..#name"), String.class)); // name of aaa - assertEquals( "name of aaa from root", aaa, getAttribute( n5, new N5URL( "?/a/aa/aaa#name" ), String.class ) ); - assertEquals( "name of aaa from root", aaa, getAttribute( n5, new N5URL( "?a/aa/aaa#name" ), String.class ) ); - assertEquals( "name of aaa from a", aaa, getAttribute( n5, aUrl.resolve( "?aa/aaa#name" ), String.class ) ); - assertEquals( "name of aaa from aa", aaa, getAttribute( n5, aaUrl.resolve( "?aaa#name" ), String.class ) ); - assertEquals( "name of aaa from aaa", aaa, getAttribute( n5, aaaUrl.resolve( "#name" ), String.class ) ); + assertEquals("name of aaa from root", aaa, getAttribute(n5, new N5URL("?/a/aa/aaa#name"), String.class)); + assertEquals("name of aaa from root", aaa, getAttribute(n5, new N5URL("?a/aa/aaa#name"), String.class)); + assertEquals("name of aaa from a", aaa, getAttribute(n5, aUrl.resolve("?aa/aaa#name"), String.class)); + assertEquals("name of aaa from aa", aaa, getAttribute(n5, aaUrl.resolve("?aaa#name"), String.class)); + assertEquals("name of aaa from aaa", aaa, getAttribute(n5, aaaUrl.resolve("#name"), String.class)); + + assertEquals("name of aaa from aaa", aaa, getAttribute(n5, aaaUrl.resolve("?./#name"), String.class)); + + assertEquals("nested list 1", (Integer)1, getAttribute(n5, new N5URL("#nestedList[0][0][0]"), Integer.class)); + assertEquals("nested list 1", (Integer)1, getAttribute(n5, new N5URL("#/nestedList/[0][0][0]"), Integer.class)); + assertEquals( + "nested list 1", + (Integer)1, + getAttribute(n5, new N5URL("#nestedList//[0]/[0]///[0]"), Integer.class)); + assertEquals( + "nested list 1", + (Integer)1, + getAttribute(n5, new N5URL("#/nestedList[0]//[0][0]"), Integer.class)); - assertEquals( "name of aaa from aaa", aaa, getAttribute( n5, aaaUrl.resolve( "?./#name" ), String.class ) ); + } - assertEquals( "nested list 1", (Integer)1, getAttribute( n5, new N5URL( "#nestedList[0][0][0]" ), Integer.class ) ); - assertEquals( "nested list 1", (Integer)1, getAttribute( n5, new N5URL( "#/nestedList/[0][0][0]" ), Integer.class ) ); - assertEquals( "nested list 1", (Integer)1, getAttribute( n5, new N5URL( "#nestedList//[0]/[0]///[0]" ), Integer.class ) ); - assertEquals( "nested list 1", (Integer)1, getAttribute( n5, new N5URL( "#/nestedList[0]//[0][0]" ), Integer.class ) ); + private T getAttribute( + final N5Reader n5, + final String url1, + final Class clazz) throws URISyntaxException, IOException { + return getAttribute(n5, url1, null, clazz); } - private < T > T getAttribute( final N5Reader n5, final String url1, Class< T > clazz ) throws URISyntaxException, IOException - { - return getAttribute( n5, url1, null, clazz ); - } + private T getAttribute( + final N5Reader n5, + final String url1, + final String url2, + final Class clazz) throws URISyntaxException, IOException { - private < T > T getAttribute( final N5Reader n5, final String url1, final String url2, Class< T > clazz ) throws URISyntaxException, IOException - { - final N5URL n5URL = url2 == null ? new N5URL( url1 ) : new N5URL( url1 ).resolve( url2 ); - return getAttribute( n5, n5URL, clazz ); + final N5URL n5URL = url2 == null ? new N5URL(url1) : new N5URL(url1).resolve(url2); + return getAttribute(n5, n5URL, clazz); } - private < T > T getAttribute( final N5Reader n5, final N5URL url1, Class< T > clazz ) throws URISyntaxException, IOException - { - return getAttribute( n5, url1, null, clazz ); + private T getAttribute( + final N5Reader n5, + final N5URL url1, + final Class clazz) throws URISyntaxException, IOException { + + return getAttribute(n5, url1, null, clazz); } - private < T > T getAttribute( final N5Reader n5, final N5URL url1, final String url2, Class< T > clazz ) throws URISyntaxException, IOException - { - final N5URL n5URL = url2 == null ? url1 : url1.resolve( url2 ); - return n5.getAttribute( n5URL.getGroupPath(), n5URL.getAttributePath(), clazz ); + private T getAttribute( + final N5Reader n5, + final N5URL url1, + final String url2, + final Class clazz) throws URISyntaxException, IOException { + + final N5URL n5URL = url2 == null ? url1 : url1.resolve(url2); + return n5.getAttribute(n5URL.getGroupPath(), n5URL.getAttributePath(), clazz); } @Test - public void testPathObject() throws IOException, URISyntaxException - { - final TestInts ints = getAttribute( n5, new N5URL( "?objs#intsKey" ), TestInts.class ); - assertEquals( testObjInts.name, ints.name ); - assertEquals( testObjInts.type, ints.type ); - assertArrayEquals( testObjInts.t(), ints.t() ); - - final TestDoubles doubles = getAttribute( n5, new N5URL( "?objs#doublesKey" ), TestDoubles.class ); - assertEquals( testObjDoubles.name, doubles.name ); - assertEquals( testObjDoubles.type, doubles.type ); - assertArrayEquals( testObjDoubles.t(), doubles.t(), 1e-9 ); - - final TestDoubles[] doubleArray = getAttribute( n5, new N5URL( "?objs#array" ), TestDoubles[].class ); - final TestDoubles doubles1 = new TestDoubles( "doubles", "doubles1", new double[] { 5.7, 4.5, 3.4 } ); - final TestDoubles doubles2 = new TestDoubles( "doubles", "doubles2", new double[] { 5.8, 4.6, 3.5 } ); - final TestDoubles doubles3 = new TestDoubles( "doubles", "doubles3", new double[] { 5.9, 4.7, 3.6 } ); - final TestDoubles doubles4 = new TestDoubles( "doubles", "doubles4", new double[] { 5.10, 4.8, 3.7 } ); - final TestDoubles[] expectedDoubles = new TestDoubles[] { doubles1, doubles2, doubles3, doubles4 }; - assertArrayEquals( expectedDoubles, doubleArray ); - - final String[] stringArray = getAttribute( n5, new N5URL( "?objs#String" ), String[].class ); - final String[] expectedString = new String[] { "This", "is", "a", "test" }; - - final Integer[] integerArray = getAttribute( n5, new N5URL( "?objs#Integer" ), Integer[].class ); - final Integer[] expectedInteger = new Integer[] { 1, 2, 3, 4 }; - - final int[] intArray = getAttribute( n5, new N5URL( "?objs#int" ), int[].class ); - final int[] expectedInt = new int[] { 1, 2, 3, 4 }; - assertArrayEquals( expectedInt, intArray ); + public void testPathObject() throws IOException, URISyntaxException { + + final TestInts ints = getAttribute(n5, new N5URL("?objs#intsKey"), TestInts.class); + assertEquals(testObjInts.name, ints.name); + assertEquals(testObjInts.type, ints.type); + assertArrayEquals(testObjInts.t(), ints.t()); + + final TestDoubles doubles = getAttribute(n5, new N5URL("?objs#doublesKey"), TestDoubles.class); + assertEquals(testObjDoubles.name, doubles.name); + assertEquals(testObjDoubles.type, doubles.type); + assertArrayEquals(testObjDoubles.t(), doubles.t(), 1e-9); + + final TestDoubles[] doubleArray = getAttribute(n5, new N5URL("?objs#array"), TestDoubles[].class); + final TestDoubles doubles1 = new TestDoubles("doubles", "doubles1", new double[]{5.7, 4.5, 3.4}); + final TestDoubles doubles2 = new TestDoubles("doubles", "doubles2", new double[]{5.8, 4.6, 3.5}); + final TestDoubles doubles3 = new TestDoubles("doubles", "doubles3", new double[]{5.9, 4.7, 3.6}); + final TestDoubles doubles4 = new TestDoubles("doubles", "doubles4", new double[]{5.10, 4.8, 3.7}); + final TestDoubles[] expectedDoubles = new TestDoubles[]{doubles1, doubles2, doubles3, doubles4}; + assertArrayEquals(expectedDoubles, doubleArray); + + final String[] stringArray = getAttribute(n5, new N5URL("?objs#String"), String[].class); + final String[] expectedString = new String[]{"This", "is", "a", "test"}; + + final Integer[] integerArray = getAttribute(n5, new N5URL("?objs#Integer"), Integer[].class); + final Integer[] expectedInteger = new Integer[]{1, 2, 3, 4}; + + final int[] intArray = getAttribute(n5, new N5URL("?objs#int"), int[].class); + final int[] expectedInt = new int[]{1, 2, 3, 4}; + assertArrayEquals(expectedInt, intArray); } - private < K, V > boolean mapsEqual( Map< K, V > a, Map< K, V > b ) - { - if ( !a.keySet().equals( b.keySet() ) ) + private boolean mapsEqual(final Map a, final Map b) { + + if (!a.keySet().equals(b.keySet())) return false; - for ( K k : a.keySet() ) - { - if ( !a.get( k ).equals( b.get( k ) ) ) + for (final K k : a.keySet()) { + if (!a.get(k).equals(b.get(k))) return false; } return true; } - private static class TestObject< T > - { + private static class TestObject { + String type; String name; T t; - public TestObject( String type, String name, T t ) - { + public TestObject(final String type, final String name, final T t) { + this.name = name; this.type = type; this.t = t; } - public T t() - { + public T t() { + return t; } @Override - public boolean equals( Object obj ) - { - - if ( obj.getClass() == this.getClass() ) - { - final TestObject< ? > otherTestObject = ( TestObject< ? > ) obj; - return Objects.equals( name, otherTestObject.name ) - && Objects.equals( type, otherTestObject.type ) - && Objects.equals( t, otherTestObject.t ); + public boolean equals(final Object obj) { + + if (obj.getClass() == this.getClass()) { + final TestObject otherTestObject = (TestObject)obj; + return Objects.equals(name, otherTestObject.name) + && Objects.equals(type, otherTestObject.type) + && Objects.equals(t, otherTestObject.t); } return false; } } - public static class TestDoubles extends TestObject< double[] > - { - public TestDoubles( String type, String name, double[] t ) - { - super( type, name, t ); + public static class TestDoubles extends TestObject { + + public TestDoubles(final String type, final String name, final double[] t) { + + super(type, name, t); } @Override - public boolean equals( Object obj ) - { - - if ( obj.getClass() == this.getClass() ) - { - final TestDoubles otherTestObject = ( TestDoubles ) obj; - return Objects.equals( name, otherTestObject.name ) - && Objects.equals( type, otherTestObject.type ) - && Arrays.equals( t, otherTestObject.t ); + public boolean equals(final Object obj) { + + if (obj.getClass() == this.getClass()) { + final TestDoubles otherTestObject = (TestDoubles)obj; + return Objects.equals(name, otherTestObject.name) + && Objects.equals(type, otherTestObject.type) + && Arrays.equals(t, otherTestObject.t); } return false; } } - private static class TestInts extends TestObject< int[] > - { - public TestInts( String type, String name, int[] t ) - { - super( type, name, t ); + private static class TestInts extends TestObject { + + public TestInts(final String type, final String name, final int[] t) { + + super(type, name, t); } @Override - public boolean equals( Object obj ) - { - - if ( obj.getClass() == this.getClass() ) - { - final TestInts otherTestObject = ( TestInts ) obj; - return Objects.equals( name, otherTestObject.name ) - && Objects.equals( type, otherTestObject.type ) - && Arrays.equals( t, otherTestObject.t ); + public boolean equals(final Object obj) { + + if (obj.getClass() == this.getClass()) { + final TestInts otherTestObject = (TestInts)obj; + return Objects.equals(name, otherTestObject.name) + && Objects.equals(type, otherTestObject.type) + && Arrays.equals(t, otherTestObject.t); } return false; } From 63bf5f2e16ed765bc469dfa7f9a6696bb8c4ab6e Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Mon, 12 Jun 2023 09:53:11 -0400 Subject: [PATCH 210/243] fix: n5 on windows --- .../n5/CachedGsonKeyValueN5Reader.java | 9 ++-- .../n5/CachedGsonKeyValueN5Writer.java | 2 +- .../n5/FileSystemKeyValueAccess.java | 14 ++++-- .../saalfeldlab/n5/GsonKeyValueN5Reader.java | 4 +- .../saalfeldlab/n5/GsonKeyValueN5Writer.java | 2 +- .../saalfeldlab/n5/KeyValueAccess.java | 13 ++++-- .../saalfeldlab/n5/AbstractN5Test.java | 20 ++++---- .../n5/FileSystemKeyValueAccessTest.java | 46 +++++++++---------- .../org/janelia/saalfeldlab/n5/N5FSTest.java | 3 +- .../org/janelia/saalfeldlab/n5/UriTest.java | 19 +++----- 10 files changed, 68 insertions(+), 64 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Reader.java index 29133446..982a8764 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Reader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Reader.java @@ -245,14 +245,13 @@ default String getDataBlockPath( final String normalPath, final long... gridPosition) { - final String[] components = new String[gridPosition.length + 2]; - components[0] = getURI().getPath(); - components[1] = normalPath; - int i = 1; + final String[] components = new String[gridPosition.length + 1]; + components[0] = normalPath; + int i = 0; for (final long p : gridPosition) components[++i] = Long.toString(p); - return getKeyValueAccess().compose(components); + return getKeyValueAccess().compose(getURI(), components); } /** diff --git a/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Writer.java index ae7ee209..e6cdfee0 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Writer.java @@ -88,7 +88,7 @@ else if (getCache().exists(normalPath, N5KeyValueReader.ATTRIBUTES_JSON)) { } for (final String child : pathParts) { - final String childPath = getKeyValueAccess().compose(parent, child); + final String childPath = parent.isEmpty() ? child : parent + "/" + child; getCache().initializeNonemptyCache(childPath, N5KeyValueReader.ATTRIBUTES_JSON); getCache().updateCacheInfo(childPath, N5KeyValueReader.ATTRIBUTES_JSON); diff --git a/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java index cd165823..7e7206bc 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java @@ -280,8 +280,16 @@ public String normalize(final String path) { public URI uri(final String normalPath) throws URISyntaxException { // normalize make absolute the scheme specific part only - final URI uri = new URI(normalPath); - return new URI("file", normalize(new File(uri.getSchemeSpecificPart()).getAbsolutePath()), uri.getFragment()); + return new File(normalPath).toURI().normalize(); + + } + + @Override + public String compose(final URI uri, final String... components) { + final String[] uriComps = new String[components.length+1]; + System.arraycopy(components, 0, uriComps, 1, components.length); + uriComps[0] = fileSystem.provider().getPath(uri).toString(); + return compose(uriComps); } @Override @@ -291,7 +299,7 @@ public String compose(final String... components) { return null; if (components.length == 1) return fileSystem.getPath(components[0]).toString(); - return fileSystem.getPath(components[0], Arrays.copyOfRange(components, 1, components.length)).toString(); + return fileSystem.getPath(components[0], Arrays.copyOfRange(components, 1, components.length)).normalize().toString(); } @Override diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java index 3b0d4b41..96458354 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java @@ -156,7 +156,7 @@ default String getDataBlockPath( */ default String groupPath(final String normalGroupPath) { - return getKeyValueAccess().compose(getURI().getPath(), normalGroupPath); + return getKeyValueAccess().compose(getURI(), normalGroupPath); } /** @@ -169,6 +169,6 @@ default String groupPath(final String normalGroupPath) { */ default String attributesPath(final String normalPath) { - return getKeyValueAccess().compose(getURI().getPath(), normalPath, N5KeyValueReader.ATTRIBUTES_JSON); + return getKeyValueAccess().compose(getURI(), normalPath, N5KeyValueReader.ATTRIBUTES_JSON); } } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java index 3bdbf067..b9c66158 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java @@ -159,7 +159,7 @@ default void setAttributes( default boolean removeAttribute(final String groupPath, final String attributePath) throws N5Exception { final String normalPath = N5URL.normalizeGroupPath(groupPath); - final String absoluteNormalPath = getKeyValueAccess().compose(getURI().getPath(), normalPath); + final String absoluteNormalPath = getKeyValueAccess().compose(getURI(), normalPath); final String normalKey = N5URL.normalizeAttributePath(attributePath); if (!getKeyValueAccess().isDirectory(absoluteNormalPath)) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccess.java index 6d3b8a5f..77cf5297 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccess.java @@ -51,12 +51,19 @@ public interface KeyValueAccess { public String[] components(final String path); /** - * Compose a path from components. + * Compose a path from a base uri and subsequent components. * - * @param components - * the path components + * @param uri the base path uri + * @param components the path components * @return the path */ + public default String compose(final URI uri, final String... components) { + final String[] uriComponents = new String[components.length+1]; + System.arraycopy(components, 0, uriComponents, 1, components.length); + uriComponents[0] = uri.getPath(); + return compose(uriComponents); + } + public String compose(final String... components); /** diff --git a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java index 6b7f234f..fa97cf15 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java @@ -164,16 +164,14 @@ public static void rampDownAfterClass() throws IOException { @Test public void testCreateGroup() { - try { - n5.createGroup(groupName); - } catch (final N5Exception e) { - fail(e.getMessage()); - } - + n5.createGroup(groupName); final Path groupPath = Paths.get(groupName); - for (int i = 0; i < groupPath.getNameCount(); ++i) - if (!n5.exists(groupPath.subpath(0, i + 1).toString())) - fail("Group does not exist"); + String subGroup = ""; + for (int i = 0; i < groupPath.getNameCount(); ++i) { + subGroup = subGroup + "/" + groupPath.getName(i); + if (!n5.exists(subGroup)) + fail("Group does not exist: " + subGroup); + } } @Test @@ -789,7 +787,7 @@ public void testRemoveContainer() throws IOException, URISyntaxException { String location; try (final N5Writer n5 = createN5Writer()) { - location = n5.getURI().toString(); + location = n5.getURI().getPath(); assertNotNull(createN5Reader(location)); n5.remove(); assertThrows(Exception.class, () -> createN5Reader(location)); @@ -1074,7 +1072,7 @@ public void testVersion() throws NumberFormatException, IOException, URISyntaxEx assertFalse(N5Reader.VERSION.isCompatible(version)); assertThrows(N5Exception.N5IOException.class, () -> { - final String containerPath = writer.getURI().toString(); + final String containerPath = writer.getURI().getPath(); createN5Writer(containerPath); }); diff --git a/src/test/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccessTest.java b/src/test/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccessTest.java index a95f15b7..e902eb88 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccessTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccessTest.java @@ -6,6 +6,7 @@ import static org.junit.Assert.assertArrayEquals; import java.nio.file.FileSystems; +import java.nio.file.Paths; import java.util.Arrays; import org.junit.BeforeClass; @@ -18,34 +19,32 @@ */ public class FileSystemKeyValueAccessTest { - private static String[] testPaths = new String[] { - "/test/path/file", - "test/path/file", - "/test/path/file/", - "test/path/file/", - "/file", - "file", - "/file/", - "file/", - "/", - "" -// "", -// Paths.get("C:", "test", "path", "file").toString() + private static String root = FileSystems.getDefault().getRootDirectories().iterator().next().toString(); + private static String separator = FileSystems.getDefault().getSeparator(); + private static String[] testPaths = new String[]{ + Paths.get(root, "test", "path", "file").toString(), + Paths.get("test", "path", "file").toString(), + Paths.get(root, "test", "path", "file", separator).toString(), + Paths.get("test", "path", "file", separator).toString(), + Paths.get(root, "file").toString(), + Paths.get("file").toString(), + Paths.get(root, "file", separator).toString(), + Paths.get("file", separator).toString(), + Paths.get(root).toString(), + Paths.get("").toString() }; private static String[][] testPathComponents = new String[][] { - {"/", "test", "path", "file"}, + {root, "test", "path", "file"}, {"test", "path", "file"}, - {"/", "test", "path", "file"}, + {root, "test", "path", "file"}, {"test", "path", "file"}, - {"/", "file"}, + {root, "file"}, {"file"}, - {"/", "file"}, + {root, "file"}, {"file"}, - {"/"}, + {root}, {""} -// {""}, -// {"C:", "test", "path", "file"} }; /** @@ -59,11 +58,12 @@ public void testComponents() { final FileSystemKeyValueAccess access = new FileSystemKeyValueAccess(FileSystems.getDefault()); - for (int i = 0; i < testPaths.length; ++i) { + for (int i = 0; i < testPaths.length; ++i) { - System.out.println(String.format("%d: %s -> %s", i, testPaths[i], Arrays.toString(access.components(testPaths[i])))); + final String[] components = access.components(testPaths[i]); + System.out.println(String.format("%d: %s -> %s", i, testPaths[i], Arrays.toString(components))); - assertArrayEquals(testPathComponents[i], access.components(testPaths[i])); + assertArrayEquals(testPathComponents[i], components); } } } diff --git a/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java b/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java index 1ffe759d..4d91cde7 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java @@ -82,8 +82,7 @@ private static String tempN5PathName() { @Override protected String tempN5Location() throws URISyntaxException { - final String basePath = tempN5PathName(); - return new URI("file", null, basePath, null).toString(); + return tempN5PathName(); } @Override diff --git a/src/test/java/org/janelia/saalfeldlab/n5/UriTest.java b/src/test/java/org/janelia/saalfeldlab/n5/UriTest.java index b94c0413..69c578ce 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/UriTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/UriTest.java @@ -1,7 +1,6 @@ package org.janelia.saalfeldlab.n5; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.fail; import java.net.URI; import java.net.URISyntaxException; @@ -30,26 +29,20 @@ public void before() { } @Test - public void testUriParsing() { + public void testUriParsing() throws URISyntaxException { final URI uri = n5.getURI(); - try { assertEquals("Container URI must contain scheme", "file", uri.getScheme()); - assertEquals( - "Container URI must be absolute", + assertEquals("Container URI must be absolute", Paths.get(relativePath).toAbsolutePath().toString(), - uri.getPath()); + Paths.get(uri).toAbsolutePath().toString()); assertEquals("Container URI must be normalized 1", uri, kva.uri(relativeAbnormalPath)); assertEquals("Container URI must be normalized 2", uri, kva.uri(relativeAbnormalPath2)); - assertEquals("Container URI must be normalized 3", uri, kva.uri("file:" + relativePath)); - assertEquals("Container URI must be normalized 4", uri, kva.uri("file:" + relativeAbnormalPath)); - assertEquals("Container URI must be normalized 5", uri, kva.uri("file:" + relativeAbnormalPath2)); - - } catch (final URISyntaxException e) { - fail(e.getMessage()); - } +// assertEquals("Container URI must be normalized 3", uri, kva.uri("file:" + relativePath)); +// assertEquals("Container URI must be normalized 4", uri, kva.uri("file:" + relativeAbnormalPath)); +// assertEquals("Container URI must be normalized 5", uri, kva.uri("file:" + relativeAbnormalPath2)); } From 387d8c0d8b91ea08fed971896736749915635ba4 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Mon, 12 Jun 2023 15:03:00 -0400 Subject: [PATCH 211/243] refactor: N5URL -> N5URI --- .../n5/CachedGsonKeyValueN5Reader.java | 22 +-- .../n5/CachedGsonKeyValueN5Writer.java | 10 +- .../saalfeldlab/n5/GsonKeyValueN5Reader.java | 6 +- .../saalfeldlab/n5/GsonKeyValueN5Writer.java | 22 +-- .../janelia/saalfeldlab/n5/GsonN5Reader.java | 10 +- .../org/janelia/saalfeldlab/n5/GsonUtils.java | 8 +- .../saalfeldlab/n5/{N5URL.java => N5URI.java} | 77 ++++++----- .../org/janelia/saalfeldlab/n5/N5Writer.java | 6 +- .../saalfeldlab/n5/AbstractN5Test.java | 4 +- .../org/janelia/saalfeldlab/n5/N5URLTest.java | 130 +++++++++--------- .../saalfeldlab/n5/url/UrlAttributeTest.java | 104 +++++++------- 11 files changed, 203 insertions(+), 196 deletions(-) rename src/main/java/org/janelia/saalfeldlab/n5/{N5URL.java => N5URI.java} (88%) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Reader.java index 982a8764..d23c7614 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Reader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Reader.java @@ -62,7 +62,7 @@ default JsonElement getAttributesFromContainer(final String normalPathName, fina @Override default DatasetAttributes getDatasetAttributes(final String pathName) { - final String normalPath = N5URL.normalizeGroupPath(pathName); + final String normalPath = N5URI.normalizeGroupPath(pathName); final JsonElement attributes; if (!datasetExists(pathName)) @@ -79,7 +79,7 @@ default DatasetAttributes getDatasetAttributes(final String pathName) { default DatasetAttributes normalGetDatasetAttributes(final String pathName) throws N5Exception.N5IOException { - final String normalPath = N5URL.normalizeGroupPath(pathName); + final String normalPath = N5URI.normalizeGroupPath(pathName); final JsonElement attributes = GsonKeyValueN5Reader.super.getAttributes(normalPath); return createDatasetAttributes(attributes); } @@ -90,8 +90,8 @@ default T getAttribute( final String key, final Class clazz) throws N5Exception { - final String normalPathName = N5URL.normalizeGroupPath(pathName); - final String normalizedAttributePath = N5URL.normalizeAttributePath(key); + final String normalPathName = N5URI.normalizeGroupPath(pathName); + final String normalizedAttributePath = N5URI.normalizeAttributePath(key); final JsonElement attributes; if (cacheMeta()) { @@ -108,8 +108,8 @@ default T getAttribute( final String key, final Type type) throws N5Exception { - final String normalPathName = N5URL.normalizeGroupPath(pathName); - final String normalizedAttributePath = N5URL.normalizeAttributePath(key); + final String normalPathName = N5URI.normalizeGroupPath(pathName); + final String normalizedAttributePath = N5URI.normalizeAttributePath(key); JsonElement attributes; if (cacheMeta()) { attributes = getCache().getAttributes(normalPathName, N5KeyValueReader.ATTRIBUTES_JSON); @@ -122,7 +122,7 @@ default T getAttribute( @Override default boolean exists(final String pathName) { - final String normalPathName = N5URL.normalizeGroupPath(pathName); + final String normalPathName = N5URI.normalizeGroupPath(pathName); if (cacheMeta()) return getCache().isGroup(normalPathName, N5KeyValueReader.ATTRIBUTES_JSON); else { @@ -142,7 +142,7 @@ default boolean existsFromContainer(final String normalPathName, final String no @Override default boolean groupExists(final String pathName) { - final String normalPathName = N5URL.normalizeGroupPath(pathName); + final String normalPathName = N5URI.normalizeGroupPath(pathName); if (cacheMeta()) return getCache().isGroup(normalPathName, null); else { @@ -165,7 +165,7 @@ default boolean isGroupFromAttributes(final String normalCacheKey, final JsonEle @Override default boolean datasetExists(final String pathName) throws N5Exception.N5IOException { - final String normalPathName = N5URL.normalizeGroupPath(pathName); + final String normalPathName = N5URI.normalizeGroupPath(pathName); if (cacheMeta()) { return getCache().isDataset(normalPathName, N5KeyValueReader.ATTRIBUTES_JSON); } @@ -195,7 +195,7 @@ default boolean isDatasetFromAttributes(final String normalCacheKey, final JsonE @Override default JsonElement getAttributes(final String pathName) throws N5Exception.N5IOException { - final String groupPath = N5URL.normalizeGroupPath(pathName); + final String groupPath = N5URI.normalizeGroupPath(pathName); /* If cached, return the cache */ if (cacheMeta()) { @@ -208,7 +208,7 @@ default JsonElement getAttributes(final String pathName) throws N5Exception.N5IO @Override default String[] list(final String pathName) throws N5Exception.N5IOException { - final String normalPath = N5URL.normalizeGroupPath(pathName); + final String normalPath = N5URI.normalizeGroupPath(pathName); if (cacheMeta()) { return getCache().list(normalPath); } else { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Writer.java index e6cdfee0..a54023b9 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Writer.java @@ -53,7 +53,7 @@ default void setVersion(final String path) throws N5Exception { @Override default void createGroup(final String path) throws N5Exception { - final String normalPath = N5URL.normalizeGroupPath(path); + final String normalPath = N5URI.normalizeGroupPath(path); // TODO: John document this! // if you are a group, avoid hitting the backend // if something exists, be safe @@ -82,7 +82,7 @@ else if (getCache().exists(normalPath, N5KeyValueReader.ATTRIBUTES_JSON)) { // check all nodes that are parents of the added node, if they have // a children set, add the new child to it String[] pathParts = getKeyValueAccess().components(normalPath); - String parent = N5URL.normalizeGroupPath("/"); + String parent = N5URI.normalizeGroupPath("/"); if (pathParts.length == 0) { pathParts = new String[]{""}; } @@ -145,8 +145,8 @@ default boolean remove(final String path) throws N5Exception { * normalizeGroupPath again the below duplicates code, but avoids extra * work */ - final String normalPath = N5URL.normalizeGroupPath(path); - final String groupPath = groupPath(normalPath); + final String normalPath = N5URI.normalizeGroupPath(path); + final String groupPath = absoluteGroupPath(normalPath); try { if (getKeyValueAccess().isDirectory(groupPath)) getKeyValueAccess().delete(groupPath); @@ -158,7 +158,7 @@ default boolean remove(final String path) throws N5Exception { final String[] pathParts = getKeyValueAccess().components(normalPath); final String parent; if (pathParts.length <= 1) { - parent = N5URL.normalizeGroupPath("/"); + parent = N5URI.normalizeGroupPath("/"); } else { final int parentPathLength = pathParts.length - 1; parent = getKeyValueAccess().compose(Arrays.copyOf(pathParts, parentPathLength)); diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java index 96458354..69adf954 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java @@ -50,7 +50,7 @@ default boolean groupExists(final String normalPath) { @Override default boolean exists(final String pathName) { - final String normalPath = N5URL.normalizeGroupPath(pathName); + final String normalPath = N5URI.normalizeGroupPath(pathName); return groupExists(normalPath) || datasetExists(normalPath); } @@ -72,8 +72,8 @@ default boolean datasetExists(final String pathName) throws N5Exception { @Override default JsonElement getAttributes(final String pathName) throws N5Exception { - final String groupPath = N5URL.normalizeGroupPath(pathName); - final String attributesPath = attributesPath(groupPath); + final String groupPath = N5URI.normalizeGroupPath(pathName); + final String attributesPath = absoluteAttributesPath(groupPath); if (!getKeyValueAccess().isFile(attributesPath)) return null; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java index b9c66158..c9c26a65 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java @@ -69,7 +69,7 @@ static String initializeContainer( @Override default void createGroup(final String path) throws N5Exception { - final String normalPath = N5URL.normalizeGroupPath(path); + final String normalPath = N5URI.normalizeGroupPath(path); try { getKeyValueAccess().createDirectories(groupPath(normalPath)); } catch (final IOException e) { @@ -106,7 +106,7 @@ default void setAttributes( final String path, final JsonElement attributes) throws N5Exception { - final String normalPath = N5URL.normalizeGroupPath(path); + final String normalPath = N5URI.normalizeGroupPath(path); if (!exists(normalPath)) throw new N5IOException("" + normalPath + " is not a group or dataset."); @@ -148,7 +148,7 @@ default void setAttributes( final String path, final Map attributes) throws N5Exception { - final String normalPath = N5URL.normalizeGroupPath(path); + final String normalPath = N5URI.normalizeGroupPath(path); if (!exists(normalPath)) throw new N5IOException("" + normalPath + " is not a group or dataset."); @@ -158,9 +158,9 @@ default void setAttributes( @Override default boolean removeAttribute(final String groupPath, final String attributePath) throws N5Exception { - final String normalPath = N5URL.normalizeGroupPath(groupPath); + final String normalPath = N5URI.normalizeGroupPath(groupPath); final String absoluteNormalPath = getKeyValueAccess().compose(getURI(), normalPath); - final String normalKey = N5URL.normalizeAttributePath(attributePath); + final String normalKey = N5URI.normalizeAttributePath(attributePath); if (!getKeyValueAccess().isDirectory(absoluteNormalPath)) return false; @@ -181,8 +181,8 @@ default boolean removeAttribute(final String groupPath, final String attributePa @Override default T removeAttribute(final String pathName, final String key, final Class cls) throws N5Exception { - final String normalPath = N5URL.normalizeGroupPath(pathName); - final String normalKey = N5URL.normalizeAttributePath(key); + final String normalPath = N5URI.normalizeGroupPath(pathName); + final String normalKey = N5URI.normalizeAttributePath(key); final JsonElement attributes = getAttributes(normalPath); final T obj = GsonUtils.removeAttribute(attributes, normalKey, cls, getGson()); @@ -195,10 +195,10 @@ default T removeAttribute(final String pathName, final String key, final Cla @Override default boolean removeAttributes(final String pathName, final List attributes) throws N5Exception { - final String normalPath = N5URL.normalizeGroupPath(pathName); + final String normalPath = N5URI.normalizeGroupPath(pathName); boolean removed = false; for (final String attribute : attributes) { - final String normalKey = N5URL.normalizeAttributePath(attribute); + final String normalKey = N5URI.normalizeAttributePath(attribute); removed |= removeAttribute(normalPath, normalKey); } return removed; @@ -223,8 +223,8 @@ default void writeBlock( @Override default boolean remove(final String path) throws N5Exception { - final String normalPath = N5URL.normalizeGroupPath(path); - final String groupPath = groupPath(normalPath); + final String normalPath = N5URI.normalizeGroupPath(path); + final String groupPath = absoluteGroupPath(normalPath); try { if (getKeyValueAccess().isDirectory(groupPath)) getKeyValueAccess().delete(groupPath); diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonN5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonN5Reader.java index b3295778..dbec5973 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonN5Reader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonN5Reader.java @@ -49,7 +49,7 @@ default Map> listAttributes(final String pathName) throws N5Exc @Override default DatasetAttributes getDatasetAttributes(final String pathName) throws N5Exception { - final String normalPath = N5URL.normalizeGroupPath(pathName); + final String normalPath = N5URI.normalizeGroupPath(pathName); final JsonElement attributes = getAttributes(normalPath); return createDatasetAttributes(attributes); } @@ -85,8 +85,8 @@ default DatasetAttributes createDatasetAttributes(final JsonElement attributes) @Override default T getAttribute(final String pathName, final String key, final Class clazz) throws N5Exception { - final String normalPathName = N5URL.normalizeGroupPath(pathName); - final String normalizedAttributePath = N5URL.normalizeAttributePath(key); + final String normalPathName = N5URI.normalizeGroupPath(pathName); + final String normalizedAttributePath = N5URI.normalizeAttributePath(key); final JsonElement attributes = getAttributes(normalPathName); return GsonUtils.readAttribute(attributes, normalizedAttributePath, clazz, getGson()); @@ -95,8 +95,8 @@ default T getAttribute(final String pathName, final String key, final Class< @Override default T getAttribute(final String pathName, final String key, final Type type) throws N5Exception { - final String normalPathName = N5URL.normalizeGroupPath(pathName); - final String normalizedAttributePath = N5URL.normalizeAttributePath(key); + final String normalPathName = N5URI.normalizeGroupPath(pathName); + final String normalizedAttributePath = N5URI.normalizeAttributePath(key); final JsonElement attributes = getAttributes(normalPathName); return GsonUtils.readAttribute(attributes, normalizedAttributePath, type, getGson()); } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonUtils.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonUtils.java index 5ba8e596..353ae550 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonUtils.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonUtils.java @@ -169,7 +169,7 @@ static JsonElement getAttribute(JsonElement root, final String normalizedAttribu final JsonObject jsonObject = root.getAsJsonObject(); root = jsonObject.get(pathPartWithoutEscapeCharacters); } else { - final Matcher matcher = N5URL.ARRAY_INDEX.matcher(pathPart); + final Matcher matcher = N5URI.ARRAY_INDEX.matcher(pathPart); if (root != null && root.isJsonArray() && matcher.matches()) { final int index = Integer.parseInt(matcher.group().replace("[", "").replace("]", "")); final JsonArray jsonArray = root.getAsJsonArray(); @@ -477,7 +477,7 @@ static JsonElement removeAttribute(JsonElement root, final String normalizedAttr jsonObject.remove(pathPartWithoutEscapeCharacters); } } else { - final Matcher matcher = N5URL.ARRAY_INDEX.matcher(pathPart); + final Matcher matcher = N5URI.ARRAY_INDEX.matcher(pathPart); if (root != null && root.isJsonArray() && matcher.matches()) { final int index = Integer.parseInt(matcher.group().replace("[", "").replace("]", "")); final JsonArray jsonArray = root.getAsJsonArray(); @@ -556,7 +556,7 @@ static void writeAttributes( static JsonElement insertAttributes(JsonElement root, final Map attributes, final Gson gson) { for (final Map.Entry attribute : attributes.entrySet()) { - root = insertAttribute(root, N5URL.normalizeAttributePath(attribute.getKey()), attribute.getValue(), gson); + root = insertAttribute(root, N5URI.normalizeAttributePath(attribute.getKey()), attribute.getValue(), gson); } return root; } @@ -567,7 +567,7 @@ static JsonElement insertAttribute( final T attribute, final Gson gson) { - LinkedAttributePathToken pathToken = N5URL.getAttributePathTokens(normalizedAttributePath); + LinkedAttributePathToken pathToken = N5URI.getAttributePathTokens(normalizedAttributePath); /* No path to traverse or build; just return the value */ if (pathToken == null) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java b/src/main/java/org/janelia/saalfeldlab/n5/N5URI.java similarity index 88% rename from src/main/java/org/janelia/saalfeldlab/n5/N5URL.java rename to src/main/java/org/janelia/saalfeldlab/n5/N5URI.java index 90d991b0..52001450 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5URL.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5URI.java @@ -17,7 +17,7 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; -public class N5URL { +public class N5URI { private static final Charset UTF8 = Charset.forName("UTF-8"); public static final Pattern ARRAY_INDEX = Pattern.compile("\\[([0-9]+)]"); @@ -27,12 +27,12 @@ public class N5URL { private final String group; private final String attribute; - public N5URL(final String uri) throws URISyntaxException { + public N5URI(final String uri) throws URISyntaxException { this(encodeAsUri(uri)); } - public N5URL(final URI uri) { + public N5URI(final URI uri) { this.uri = uri; scheme = uri.getScheme() == null ? null : uri.getScheme(); @@ -87,9 +87,9 @@ public String normalizeAttributePath() { } /** - * Parse this {@link N5URL} as a {@link LinkedAttributePathToken}. + * Parse this {@link N5URI} as a {@link LinkedAttributePathToken}. * - * @see N5URL#getAttributePathTokens(String) + * @see N5URI#getAttributePathTokens(String) * @return the linked attribute path token */ public LinkedAttributePathToken getAttributePathTokens() { @@ -178,11 +178,11 @@ private String getSchemeSpecificPartWithoutQuery() { } /** - * N5URL is always considered absolute if a scheme is provided. - * If no scheme is provided, the N5URL is absolute if it starts with either + * N5URI is always considered absolute if a scheme is provided. + * If no scheme is provided, the N5URI is absolute if it starts with either * "/" or "[A-Z]:" * - * @return if the path for this N5URL is absolute + * @return if the path for this N5URI is absolute */ public boolean isAbsolute() { @@ -197,18 +197,18 @@ public boolean isAbsolute() { } /** - * Generate a new N5URL which is the result of resolving {@link N5URL - * relativeN5Url} to this {@link N5URL}. - * If relativeN5Url is not relative to this N5URL, then the resulting N5URL + * Generate a new N5URI which is the result of resolving {@link N5URI + * relativeN5Url} to this {@link N5URI}. + * If relativeN5Url is not relative to this N5URI, then the resulting N5URI * is equivalent to relativeN5Url. * * @param relativeN5Url - * N5URL to resolve against ourselves + * N5URI to resolve against ourselves * @return the result of the resolution. * @throws URISyntaxException * if the uri is malformed */ - public N5URL resolve(final N5URL relativeN5Url) throws URISyntaxException { + public N5URI resolve(final N5URI relativeN5Url) throws URISyntaxException { final URI thisUri = uri; final URI relativeUri = relativeN5Url.uri; @@ -229,7 +229,7 @@ public N5URL resolve(final N5URL relativeN5Url) throws URISyntaxException { .append(relativeUri.getPath()) .append(relativeN5Url.getGroupPart()) .append(relativeN5Url.getAttributePart()); - return new N5URL(newUri.toString()); + return new N5URI(newUri.toString()); } final String thisAuthority = thisUri.getAuthority(); if (thisAuthority != null) { @@ -245,7 +245,7 @@ public N5URL resolve(final N5URL relativeN5Url) throws URISyntaxException { .append(path) .append(relativeN5Url.getGroupPart()) .append(relativeN5Url.getAttributePart()); - return new N5URL(newUri.toString()); + return new N5URI(newUri.toString()); } newUri.append(thisUri.getPath()); @@ -258,7 +258,7 @@ public N5URL resolve(final N5URL relativeN5Url) throws URISyntaxException { newUri.append(relativeN5Url.getGroupPart()); } newUri.append(relativeN5Url.getAttributePart()); - return new N5URL(newUri.toString()); + return new N5URI(newUri.toString()); } newUri.append(this.getGroupPart()); @@ -270,17 +270,17 @@ public N5URL resolve(final N5URL relativeN5Url) throws URISyntaxException { newUri.append(relativeN5Url.getAttributePart()); } - return new N5URL(newUri.toString()); + return new N5URI(newUri.toString()); } newUri.append(this.getAttributePart()); - return new N5URL(newUri.toString()); + return new N5URI(newUri.toString()); } /** - * Generate a new N5URL which is the result of resolving {@link URI - * relativeUri} to this {@link N5URL}. - * If relativeUri is not relative to this N5URL, then the resulting N5URL is + * Generate a new N5URI which is the result of resolving {@link URI + * relativeUri} to this {@link N5URI}. + * If relativeUri is not relative to this N5URI, then the resulting N5URI is * equivalent to relativeUri. * * @param relativeUri @@ -289,15 +289,15 @@ public N5URL resolve(final N5URL relativeN5Url) throws URISyntaxException { * @throws URISyntaxException * if the uri is malformed */ - public N5URL resolve(final URI relativeUri) throws URISyntaxException { + public N5URI resolve(final URI relativeUri) throws URISyntaxException { - return resolve(new N5URL(relativeUri)); + return resolve(new N5URI(relativeUri)); } /** - * Generate a new N5URL which is the result of resolving {@link String - * relativeString} to this {@link N5URL} - * If relativeString is not relative to this N5URL, then the resulting N5URL + * Generate a new N5URI which is the result of resolving {@link String + * relativeString} to this {@link N5URI} + * If relativeString is not relative to this N5URI, then the resulting N5URI * is equivalent to relativeString. * * @param relativeString @@ -306,20 +306,24 @@ public N5URL resolve(final URI relativeUri) throws URISyntaxException { * @throws URISyntaxException * if the uri is malformed */ - public N5URL resolve(final String relativeString) throws URISyntaxException { + public N5URI resolve(final String relativeString) throws URISyntaxException { - return resolve(new N5URL(relativeString)); + return resolve(new N5URI(relativeString)); } /** - * Normalize a path, resulting in removal of redundant "/", "./", and + * Normalize a POSIX path, resulting in removal of redundant "/", "./", and * resolution of relative "../". + *

    + * NOTE: currently a private helper method only used by {@link N5URI#normalizeGroupPath(String)}. + * It's safe to do in that case since relative group paths should always be POSIX compliant. + * A new helper method to understand other path types (e.g. Windows) may be necessary eventually. * * @param path * to normalize * @return the normalized path */ - public static String normalizePath(String path) { + private static String normalizePath(String path) { path = path == null ? "" : path; final char[] pathChars = path.toCharArray(); @@ -403,7 +407,7 @@ public static String normalizeGroupPath(final String path) { * Alternatively, could do something like the below in every * KeyValueReader implementation * - * return keyValueAccess.relativize( N5URL.normalizeGroupPath(path), + * return keyValueAccess.relativize( N5URI.normalizeGroupPath(path), * basePath); * * has to be in the implementations, since KeyValueAccess doesn't have a @@ -553,7 +557,7 @@ public static URI encodeAsUri(final String uri) throws URISyntaxException { } /** - * Generate an {@link N5URL} from a container, group, and attribute + * Generate an {@link N5URI} from a container, group, and attribute * * @param container * of the N5Url @@ -561,11 +565,11 @@ public static URI encodeAsUri(final String uri) throws URISyntaxException { * of the N5Url * @param attribute * of the N5Url - * @return the {@link N5URL} + * @return the {@link N5URI} * @throws URISyntaxException * if the uri is malformed */ - public static N5URL from( + public static N5URI from( final String container, final String group, final String attribute) throws URISyntaxException { @@ -573,7 +577,7 @@ public static N5URL from( final String containerPart = container != null ? container : ""; final String groupPart = group != null ? "?" + group : ""; final String attributePart = attribute != null ? "#" + attribute : ""; - return new N5URL(containerPart + groupPart + attributePart); + return new N5URI(containerPart + groupPart + attributePart); } /** @@ -581,6 +585,7 @@ public static N5URL from( * * @see URI#decode(char) */ + @SuppressWarnings("JavadocReference") private static int decode(final char c) { if ((c >= '0') && (c <= '9')) @@ -598,6 +603,7 @@ private static int decode(final char c) { * * @see URI#decode(char, char) */ + @SuppressWarnings("JavadocReference") private static byte decode(final char c1, final char c2) { return (byte)(((decode(c1) & 0xf) << 4) @@ -620,6 +626,7 @@ private static byte decode(final char c1, final char c2) { * * @see URI#decode(char, char) */ + @SuppressWarnings("JavadocReference") private static String decodeFragment(final String rawFragment) { if (rawFragment == null) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java index 172580a9..a7c5a000 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java @@ -115,10 +115,10 @@ void setAttributes( */ default boolean removeAttributes(final String groupPath, final List attributePaths) throws N5Exception { - final String normalPath = N5URL.normalizeGroupPath(groupPath); + final String normalPath = N5URI.normalizeGroupPath(groupPath); boolean removed = false; for (final String attribute : attributePaths) { - final String normalKey = N5URL.normalizeAttributePath(attribute); + final String normalKey = N5URI.normalizeAttributePath(attribute); removed |= removeAttribute(normalPath, attribute); } return removed; @@ -201,7 +201,7 @@ default void createDataset( final String datasetPath, final DatasetAttributes datasetAttributes) throws N5Exception { - final String normalPath = N5URL.normalizeGroupPath(datasetPath); + final String normalPath = N5URI.normalizeGroupPath(datasetPath); createGroup(normalPath); setDatasetAttributes(normalPath, datasetAttributes); } diff --git a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java index fa97cf15..ab721c47 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java @@ -1185,8 +1185,8 @@ protected static void addAndTest(final N5Writer writer, final ArrayList { try { - final String normalizedTestKey = N5URL.from(null, "", test.attributePath).normalizeAttributePath().replaceAll("^/", ""); - final String normalizedTestDataKey = N5URL.from(null, "", testData.attributePath).normalizeAttributePath().replaceAll("^/", ""); + final String normalizedTestKey = N5URI.from(null, "", test.attributePath).normalizeAttributePath().replaceAll("^/", ""); + final String normalizedTestDataKey = N5URI.from(null, "", testData.attributePath).normalizeAttributePath().replaceAll("^/", ""); return normalizedTestKey.equals(normalizedTestDataKey); } catch (final URISyntaxException e) { throw new RuntimeException(e); diff --git a/src/test/java/org/janelia/saalfeldlab/n5/N5URLTest.java b/src/test/java/org/janelia/saalfeldlab/n5/N5URLTest.java index a88af4ff..f8e626fe 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/N5URLTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/N5URLTest.java @@ -12,77 +12,77 @@ public class N5URLTest { @Test public void testAttributePath() { - assertEquals("/a/b/c/d/e", N5URL.normalizeAttributePath("/a/b/c/d/e")); - assertEquals("/a/b/c/d/e", N5URL.normalizeAttributePath("/a/b/c/d/e/")); - assertEquals("/", N5URL.normalizeAttributePath("/")); - assertEquals("", N5URL.normalizeAttributePath("")); - assertEquals("/", N5URL.normalizeAttributePath("/a/..")); - assertEquals("/", N5URL.normalizeAttributePath("/a/../b/../c/d/../..")); - assertEquals("/", N5URL.normalizeAttributePath("/a/../b/../c/d/../..")); - assertEquals("/", N5URL.normalizeAttributePath("/a/../b/../c/d/../..")); - assertEquals("/", N5URL.normalizeAttributePath("/./././././")); - assertEquals("/", N5URL.normalizeAttributePath("/./././././.")); - assertEquals("", N5URL.normalizeAttributePath("./././././")); - assertEquals("a.", N5URL.normalizeAttributePath("./a././././")); - assertEquals("\\\\.", N5URL.normalizeAttributePath("./a./../\\\\././.")); - assertEquals("a./\\\\.", N5URL.normalizeAttributePath("./a./\\\\././.")); - assertEquals("/a./\\\\.", N5URL.normalizeAttributePath("/./a./\\\\././.")); - - assertEquals("/a/[0]/b/[0]", N5URL.normalizeAttributePath("/a/[0]/b[0]/")); - assertEquals("[0]", N5URL.normalizeAttributePath("[0]")); - assertEquals("/[0]", N5URL.normalizeAttributePath("/[0]/")); - assertEquals("/a/[0]", N5URL.normalizeAttributePath("/a[0]/")); - assertEquals("/[0]/b", N5URL.normalizeAttributePath("/[0]b/")); - - assertEquals("[b]", N5URL.normalizeAttributePath("[b]")); - assertEquals("a[b]c", N5URL.normalizeAttributePath("a[b]c")); - assertEquals("a[bc", N5URL.normalizeAttributePath("a[bc")); - assertEquals("ab]c", N5URL.normalizeAttributePath("ab]c")); - assertEquals("a[b00]c", N5URL.normalizeAttributePath("a[b00]c")); - - assertEquals("[ ]", N5URL.normalizeAttributePath("[ ]")); - assertEquals("[\n]", N5URL.normalizeAttributePath("[\n]")); - assertEquals("[\t]", N5URL.normalizeAttributePath("[\t]")); - assertEquals("[\r\n]", N5URL.normalizeAttributePath("[\r\n]")); - assertEquals("[ ][\n][\t][ \t \n \r\n][\r\n]", N5URL.normalizeAttributePath("[ ][\n][\t][ \t \n \r\n][\r\n]")); - assertEquals("[\\]", N5URL.normalizeAttributePath("[\\]")); - - assertEquals("let's/try/a/real/case/with spaces", N5URL.normalizeAttributePath("let's/try/a/real/case/with spaces/")); - assertEquals("let's/try/a/real/case/with spaces", N5URL.normalizeAttributePath("let's/try/a/real/////case////with spaces/")); - assertEquals("../first/relative/a/wd/.w/asd", N5URL.normalizeAttributePath("../first/relative/test/../a/b/.././wd///.w/asd")); - assertEquals("..", N5URL.normalizeAttributePath("../result/../only/../single/..")); - assertEquals("../..", N5URL.normalizeAttributePath("../result/../multiple/../..")); - - String normalizedPath = N5URL.normalizeAttributePath("let's/try/a/some/////with/ / //white spaces/"); - assertEquals("Normalizing a normal path should be the identity", normalizedPath, N5URL.normalizeAttributePath(normalizedPath)); + assertEquals("/a/b/c/d/e", N5URI.normalizeAttributePath("/a/b/c/d/e")); + assertEquals("/a/b/c/d/e", N5URI.normalizeAttributePath("/a/b/c/d/e/")); + assertEquals("/", N5URI.normalizeAttributePath("/")); + assertEquals("", N5URI.normalizeAttributePath("")); + assertEquals("/", N5URI.normalizeAttributePath("/a/..")); + assertEquals("/", N5URI.normalizeAttributePath("/a/../b/../c/d/../..")); + assertEquals("/", N5URI.normalizeAttributePath("/a/../b/../c/d/../..")); + assertEquals("/", N5URI.normalizeAttributePath("/a/../b/../c/d/../..")); + assertEquals("/", N5URI.normalizeAttributePath("/./././././")); + assertEquals("/", N5URI.normalizeAttributePath("/./././././.")); + assertEquals("", N5URI.normalizeAttributePath("./././././")); + assertEquals("a.", N5URI.normalizeAttributePath("./a././././")); + assertEquals("\\\\.", N5URI.normalizeAttributePath("./a./../\\\\././.")); + assertEquals("a./\\\\.", N5URI.normalizeAttributePath("./a./\\\\././.")); + assertEquals("/a./\\\\.", N5URI.normalizeAttributePath("/./a./\\\\././.")); + + assertEquals("/a/[0]/b/[0]", N5URI.normalizeAttributePath("/a/[0]/b[0]/")); + assertEquals("[0]", N5URI.normalizeAttributePath("[0]")); + assertEquals("/[0]", N5URI.normalizeAttributePath("/[0]/")); + assertEquals("/a/[0]", N5URI.normalizeAttributePath("/a[0]/")); + assertEquals("/[0]/b", N5URI.normalizeAttributePath("/[0]b/")); + + assertEquals("[b]", N5URI.normalizeAttributePath("[b]")); + assertEquals("a[b]c", N5URI.normalizeAttributePath("a[b]c")); + assertEquals("a[bc", N5URI.normalizeAttributePath("a[bc")); + assertEquals("ab]c", N5URI.normalizeAttributePath("ab]c")); + assertEquals("a[b00]c", N5URI.normalizeAttributePath("a[b00]c")); + + assertEquals("[ ]", N5URI.normalizeAttributePath("[ ]")); + assertEquals("[\n]", N5URI.normalizeAttributePath("[\n]")); + assertEquals("[\t]", N5URI.normalizeAttributePath("[\t]")); + assertEquals("[\r\n]", N5URI.normalizeAttributePath("[\r\n]")); + assertEquals("[ ][\n][\t][ \t \n \r\n][\r\n]", N5URI.normalizeAttributePath("[ ][\n][\t][ \t \n \r\n][\r\n]")); + assertEquals("[\\]", N5URI.normalizeAttributePath("[\\]")); + + assertEquals("let's/try/a/real/case/with spaces", N5URI.normalizeAttributePath("let's/try/a/real/case/with spaces/")); + assertEquals("let's/try/a/real/case/with spaces", N5URI.normalizeAttributePath("let's/try/a/real/////case////with spaces/")); + assertEquals("../first/relative/a/wd/.w/asd", N5URI.normalizeAttributePath("../first/relative/test/../a/b/.././wd///.w/asd")); + assertEquals("..", N5URI.normalizeAttributePath("../result/../only/../single/..")); + assertEquals("../..", N5URI.normalizeAttributePath("../result/../multiple/../..")); + + String normalizedPath = N5URI.normalizeAttributePath("let's/try/a/some/////with/ / //white spaces/"); + assertEquals("Normalizing a normal path should be the identity", normalizedPath, N5URI.normalizeAttributePath(normalizedPath)); } @Test public void testEscapedAttributePaths() { - assertEquals("\\/a\\/b\\/c\\/d\\/e", N5URL.normalizeAttributePath("\\/a\\/b\\/c\\/d\\/e")); - assertEquals("/a\\\\/b/c", N5URL.normalizeAttributePath("/a\\\\/b/c")); - assertEquals("a[b]\\[10]", N5URL.normalizeAttributePath("a[b]\\[10]")); - assertEquals("\\[10]", N5URL.normalizeAttributePath("\\[10]")); - assertEquals("a/[0]/\\[10]b", N5URL.normalizeAttributePath("a[0]\\[10]b")); - assertEquals("[\\[10]\\[20]]", N5URL.normalizeAttributePath("[\\[10]\\[20]]")); - assertEquals("[\\[10]/[20]/]", N5URL.normalizeAttributePath("[\\[10][20]]")); - assertEquals("\\/", N5URL.normalizeAttributePath("\\/")); + assertEquals("\\/a\\/b\\/c\\/d\\/e", N5URI.normalizeAttributePath("\\/a\\/b\\/c\\/d\\/e")); + assertEquals("/a\\\\/b/c", N5URI.normalizeAttributePath("/a\\\\/b/c")); + assertEquals("a[b]\\[10]", N5URI.normalizeAttributePath("a[b]\\[10]")); + assertEquals("\\[10]", N5URI.normalizeAttributePath("\\[10]")); + assertEquals("a/[0]/\\[10]b", N5URI.normalizeAttributePath("a[0]\\[10]b")); + assertEquals("[\\[10]\\[20]]", N5URI.normalizeAttributePath("[\\[10]\\[20]]")); + assertEquals("[\\[10]/[20]/]", N5URI.normalizeAttributePath("[\\[10][20]]")); + assertEquals("\\/", N5URI.normalizeAttributePath("\\/")); } @Test public void testIsAbsolute() throws URISyntaxException { /* Always true if scheme provided */ - assertTrue(new N5URL("file:///a/b/c").isAbsolute()); - assertTrue(new N5URL("file://C:\\\\a\\\\b\\\\c").isAbsolute()); + assertTrue(new N5URI("file:///a/b/c").isAbsolute()); + assertTrue(new N5URI("file://C:\\\\a\\\\b\\\\c").isAbsolute()); /* Unix Paths*/ - assertTrue(new N5URL("/a/b/c").isAbsolute()); - assertFalse(new N5URL("a/b/c").isAbsolute()); + assertTrue(new N5URI("/a/b/c").isAbsolute()); + assertFalse(new N5URI("a/b/c").isAbsolute()); /* Windows Paths*/ - assertTrue(new N5URL("C:\\\\a\\\\b\\\\c").isAbsolute()); - assertFalse(new N5URL("a\\\\b\\\\c").isAbsolute()); + assertTrue(new N5URI("C:\\\\a\\\\b\\\\c").isAbsolute()); + assertFalse(new N5URI("a\\\\b\\\\c").isAbsolute()); } @Test @@ -90,27 +90,27 @@ public void testGetRelative() throws URISyntaxException { assertEquals( "/a/b/c/d?e#f", - new N5URL("/a/b/c").resolve("d?e#f").toString()); + new N5URI("/a/b/c").resolve("d?e#f").toString()); assertEquals( "/d?e#f", - new N5URL("/a/b/c").resolve("/d?e#f").toString()); + new N5URI("/a/b/c").resolve("/d?e#f").toString()); assertEquals( "file:/a/b/c", - new N5URL("s3://janelia-cosem-datasets/jrc_hela-3/jrc_hela-3.n5").resolve("file:/a/b/c").toString()); + new N5URI("s3://janelia-cosem-datasets/jrc_hela-3/jrc_hela-3.n5").resolve("file:/a/b/c").toString()); assertEquals( "s3://janelia-cosem-datasets/a/b/c", - new N5URL("s3://janelia-cosem-datasets/jrc_hela-3/jrc_hela-3.n5").resolve("/a/b/c").toString()); + new N5URI("s3://janelia-cosem-datasets/jrc_hela-3/jrc_hela-3.n5").resolve("/a/b/c").toString()); assertEquals( "s3://janelia-cosem-datasets/a/b/c?d/e#f/g", - new N5URL("s3://janelia-cosem-datasets/jrc_hela-3/jrc_hela-3.n5").resolve("/a/b/c?d/e#f/g").toString()); + new N5URI("s3://janelia-cosem-datasets/jrc_hela-3/jrc_hela-3.n5").resolve("/a/b/c?d/e#f/g").toString()); assertEquals( "s3://janelia-cosem-datasets/jrc_hela-3/jrc_hela-3.n5/a/b/c?d/e#f/g", - new N5URL("s3://janelia-cosem-datasets/jrc_hela-3/jrc_hela-3.n5").resolve("a/b/c?d/e#f/g").toString()); + new N5URI("s3://janelia-cosem-datasets/jrc_hela-3/jrc_hela-3.n5").resolve("a/b/c?d/e#f/g").toString()); assertEquals( "s3://janelia-cosem-datasets/jrc_hela-3/jrc_hela-3.n5?d/e#f/g", - new N5URL("s3://janelia-cosem-datasets/jrc_hela-3/jrc_hela-3.n5").resolve("?d/e#f/g").toString()); + new N5URI("s3://janelia-cosem-datasets/jrc_hela-3/jrc_hela-3.n5").resolve("?d/e#f/g").toString()); assertEquals( "s3://janelia-cosem-datasets/jrc_hela-3/jrc_hela-3.n5#f/g", - new N5URL("s3://janelia-cosem-datasets/jrc_hela-3/jrc_hela-3.n5").resolve("#f/g").toString()); + new N5URI("s3://janelia-cosem-datasets/jrc_hela-3/jrc_hela-3.n5").resolve("#f/g").toString()); } } \ No newline at end of file diff --git a/src/test/java/org/janelia/saalfeldlab/n5/url/UrlAttributeTest.java b/src/test/java/org/janelia/saalfeldlab/n5/url/UrlAttributeTest.java index b748ca07..525fd964 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/url/UrlAttributeTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/url/UrlAttributeTest.java @@ -17,7 +17,7 @@ import org.janelia.saalfeldlab.n5.N5FSWriter; import org.janelia.saalfeldlab.n5.N5Reader; -import org.janelia.saalfeldlab.n5.N5URL; +import org.janelia.saalfeldlab.n5.N5URI; import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -60,48 +60,48 @@ public void before() { public void testRootAttributes() throws URISyntaxException, IOException { // get - final Map everything = getAttribute(n5, new N5URL(""), Map.class); + final Map everything = getAttribute(n5, new N5URI(""), Map.class); final String version = "2.6.1"; // the n5 version at the time the test // data were created assertEquals("empty url", version, everything.get("n5")); - final Map everything2 = getAttribute(n5, new N5URL("#/"), Map.class); + final Map everything2 = getAttribute(n5, new N5URI("#/"), Map.class); assertEquals("root attribute", version, everything2.get("n5")); - assertEquals("url to attribute", "bar", getAttribute(n5, new N5URL("#foo"), String.class)); - assertEquals("url to attribute absolute", "bar", getAttribute(n5, new N5URL("#/foo"), String.class)); + assertEquals("url to attribute", "bar", getAttribute(n5, new N5URI("#foo"), String.class)); + assertEquals("url to attribute absolute", "bar", getAttribute(n5, new N5URI("#/foo"), String.class)); - assertEquals("#foo", "bar", getAttribute(n5, new N5URL("#foo"), String.class)); - assertEquals("#/foo", "bar", getAttribute(n5, new N5URL("#/foo"), String.class)); - assertEquals("?#foo", "bar", getAttribute(n5, new N5URL("?#foo"), String.class)); - assertEquals("?#/foo", "bar", getAttribute(n5, new N5URL("?#/foo"), String.class)); - assertEquals("?/#/foo", "bar", getAttribute(n5, new N5URL("?/#/foo"), String.class)); - assertEquals("?/.#/foo", "bar", getAttribute(n5, new N5URL("?/.#/foo"), String.class)); - assertEquals("?./#/foo", "bar", getAttribute(n5, new N5URL("?./#/foo"), String.class)); - assertEquals("?.#foo", "bar", getAttribute(n5, new N5URL("?.#foo"), String.class)); - assertEquals("?/a/..#foo", "bar", getAttribute(n5, new N5URL("?/a/..#foo"), String.class)); - assertEquals("?/a/../.#foo", "bar", getAttribute(n5, new N5URL("?/a/../.#foo"), String.class)); + assertEquals("#foo", "bar", getAttribute(n5, new N5URI("#foo"), String.class)); + assertEquals("#/foo", "bar", getAttribute(n5, new N5URI("#/foo"), String.class)); + assertEquals("?#foo", "bar", getAttribute(n5, new N5URI("?#foo"), String.class)); + assertEquals("?#/foo", "bar", getAttribute(n5, new N5URI("?#/foo"), String.class)); + assertEquals("?/#/foo", "bar", getAttribute(n5, new N5URI("?/#/foo"), String.class)); + assertEquals("?/.#/foo", "bar", getAttribute(n5, new N5URI("?/.#/foo"), String.class)); + assertEquals("?./#/foo", "bar", getAttribute(n5, new N5URI("?./#/foo"), String.class)); + assertEquals("?.#foo", "bar", getAttribute(n5, new N5URI("?.#foo"), String.class)); + assertEquals("?/a/..#foo", "bar", getAttribute(n5, new N5URI("?/a/..#foo"), String.class)); + assertEquals("?/a/../.#foo", "bar", getAttribute(n5, new N5URI("?/a/../.#foo"), String.class)); /* whitespace-encoding-necesary, fragment-only test */ - assertEquals("#f o o", "b a r", getAttribute(n5, new N5URL("#f o o"), String.class)); + assertEquals("#f o o", "b a r", getAttribute(n5, new N5URI("#f o o"), String.class)); - Assert.assertArrayEquals("url list", list, getAttribute(n5, new N5URL("#list"), int[].class)); + Assert.assertArrayEquals("url list", list, getAttribute(n5, new N5URI("#list"), int[].class)); // list - assertEquals("url list[0]", list[0], (int)getAttribute(n5, new N5URL("#list[0]"), Integer.class)); - assertEquals("url list[1]", list[1], (int)getAttribute(n5, new N5URL("#list[1]"), Integer.class)); - assertEquals("url list[2]", list[2], (int)getAttribute(n5, new N5URL("#list[2]"), Integer.class)); + assertEquals("url list[0]", list[0], (int)getAttribute(n5, new N5URI("#list[0]"), Integer.class)); + assertEquals("url list[1]", list[1], (int)getAttribute(n5, new N5URI("#list[1]"), Integer.class)); + assertEquals("url list[2]", list[2], (int)getAttribute(n5, new N5URI("#list[2]"), Integer.class)); - assertEquals("url list[3]", list[3], (int)getAttribute(n5, new N5URL("#list[3]"), Integer.class)); - assertEquals("url list/[3]", list[3], (int)getAttribute(n5, new N5URL("#list/[3]"), Integer.class)); - assertEquals("url list//[3]", list[3], (int)getAttribute(n5, new N5URL("#list//[3]"), Integer.class)); - assertEquals("url //list//[3]", list[3], (int)getAttribute(n5, new N5URL("#//list//[3]"), Integer.class)); - assertEquals("url //list//[3]//", list[3], (int)getAttribute(n5, new N5URL("#//list////[3]//"), Integer.class)); + assertEquals("url list[3]", list[3], (int)getAttribute(n5, new N5URI("#list[3]"), Integer.class)); + assertEquals("url list/[3]", list[3], (int)getAttribute(n5, new N5URI("#list/[3]"), Integer.class)); + assertEquals("url list//[3]", list[3], (int)getAttribute(n5, new N5URI("#list//[3]"), Integer.class)); + assertEquals("url //list//[3]", list[3], (int)getAttribute(n5, new N5URI("#//list//[3]"), Integer.class)); + assertEquals("url //list//[3]//", list[3], (int)getAttribute(n5, new N5URI("#//list////[3]//"), Integer.class)); // object - assertTrue("url object", mapsEqual(obj, getAttribute(n5, new N5URL("#object"), Map.class))); - assertEquals("url object/a", "aa", getAttribute(n5, new N5URL("#object/a"), String.class)); - assertEquals("url object/b", "bb", getAttribute(n5, new N5URL("#object/b"), String.class)); + assertTrue("url object", mapsEqual(obj, getAttribute(n5, new N5URI("#object"), Map.class))); + assertEquals("url object/a", "aa", getAttribute(n5, new N5URI("#object/a"), String.class)); + assertEquals("url object/b", "bb", getAttribute(n5, new N5URI("#object/b"), String.class)); } @Test @@ -111,20 +111,20 @@ public void testPathAttributes() throws URISyntaxException, IOException { final String aa = "aa"; final String aaa = "aaa"; - final N5URL aUrl = new N5URL("?/a"); - final N5URL aaUrl = new N5URL("?/a/aa"); - final N5URL aaaUrl = new N5URL("?/a/aa/aaa"); + final N5URI aUrl = new N5URI("?/a"); + final N5URI aaUrl = new N5URI("?/a/aa"); + final N5URI aaaUrl = new N5URI("?/a/aa/aaa"); // name of a assertEquals("name of a from root", a, getAttribute(n5, "?/a#name", String.class)); - assertEquals("name of a from root", a, getAttribute(n5, new N5URL("?a#name"), String.class)); - assertEquals("name of a from a", a, getAttribute(n5, aUrl.resolve(new N5URL("?/a#name")), String.class)); + assertEquals("name of a from root", a, getAttribute(n5, new N5URI("?a#name"), String.class)); + assertEquals("name of a from a", a, getAttribute(n5, aUrl.resolve(new N5URI("?/a#name")), String.class)); assertEquals("name of a from aa", a, getAttribute(n5, aaUrl.resolve("?..#name"), String.class)); - assertEquals("name of a from aaa", a, getAttribute(n5, aaaUrl.resolve(new N5URL("?../..#name")), String.class)); + assertEquals("name of a from aaa", a, getAttribute(n5, aaaUrl.resolve(new N5URI("?../..#name")), String.class)); // name of aa - assertEquals("name of aa from root", aa, getAttribute(n5, new N5URL("?/a/aa#name"), String.class)); - assertEquals("name of aa from root", aa, getAttribute(n5, new N5URL("?a/aa#name"), String.class)); + assertEquals("name of aa from root", aa, getAttribute(n5, new N5URI("?/a/aa#name"), String.class)); + assertEquals("name of aa from root", aa, getAttribute(n5, new N5URI("?a/aa#name"), String.class)); assertEquals("name of aa from a", aa, getAttribute(n5, aUrl.resolve("?aa#name"), String.class)); assertEquals("name of aa from aa", aa, getAttribute(n5, aaUrl.resolve("?./#name"), String.class)); @@ -133,24 +133,24 @@ public void testPathAttributes() throws URISyntaxException, IOException { assertEquals("name of aa from aaa", aa, getAttribute(n5, aaaUrl.resolve("?..#name"), String.class)); // name of aaa - assertEquals("name of aaa from root", aaa, getAttribute(n5, new N5URL("?/a/aa/aaa#name"), String.class)); - assertEquals("name of aaa from root", aaa, getAttribute(n5, new N5URL("?a/aa/aaa#name"), String.class)); + assertEquals("name of aaa from root", aaa, getAttribute(n5, new N5URI("?/a/aa/aaa#name"), String.class)); + assertEquals("name of aaa from root", aaa, getAttribute(n5, new N5URI("?a/aa/aaa#name"), String.class)); assertEquals("name of aaa from a", aaa, getAttribute(n5, aUrl.resolve("?aa/aaa#name"), String.class)); assertEquals("name of aaa from aa", aaa, getAttribute(n5, aaUrl.resolve("?aaa#name"), String.class)); assertEquals("name of aaa from aaa", aaa, getAttribute(n5, aaaUrl.resolve("#name"), String.class)); assertEquals("name of aaa from aaa", aaa, getAttribute(n5, aaaUrl.resolve("?./#name"), String.class)); - assertEquals("nested list 1", (Integer)1, getAttribute(n5, new N5URL("#nestedList[0][0][0]"), Integer.class)); - assertEquals("nested list 1", (Integer)1, getAttribute(n5, new N5URL("#/nestedList/[0][0][0]"), Integer.class)); + assertEquals("nested list 1", (Integer)1, getAttribute(n5, new N5URI("#nestedList[0][0][0]"), Integer.class)); + assertEquals("nested list 1", (Integer)1, getAttribute(n5, new N5URI("#/nestedList/[0][0][0]"), Integer.class)); assertEquals( "nested list 1", (Integer)1, - getAttribute(n5, new N5URL("#nestedList//[0]/[0]///[0]"), Integer.class)); + getAttribute(n5, new N5URI("#nestedList//[0]/[0]///[0]"), Integer.class)); assertEquals( "nested list 1", (Integer)1, - getAttribute(n5, new N5URL("#/nestedList[0]//[0][0]"), Integer.class)); + getAttribute(n5, new N5URI("#/nestedList[0]//[0][0]"), Integer.class)); } @@ -168,13 +168,13 @@ private T getAttribute( final String url2, final Class clazz) throws URISyntaxException, IOException { - final N5URL n5URL = url2 == null ? new N5URL(url1) : new N5URL(url1).resolve(url2); + final N5URI n5URL = url2 == null ? new N5URI(url1) : new N5URI(url1).resolve(url2); return getAttribute(n5, n5URL, clazz); } private T getAttribute( final N5Reader n5, - final N5URL url1, + final N5URI url1, final Class clazz) throws URISyntaxException, IOException { return getAttribute(n5, url1, null, clazz); @@ -182,28 +182,28 @@ private T getAttribute( private T getAttribute( final N5Reader n5, - final N5URL url1, + final N5URI url1, final String url2, final Class clazz) throws URISyntaxException, IOException { - final N5URL n5URL = url2 == null ? url1 : url1.resolve(url2); + final N5URI n5URL = url2 == null ? url1 : url1.resolve(url2); return n5.getAttribute(n5URL.getGroupPath(), n5URL.getAttributePath(), clazz); } @Test public void testPathObject() throws IOException, URISyntaxException { - final TestInts ints = getAttribute(n5, new N5URL("?objs#intsKey"), TestInts.class); + final TestInts ints = getAttribute(n5, new N5URI("?objs#intsKey"), TestInts.class); assertEquals(testObjInts.name, ints.name); assertEquals(testObjInts.type, ints.type); assertArrayEquals(testObjInts.t(), ints.t()); - final TestDoubles doubles = getAttribute(n5, new N5URL("?objs#doublesKey"), TestDoubles.class); + final TestDoubles doubles = getAttribute(n5, new N5URI("?objs#doublesKey"), TestDoubles.class); assertEquals(testObjDoubles.name, doubles.name); assertEquals(testObjDoubles.type, doubles.type); assertArrayEquals(testObjDoubles.t(), doubles.t(), 1e-9); - final TestDoubles[] doubleArray = getAttribute(n5, new N5URL("?objs#array"), TestDoubles[].class); + final TestDoubles[] doubleArray = getAttribute(n5, new N5URI("?objs#array"), TestDoubles[].class); final TestDoubles doubles1 = new TestDoubles("doubles", "doubles1", new double[]{5.7, 4.5, 3.4}); final TestDoubles doubles2 = new TestDoubles("doubles", "doubles2", new double[]{5.8, 4.6, 3.5}); final TestDoubles doubles3 = new TestDoubles("doubles", "doubles3", new double[]{5.9, 4.7, 3.6}); @@ -211,13 +211,13 @@ public void testPathObject() throws IOException, URISyntaxException { final TestDoubles[] expectedDoubles = new TestDoubles[]{doubles1, doubles2, doubles3, doubles4}; assertArrayEquals(expectedDoubles, doubleArray); - final String[] stringArray = getAttribute(n5, new N5URL("?objs#String"), String[].class); + final String[] stringArray = getAttribute(n5, new N5URI("?objs#String"), String[].class); final String[] expectedString = new String[]{"This", "is", "a", "test"}; - final Integer[] integerArray = getAttribute(n5, new N5URL("?objs#Integer"), Integer[].class); + final Integer[] integerArray = getAttribute(n5, new N5URI("?objs#Integer"), Integer[].class); final Integer[] expectedInteger = new Integer[]{1, 2, 3, 4}; - final int[] intArray = getAttribute(n5, new N5URL("?objs#int"), int[].class); + final int[] intArray = getAttribute(n5, new N5URI("?objs#int"), int[].class); final int[] expectedInt = new int[]{1, 2, 3, 4}; assertArrayEquals(expectedInt, intArray); } From 6e36f421adb0214d28204ce10faa6be8328f313f Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Mon, 12 Jun 2023 15:03:05 -0400 Subject: [PATCH 212/243] refactor: explicitly indicate absolute group/attribute/datablock --- .../saalfeldlab/n5/CachedGsonKeyValueN5Reader.java | 6 +++--- .../saalfeldlab/n5/CachedGsonKeyValueN5Writer.java | 2 +- .../janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java | 12 ++++++------ .../janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java | 8 ++++---- .../org/janelia/saalfeldlab/n5/N5CachedFSTest.java | 4 ++-- 5 files changed, 16 insertions(+), 16 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Reader.java index d23c7614..49264f0d 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Reader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Reader.java @@ -134,9 +134,9 @@ default boolean exists(final String pathName) { default boolean existsFromContainer(final String normalPathName, final String normalCacheKey) { if (normalCacheKey == null) - return getKeyValueAccess().isDirectory(groupPath(normalPathName)); + return getKeyValueAccess().isDirectory(absoluteGroupPath(normalPathName)); else - return getKeyValueAccess().isFile(getKeyValueAccess().compose(groupPath(normalPathName), normalCacheKey)); + return getKeyValueAccess().isFile(getKeyValueAccess().compose(absoluteGroupPath(normalPathName), normalCacheKey)); } @Override @@ -241,7 +241,7 @@ default String[] listFromContainer(final String normalPathName) { * @return */ @Override - default String getDataBlockPath( + default String absoluteDataBlockPath( final String normalPath, final long... gridPosition) { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Writer.java index a54023b9..495883a3 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Writer.java @@ -73,7 +73,7 @@ else if (getCache().exists(normalPath, N5KeyValueReader.ATTRIBUTES_JSON)) { * work */ try { - getKeyValueAccess().createDirectories(groupPath(normalPath)); + getKeyValueAccess().createDirectories(absoluteGroupPath(normalPath)); } catch (final IOException e) { throw new N5Exception.N5IOException("Failed to create group " + path, e); } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java index 69adf954..03366f22 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java @@ -44,7 +44,7 @@ public interface GsonKeyValueN5Reader extends GsonN5Reader { default boolean groupExists(final String normalPath) { - return getKeyValueAccess().isDirectory(groupPath(normalPath)); + return getKeyValueAccess().isDirectory(absoluteGroupPath(normalPath)); } @Override @@ -92,7 +92,7 @@ default DataBlock readBlock( final DatasetAttributes datasetAttributes, final long... gridPosition) throws N5Exception { - final String path = getDataBlockPath(N5URL.normalizeGroupPath(pathName), gridPosition); + final String path = absoluteDataBlockPath(N5URI.normalizeGroupPath(pathName), gridPosition); if (!getKeyValueAccess().isFile(path)) return null; @@ -109,7 +109,7 @@ default DataBlock readBlock( default String[] list(final String pathName) throws N5Exception { try { - return getKeyValueAccess().listDirectories(groupPath(pathName)); + return getKeyValueAccess().listDirectories(absoluteGroupPath(pathName)); } catch (final IOException e) { throw new N5IOException("Cannot list directories for group " + pathName, e); } @@ -132,7 +132,7 @@ default String[] list(final String pathName) throws N5Exception { * @param gridPosition * @return */ - default String getDataBlockPath( + default String absoluteDataBlockPath( final String normalPath, final long... gridPosition) { @@ -154,7 +154,7 @@ default String getDataBlockPath( * normalized group path without leading slash * @return the absolute path to the group */ - default String groupPath(final String normalGroupPath) { + default String absoluteGroupPath(final String normalGroupPath) { return getKeyValueAccess().compose(getURI(), normalGroupPath); } @@ -167,7 +167,7 @@ default String groupPath(final String normalGroupPath) { * normalized group path without leading slash * @return the absolute path to the attributes */ - default String attributesPath(final String normalPath) { + default String absoluteAttributesPath(final String normalPath) { return getKeyValueAccess().compose(getURI(), normalPath, N5KeyValueReader.ATTRIBUTES_JSON); } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java index c9c26a65..9c4fd3f1 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java @@ -71,7 +71,7 @@ default void createGroup(final String path) throws N5Exception { final String normalPath = N5URI.normalizeGroupPath(path); try { - getKeyValueAccess().createDirectories(groupPath(normalPath)); + getKeyValueAccess().createDirectories(absoluteGroupPath(normalPath)); } catch (final IOException e) { throw new N5Exception.N5IOException("Failed to create group " + path, e); } @@ -94,7 +94,7 @@ default void writeAttributes( final String normalGroupPath, final JsonElement attributes) throws N5Exception { - try (final LockedChannel lock = getKeyValueAccess().lockForWriting(attributesPath(normalGroupPath))) { + try (final LockedChannel lock = getKeyValueAccess().lockForWriting(absoluteAttributesPath(normalGroupPath))) { GsonUtils.writeAttributes(lock.newWriter(), attributes, getGson()); } catch (final IOException e) { throw new N5Exception.N5IOException("Failed to write attributes into " + normalGroupPath, e); @@ -210,7 +210,7 @@ default void writeBlock( final DatasetAttributes datasetAttributes, final DataBlock dataBlock) throws N5Exception { - final String blockPath = getDataBlockPath(N5URL.normalizeGroupPath(path), dataBlock.getGridPosition()); + final String blockPath = absoluteDataBlockPath(N5URI.normalizeGroupPath(path), dataBlock.getGridPosition()); try (final LockedChannel lock = getKeyValueAccess().lockForWriting(blockPath)) { DefaultBlockWriter.writeBlock(lock.newOutputStream(), datasetAttributes, dataBlock); } catch (final IOException e) { @@ -241,7 +241,7 @@ default boolean deleteBlock( final String path, final long... gridPosition) throws N5Exception { - final String blockPath = getDataBlockPath(N5URL.normalizeGroupPath(path), gridPosition); + final String blockPath = absoluteDataBlockPath(N5URI.normalizeGroupPath(path), gridPosition); try { if (getKeyValueAccess().isFile(blockPath)) getKeyValueAccess().delete(blockPath); diff --git a/src/test/java/org/janelia/saalfeldlab/n5/N5CachedFSTest.java b/src/test/java/org/janelia/saalfeldlab/n5/N5CachedFSTest.java index dcf4b795..e9a04e00 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/N5CachedFSTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/N5CachedFSTest.java @@ -55,7 +55,7 @@ public void cacheTest() throws IOException, URISyntaxException { * The only possible way for the test to succeed is if it never again attempts to read the file, and relies on the cache. */ try (N5KeyValueWriter n5 = (N5KeyValueWriter) createN5Writer()) { final String cachedGroup = "cachedGroup"; - final String attributesPath = n5.attributesPath(cachedGroup); + final String attributesPath = n5.absoluteAttributesPath(cachedGroup); final ArrayList> tests = new ArrayList<>(); @@ -70,7 +70,7 @@ public void cacheTest() throws IOException, URISyntaxException { try (N5KeyValueWriter n5 = (N5KeyValueWriter)createN5Writer(tempN5Location(), false)) { final String cachedGroup = "cachedGroup"; - final String attributesPath = n5.attributesPath(cachedGroup); + final String attributesPath = n5.absoluteAttributesPath(cachedGroup); final ArrayList> tests = new ArrayList<>(); n5.createGroup(cachedGroup); From f8801c0f61c8e11799ddc022bf74537c9d828beb Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Mon, 12 Jun 2023 15:04:51 -0400 Subject: [PATCH 213/243] feat(test): refine expected uri test behavior --- .../java/org/janelia/saalfeldlab/n5/UriTest.java | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/UriTest.java b/src/test/java/org/janelia/saalfeldlab/n5/UriTest.java index 69c578ce..906bed3d 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/UriTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/UriTest.java @@ -35,15 +35,12 @@ public void testUriParsing() throws URISyntaxException { assertEquals("Container URI must contain scheme", "file", uri.getScheme()); assertEquals("Container URI must be absolute", - Paths.get(relativePath).toAbsolutePath().toString(), - Paths.get(uri).toAbsolutePath().toString()); - - assertEquals("Container URI must be normalized 1", uri, kva.uri(relativeAbnormalPath)); - assertEquals("Container URI must be normalized 2", uri, kva.uri(relativeAbnormalPath2)); -// assertEquals("Container URI must be normalized 3", uri, kva.uri("file:" + relativePath)); -// assertEquals("Container URI must be normalized 4", uri, kva.uri("file:" + relativeAbnormalPath)); -// assertEquals("Container URI must be normalized 5", uri, kva.uri("file:" + relativeAbnormalPath2)); + uri.getPath(), + Paths.get(relativePath).toAbsolutePath().toUri().normalize().getPath()); + assertEquals("Container URI must be normalized 1", uri, kva.uri(relativePath)); + assertEquals("Container URI must be normalized 2", uri, kva.uri(relativeAbnormalPath)); + assertEquals("Container URI must be normalized 3", uri, kva.uri(relativeAbnormalPath2)); } } From 68c1642a09a1caa3d085b07dae83d8ec0ed2de54 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Mon, 12 Jun 2023 15:04:56 -0400 Subject: [PATCH 214/243] feat: handle valid URIs as input --- .../org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java | 4 +++- src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java | 3 ++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java index 7e7206bc..cef30f6d 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java @@ -280,7 +280,9 @@ public String normalize(final String path) { public URI uri(final String normalPath) throws URISyntaxException { // normalize make absolute the scheme specific part only - return new File(normalPath).toURI().normalize(); + final URI normalUri = URI.create(normalPath); + if (normalUri.isAbsolute()) return normalUri.normalize(); + else return new File(normalPath).toURI().normalize(); } diff --git a/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java b/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java index 4d91cde7..b6ca640e 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java @@ -82,7 +82,8 @@ private static String tempN5PathName() { @Override protected String tempN5Location() throws URISyntaxException { - return tempN5PathName(); + final String basePath = new File(tempN5PathName()).toURI().normalize().getPath(); + return new URI("file", null, basePath, null).toString(); } @Override From 4e6a2e6f57f80165261cf0681f9c81cc34d21588 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Tue, 13 Jun 2023 09:49:05 -0400 Subject: [PATCH 215/243] fix: support uri for new N5 reader/writer --- src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java index ab721c47..677f9102 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java @@ -787,7 +787,7 @@ public void testRemoveContainer() throws IOException, URISyntaxException { String location; try (final N5Writer n5 = createN5Writer()) { - location = n5.getURI().getPath(); + location = n5.getURI().toString(); assertNotNull(createN5Reader(location)); n5.remove(); assertThrows(Exception.class, () -> createN5Reader(location)); From 6d7a585d887b14dedbcf2034df4fd5e81f75eebf Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Tue, 13 Jun 2023 10:09:16 -0400 Subject: [PATCH 216/243] test: let createN5Reader/Writer handle URIs --- src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java index 677f9102..c86c8d78 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java @@ -1072,7 +1072,7 @@ public void testVersion() throws NumberFormatException, IOException, URISyntaxEx assertFalse(N5Reader.VERSION.isCompatible(version)); assertThrows(N5Exception.N5IOException.class, () -> { - final String containerPath = writer.getURI().getPath(); + final String containerPath = writer.getURI().toString(); createN5Writer(containerPath); }); From 25f45d3a8abfc08ab626fa9634977456652b1951 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Tue, 13 Jun 2023 11:43:30 -0400 Subject: [PATCH 217/243] refactor: move groupPath to GsonKeyValueN5Reader * where more downstream implementations can use it --- .../janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java | 12 ++++++++++++ .../org/janelia/saalfeldlab/n5/N5KeyValueReader.java | 8 -------- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java index 03366f22..315d224f 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java @@ -27,6 +27,7 @@ import java.io.IOException; import java.util.Arrays; +import java.util.stream.Stream; import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; @@ -47,6 +48,17 @@ default boolean groupExists(final String normalPath) { return getKeyValueAccess().isDirectory(absoluteGroupPath(normalPath)); } + @Override + default String groupPath(final String... nodes) { + + // alternatively call compose twice, once with this functions inputs, + // then pass the result to the other groupPath method + // this impl assumes streams and array building are less expensive than + // keyValueAccess composition (may not always be true) + return getKeyValueAccess() + .compose(Stream.concat(Stream.of(getURI().getPath()), Arrays.stream(nodes)).toArray(String[]::new)); + } + @Override default boolean exists(final String pathName) { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java index 2193797a..7d68474b 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java @@ -27,8 +27,6 @@ import java.net.URI; import java.net.URISyntaxException; -import java.util.Arrays; -import java.util.stream.Stream; import org.janelia.saalfeldlab.n5.cache.N5JsonCache; @@ -168,10 +166,4 @@ public N5JsonCache getCache() { return this.cache; } - @Override - public String groupPath(final String... nodes) { - - return keyValueAccess - .compose(Stream.concat(Stream.of(getURI().getPath()), Arrays.stream(nodes)).toArray(String[]::new)); - } } From 694920980480bcea4596a5c25eda4a76b95ff900 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Tue, 13 Jun 2023 12:03:14 -0400 Subject: [PATCH 218/243] fix: more windows path issues --- .../saalfeldlab/n5/FileSystemKeyValueAccess.java | 10 +++++++--- .../janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java | 3 +-- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java index cef30f6d..19792b57 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java @@ -280,9 +280,13 @@ public String normalize(final String path) { public URI uri(final String normalPath) throws URISyntaxException { // normalize make absolute the scheme specific part only - final URI normalUri = URI.create(normalPath); - if (normalUri.isAbsolute()) return normalUri.normalize(); - else return new File(normalPath).toURI().normalize(); + try { + final URI normalUri = URI.create(normalPath); + if (normalUri.isAbsolute()) return normalUri.normalize(); + } catch (IllegalArgumentException e) { + return new File(normalPath).toURI().normalize(); + } + return new File(normalPath).toURI().normalize(); } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java index 315d224f..2e824bb6 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java @@ -55,8 +55,7 @@ default String groupPath(final String... nodes) { // then pass the result to the other groupPath method // this impl assumes streams and array building are less expensive than // keyValueAccess composition (may not always be true) - return getKeyValueAccess() - .compose(Stream.concat(Stream.of(getURI().getPath()), Arrays.stream(nodes)).toArray(String[]::new)); + return getKeyValueAccess().compose(getURI(), nodes); } @Override From d14fe554178b180cea4aeba50013c245c0b39b8b Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Wed, 14 Jun 2023 13:03:54 -0400 Subject: [PATCH 219/243] doc/chore: add documentation and minor clean up --- .../n5/CachedGsonKeyValueN5Writer.java | 18 +++---- .../saalfeldlab/n5/GsonKeyValueN5Reader.java | 1 - .../janelia/saalfeldlab/n5/GsonN5Reader.java | 3 +- .../org/janelia/saalfeldlab/n5/N5Reader.java | 51 ++++++++----------- .../org/janelia/saalfeldlab/n5/N5URI.java | 6 +++ .../saalfeldlab/n5/cache/N5JsonCache.java | 6 +++ .../n5/cache/N5JsonCacheableContainer.java | 15 ++++-- 7 files changed, 50 insertions(+), 50 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Writer.java index 495883a3..1c6fee59 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Writer.java @@ -54,9 +54,9 @@ default void setVersion(final String path) throws N5Exception { default void createGroup(final String path) throws N5Exception { final String normalPath = N5URI.normalizeGroupPath(path); - // TODO: John document this! - // if you are a group, avoid hitting the backend - // if something exists, be safe + // avoid hitting the backend if this path is already a group according to the cache + // else if exists is true (then a dataset is present) so throw an exception to avoid + // overwriting / invalidating existing data if (cacheMeta()) { if (getCache().isGroup(normalPath, N5KeyValueReader.ATTRIBUTES_JSON)) return; @@ -67,10 +67,8 @@ else if (getCache().exists(normalPath, N5KeyValueReader.ATTRIBUTES_JSON)) { // N5Writer.super.createGroup(path); /* - * the 6 lines below duplicate the single line above but would have to - * call - * normalizeGroupPath again the below duplicates code, but avoids extra - * work + * the lines below duplicate the single line above but would have to call + * normalizeGroupPath again the below duplicates code, but avoids extra work */ try { getKeyValueAccess().createDirectories(absoluteGroupPath(normalPath)); @@ -140,10 +138,8 @@ default boolean remove(final String path) throws N5Exception { // GsonKeyValueN5Writer.super.remove(path) /* - * the 8 lines below duplicate the single line above but would have to - * call - * normalizeGroupPath again the below duplicates code, but avoids extra - * work + * the lines below duplicate the single line above but would have to call + * normalizeGroupPath again the below duplicates code, but avoids extra work */ final String normalPath = N5URI.normalizeGroupPath(path); final String groupPath = absoluteGroupPath(normalPath); diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java index 2e824bb6..907c0f25 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java @@ -27,7 +27,6 @@ import java.io.IOException; import java.util.Arrays; -import java.util.stream.Stream; import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonN5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonN5Reader.java index dbec5973..0df07b35 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonN5Reader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonN5Reader.java @@ -25,7 +25,6 @@ */ package org.janelia.saalfeldlab.n5; -import java.io.IOException; import java.lang.reflect.Type; import java.util.Map; @@ -107,7 +106,7 @@ default T getAttribute(final String pathName, final String key, final Type t * @param pathName * group path * @return - * @throws IOException + * @throws N5Exception */ JsonElement getAttributes(final String pathName) throws N5Exception; } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java index 13637985..d034dd72 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java @@ -339,7 +339,7 @@ default T readSerializedBlock( boolean exists(final String pathName); /** - * Test whether a dataset exists. + * Test whether a dataset exists at a given path. * * @param pathName * dataset path @@ -364,7 +364,7 @@ default boolean datasetExists(final String pathName) throws N5Exception { String[] list(final String pathName) throws N5Exception; /** - * Recursively list all groups (including datasets) in the given group. + * Recursively list all groups and datasets in the given path. * Only paths that satisfy the provided filter will be included, but the * children of paths that were excluded may be included (filter does not * apply to the subtree). @@ -373,7 +373,7 @@ default boolean datasetExists(final String pathName) throws N5Exception { * base group path * @param filter * filter for children to be included - * @return list of groups + * @return list of child groups and datasets * @throws N5Exception * the exception */ @@ -396,11 +396,11 @@ default String[] deepList( } /** - * Recursively list all groups (including datasets) in the given group. + * Recursively list all groups and datasets in the given path. * * @param pathName * base group path - * @return list of groups + * @return list of groups and datasets * @throws N5Exception * the exception */ @@ -410,8 +410,7 @@ default String[] deepList(final String pathName) throws N5Exception { } /** - * Recursively list all datasets in the given group. Only paths that satisfy - * the + * Recursively list all datasets in the given path. Only paths that satisfy the * provided filter will be included, but the children of paths that were * excluded may be included (filter does not apply to the subtree). * @@ -431,19 +430,14 @@ default String[] deepList(final String pathName) throws N5Exception { * } * *

    - * but will execute {@link #datasetExists(String)} only once per node. This - * can - * be relevant for performance on high latency backends such as cloud - * stores. + * but will execute {@link #datasetExists(String)} only once per node. This can + * be relevant for performance on high latency backends such as cloud stores. *

    * - * @param pathName - * base group path - * @param filter - * filter for datasets to be included + * @param pathName base group path + * @param filter filter for datasets to be included * @return list of groups - * @throws N5Exception - * the exception + * @throws N5Exception the exception */ default String[] deepListDatasets( final String pathName, @@ -464,7 +458,7 @@ default String[] deepListDatasets( } /** - * Recursively list all including datasets in the given group. + * Recursively list all including datasets in the given path. * *

    * This method delivers the same results as @@ -500,23 +494,18 @@ default String[] deepListDatasets(final String pathName) throws N5Exception { } /** - * Helper method to recursively list all groups. This method is not part - * of the public API and is accessible only because Java 8 does not support - * private interface methods yet. + * Helper method to recursively list all groups and datasets. This method is not part of the + * public API and is accessible only because Java 8 does not support private + * interface methods yet. * * TODO make private when committing to Java versions newer than 8 * - * @param n5 - * the n5 reader - * @param pathName - * the base group path - * @param datasetsOnly - * true if only dataset paths should be returned - * @param filter - * a dataset filter + * @param n5 the n5 reader + * @param pathName the base group path + * @param datasetsOnly true if only dataset paths should be returned + * @param filter a dataset filter * @return the list of all children - * @throws N5Exception - * the exception + * @throws N5Exception the exception */ static ArrayList deepList( final N5Reader n5, diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5URI.java b/src/main/java/org/janelia/saalfeldlab/n5/N5URI.java index 52001450..004c0de3 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5URI.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5URI.java @@ -17,6 +17,12 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; +/** + * A {@link URI} for N5 containers, groups, datasets, and attributes. + *

    + * Container paths are stored in the URI path. Group / dataset paths are stored in the URI query, + * and attribute paths are stored in the URI fragment. + */ public class N5URI { private static final Charset UTF8 = Charset.forName("UTF-8"); diff --git a/src/main/java/org/janelia/saalfeldlab/n5/cache/N5JsonCache.java b/src/main/java/org/janelia/saalfeldlab/n5/cache/N5JsonCache.java index ffca797b..20dbffae 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/cache/N5JsonCache.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/cache/N5JsonCache.java @@ -8,6 +8,12 @@ import com.google.gson.JsonElement; +/* + * A cache containing JSON attributes and children for groups and + * datasets stored in N5 containers. Used by {@link CachedGsonKeyValueN5Reader} + * and {@link CachedGsonKeyValueN5Writer}. + * + */ public class N5JsonCache { public static final N5CacheInfo emptyCacheInfo = new N5CacheInfo(); diff --git a/src/main/java/org/janelia/saalfeldlab/n5/cache/N5JsonCacheableContainer.java b/src/main/java/org/janelia/saalfeldlab/n5/cache/N5JsonCacheableContainer.java index e8d59301..8c598ec9 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/cache/N5JsonCacheableContainer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/cache/N5JsonCacheableContainer.java @@ -10,7 +10,7 @@ * An N5 container whose structure and attributes can be cached. *

    * Implementations of interface methods must explicitly query the backing - * storage. Cached implmentations (e.g {@link CachedGsonKeyValueN5Reader}) call + * storage unless noted otherwise. Cached implementations (e.g {@link CachedGsonKeyValueN5Reader}) call * these methods to update their {@link N5JsonCache}. Corresponding * {@link N5Reader} methods should use the cache, if present. */ @@ -60,8 +60,11 @@ public interface N5JsonCacheableContainer { boolean isDatasetFromContainer(final String normalPathName); /** - * If this method is called, its parent path must exist, - * and normalCacheKey must exist, with contents given by attributes. + * + * Returns true if a path is a group, given that the the given attributes exist + * for the given cache key. + *

    + * Should not call the backing storage. * * @param normalCacheKey * the cache key @@ -72,8 +75,10 @@ public interface N5JsonCacheableContainer { boolean isGroupFromAttributes(final String normalCacheKey, final JsonElement attributes); /** - * If this method is called, its parent path must exist, - * and normalCacheKey must exist, with contents given by attributes. + * Returns true if a path is a dataset, given that the the given attributes exist + * for the given cache key. + *

    + * Should not call the backing storage. * * @param normalCacheKey * the cache key From 821ee0a7fb560469453726b88317f6955c3bd45d Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Thu, 15 Jun 2023 14:52:14 -0400 Subject: [PATCH 220/243] throw N5Exception if list is called on a path that isn't a group --- src/main/java/org/janelia/saalfeldlab/n5/cache/N5JsonCache.java | 2 +- src/test/java/org/janelia/saalfeldlab/n5/N5CachedFSTest.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/cache/N5JsonCache.java b/src/main/java/org/janelia/saalfeldlab/n5/cache/N5JsonCache.java index 20dbffae..873d8b5b 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/cache/N5JsonCache.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/cache/N5JsonCache.java @@ -132,7 +132,7 @@ public String[] list(final String normalPathKey) { cacheInfo = getCacheInfo(normalPathKey); } if (cacheInfo == emptyCacheInfo) - return null; + throw new N5Exception.N5IOException(normalPathKey + " is not a valid group"); if (cacheInfo.children == null) addChild(cacheInfo, normalPathKey); diff --git a/src/test/java/org/janelia/saalfeldlab/n5/N5CachedFSTest.java b/src/test/java/org/janelia/saalfeldlab/n5/N5CachedFSTest.java index e9a04e00..c7a4aada 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/N5CachedFSTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/N5CachedFSTest.java @@ -274,7 +274,7 @@ public static void cacheBehaviorHelper(final TrackingStorage n5) throws IOExcept assertEquals(expectedAttributeCount, n5.getAttrCallCount()); assertEquals(expectedListCount, n5.getListCallCount()); - n5.list(nonExistentGroup); + assertThrows(N5Exception.class, () -> n5.list(nonExistentGroup)); assertEquals(expectedExistCount, n5.getExistCallCount()); assertEquals(expectedGroupCount, n5.getGroupCallCount()); assertEquals(expectedDatasetCount, n5.getDatasetCallCount()); From ac52633d88aa1ed015ebf23e650dfa58dfe07dae Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Thu, 15 Jun 2023 14:57:49 -0400 Subject: [PATCH 221/243] test: that N5Exception is thrown if list is called on a path that isn't a group --- src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java index c86c8d78..941f3b23 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java @@ -825,6 +825,11 @@ public void testList() throws IOException, URISyntaxException { assertArrayEquals(new String[]{"test"}, listN5.list("")); assertArrayEquals(new String[]{"test"}, listN5.list("/")); + // calling list on a non-existant group throws an exception + assertThrows(N5Exception.class, () -> { + listN5.list("this-group-does-not-exist"); + }); + } catch (final IOException e) { fail(e.getMessage()); } From 4efd7410e85097b7e43604338f5df2f34da5f6fa Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Thu, 15 Jun 2023 15:25:04 -0400 Subject: [PATCH 222/243] doc: when N5Reader methods throw exceptions --- .../org/janelia/saalfeldlab/n5/N5Reader.java | 30 ++++++++++--------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java index d034dd72..f2e2cf83 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java @@ -345,7 +345,7 @@ default T readSerializedBlock( * dataset path * @return true if a dataset exists * @throws N5Exception - * the exception + * an exception is thrown if the existence of a dataset at the given path cannot be determined. */ default boolean datasetExists(final String pathName) throws N5Exception { @@ -359,7 +359,7 @@ default boolean datasetExists(final String pathName) throws N5Exception { * group path * @return list of children * @throws N5Exception - * the exception + * an exception is thrown if pathName is not a valid group */ String[] list(final String pathName) throws N5Exception; @@ -375,7 +375,7 @@ default boolean datasetExists(final String pathName) throws N5Exception { * filter for children to be included * @return list of child groups and datasets * @throws N5Exception - * the exception + * an exception is thrown if pathName is not a valid group */ default String[] deepList( final String pathName, @@ -402,7 +402,7 @@ default String[] deepList( * base group path * @return list of groups and datasets * @throws N5Exception - * the exception + * an exception is thrown if pathName is not a valid group */ default String[] deepList(final String pathName) throws N5Exception { @@ -437,7 +437,8 @@ default String[] deepList(final String pathName) throws N5Exception { * @param pathName base group path * @param filter filter for datasets to be included * @return list of groups - * @throws N5Exception the exception + * @throws N5Exception + * an exception is thrown if pathName is not a valid group */ default String[] deepListDatasets( final String pathName, @@ -486,7 +487,7 @@ default String[] deepListDatasets( * base group path * @return list of groups * @throws N5Exception - * the exception + * an exception is thrown if pathName is not a valid group */ default String[] deepListDatasets(final String pathName) throws N5Exception { @@ -505,7 +506,8 @@ default String[] deepListDatasets(final String pathName) throws N5Exception { * @param datasetsOnly true if only dataset paths should be returned * @param filter a dataset filter * @return the list of all children - * @throws N5Exception the exception + * @throws N5Exception + * an exception is thrown if pathName is not a valid group */ static ArrayList deepList( final N5Reader n5, @@ -545,7 +547,7 @@ static ArrayList deepList( * executor service * @return list of datasets * @throws N5Exception - * the exception + * an exception is thrown if pathName is not a valid group * @throws ExecutionException * the execution exception * @throws InterruptedException @@ -582,11 +584,11 @@ default String[] deepList( * executor service * @return list of groups * @throws N5Exception - * the exception + * an exception is thrown if pathName is not a valid group * @throws ExecutionException * the execution exception * @throws InterruptedException - * the interrupted exception + * this exception is thrown if execution is interrupted */ default String[] deepList( final String pathName, @@ -632,11 +634,11 @@ default String[] deepList( * executor service * @return list of datasets * @throws N5Exception - * the exception + * an exception is thrown if pathName is not a valid group * @throws ExecutionException * the execution exception * @throws InterruptedException - * the interrupted exception + * this exception is thrown if execution is interrupted */ default String[] deepListDatasets( final String pathName, @@ -691,11 +693,11 @@ default String[] deepListDatasets( * executor service * @return list of groups * @throws N5Exception - * the exception + * an exception is thrown if pathName is not a valid group * @throws ExecutionException * the execution exception * @throws InterruptedException - * the interrupted exception + * this exception is thrown if execution is interrupted */ default String[] deepListDatasets( final String pathName, From a0325adb1d03f93e6fd26c53cc85f137ee793254 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Fri, 16 Jun 2023 10:21:58 -0400 Subject: [PATCH 223/243] ci: run tests on windows and ubuntu --- .github/workflows/platform-test.yml | 30 +++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 .github/workflows/platform-test.yml diff --git a/.github/workflows/platform-test.yml b/.github/workflows/platform-test.yml new file mode 100644 index 00000000..2b02efef --- /dev/null +++ b/.github/workflows/platform-test.yml @@ -0,0 +1,30 @@ +name: test + +on: + push: + branches: + - master + tags: + - "*-[0-9]+.*" + pull_request: + branches: + - master + workflow_dispatch: + +jobs: + build: + strategy: + matrix: + os: [ubuntu-latest, windows-latest] + runs-on: ${{ matrix.os }} + + steps: + - uses: actions/checkout@v2 + - name: Set up Java + uses: actions/setup-java@v2 + with: + java-version: '8' + distribution: 'zulu' + cache: 'maven' + - name: Maven Test + run: mvn -B clean test --file pom.xml From e1eebd694995413d22e23a59a44b3ed5a561770b Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Fri, 16 Jun 2023 10:06:22 -0400 Subject: [PATCH 224/243] ci: build on windows also --- .github/workflows/build-main.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-main.yml b/.github/workflows/build-main.yml index 340ad68a..f77cfd03 100644 --- a/.github/workflows/build-main.yml +++ b/.github/workflows/build-main.yml @@ -9,7 +9,10 @@ on: jobs: build: - runs-on: ubuntu-latest + strategy: + matrix: + os: [ubuntu-latest, windows-latest] + runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v2 From 3fdde34c7a7e2e01a5d3c60d0c272ce7d1dd716e Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Tue, 20 Jun 2023 13:21:31 -0400 Subject: [PATCH 225/243] feat!: throw exceptions when attempting to parse an attribute with an incompatible type --- .../n5/CachedGsonKeyValueN5Reader.java | 13 ++- .../saalfeldlab/n5/GsonKeyValueN5Writer.java | 8 +- .../janelia/saalfeldlab/n5/GsonN5Reader.java | 53 +++++---- .../org/janelia/saalfeldlab/n5/GsonUtils.java | 28 ++--- .../janelia/saalfeldlab/n5/N5Exception.java | 38 +++++++ .../saalfeldlab/n5/AbstractN5Test.java | 102 +++++++++--------- 6 files changed, 152 insertions(+), 90 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Reader.java index 49264f0d..9539a7ea 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Reader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Reader.java @@ -28,6 +28,7 @@ import java.io.IOException; import java.lang.reflect.Type; +import com.google.gson.JsonSyntaxException; import org.janelia.saalfeldlab.n5.cache.N5JsonCache; import org.janelia.saalfeldlab.n5.cache.N5JsonCacheableContainer; @@ -99,7 +100,11 @@ default T getAttribute( } else { attributes = GsonKeyValueN5Reader.super.getAttributes(normalPathName); } - return GsonUtils.readAttribute(attributes, normalizedAttributePath, clazz, getGson()); + try { + return GsonUtils.readAttribute(attributes, normalizedAttributePath, clazz, getGson()); + } catch (JsonSyntaxException | NumberFormatException | ClassCastException e) { + throw new N5Exception.N5ClassCastException(e); + } } @Override @@ -116,7 +121,11 @@ default T getAttribute( } else { attributes = GsonKeyValueN5Reader.super.getAttributes(normalPathName); } - return GsonUtils.readAttribute(attributes, normalizedAttributePath, type, getGson()); + try { + return GsonUtils.readAttribute(attributes, normalizedAttributePath, type, getGson()); + } catch (JsonSyntaxException | NumberFormatException | ClassCastException e) { + throw new N5Exception.N5ClassCastException(e); + } } @Override diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java index 9c4fd3f1..6f3ecd82 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java @@ -30,6 +30,7 @@ import java.util.List; import java.util.Map; +import com.google.gson.JsonSyntaxException; import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; import com.google.gson.Gson; @@ -185,7 +186,12 @@ default T removeAttribute(final String pathName, final String key, final Cla final String normalKey = N5URI.normalizeAttributePath(key); final JsonElement attributes = getAttributes(normalPath); - final T obj = GsonUtils.removeAttribute(attributes, normalKey, cls, getGson()); + final T obj; + try { + obj = GsonUtils.removeAttribute(attributes, normalKey, cls, getGson()); + } catch (JsonSyntaxException | NumberFormatException | ClassCastException e) { + throw new N5Exception.N5ClassCastException(e); + } if (obj != null) { writeAttributes(normalPath, attributes); } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonN5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonN5Reader.java index 0df07b35..bfadb345 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonN5Reader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonN5Reader.java @@ -30,6 +30,7 @@ import com.google.gson.Gson; import com.google.gson.JsonElement; +import com.google.gson.JsonSyntaxException; /** * {@link N5Reader} with JSON attributes parsed with {@link Gson}. @@ -55,30 +56,30 @@ default DatasetAttributes getDatasetAttributes(final String pathName) throws N5E default DatasetAttributes createDatasetAttributes(final JsonElement attributes) { - final long[] dimensions = GsonUtils - .readAttribute(attributes, DatasetAttributes.DIMENSIONS_KEY, long[].class, getGson()); - if (dimensions == null) { - return null; - } - - final DataType dataType = GsonUtils - .readAttribute(attributes, DatasetAttributes.DATA_TYPE_KEY, DataType.class, getGson()); - if (dataType == null) { - return null; - } + try { + final long[] dimensions = GsonUtils.readAttribute(attributes, DatasetAttributes.DIMENSIONS_KEY, long[].class, getGson()); + if (dimensions == null) { + return null; + } - final int[] blockSize = GsonUtils - .readAttribute(attributes, DatasetAttributes.BLOCK_SIZE_KEY, int[].class, getGson()); + final DataType dataType = GsonUtils.readAttribute(attributes, DatasetAttributes.DATA_TYPE_KEY, DataType.class, getGson()); + if (dataType == null) { + return null; + } - final Compression compression = GsonUtils - .readAttribute(attributes, DatasetAttributes.COMPRESSION_KEY, Compression.class, getGson()); + final int[] blockSize = GsonUtils.readAttribute(attributes, DatasetAttributes.BLOCK_SIZE_KEY, int[].class, getGson()); + final Compression compression = GsonUtils.readAttribute(attributes, DatasetAttributes.COMPRESSION_KEY, Compression.class, getGson()); - /* version 0 */ - final String compressionVersion0Name = compression == null - ? GsonUtils.readAttribute(attributes, DatasetAttributes.compressionTypeKey, String.class, getGson()) - : null; + /* version 0 */ + final String compressionVersion0Name = compression == null + ? GsonUtils.readAttribute(attributes, DatasetAttributes.compressionTypeKey, String.class, getGson()) + : null; - return DatasetAttributes.from(dimensions, dataType, blockSize, compression, compressionVersion0Name); + return DatasetAttributes.from(dimensions, dataType, blockSize, compression, compressionVersion0Name); + } catch (JsonSyntaxException | NumberFormatException | ClassCastException e) { + /* We cannot create a dataset, so return null. */ + return null; + } } @Override @@ -88,7 +89,11 @@ default T getAttribute(final String pathName, final String key, final Class< final String normalizedAttributePath = N5URI.normalizeAttributePath(key); final JsonElement attributes = getAttributes(normalPathName); - return GsonUtils.readAttribute(attributes, normalizedAttributePath, clazz, getGson()); + try { + return GsonUtils.readAttribute(attributes, normalizedAttributePath, clazz, getGson()); + } catch (JsonSyntaxException | NumberFormatException | ClassCastException e) { + throw new N5Exception.N5ClassCastException(e); + } } @Override @@ -97,7 +102,11 @@ default T getAttribute(final String pathName, final String key, final Type t final String normalPathName = N5URI.normalizeGroupPath(pathName); final String normalizedAttributePath = N5URI.normalizeAttributePath(key); final JsonElement attributes = getAttributes(normalPathName); - return GsonUtils.readAttribute(attributes, normalizedAttributePath, type, getGson()); + try { + return GsonUtils.readAttribute(attributes, normalizedAttributePath, type, getGson()); + } catch (JsonSyntaxException | NumberFormatException | ClassCastException e) { + throw new N5Exception.N5ClassCastException(e); + } } /** diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonUtils.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonUtils.java index 353ae550..eda92a70 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonUtils.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonUtils.java @@ -44,7 +44,7 @@ import com.google.gson.reflect.TypeToken; /** - * {@link N5Reader} for JSON attributes parsed by {@link Gson}. + * Utility class for working with JSON. * * @author Stephan Saalfeld */ @@ -77,7 +77,7 @@ static T readAttribute( final JsonElement root, final String normalizedAttributePath, final Class cls, - final Gson gson) { + final Gson gson) throws JsonSyntaxException, NumberFormatException, ClassCastException { return readAttribute(root, normalizedAttributePath, TypeToken.get(cls).getType(), gson); } @@ -86,7 +86,7 @@ static T readAttribute( final JsonElement root, final String normalizedAttributePath, final Type type, - final Gson gson) { + final Gson gson) throws JsonSyntaxException, NumberFormatException, ClassCastException { final JsonElement attribute = getAttribute(root, normalizedAttributePath); return parseAttributeElement(attribute, gson, type); @@ -106,7 +106,7 @@ static T readAttribute( * @return the deserialized attribute object, or {@code null} if * {@code attribute} cannot deserialize to {@code T} */ - static T parseAttributeElement(final JsonElement attribute, final Gson gson, final Type type) { + static T parseAttributeElement(final JsonElement attribute, final Gson gson, final Type type) throws JsonSyntaxException, NumberFormatException, ClassCastException { if (attribute == null) return null; @@ -128,20 +128,20 @@ static T parseAttributeElement(final JsonElement attribute, final Gson gson, return retArray; } catch (final JsonSyntaxException e) { if (type == String.class) + //noinspection unchecked return (T)gson.toJson(attribute); - return null; - } catch (final NumberFormatException nfe) { - return null; } } try { - return gson.fromJson(attribute, type); + final T parsedResult = gson.fromJson(attribute, type); + if (parsedResult == null) + throw new ClassCastException("Cannot parse json as type " + type.getTypeName()); + return parsedResult; } catch (final JsonSyntaxException e) { if (type == String.class) + //noinspection unchecked return (T)gson.toJson(attribute); - return null; - } catch (final NumberFormatException nfe) { - return null; + throw e; } } @@ -388,7 +388,7 @@ static T removeAttribute( final JsonElement root, final String normalizedAttributePath, final Class cls, - final Gson gson) throws IOException { + final Gson gson) throws JsonSyntaxException, NumberFormatException, ClassCastException, IOException { final T removed = removeAttribute(root, normalizedAttributePath, cls, gson); if (removed != null) { @@ -411,7 +411,7 @@ static boolean removeAttribute( final Writer writer, final JsonElement root, final String normalizedAttributePath, - final Gson gson) throws IOException { + final Gson gson) throws JsonSyntaxException, NumberFormatException, ClassCastException, IOException { final JsonElement removed = removeAttribute(root, normalizedAttributePath, JsonElement.class, gson); if (removed != null) { @@ -440,7 +440,7 @@ static T removeAttribute( final JsonElement root, final String normalizedAttributePath, final Class cls, - final Gson gson) { + final Gson gson) throws JsonSyntaxException, NumberFormatException, ClassCastException { final T attribute = GsonUtils.readAttribute(root, normalizedAttributePath, cls, gson); if (attribute != null) { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5Exception.java b/src/main/java/org/janelia/saalfeldlab/n5/N5Exception.java index a2dea42b..819e0ba5 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5Exception.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5Exception.java @@ -59,4 +59,42 @@ protected N5IOException( super(message, cause, enableSuppression, writableStackTrace); } } + + /** + * This excpetion represents the situation when an attribute is requested by key as a specific Class, + * that attribute key does exist, but is not parseable as the desired Class + */ + public static class N5ClassCastException extends N5Exception { + + + public N5ClassCastException(final Class cls) { + + super("Cannot cast as class " + cls.getName()); + } + + public N5ClassCastException(final String message) { + + super(message); + } + + public N5ClassCastException(final String message, final Throwable cause) { + + super(message, cause); + } + + public N5ClassCastException(final Throwable cause) { + + super(cause); + } + + protected N5ClassCastException( + final String message, + final Throwable cause, + final boolean enableSuppression, + final boolean writableStackTrace) { + + super(message, cause, enableSuppression, writableStackTrace); + } + } + } diff --git a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java index 941f3b23..273a54c5 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java @@ -88,7 +88,7 @@ public abstract class AbstractN5Test { static protected N5Writer n5; - protected abstract String tempN5Location() throws URISyntaxException; + protected abstract String tempN5Location() throws URISyntaxException, IOException; protected N5Writer createN5Writer() throws IOException, URISyntaxException { return createN5Writer(tempN5Location()); @@ -454,11 +454,11 @@ public void testAttributeParsingPrimitive() throws IOException, URISyntaxExcepti /* Test parsing of int, int[], double, double[], String, and String[] types * - * All types should be parseable as JsonElements + * All types are parseable as JsonElements or String * - * ints should be parseable as doubles and Strings - * doubles should be parseable as doubles and Strings - * Strings sould be parsable as Strings + * ints are also parseable as doubles and Strings + * doubles are also parseable as ints and Strings + * Strings should be parsable as Strings * * int[]s should be parseable as double[]s and String[]s * double[]s should be parseable as double[]s and String[]s @@ -466,58 +466,58 @@ public void testAttributeParsingPrimitive() throws IOException, URISyntaxExcepti */ n5.setAttribute(groupName, "key", "value"); - assertNull(n5.getAttribute(groupName, "key", Integer.class )); - assertNull(n5.getAttribute(groupName, "key", int[].class )); - assertNull(n5.getAttribute(groupName, "key", Double.class )); - assertNull(n5.getAttribute(groupName, "key", double[].class )); - assertNotNull(n5.getAttribute(groupName, "key", String.class )); - assertNull(n5.getAttribute(groupName, "key", String[].class )); - assertNotNull(n5.getAttribute(groupName, "key", JsonElement.class )); - - n5.setAttribute(groupName, "key", new String[]{"value" }); - assertNull(n5.getAttribute(groupName, "key", Integer.class )); - assertNull(n5.getAttribute(groupName, "key", int[].class )); - assertNull(n5.getAttribute(groupName, "key", Double.class )); - assertNull(n5.getAttribute(groupName, "key", double[].class )); - assertNotNull(n5.getAttribute(groupName, "key", String.class )); - assertNotNull(n5.getAttribute(groupName, "key", String[].class )); - assertNotNull(n5.getAttribute(groupName, "key", JsonElement.class )); + assertEquals("value", n5.getAttribute(groupName, "key", String.class)); + assertNotNull(n5.getAttribute(groupName, "key", JsonElement.class)); + assertThrows(N5ClassCastException.class, () -> n5.getAttribute(groupName, "key", Integer.class)); + assertThrows(N5ClassCastException.class, () -> n5.getAttribute(groupName, "key", int[].class)); + assertThrows(N5ClassCastException.class, () -> n5.getAttribute(groupName, "key", Double.class)); + assertThrows(N5ClassCastException.class, () -> n5.getAttribute(groupName, "key", double[].class)); + assertThrows(N5ClassCastException.class, () -> n5.getAttribute(groupName, "key", String[].class)); + + n5.setAttribute(groupName, "key", new String[]{"value"}); + assertArrayEquals(new String[]{"value"}, n5.getAttribute(groupName, "key", String[].class)); + assertEquals(JsonParser.parseString("[\"value\"]"), JsonParser.parseString(n5.getAttribute(groupName, "key", String.class))); + assertNotNull(n5.getAttribute(groupName, "key", JsonElement.class)); + assertThrows(N5ClassCastException.class, () -> n5.getAttribute(groupName, "key", Integer.class)); + assertThrows(N5ClassCastException.class, () -> n5.getAttribute(groupName, "key", int[].class)); + assertThrows(N5ClassCastException.class, () -> n5.getAttribute(groupName, "key", Double.class)); + assertThrows(N5ClassCastException.class, () -> n5.getAttribute(groupName, "key", double[].class)); n5.setAttribute(groupName, "key", 1); - assertNotNull(n5.getAttribute(groupName, "key", Integer.class )); - assertNull(n5.getAttribute(groupName, "key", int[].class )); - assertNotNull(n5.getAttribute(groupName, "key", Double.class )); - assertNull(n5.getAttribute(groupName, "key", double[].class )); - assertNotNull(n5.getAttribute(groupName, "key", String.class )); - assertNull(n5.getAttribute(groupName, "key", String[].class )); - assertNotNull(n5.getAttribute(groupName, "key", JsonElement.class )); + assertEquals(1, (long)n5.getAttribute(groupName, "key", Integer.class)); + assertEquals(1.0, n5.getAttribute(groupName, "key", Double.class), .00001); + assertEquals("1", n5.getAttribute(groupName, "key", String.class)); + assertNotNull(n5.getAttribute(groupName, "key", JsonElement.class)); + assertThrows(N5ClassCastException.class, () -> n5.getAttribute(groupName, "key", int[].class)); + assertThrows(N5ClassCastException.class, () -> n5.getAttribute(groupName, "key", double[].class)); + assertThrows(N5ClassCastException.class, () -> n5.getAttribute(groupName, "key", String[].class)); n5.setAttribute(groupName, "key", new int[]{2, 3}); - assertNull(n5.getAttribute(groupName, "key", Integer.class )); - assertNotNull(n5.getAttribute(groupName, "key", int[].class )); - assertNull(n5.getAttribute(groupName, "key", Double.class )); - assertNotNull(n5.getAttribute(groupName, "key", double[].class )); - assertNotNull(n5.getAttribute(groupName, "key", String.class )); - assertNotNull(n5.getAttribute(groupName, "key", String[].class )); - assertNotNull(n5.getAttribute(groupName, "key", JsonElement.class )); + assertArrayEquals(new int[]{2, 3}, n5.getAttribute(groupName, "key", int[].class)); + assertArrayEquals(new double[]{2.0, 3.0}, n5.getAttribute(groupName, "key", double[].class), .00001); + assertEquals(JsonParser.parseString("[2,3]"), JsonParser.parseString(n5.getAttribute(groupName, "key", String.class))); + assertArrayEquals(new String[]{"2", "3"}, n5.getAttribute(groupName, "key", String[].class)); + assertNotNull(n5.getAttribute(groupName, "key", JsonElement.class)); + assertThrows(N5ClassCastException.class, () -> n5.getAttribute(groupName, "key", Integer.class)); + assertThrows(N5ClassCastException.class, () -> n5.getAttribute(groupName, "key", Double.class)); n5.setAttribute(groupName, "key", 0.1); -// assertNull(n5.getAttribute(groupName, "key", Integer.class )); // TODO returns 0, is this right - assertNull(n5.getAttribute(groupName, "key", int[].class )); - assertNotNull(n5.getAttribute(groupName, "key", Double.class )); - assertNull(n5.getAttribute(groupName, "key", double[].class )); - assertNotNull(n5.getAttribute(groupName, "key", String.class )); - assertNull(n5.getAttribute(groupName, "key", String[].class )); - assertNotNull(n5.getAttribute(groupName, "key", JsonElement.class )); + assertEquals(0, (long)n5.getAttribute(groupName, "key", Integer.class)); + assertEquals(0.1, n5.getAttribute(groupName, "key", Double.class), .00001); + assertEquals("0.1", n5.getAttribute(groupName, "key", String.class)); + assertNotNull(n5.getAttribute(groupName, "key", JsonElement.class)); + assertThrows(N5ClassCastException.class, () -> n5.getAttribute(groupName, "key", int[].class)); + assertThrows(N5ClassCastException.class, () -> n5.getAttribute(groupName, "key", double[].class)); + assertThrows(N5ClassCastException.class, () -> n5.getAttribute(groupName, "key", String[].class)); n5.setAttribute(groupName, "key", new double[]{0.2, 0.3}); - assertNull(n5.getAttribute(groupName, "key", Integer.class )); -// assertNull(n5.getAttribute(groupName, "key", int[].class )); // TODO returns not null, is this right? - assertNull(n5.getAttribute(groupName, "key", Double.class )); - assertNotNull(n5.getAttribute(groupName, "key", double[].class )); - assertNotNull(n5.getAttribute(groupName, "key", String.class )); - assertNotNull(n5.getAttribute(groupName, "key", String[].class )); - assertNotNull(n5.getAttribute(groupName, "key", JsonElement.class )); + assertArrayEquals(new int[]{0, 0}, n5.getAttribute(groupName, "key", int[].class)); // TODO returns not null, is this right? + assertArrayEquals(new double[]{0.2, 0.3}, n5.getAttribute(groupName, "key", double[].class), .00001); + assertEquals(JsonParser.parseString("[0.2,0.3]"), JsonParser.parseString(n5.getAttribute(groupName, "key", String.class))); + assertArrayEquals(new String[]{"0.2", "0.3"}, n5.getAttribute(groupName, "key", String[].class)); + assertNotNull(n5.getAttribute(groupName, "key", JsonElement.class)); + assertThrows(N5ClassCastException.class, () -> n5.getAttribute(groupName, "key", Integer.class)); + assertThrows(N5ClassCastException.class, () -> n5.getAttribute(groupName, "key", Double.class)); } } @@ -623,7 +623,7 @@ public void testNullAttributes() throws IOException, URISyntaxException { writer.setAttribute(groupName, "existingValue", 1); assertEquals((Integer)1, writer.getAttribute(groupName, "existingValue", Integer.class)); writer.setAttribute(groupName, "existingValue", null); - assertNull(writer.getAttribute(groupName, "existingValue", Integer.class)); + assertThrows(N5ClassCastException.class, () -> writer.getAttribute(groupName, "existingValue", Integer.class)); assertEquals(JsonNull.INSTANCE, writer.getAttribute(groupName, "existingValue", JsonElement.class)); } @@ -689,7 +689,7 @@ public void testRemoveAttributes() throws IOException, URISyntaxException { writer.setAttribute("", "a/b/c", 100); assertEquals((Integer)100, writer.getAttribute("", "a/b/c", Integer.class)); /* Remove Test with incorrect Type */ - assertNull(writer.removeAttribute("", "a/b/c", Boolean.class)); + assertThrows(N5ClassCastException.class, () -> writer.removeAttribute("", "a/b/c", Boolean.class)); final Integer abcInteger = writer.removeAttribute("", "a/b/c", Integer.class); assertEquals((Integer)100, abcInteger); assertNull(writer.getAttribute("", "a/b/c", Integer.class)); From a3de0af7d072e80eaa94a963fa43e6be5511c06f Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Tue, 20 Jun 2023 13:21:54 -0400 Subject: [PATCH 226/243] doc: cleanup --- .../java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java | 2 +- src/main/java/org/janelia/saalfeldlab/n5/GsonN5Writer.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java index 907c0f25..66dbcc52 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java @@ -77,7 +77,7 @@ default boolean datasetExists(final String pathName) throws N5Exception { * @param pathName * group path * @return - * @throws IOException + * @throws N5Exception if the attributes cannot be read */ @Override default JsonElement getAttributes(final String pathName) throws N5Exception { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonN5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonN5Writer.java index 74291514..f0030441 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonN5Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonN5Writer.java @@ -36,7 +36,7 @@ public interface GsonN5Writer extends GsonN5Reader, N5Writer { /** * Set the attributes of a group. This result of this method is equivalent - * with {@link #setAttribute(groupPath, "/", attributes)}. + * with {@link N5Writer#setAttribute(String, String, Object) N5Writer#setAttribute(groupPath, "/", attributes)}. * * @param groupPath * to write the attributes to From 0ce652b8eedb3db78dc223131adebe8dc591b8c3 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Tue, 20 Jun 2023 13:21:59 -0400 Subject: [PATCH 227/243] style: whitespace, imports --- .../saalfeldlab/n5/AbstractN5Test.java | 57 ++++++++++--------- 1 file changed, 30 insertions(+), 27 deletions(-) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java index 273a54c5..94e67d24 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java @@ -25,14 +25,18 @@ */ package org.janelia.saalfeldlab.n5; -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertThrows; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; +import com.google.gson.GsonBuilder; +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonNull; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import com.google.gson.reflect.TypeToken; +import org.janelia.saalfeldlab.n5.N5Exception.N5ClassCastException; +import org.janelia.saalfeldlab.n5.N5Reader.Version; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.Test; import java.io.IOException; import java.net.URISyntaxException; @@ -49,17 +53,14 @@ import java.util.concurrent.Executors; import java.util.function.Predicate; -import org.janelia.saalfeldlab.n5.N5Reader.Version; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.Test; - -import com.google.gson.GsonBuilder; -import com.google.gson.JsonArray; -import com.google.gson.JsonElement; -import com.google.gson.JsonNull; -import com.google.gson.JsonObject; -import com.google.gson.reflect.TypeToken; +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; /** * Abstract base class for testing N5 functionality. @@ -91,10 +92,12 @@ public abstract class AbstractN5Test { protected abstract String tempN5Location() throws URISyntaxException, IOException; protected N5Writer createN5Writer() throws IOException, URISyntaxException { + return createN5Writer(tempN5Location()); } protected N5Writer createN5Writer(final String location) throws IOException, URISyntaxException { + return createN5Writer(location, new GsonBuilder()); } @@ -997,7 +1000,7 @@ public void testExists() throws IOException, URISyntaxException { final String groupName2 = groupName + "-2"; final String datasetName2 = datasetName + "-2"; final String notExists = groupName + "-notexists"; - try (N5Writer n5 = createN5Writer()){ + try (N5Writer n5 = createN5Writer()) { n5.createDataset(datasetName2, dimensions, blockSize, DataType.UINT64, new RawCompression()); assertTrue(n5.exists(datasetName2)); assertTrue(n5.datasetExists(datasetName2)); @@ -1419,23 +1422,23 @@ private String readAttributesAsString(final String group) { n5.createGroup(groupName); n5.setAttribute(groupName, "/", "String"); - final JsonElement stringPrimitive = n5.getAttribute(groupName, "/", JsonElement.class); + final JsonElement stringPrimitive = n5.getAttribute(groupName, "/", JsonElement.class); assertTrue(stringPrimitive.isJsonPrimitive()); assertEquals("String", stringPrimitive.getAsString()); n5.setAttribute(groupName, "/", 0); - final JsonElement intPrimitive = n5.getAttribute(groupName, "/", JsonElement.class); + final JsonElement intPrimitive = n5.getAttribute(groupName, "/", JsonElement.class); assertTrue(intPrimitive.isJsonPrimitive()); assertEquals(0, intPrimitive.getAsInt()); n5.setAttribute(groupName, "/", true); - final JsonElement booleanPrimitive = n5.getAttribute(groupName, "/", JsonElement.class); + final JsonElement booleanPrimitive = n5.getAttribute(groupName, "/", JsonElement.class); assertTrue(booleanPrimitive.isJsonPrimitive()); assertEquals(true, booleanPrimitive.getAsBoolean()); - n5.setAttribute(groupName, "/", null); - final JsonElement jsonNull = n5.getAttribute(groupName, "/", JsonElement.class); + n5.setAttribute(groupName, "/", null); + final JsonElement jsonNull = n5.getAttribute(groupName, "/", JsonElement.class); assertTrue(jsonNull.isJsonNull()); assertEquals(JsonNull.INSTANCE, jsonNull); - n5.setAttribute(groupName, "[5]", "array"); - final JsonElement rootJsonArray = n5.getAttribute(groupName, "/", JsonElement.class); + n5.setAttribute(groupName, "[5]", "array"); + final JsonElement rootJsonArray = n5.getAttribute(groupName, "/", JsonElement.class); assertTrue(rootJsonArray.isJsonArray()); final JsonArray rootArray = rootJsonArray.getAsJsonArray(); assertEquals("array", rootArray.get(5).getAsString()); From 2513c1b720848edff6f974452c0332c99b35d411 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Tue, 20 Jun 2023 13:22:03 -0400 Subject: [PATCH 228/243] feat(test): add write and remove cache tracking tests --- .../saalfeldlab/n5/N5CachedFSTest.java | 72 ++++++++++++++++++- 1 file changed, 70 insertions(+), 2 deletions(-) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/N5CachedFSTest.java b/src/test/java/org/janelia/saalfeldlab/n5/N5CachedFSTest.java index c7a4aada..3c9aaa09 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/N5CachedFSTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/N5CachedFSTest.java @@ -145,6 +145,7 @@ public static void cacheBehaviorHelper(final TrackingStorage n5) throws IOExcept final int expectedDatasetCount = 0; final int expectedAttributeCount = 0; int expectedListCount = 0; + int expectedWriteAttributeCount = 0; boolean exists = n5.exists(groupA); boolean groupExists = n5.groupExists(groupA); @@ -156,9 +157,11 @@ public static void cacheBehaviorHelper(final TrackingStorage n5) throws IOExcept assertEquals(expectedGroupCount, n5.getGroupCallCount()); assertEquals(expectedDatasetCount, n5.getDatasetCallCount()); assertEquals(expectedAttributeCount, n5.getAttrCallCount()); + assertEquals(expectedWriteAttributeCount, n5.getWriteAttrCallCount()); n5.createGroup(groupA); assertEquals(expectedAttributeCount, n5.getAttrCallCount()); + assertEquals(expectedWriteAttributeCount, n5.getWriteAttrCallCount()); // group B exists = n5.exists(groupB); @@ -171,6 +174,7 @@ public static void cacheBehaviorHelper(final TrackingStorage n5) throws IOExcept assertEquals(expectedGroupCount, n5.getGroupCallCount()); assertEquals(expectedDatasetCount, n5.getDatasetCallCount()); assertEquals(expectedAttributeCount, n5.getAttrCallCount()); + assertEquals(expectedWriteAttributeCount, n5.getWriteAttrCallCount()); exists = n5.exists(groupA); groupExists = n5.groupExists(groupA); @@ -182,6 +186,7 @@ public static void cacheBehaviorHelper(final TrackingStorage n5) throws IOExcept assertEquals(expectedGroupCount, n5.getGroupCallCount()); assertEquals(expectedDatasetCount, n5.getDatasetCallCount()); assertEquals(expectedAttributeCount, n5.getAttrCallCount()); + assertEquals(expectedWriteAttributeCount, n5.getWriteAttrCallCount()); final String cachedGroup = "cachedGroup"; // should not check existence when creating a group @@ -192,6 +197,7 @@ public static void cacheBehaviorHelper(final TrackingStorage n5) throws IOExcept assertEquals(expectedDatasetCount, n5.getDatasetCallCount()); assertEquals(expectedAttributeCount, n5.getAttrCallCount()); assertEquals(expectedListCount, n5.getListCallCount()); + assertEquals(expectedWriteAttributeCount, n5.getWriteAttrCallCount()); // should not check existence when this instance created a group n5.exists(cachedGroup); @@ -202,16 +208,16 @@ public static void cacheBehaviorHelper(final TrackingStorage n5) throws IOExcept assertEquals(expectedDatasetCount, n5.getDatasetCallCount()); assertEquals(expectedAttributeCount, n5.getAttrCallCount()); assertEquals(expectedListCount, n5.getListCallCount()); + assertEquals(expectedWriteAttributeCount, n5.getWriteAttrCallCount()); // should not read attributes from container when setting them - System.out.println(n5.getAttrCallCount()); n5.setAttribute(cachedGroup, "one", 1); - System.out.println(n5.getAttrCallCount()); assertEquals(expectedExistCount, n5.getExistCallCount()); assertEquals(expectedGroupCount, n5.getGroupCallCount()); assertEquals(expectedDatasetCount, n5.getDatasetCallCount()); assertEquals(expectedAttributeCount, n5.getAttrCallCount()); assertEquals(expectedListCount, n5.getListCallCount()); + assertEquals(++expectedWriteAttributeCount, n5.getWriteAttrCallCount()); n5.setAttribute(cachedGroup, "two", 2); assertEquals(expectedExistCount, n5.getExistCallCount()); @@ -219,6 +225,39 @@ public static void cacheBehaviorHelper(final TrackingStorage n5) throws IOExcept assertEquals(expectedDatasetCount, n5.getDatasetCallCount()); assertEquals(expectedAttributeCount, n5.getAttrCallCount()); assertEquals(expectedListCount, n5.getListCallCount()); + assertEquals(++expectedWriteAttributeCount, n5.getWriteAttrCallCount()); + + n5.removeAttribute(cachedGroup, "one"); + assertEquals(expectedExistCount, n5.getExistCallCount()); + assertEquals(expectedGroupCount, n5.getGroupCallCount()); + assertEquals(expectedDatasetCount, n5.getDatasetCallCount()); + assertEquals(expectedAttributeCount, n5.getAttrCallCount()); + assertEquals(expectedListCount, n5.getListCallCount()); + assertEquals(++expectedWriteAttributeCount, n5.getWriteAttrCallCount()); + + n5.removeAttribute(cachedGroup, "one"); + assertEquals(expectedExistCount, n5.getExistCallCount()); + assertEquals(expectedGroupCount, n5.getGroupCallCount()); + assertEquals(expectedDatasetCount, n5.getDatasetCallCount()); + assertEquals(expectedAttributeCount, n5.getAttrCallCount()); + assertEquals(expectedListCount, n5.getListCallCount()); + assertEquals(expectedWriteAttributeCount, n5.getWriteAttrCallCount()); + + n5.removeAttribute(cachedGroup, "cow"); + assertEquals(expectedExistCount, n5.getExistCallCount()); + assertEquals(expectedGroupCount, n5.getGroupCallCount()); + assertEquals(expectedDatasetCount, n5.getDatasetCallCount()); + assertEquals(expectedAttributeCount, n5.getAttrCallCount()); + assertEquals(expectedListCount, n5.getListCallCount()); + assertEquals(expectedWriteAttributeCount, n5.getWriteAttrCallCount()); + + n5.removeAttribute(cachedGroup, "two", Integer.class); + assertEquals(expectedExistCount, n5.getExistCallCount()); + assertEquals(expectedGroupCount, n5.getGroupCallCount()); + assertEquals(expectedDatasetCount, n5.getDatasetCallCount()); + assertEquals(expectedAttributeCount, n5.getAttrCallCount()); + assertEquals(expectedListCount, n5.getListCallCount()); + assertEquals(++expectedWriteAttributeCount, n5.getWriteAttrCallCount()); n5.list(""); assertEquals(expectedExistCount, n5.getExistCallCount()); @@ -226,6 +265,7 @@ public static void cacheBehaviorHelper(final TrackingStorage n5) throws IOExcept assertEquals(expectedDatasetCount, n5.getDatasetCallCount()); assertEquals(expectedAttributeCount, n5.getAttrCallCount()); assertEquals(++expectedListCount, n5.getListCallCount()); + assertEquals(expectedWriteAttributeCount, n5.getWriteAttrCallCount()); n5.list(cachedGroup); assertEquals(expectedExistCount, n5.getExistCallCount()); @@ -233,6 +273,7 @@ public static void cacheBehaviorHelper(final TrackingStorage n5) throws IOExcept assertEquals(expectedDatasetCount, n5.getDatasetCallCount()); assertEquals(expectedAttributeCount, n5.getAttrCallCount()); assertEquals(++expectedListCount, n5.getListCallCount()); + assertEquals(expectedWriteAttributeCount, n5.getWriteAttrCallCount()); /* * Check existence for groups that have not been made by this reader but isGroup @@ -252,6 +293,7 @@ public static void cacheBehaviorHelper(final TrackingStorage n5) throws IOExcept assertEquals(expectedDatasetCount, n5.getDatasetCallCount()); assertEquals(expectedAttributeCount, n5.getAttrCallCount()); assertEquals(expectedListCount, n5.getListCallCount()); + assertEquals(expectedWriteAttributeCount, n5.getWriteAttrCallCount()); n5.groupExists(nonExistentGroup); assertEquals(expectedExistCount, n5.getExistCallCount()); @@ -259,6 +301,7 @@ public static void cacheBehaviorHelper(final TrackingStorage n5) throws IOExcept assertEquals(expectedDatasetCount, n5.getDatasetCallCount()); assertEquals(expectedAttributeCount, n5.getAttrCallCount()); assertEquals(expectedListCount, n5.getListCallCount()); + assertEquals(expectedWriteAttributeCount, n5.getWriteAttrCallCount()); n5.datasetExists(nonExistentGroup); assertEquals(expectedExistCount, n5.getExistCallCount()); @@ -266,6 +309,7 @@ public static void cacheBehaviorHelper(final TrackingStorage n5) throws IOExcept assertEquals(expectedDatasetCount, n5.getDatasetCallCount()); assertEquals(expectedAttributeCount, n5.getAttrCallCount()); assertEquals(expectedListCount, n5.getListCallCount()); + assertEquals(expectedWriteAttributeCount, n5.getWriteAttrCallCount()); n5.getAttributes(nonExistentGroup); assertEquals(expectedExistCount, n5.getExistCallCount()); @@ -273,6 +317,7 @@ public static void cacheBehaviorHelper(final TrackingStorage n5) throws IOExcept assertEquals(expectedDatasetCount, n5.getDatasetCallCount()); assertEquals(expectedAttributeCount, n5.getAttrCallCount()); assertEquals(expectedListCount, n5.getListCallCount()); + assertEquals(expectedWriteAttributeCount, n5.getWriteAttrCallCount()); assertThrows(N5Exception.class, () -> n5.list(nonExistentGroup)); assertEquals(expectedExistCount, n5.getExistCallCount()); @@ -280,6 +325,7 @@ public static void cacheBehaviorHelper(final TrackingStorage n5) throws IOExcept assertEquals(expectedDatasetCount, n5.getDatasetCallCount()); assertEquals(expectedAttributeCount, n5.getAttrCallCount()); assertEquals(expectedListCount, n5.getListCallCount()); + assertEquals(expectedWriteAttributeCount, n5.getWriteAttrCallCount()); final String a = "a"; final String ab = "a/b"; @@ -294,6 +340,7 @@ public static void cacheBehaviorHelper(final TrackingStorage n5) throws IOExcept assertEquals(expectedDatasetCount, n5.getDatasetCallCount()); assertEquals(expectedAttributeCount, n5.getAttrCallCount()); assertEquals(expectedListCount, n5.getListCallCount()); + assertEquals(expectedWriteAttributeCount, n5.getWriteAttrCallCount()); // ensure that backend need not be checked when testing existence of "a/b" // TODO how does this work @@ -305,6 +352,7 @@ public static void cacheBehaviorHelper(final TrackingStorage n5) throws IOExcept assertEquals(expectedDatasetCount, n5.getDatasetCallCount()); assertEquals(expectedAttributeCount, n5.getAttrCallCount()); assertEquals(expectedListCount, n5.getListCallCount()); + assertEquals(expectedWriteAttributeCount, n5.getWriteAttrCallCount()); // remove a nested group // checks for all children should not require a backend check @@ -317,6 +365,7 @@ public static void cacheBehaviorHelper(final TrackingStorage n5) throws IOExcept assertEquals(expectedDatasetCount, n5.getDatasetCallCount()); assertEquals(expectedAttributeCount, n5.getAttrCallCount()); assertEquals(expectedListCount, n5.getListCallCount()); + assertEquals(expectedWriteAttributeCount, n5.getWriteAttrCallCount()); assertFalse(n5.exists(ab)); assertFalse(n5.groupExists(ab)); @@ -326,6 +375,7 @@ public static void cacheBehaviorHelper(final TrackingStorage n5) throws IOExcept assertEquals(expectedDatasetCount, n5.getDatasetCallCount()); assertEquals(expectedAttributeCount, n5.getAttrCallCount()); assertEquals(expectedListCount, n5.getListCallCount()); + assertEquals(expectedWriteAttributeCount, n5.getWriteAttrCallCount()); assertFalse(n5.exists(abc)); assertFalse(n5.groupExists(abc)); @@ -335,21 +385,27 @@ public static void cacheBehaviorHelper(final TrackingStorage n5) throws IOExcept assertEquals(expectedDatasetCount, n5.getDatasetCallCount()); assertEquals(expectedAttributeCount, n5.getAttrCallCount()); assertEquals(expectedListCount, n5.getListCallCount()); + assertEquals(expectedWriteAttributeCount, n5.getWriteAttrCallCount()); n5.createGroup("a"); assertEquals(expectedExistCount, n5.getExistCallCount()); + assertEquals(expectedWriteAttributeCount, n5.getWriteAttrCallCount()); n5.createGroup("a/a"); assertEquals(++expectedExistCount, n5.getExistCallCount()); + assertEquals(expectedWriteAttributeCount, n5.getWriteAttrCallCount()); n5.createGroup("a/b"); assertEquals(expectedExistCount, n5.getExistCallCount()); + assertEquals(expectedWriteAttributeCount, n5.getWriteAttrCallCount()); n5.createGroup("a/c"); assertEquals(++expectedExistCount, n5.getExistCallCount()); + assertEquals(expectedWriteAttributeCount, n5.getWriteAttrCallCount()); assertArrayEquals(new String[] {"a", "b", "c"}, n5.list("a")); // call list assertEquals(expectedGroupCount, n5.getGroupCallCount()); assertEquals(expectedDatasetCount, n5.getDatasetCallCount()); assertEquals(expectedAttributeCount, n5.getAttrCallCount()); assertEquals(++expectedListCount, n5.getListCallCount()); // list incremented + assertEquals(expectedWriteAttributeCount, n5.getWriteAttrCallCount()); // remove a n5.remove("a/a"); @@ -359,6 +415,7 @@ public static void cacheBehaviorHelper(final TrackingStorage n5) throws IOExcept assertEquals(expectedDatasetCount, n5.getDatasetCallCount()); assertEquals(expectedAttributeCount, n5.getAttrCallCount()); assertEquals(expectedListCount, n5.getListCallCount()); // list NOT incremented + assertEquals(expectedWriteAttributeCount, n5.getWriteAttrCallCount()); // TODO repeat the above exercise when creating dataset } @@ -372,6 +429,7 @@ public static interface TrackingStorage extends CachedGsonKeyValueN5Writer { public int getDatasetCallCount(); public int getDatasetAttrCallCount(); public int getListCallCount(); + public int getWriteAttrCallCount(); } public static class N5TrackingStorage extends N5KeyValueWriter implements TrackingStorage { @@ -383,6 +441,7 @@ public static class N5TrackingStorage extends N5KeyValueWriter implements Tracki public int datasetCallCount = 0; public int datasetAttrCallCount = 0; public int listCallCount = 0; + public int writeAttrCallCount = 0; public N5TrackingStorage(final KeyValueAccess keyValueAccess, final String basePath, final GsonBuilder gsonBuilder, final boolean cacheAttributes) throws IOException { @@ -432,6 +491,11 @@ public String[] listFromContainer(final String key) { return super.listFromContainer(key); } + @Override public void writeAttributes(String normalGroupPath, JsonElement attributes) throws N5Exception { + writeAttrCallCount++; + super.writeAttributes(normalGroupPath, attributes); + } + @Override public int getAttrCallCount() { return attrCallCount; @@ -466,6 +530,10 @@ public int getDatasetAttrCallCount() { public int getListCallCount() { return listCallCount; } + + @Override public int getWriteAttrCallCount() { + return writeAttrCallCount; + } } } From 49243085684329baa076fcb61896fd2a0c4c2786 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Tue, 20 Jun 2023 13:22:06 -0400 Subject: [PATCH 229/243] feat(test): ensure result URI can be used to get a new reader over the same container --- .../org/janelia/saalfeldlab/n5/AbstractN5Test.java | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java index 94e67d24..20b08a2e 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java @@ -798,6 +798,16 @@ public void testRemoveContainer() throws IOException, URISyntaxException { assertThrows(Exception.class, () -> createN5Reader(location)); } + @Test + public void testUri() throws IOException, URISyntaxException { + + try (final N5Writer writer = createN5Writer()) { + try (final N5Reader reader = createN5Reader(writer.getURI().toString())) { + assertEquals(writer.getURI(), reader.getURI()); + } + } + } + @Test public void testRemoveGroup() throws IOException, URISyntaxException { From 27d92788498a72035849995f93123206bb1ab7d6 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Tue, 20 Jun 2023 13:22:10 -0400 Subject: [PATCH 230/243] fix(test): close more reader/writers --- .../org/janelia/saalfeldlab/n5/AbstractN5Test.java | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java index 20b08a2e..e7040745 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java @@ -793,9 +793,9 @@ public void testRemoveContainer() throws IOException, URISyntaxException { location = n5.getURI().toString(); assertNotNull(createN5Reader(location)); n5.remove(); - assertThrows(Exception.class, () -> createN5Reader(location)); + assertThrows(Exception.class, () -> createN5Reader(location).close()); } - assertThrows(Exception.class, () -> createN5Reader(location)); + assertThrows(Exception.class, () -> createN5Reader(location).close()); } @Test @@ -1091,7 +1091,7 @@ public void testVersion() throws NumberFormatException, IOException, URISyntaxEx assertThrows(N5Exception.N5IOException.class, () -> { final String containerPath = writer.getURI().toString(); - createN5Writer(containerPath); + createN5Writer(containerPath).close(); }); final Version compatibleVersion = new Version(N5Reader.VERSION.getMajor(), N5Reader.VERSION.getMinor(), N5Reader.VERSION.getPatch()); @@ -1125,10 +1125,7 @@ public void testReaderCreation() throws IOException, URISyntaxException { writer.removeAttribute("/", "/"); writer.setAttribute("/", N5Reader.VERSION_KEY, new Version(N5Reader.VERSION.getMajor() + 1, N5Reader.VERSION.getMinor(), N5Reader.VERSION.getPatch()).toString()); - assertThrows("Incompatible version throws error", N5Exception.N5IOException.class, - () -> { - createN5Reader(location); - }); + assertThrows("Incompatible version throws error", N5Exception.class, () -> createN5Reader(location).close()); writer.remove(); } // non-existent group should fail @@ -1136,6 +1133,7 @@ public void testReaderCreation() throws IOException, URISyntaxException { () -> { final N5Reader test = createN5Reader(location); test.list("/"); + test.close(); }); } From b637363e9df9396f0e18f5ce53308ad7c21df2f0 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Tue, 20 Jun 2023 13:22:15 -0400 Subject: [PATCH 231/243] fix(test): test was reading and testing the incorrect attribute key --- src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java index e7040745..094ca4dc 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java @@ -1295,8 +1295,8 @@ public void testAttributePaths() throws IOException, URISyntaxException { addAndTest(writer, existingTests, new TestData<>(testGroup, "/filled/string_array[4]", "e")); addAndTest(writer, existingTests, new TestData<>(testGroup, "/filled/string_array[0]", "a")); - /* We intentionally skipped index 3, it should be null */ - assertNull(writer.getAttribute(testGroup, "/filled/double_array[3]", JsonNull.class)); + /* We intentionally skipped index 3, but it should have been pre-populated with JsonNull */ + assertEquals(JsonNull.INSTANCE, writer.getAttribute(testGroup, "/filled/string_array[3]", JsonNull.class)); /* Ensure that escaping does NOT interpret the json path structure, but rather it adds the keys opaquely*/ final HashMap testAttributes = new HashMap<>(); From 33a09908a45f71b92b4d38043ff86f28024f0e54 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Tue, 20 Jun 2023 13:22:19 -0400 Subject: [PATCH 232/243] refactor(test): better test patterns --- .../saalfeldlab/n5/AbstractN5Test.java | 134 ++++++------------ 1 file changed, 46 insertions(+), 88 deletions(-) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java index 094ca4dc..e3f6d73e 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java @@ -908,12 +908,8 @@ public void testDeepList() throws IOException, URISyntaxException, ExecutionExce assertFalse("deepList stops at datasets", datasetListP3.contains(datasetName + "/0")); // test filtering - final Predicate isCalledDataset = d -> { - return d.endsWith("/dataset"); - }; - final Predicate isBorC = d -> { - return d.matches(".*/[bc]$"); - }; + final Predicate isCalledDataset = d -> d.endsWith("/dataset"); + final Predicate isBorC = d -> d.matches(".*/[bc]$"); final List datasetListFilter1 = Arrays.asList(n5.deepList(prefix, isCalledDataset)); assertTrue( @@ -944,29 +940,13 @@ public void testDeepList() throws IOException, URISyntaxException, ExecutionExce datasetListFilterD.size() == 1 && (prefix + "/" + datasetListFilterD.get(0)).equals(datasetName)); assertArrayEquals( datasetListFilterD.toArray(), - n5.deepList( - prefix, - a -> { - try { - return n5.datasetExists(a); - } catch (final N5Exception e) { - return false; - } - })); + n5.deepList(prefix, n5::datasetExists)); final List datasetListFilterDandBC = Arrays.asList(n5.deepListDatasets(prefix, isBorC)); assertTrue("deepListDatasetFilter", datasetListFilterDandBC.size() == 0); assertArrayEquals( datasetListFilterDandBC.toArray(), - n5.deepList( - prefix, - a -> { - try { - return n5.datasetExists(a) && isBorC.test(a); - } catch (final N5Exception e) { - return false; - } - })); + n5.deepList(prefix, a -> n5.datasetExists(a) && isBorC.test(a))); final List datasetListFilterDP = Arrays.asList(n5.deepListDatasets(prefix, Executors.newFixedThreadPool(2))); @@ -975,32 +955,14 @@ public void testDeepList() throws IOException, URISyntaxException, ExecutionExce datasetListFilterDP.size() == 1 && (prefix + "/" + datasetListFilterDP.get(0)).equals(datasetName)); assertArrayEquals( datasetListFilterDP.toArray(), - n5.deepList( - prefix, - a -> { - try { - return n5.datasetExists(a); - } catch (final N5Exception e) { - return false; - } - }, - Executors.newFixedThreadPool(2))); + n5.deepList(prefix, n5::datasetExists, Executors.newFixedThreadPool(2))); final List datasetListFilterDandBCP = Arrays.asList(n5.deepListDatasets(prefix, isBorC, Executors.newFixedThreadPool(2))); assertTrue("deepListDatasetFilter Parallel", datasetListFilterDandBCP.size() == 0); assertArrayEquals( datasetListFilterDandBCP.toArray(), - n5.deepList( - prefix, - a -> { - try { - return n5.datasetExists(a) && isBorC.test(a); - } catch (final N5Exception e) { - return false; - } - }, - Executors.newFixedThreadPool(2))); + n5.deepList(prefix, a -> n5.datasetExists(a) && isBorC.test(a), Executors.newFixedThreadPool(2))); } } @@ -1030,49 +992,45 @@ public void testListAttributes() { final String groupName2 = groupName + "-2"; final String datasetName2 = datasetName + "-2"; - try { - n5.createDataset(datasetName2, dimensions, blockSize, DataType.UINT64, new RawCompression()); - n5.setAttribute(datasetName2, "attr1", new double[]{1.1, 2.1, 3.1}); - n5.setAttribute(datasetName2, "attr2", new String[]{"a", "b", "c"}); - n5.setAttribute(datasetName2, "attr3", 1.1); - n5.setAttribute(datasetName2, "attr4", "a"); - n5.setAttribute(datasetName2, "attr5", new long[]{1, 2, 3}); - n5.setAttribute(datasetName2, "attr6", 1); - n5.setAttribute(datasetName2, "attr7", new double[]{1, 2, 3.1}); - n5.setAttribute(datasetName2, "attr8", new Object[]{"1", 2, 3.1}); - - Map> attributesMap = n5.listAttributes(datasetName2); - assertTrue(attributesMap.get("attr1") == double[].class); - assertTrue(attributesMap.get("attr2") == String[].class); - assertTrue(attributesMap.get("attr3") == double.class); - assertTrue(attributesMap.get("attr4") == String.class); - assertTrue(attributesMap.get("attr5") == long[].class); - assertTrue(attributesMap.get("attr6") == long.class); - assertTrue(attributesMap.get("attr7") == double[].class); - assertTrue(attributesMap.get("attr8") == Object[].class); - - n5.createGroup(groupName2); - n5.setAttribute(groupName2, "attr1", new double[]{1.1, 2.1, 3.1}); - n5.setAttribute(groupName2, "attr2", new String[]{"a", "b", "c"}); - n5.setAttribute(groupName2, "attr3", 1.1); - n5.setAttribute(groupName2, "attr4", "a"); - n5.setAttribute(groupName2, "attr5", new long[]{1, 2, 3}); - n5.setAttribute(groupName2, "attr6", 1); - n5.setAttribute(groupName2, "attr7", new double[]{1, 2, 3.1}); - n5.setAttribute(groupName2, "attr8", new Object[]{"1", 2, 3.1}); - - attributesMap = n5.listAttributes(groupName2); - assertTrue(attributesMap.get("attr1") == double[].class); - assertTrue(attributesMap.get("attr2") == String[].class); - assertTrue(attributesMap.get("attr3") == double.class); - assertTrue(attributesMap.get("attr4") == String.class); - assertTrue(attributesMap.get("attr5") == long[].class); - assertTrue(attributesMap.get("attr6") == long.class); - assertTrue(attributesMap.get("attr7") == double[].class); - assertTrue(attributesMap.get("attr8") == Object[].class); - } catch (final N5Exception e) { - fail(e.getMessage()); - } + n5.createDataset(datasetName2, dimensions, blockSize, DataType.UINT64, new RawCompression()); + n5.setAttribute(datasetName2, "attr1", new double[]{1.1, 2.1, 3.1}); + n5.setAttribute(datasetName2, "attr2", new String[]{"a", "b", "c"}); + n5.setAttribute(datasetName2, "attr3", 1.1); + n5.setAttribute(datasetName2, "attr4", "a"); + n5.setAttribute(datasetName2, "attr5", new long[]{1, 2, 3}); + n5.setAttribute(datasetName2, "attr6", 1); + n5.setAttribute(datasetName2, "attr7", new double[]{1, 2, 3.1}); + n5.setAttribute(datasetName2, "attr8", new Object[]{"1", 2, 3.1}); + + Map> attributesMap = n5.listAttributes(datasetName2); + assertTrue(attributesMap.get("attr1") == double[].class); + assertTrue(attributesMap.get("attr2") == String[].class); + assertTrue(attributesMap.get("attr3") == double.class); + assertTrue(attributesMap.get("attr4") == String.class); + assertTrue(attributesMap.get("attr5") == long[].class); + assertTrue(attributesMap.get("attr6") == long.class); + assertTrue(attributesMap.get("attr7") == double[].class); + assertTrue(attributesMap.get("attr8") == Object[].class); + + n5.createGroup(groupName2); + n5.setAttribute(groupName2, "attr1", new double[]{1.1, 2.1, 3.1}); + n5.setAttribute(groupName2, "attr2", new String[]{"a", "b", "c"}); + n5.setAttribute(groupName2, "attr3", 1.1); + n5.setAttribute(groupName2, "attr4", "a"); + n5.setAttribute(groupName2, "attr5", new long[]{1, 2, 3}); + n5.setAttribute(groupName2, "attr6", 1); + n5.setAttribute(groupName2, "attr7", new double[]{1, 2, 3.1}); + n5.setAttribute(groupName2, "attr8", new Object[]{"1", 2, 3.1}); + + attributesMap = n5.listAttributes(groupName2); + assertTrue(attributesMap.get("attr1") == double[].class); + assertTrue(attributesMap.get("attr2") == String[].class); + assertTrue(attributesMap.get("attr3") == double.class); + assertTrue(attributesMap.get("attr4") == String.class); + assertTrue(attributesMap.get("attr5") == long[].class); + assertTrue(attributesMap.get("attr6") == long.class); + assertTrue(attributesMap.get("attr7") == double[].class); + assertTrue(attributesMap.get("attr8") == Object[].class); } @Test @@ -1082,7 +1040,7 @@ public void testVersion() throws NumberFormatException, IOException, URISyntaxEx final Version n5Version = writer.getVersion(); - assertTrue(n5Version.equals(N5Reader.VERSION)); + assertEquals(n5Version, N5Reader.VERSION); final Version incompatibleVersion = new Version(N5Reader.VERSION.getMajor() + 1, N5Reader.VERSION.getMinor(), N5Reader.VERSION.getPatch()); writer.setAttribute("/", N5Reader.VERSION_KEY, incompatibleVersion.toString()); From 9347fd09f6ffc3d9c7c9a2d02af9d5efba5b1dd4 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Tue, 20 Jun 2023 14:10:16 -0400 Subject: [PATCH 233/243] Revert "BREAKING: rm default groupPath impl in N5Reader" This reverts commit ccda98e2 --- .../saalfeldlab/n5/GsonKeyValueN5Reader.java | 10 ------- .../org/janelia/saalfeldlab/n5/N5Reader.java | 26 ++++++++++++++----- 2 files changed, 20 insertions(+), 16 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java index 66dbcc52..c32fcd3c 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java @@ -47,16 +47,6 @@ default boolean groupExists(final String normalPath) { return getKeyValueAccess().isDirectory(absoluteGroupPath(normalPath)); } - @Override - default String groupPath(final String... nodes) { - - // alternatively call compose twice, once with this functions inputs, - // then pass the result to the other groupPath method - // this impl assumes streams and array building are less expensive than - // keyValueAccess composition (may not always be true) - return getKeyValueAccess().compose(getURI(), nodes); - } - @Override default boolean exists(final String pathName) { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java index f2e2cf83..c5601129 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java @@ -780,15 +780,29 @@ default String getGroupSeparator() { } /** - * Returns an absolute path to a group by concatenating all nodes with the - * node separator - * defined by {@link #getGroupSeparator()}. + * Creates a group path by concatenating all nodes with the node separator + * defined by {@link #getGroupSeparator()}. The string will not have a + * leading or trailing node separator symbol. * * @param nodes - * the group components - * @return the absolute group path + * @return */ - String groupPath(final String... nodes); + default String groupPath(final String... nodes) { + + if (nodes == null || nodes.length == 0) + return ""; + + final String groupSeparator = getGroupSeparator(); + final StringBuilder builder = new StringBuilder(nodes[0]); + + for (int i = 1; i < nodes.length; ++i) { + + builder.append(groupSeparator); + builder.append(nodes[i]); + } + + return builder.toString(); + } /** * Default implementation of {@link AutoCloseable#close()} for all From 39172a86604bf263748e8ad62e51c7c37dab0af0 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Tue, 20 Jun 2023 16:10:44 -0400 Subject: [PATCH 234/243] test: restrict delta for double equivalence --- .../java/org/janelia/saalfeldlab/n5/AbstractN5Test.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java index e3f6d73e..ffeb3e3f 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java @@ -488,7 +488,7 @@ public void testAttributeParsingPrimitive() throws IOException, URISyntaxExcepti n5.setAttribute(groupName, "key", 1); assertEquals(1, (long)n5.getAttribute(groupName, "key", Integer.class)); - assertEquals(1.0, n5.getAttribute(groupName, "key", Double.class), .00001); + assertEquals(1.0, n5.getAttribute(groupName, "key", Double.class), 1e-9); assertEquals("1", n5.getAttribute(groupName, "key", String.class)); assertNotNull(n5.getAttribute(groupName, "key", JsonElement.class)); assertThrows(N5ClassCastException.class, () -> n5.getAttribute(groupName, "key", int[].class)); @@ -497,7 +497,7 @@ public void testAttributeParsingPrimitive() throws IOException, URISyntaxExcepti n5.setAttribute(groupName, "key", new int[]{2, 3}); assertArrayEquals(new int[]{2, 3}, n5.getAttribute(groupName, "key", int[].class)); - assertArrayEquals(new double[]{2.0, 3.0}, n5.getAttribute(groupName, "key", double[].class), .00001); + assertArrayEquals(new double[]{2.0, 3.0}, n5.getAttribute(groupName, "key", double[].class), 1e-9); assertEquals(JsonParser.parseString("[2,3]"), JsonParser.parseString(n5.getAttribute(groupName, "key", String.class))); assertArrayEquals(new String[]{"2", "3"}, n5.getAttribute(groupName, "key", String[].class)); assertNotNull(n5.getAttribute(groupName, "key", JsonElement.class)); @@ -506,7 +506,7 @@ public void testAttributeParsingPrimitive() throws IOException, URISyntaxExcepti n5.setAttribute(groupName, "key", 0.1); assertEquals(0, (long)n5.getAttribute(groupName, "key", Integer.class)); - assertEquals(0.1, n5.getAttribute(groupName, "key", Double.class), .00001); + assertEquals(0.1, n5.getAttribute(groupName, "key", Double.class), 1e-9); assertEquals("0.1", n5.getAttribute(groupName, "key", String.class)); assertNotNull(n5.getAttribute(groupName, "key", JsonElement.class)); assertThrows(N5ClassCastException.class, () -> n5.getAttribute(groupName, "key", int[].class)); @@ -515,7 +515,7 @@ public void testAttributeParsingPrimitive() throws IOException, URISyntaxExcepti n5.setAttribute(groupName, "key", new double[]{0.2, 0.3}); assertArrayEquals(new int[]{0, 0}, n5.getAttribute(groupName, "key", int[].class)); // TODO returns not null, is this right? - assertArrayEquals(new double[]{0.2, 0.3}, n5.getAttribute(groupName, "key", double[].class), .00001); + assertArrayEquals(new double[]{0.2, 0.3}, n5.getAttribute(groupName, "key", double[].class), 1e-9); assertEquals(JsonParser.parseString("[0.2,0.3]"), JsonParser.parseString(n5.getAttribute(groupName, "key", String.class))); assertArrayEquals(new String[]{"0.2", "0.3"}, n5.getAttribute(groupName, "key", String[].class)); assertNotNull(n5.getAttribute(groupName, "key", JsonElement.class)); From 2261c5ad3137a5d46e0f22f060930f558658d9fe Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Thu, 22 Jun 2023 16:57:00 -0400 Subject: [PATCH 235/243] feat(test): ensure temporary containers are deleted when no longer needed --- .../saalfeldlab/n5/AbstractN5Test.java | 16 ++++-- .../saalfeldlab/n5/N5CachedFSTest.java | 15 ++++++ .../org/janelia/saalfeldlab/n5/N5FSTest.java | 53 ++++++------------- 3 files changed, 42 insertions(+), 42 deletions(-) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java index ffeb3e3f..e07e0ef3 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java @@ -87,6 +87,7 @@ public abstract class AbstractN5Test { static protected float[] floatBlock; static protected double[] doubleBlock; + //TODO: should be removed; Static n5 should not be depended on, use `createN5Wrtier` as autoclosable instaead. static protected N5Writer n5; protected abstract String tempN5Location() throws URISyntaxException, IOException; @@ -101,6 +102,7 @@ protected N5Writer createN5Writer(final String location) throws IOException, URI return createN5Writer(location, new GsonBuilder()); } + /* Tests that overide this should enusre that the `N5Writer` created will remove its container on close() */ protected abstract N5Writer createN5Writer(String location, GsonBuilder gson) throws IOException, URISyntaxException; protected N5Reader createN5Reader(final String location) throws IOException, URISyntaxException { @@ -154,12 +156,10 @@ public void setUpOnce() throws IOException, URISyntaxException { * @throws IOException */ @AfterClass - public static void rampDownAfterClass() throws IOException { + public static void rampDownAfterClass() { if (n5 != null) { - try { - assertTrue(n5.remove()); - } catch ( final Exception e ) {} + n5.remove(); n5 = null; } } @@ -628,6 +628,8 @@ public void testNullAttributes() throws IOException, URISyntaxException { writer.setAttribute(groupName, "existingValue", null); assertThrows(N5ClassCastException.class, () -> writer.getAttribute(groupName, "existingValue", Integer.class)); assertEquals(JsonNull.INSTANCE, writer.getAttribute(groupName, "existingValue", JsonElement.class)); + + writer.remove(); } /* without serializeNulls*/ @@ -782,6 +784,8 @@ public void testRemoveAttributes() throws IOException, URISyntaxException { writer.setAttribute("foo", "a", 100); writer.removeAttribute("foo", "a"); assertNull(writer.getAttribute("foo", "a", Integer.class)); + + writer.remove(); } } @@ -1049,7 +1053,9 @@ public void testVersion() throws NumberFormatException, IOException, URISyntaxEx assertThrows(N5Exception.N5IOException.class, () -> { final String containerPath = writer.getURI().toString(); - createN5Writer(containerPath).close(); + final N5Writer newWriter = createN5Writer(containerPath); + newWriter.remove(); + newWriter.close(); }); final Version compatibleVersion = new Version(N5Reader.VERSION.getMajor(), N5Reader.VERSION.getMinor(), N5Reader.VERSION.getPatch()); diff --git a/src/test/java/org/janelia/saalfeldlab/n5/N5CachedFSTest.java b/src/test/java/org/janelia/saalfeldlab/n5/N5CachedFSTest.java index 3c9aaa09..330bc1c0 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/N5CachedFSTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/N5CachedFSTest.java @@ -49,6 +49,17 @@ protected N5Reader createN5Reader(final String location, final GsonBuilder gson, return new N5FSReader(location, gson, cache); } + @Override protected N5Writer createN5Writer() throws IOException, URISyntaxException { + + return new N5FSWriter(tempN5Location(), new GsonBuilder(), true) { + @Override public void close() { + + super.close(); + remove(); + } + }; + } + @Test public void cacheTest() throws IOException, URISyntaxException { /* Test the cache by setting many attributes, then manually deleting the underlying file. @@ -80,6 +91,7 @@ public void cacheTest() throws IOException, URISyntaxException { Files.delete(Paths.get(attributesPath)); assertThrows(AssertionError.class, () -> runTests(n5, tests)); + n5.remove(); } } @@ -118,6 +130,8 @@ public void cacheGroupDatasetTest() throws IOException, URISyntaxException { assertNotNull(w1.getDatasetAttributes(datasetName)); assertNull(w2.getDatasetAttributes(datasetName)); + + w1.remove(); } } @@ -130,6 +144,7 @@ public void cacheBehaviorTest() throws IOException, URISyntaxException { new GsonBuilder(), true)) { cacheBehaviorHelper(n5); + n5.remove(); } } diff --git a/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java b/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java index b6ca640e..9b912c2e 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java @@ -62,18 +62,12 @@ public class N5FSTest extends AbstractN5Test { private static FileSystemKeyValueAccess access = new FileSystemKeyValueAccess(FileSystems.getDefault()); - private static Set tmpFiles = new HashSet<>(); - - private static String testDirPath = tempN5PathName(); - private static String tempN5PathName() { try { final File tmpFile = Files.createTempDirectory("n5-test-").toFile(); tmpFile.deleteOnExit(); - final String tmpPath = tmpFile.getCanonicalPath(); - tmpFiles.add(tmpPath); - return tmpPath; + return tmpFile.getCanonicalPath(); } catch (final Exception e) { throw new RuntimeException(e); } @@ -86,6 +80,17 @@ protected String tempN5Location() throws URISyntaxException { return new URI("file", null, basePath, null).toString(); } + @Override protected N5Writer createN5Writer() throws IOException, URISyntaxException { + + return new N5FSWriter(tempN5Location(), new GsonBuilder()) { + @Override public void close() { + + super.close(); + remove(); + } + }; + } + @Override protected N5Writer createN5Writer( final String location, @@ -102,16 +107,6 @@ protected N5Reader createN5Reader( return new N5FSReader(location, gson); } - @AfterClass - public static void cleanup() { - - for (final String tmpFile : tmpFiles) { - try { - FileUtils.deleteDirectory(new File(tmpFile)); - } catch (final Exception e) {} - } - } - @Test public void customObjectTest() throws IOException { @@ -147,11 +142,7 @@ public void customObjectTest() throws IOException { // @Test public void testReadLock() throws IOException, InterruptedException { - final Path path = Paths.get(testDirPath, "lock"); - try { - Files.delete(path); - } catch (final IOException e) {} - + final Path path = Paths.get(tempN5PathName(), "lock"); LockedChannel lock = access.lockForWriting(path); lock.close(); lock = access.lockForReading(path); @@ -184,11 +175,7 @@ public void testReadLock() throws IOException, InterruptedException { // @Test public void testWriteLock() throws IOException { - final Path path = Paths.get(testDirPath, "lock"); - try { - Files.delete(path); - } catch (final IOException e) {} - + final Path path = Paths.get(tempN5PathName(), "lock"); final LockedChannel lock = access.lockForWriting(path); System.out.println("locked"); @@ -221,11 +208,7 @@ public void testLockReleaseByReader() throws IOException { System.out.println("Testing lock release by Reader."); - final Path path = Paths.get(testDirPath, "lock"); - try { - Files.delete(path); - } catch (final IOException e) {} - + final Path path = Paths.get(tempN5PathName(), "lock"); final LockedChannel lock = access.lockForWriting(path); lock.newReader().close(); @@ -257,11 +240,7 @@ public void testLockReleaseByInputStream() throws IOException { System.out.println("Testing lock release by InputStream."); - final Path path = Paths.get(testDirPath, "lock"); - try { - Files.delete(path); - } catch (final IOException e) {} - + final Path path = Paths.get(tempN5PathName(), "lock"); final LockedChannel lock = access.lockForWriting(path); lock.newInputStream().close(); From 51b2e24bd5f24984fa167a157fc8664779900f25 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Thu, 22 Jun 2023 17:30:10 -0400 Subject: [PATCH 236/243] feat(test): don't use static n5 container --- .../saalfeldlab/n5/AbstractN5Test.java | 180 +++++++++--------- 1 file changed, 89 insertions(+), 91 deletions(-) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java index e07e0ef3..656bf4cd 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java @@ -165,15 +165,17 @@ public static void rampDownAfterClass() { } @Test - public void testCreateGroup() { - - n5.createGroup(groupName); - final Path groupPath = Paths.get(groupName); - String subGroup = ""; - for (int i = 0; i < groupPath.getNameCount(); ++i) { - subGroup = subGroup + "/" + groupPath.getName(i); - if (!n5.exists(subGroup)) - fail("Group does not exist: " + subGroup); + public void testCreateGroup() throws IOException, URISyntaxException { + + try (N5Writer n5 = createN5Writer()) { + n5.createGroup(groupName); + final Path groupPath = Paths.get(groupName); + String subGroup = ""; + for (int i = 0; i < groupPath.getNameCount(); ++i) { + subGroup = subGroup + "/" + groupPath.getName(i); + if (!n5.exists(subGroup)) + fail("Group does not exist: " + subGroup); + } } } @@ -992,49 +994,53 @@ public void testExists() throws IOException, URISyntaxException { } @Test - public void testListAttributes() { + public void testListAttributes() throws IOException, URISyntaxException { + + try (N5Writer n5 = createN5Writer()) { + final String groupName2 = groupName + "-2"; + final String datasetName2 = datasetName + "-2"; + n5.createDataset(datasetName2, dimensions, blockSize, DataType.UINT64, new RawCompression()); + n5.setAttribute(datasetName2, "attr1", new double[]{1.1, 2.1, 3.1}); + n5.setAttribute(datasetName2, "attr2", new String[]{"a", "b", "c"}); + n5.setAttribute(datasetName2, "attr3", 1.1); + n5.setAttribute(datasetName2, "attr4", "a"); + n5.setAttribute(datasetName2, "attr5", new long[]{1, 2, 3}); + n5.setAttribute(datasetName2, "attr6", 1); + n5.setAttribute(datasetName2, "attr7", new double[]{1, 2, 3.1}); + n5.setAttribute(datasetName2, "attr8", new Object[]{"1", 2, 3.1}); + + Map> attributesMap = n5.listAttributes(datasetName2); + assertTrue(attributesMap.get("attr1") == double[].class); + assertTrue(attributesMap.get("attr2") == String[].class); + assertTrue(attributesMap.get("attr3") == double.class); + assertTrue(attributesMap.get("attr4") == String.class); + assertTrue(attributesMap.get("attr5") == long[].class); + assertTrue(attributesMap.get("attr6") == long.class); + assertTrue(attributesMap.get("attr7") == double[].class); + assertTrue(attributesMap.get("attr8") == Object[].class); + + n5.createGroup(groupName2); + n5.setAttribute(groupName2, "attr1", new double[]{1.1, 2.1, 3.1}); + n5.setAttribute(groupName2, "attr2", new String[]{"a", "b", "c"}); + n5.setAttribute(groupName2, "attr3", 1.1); + n5.setAttribute(groupName2, "attr4", "a"); + n5.setAttribute(groupName2, "attr5", new long[]{1, 2, 3}); + n5.setAttribute(groupName2, "attr6", 1); + n5.setAttribute(groupName2, "attr7", new double[]{1, 2, 3.1}); + n5.setAttribute(groupName2, "attr8", new Object[]{"1", 2, 3.1}); + + attributesMap = n5.listAttributes(groupName2); + assertTrue(attributesMap.get("attr1") == double[].class); + assertTrue(attributesMap.get("attr2") == String[].class); + assertTrue(attributesMap.get("attr3") == double.class); + assertTrue(attributesMap.get("attr4") == String.class); + assertTrue(attributesMap.get("attr5") == long[].class); + assertTrue(attributesMap.get("attr6") == long.class); + assertTrue(attributesMap.get("attr7") == double[].class); + assertTrue(attributesMap.get("attr8") == Object[].class); + + } - final String groupName2 = groupName + "-2"; - final String datasetName2 = datasetName + "-2"; - n5.createDataset(datasetName2, dimensions, blockSize, DataType.UINT64, new RawCompression()); - n5.setAttribute(datasetName2, "attr1", new double[]{1.1, 2.1, 3.1}); - n5.setAttribute(datasetName2, "attr2", new String[]{"a", "b", "c"}); - n5.setAttribute(datasetName2, "attr3", 1.1); - n5.setAttribute(datasetName2, "attr4", "a"); - n5.setAttribute(datasetName2, "attr5", new long[]{1, 2, 3}); - n5.setAttribute(datasetName2, "attr6", 1); - n5.setAttribute(datasetName2, "attr7", new double[]{1, 2, 3.1}); - n5.setAttribute(datasetName2, "attr8", new Object[]{"1", 2, 3.1}); - - Map> attributesMap = n5.listAttributes(datasetName2); - assertTrue(attributesMap.get("attr1") == double[].class); - assertTrue(attributesMap.get("attr2") == String[].class); - assertTrue(attributesMap.get("attr3") == double.class); - assertTrue(attributesMap.get("attr4") == String.class); - assertTrue(attributesMap.get("attr5") == long[].class); - assertTrue(attributesMap.get("attr6") == long.class); - assertTrue(attributesMap.get("attr7") == double[].class); - assertTrue(attributesMap.get("attr8") == Object[].class); - - n5.createGroup(groupName2); - n5.setAttribute(groupName2, "attr1", new double[]{1.1, 2.1, 3.1}); - n5.setAttribute(groupName2, "attr2", new String[]{"a", "b", "c"}); - n5.setAttribute(groupName2, "attr3", 1.1); - n5.setAttribute(groupName2, "attr4", "a"); - n5.setAttribute(groupName2, "attr5", new long[]{1, 2, 3}); - n5.setAttribute(groupName2, "attr6", 1); - n5.setAttribute(groupName2, "attr7", new double[]{1, 2, 3.1}); - n5.setAttribute(groupName2, "attr8", new Object[]{"1", 2, 3.1}); - - attributesMap = n5.listAttributes(groupName2); - assertTrue(attributesMap.get("attr1") == double[].class); - assertTrue(attributesMap.get("attr2") == String[].class); - assertTrue(attributesMap.get("attr3") == double.class); - assertTrue(attributesMap.get("attr4") == String.class); - assertTrue(attributesMap.get("attr5") == long[].class); - assertTrue(attributesMap.get("attr6") == long.class); - assertTrue(attributesMap.get("attr7") == double[].class); - assertTrue(attributesMap.get("attr8") == Object[].class); } @Test @@ -1102,36 +1108,38 @@ public void testReaderCreation() throws IOException, URISyntaxException { } @Test - public void testDelete() throws IOException { - - final String datasetName = AbstractN5Test.datasetName + "-test-delete"; - n5.createDataset(datasetName, dimensions, blockSize, DataType.UINT8, new RawCompression()); - final DatasetAttributes attributes = n5.getDatasetAttributes(datasetName); - final long[] position1 = {0, 0, 0}; - final long[] position2 = {0, 1, 2}; - - // no blocks should exist to begin with - assertTrue(testDeleteIsBlockDeleted(n5.readBlock(datasetName, attributes, position1))); - assertTrue(testDeleteIsBlockDeleted(n5.readBlock(datasetName, attributes, position2))); - - final ByteArrayDataBlock dataBlock = new ByteArrayDataBlock(blockSize, position1, byteBlock); - n5.writeBlock(datasetName, attributes, dataBlock); - - // block should exist at position1 but not at position2 - final DataBlock readBlock = n5.readBlock(datasetName, attributes, position1); - assertNotNull(readBlock); - assertTrue(readBlock instanceof ByteArrayDataBlock); - assertArrayEquals(byteBlock, ((ByteArrayDataBlock)readBlock).getData()); - assertTrue(testDeleteIsBlockDeleted(n5.readBlock(datasetName, attributes, position2))); - - // deletion should report true in all cases - assertTrue(n5.deleteBlock(datasetName, position1)); - assertTrue(n5.deleteBlock(datasetName, position1)); - assertTrue(n5.deleteBlock(datasetName, position2)); - - // no block should exist anymore - assertTrue(testDeleteIsBlockDeleted(n5.readBlock(datasetName, attributes, position1))); - assertTrue(testDeleteIsBlockDeleted(n5.readBlock(datasetName, attributes, position2))); + public void testDelete() throws IOException, URISyntaxException { + + try (N5Writer n5 = createN5Writer()) { + final String datasetName = AbstractN5Test.datasetName + "-test-delete"; + n5.createDataset(datasetName, dimensions, blockSize, DataType.UINT8, new RawCompression()); + final DatasetAttributes attributes = n5.getDatasetAttributes(datasetName); + final long[] position1 = {0, 0, 0}; + final long[] position2 = {0, 1, 2}; + + // no blocks should exist to begin with + assertTrue(testDeleteIsBlockDeleted(n5.readBlock(datasetName, attributes, position1))); + assertTrue(testDeleteIsBlockDeleted(n5.readBlock(datasetName, attributes, position2))); + + final ByteArrayDataBlock dataBlock = new ByteArrayDataBlock(blockSize, position1, byteBlock); + n5.writeBlock(datasetName, attributes, dataBlock); + + // block should exist at position1 but not at position2 + final DataBlock readBlock = n5.readBlock(datasetName, attributes, position1); + assertNotNull(readBlock); + assertTrue(readBlock instanceof ByteArrayDataBlock); + assertArrayEquals(byteBlock, ((ByteArrayDataBlock)readBlock).getData()); + assertTrue(testDeleteIsBlockDeleted(n5.readBlock(datasetName, attributes, position2))); + + // deletion should report true in all cases + assertTrue(n5.deleteBlock(datasetName, position1)); + assertTrue(n5.deleteBlock(datasetName, position1)); + assertTrue(n5.deleteBlock(datasetName, position2)); + + // no block should exist anymore + assertTrue(testDeleteIsBlockDeleted(n5.readBlock(datasetName, attributes, position1))); + assertTrue(testDeleteIsBlockDeleted(n5.readBlock(datasetName, attributes, position2))); + } } protected boolean testDeleteIsBlockDeleted(final DataBlock dataBlock) { @@ -1375,16 +1383,6 @@ private String jsonKeyVal(final String key, final String val) { return String.format("\"%s\":\"%s\"", key, val); } - private String readAttributesAsString(final String group) { - - final String basePath = ((N5FSWriter)n5).getURI().getSchemeSpecificPart(); - try { - return new String(Files.readAllBytes(Paths.get(basePath, group, "attributes.json"))); - } catch (final IOException e) { - } - return null; - } - @Test public void testRootLeaves() throws IOException, URISyntaxException { From 60e2ae8334a8691427efb1ab5475077f5105e8d2 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Fri, 23 Jun 2023 09:11:41 -0400 Subject: [PATCH 237/243] fix: normalize attr path in rmAttributes --- src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java index a7c5a000..a92c5ac5 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java @@ -118,8 +118,7 @@ default boolean removeAttributes(final String groupPath, final List attr final String normalPath = N5URI.normalizeGroupPath(groupPath); boolean removed = false; for (final String attribute : attributePaths) { - final String normalKey = N5URI.normalizeAttributePath(attribute); - removed |= removeAttribute(normalPath, attribute); + removed |= removeAttribute(normalPath, N5URI.normalizeAttributePath(attribute)); } return removed; } From cfaffae0ba243c407075310292c675766c965285 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Fri, 23 Jun 2023 09:11:52 -0400 Subject: [PATCH 238/243] doc: typo fixes --- .../saalfeldlab/n5/CachedGsonKeyValueN5Writer.java | 2 +- .../janelia/saalfeldlab/n5/N5KeyValueReader.java | 14 +++++++------- .../java/org/janelia/saalfeldlab/n5/N5Writer.java | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Writer.java index 1c6fee59..d4863c82 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Writer.java @@ -123,7 +123,7 @@ default void writeAndCacheAttributes( * The output is identical to the input if: * - serializeNulls is true * - no null values are present - * - cacheing is turned off + * - caching is turned off */ if (!getGson().serializeNulls()) { nullRespectingAttributes = getGson().toJsonTree(attributes); diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java index 7d68474b..29308ac9 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java @@ -94,13 +94,13 @@ public N5KeyValueReader( * @param basePathN5 * base path * @param gsonBuilder - * @param cacheMetacache - * attributes and meta data - * Setting this to true avoidsfrequent reading and parsing of - * JSON encoded attributes andother meta data that requires - * accessing the store. This ismost interesting for high latency - * backends. Changes of cachedattributes and meta data by an - * independent writer will not betracked. + * @param cacheMeta + * cache attributes and meta data + * Setting this to true avoids frequent reading and parsing of + * JSON encoded attributes and other meta data that requires + * accessing the store. This is most interesting for high latency + * backends. Changes of cached attributes and meta data by an + * independent writer will not be tracked. * * @throws N5Exception * if the base path cannot be read or does not exist, if the N5 diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java index a92c5ac5..24ac30ea 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5Writer.java @@ -127,7 +127,7 @@ default boolean removeAttributes(final String groupPath, final List attr * Sets mandatory dataset attributes. * * @param datasetPath dataset path - * @param datasetAttributes the dataset attributse + * @param datasetAttributes the dataset attributes * @throws N5Exception the exception */ default void setDatasetAttributes( From eed9042f871e285d120c6974562a8913d50d8b9e Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Fri, 23 Jun 2023 10:35:15 -0400 Subject: [PATCH 239/243] feat(test): don't use static n5 --- .../saalfeldlab/n5/AbstractN5Test.java | 3 +-- .../org/janelia/saalfeldlab/n5/N5FSTest.java | 21 +++++++++++-------- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java index 656bf4cd..c8e6ca29 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java @@ -40,7 +40,6 @@ import java.io.IOException; import java.net.URISyntaxException; -import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; @@ -87,7 +86,7 @@ public abstract class AbstractN5Test { static protected float[] floatBlock; static protected double[] doubleBlock; - //TODO: should be removed; Static n5 should not be depended on, use `createN5Wrtier` as autoclosable instaead. + //TODO: should be removed; Static n5 should not be depended on, use `createN5Writer` as autoclosable instaead. static protected N5Writer n5; protected abstract String tempN5Location() throws URISyntaxException, IOException; diff --git a/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java b/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java index 9b912c2e..41abb9f7 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/N5FSTest.java @@ -108,7 +108,7 @@ protected N5Reader createN5Reader( } @Test - public void customObjectTest() throws IOException { + public void customObjectTest() throws IOException, URISyntaxException { final String testGroup = "test"; final ArrayList> existingTests = new ArrayList<>(); @@ -129,14 +129,17 @@ public void customObjectTest() throws IOException { "doubles", "doubles4", new double[]{5.10, 4.8, 3.7}); - n5.createGroup(testGroup); - addAndTest(n5, existingTests, new TestData<>(testGroup, "/doubles[1]", doubles1)); - addAndTest(n5, existingTests, new TestData<>(testGroup, "/doubles[2]", doubles2)); - addAndTest(n5, existingTests, new TestData<>(testGroup, "/doubles[3]", doubles3)); - addAndTest(n5, existingTests, new TestData<>(testGroup, "/doubles[4]", doubles4)); - - /* Test overwrite custom */ - addAndTest(n5, existingTests, new TestData<>(testGroup, "/doubles[1]", doubles4)); + + try (N5Writer n5 = createN5Writer()) { + n5.createGroup(testGroup); + addAndTest(n5, existingTests, new TestData<>(testGroup, "/doubles[1]", doubles1)); + addAndTest(n5, existingTests, new TestData<>(testGroup, "/doubles[2]", doubles2)); + addAndTest(n5, existingTests, new TestData<>(testGroup, "/doubles[3]", doubles3)); + addAndTest(n5, existingTests, new TestData<>(testGroup, "/doubles[4]", doubles4)); + + /* Test overwrite custom */ + addAndTest(n5, existingTests, new TestData<>(testGroup, "/doubles[1]", doubles4)); + } } // @Test From a6ab3f20a4df00b4913d19c433c7abddb73aa004 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Fri, 23 Jun 2023 11:53:13 -0400 Subject: [PATCH 240/243] feat(test): remove static n5 reference --- .../saalfeldlab/n5/AbstractN5Test.java | 20 ------------------- 1 file changed, 20 deletions(-) diff --git a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java index c8e6ca29..21514375 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/AbstractN5Test.java @@ -86,9 +86,6 @@ public abstract class AbstractN5Test { static protected float[] floatBlock; static protected double[] doubleBlock; - //TODO: should be removed; Static n5 should not be depended on, use `createN5Writer` as autoclosable instaead. - static protected N5Writer n5; - protected abstract String tempN5Location() throws URISyntaxException, IOException; protected N5Writer createN5Writer() throws IOException, URISyntaxException { @@ -129,11 +126,6 @@ protected Compression[] getCompressions() { @Before public void setUpOnce() throws IOException, URISyntaxException { - if (n5 != null) - return; - - n5 = createN5Writer(); - final Random rnd = new Random(); byteBlock = new byte[blockSize[0] * blockSize[1] * blockSize[2]]; shortBlock = new short[blockSize[0] * blockSize[1] * blockSize[2]]; @@ -151,18 +143,6 @@ public void setUpOnce() throws IOException, URISyntaxException { } } - /** - * @throws IOException - */ - @AfterClass - public static void rampDownAfterClass() { - - if (n5 != null) { - n5.remove(); - n5 = null; - } - } - @Test public void testCreateGroup() throws IOException, URISyntaxException { From 8bc9a2578836e3a61dc443b42f8c79e3f6930b49 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Tue, 27 Jun 2023 10:46:31 -0400 Subject: [PATCH 241/243] docs: remove some javadoc warnings --- .../n5/CachedGsonKeyValueN5Reader.java | 32 +++++-------------- .../org/janelia/saalfeldlab/n5/DataBlock.java | 2 +- .../saalfeldlab/n5/DefaultBlockWriter.java | 2 +- .../saalfeldlab/n5/GsonKeyValueN5Reader.java | 6 ++-- .../saalfeldlab/n5/GsonKeyValueN5Writer.java | 9 +++--- .../janelia/saalfeldlab/n5/GsonN5Reader.java | 6 ++-- .../janelia/saalfeldlab/n5/GsonN5Writer.java | 2 +- .../janelia/saalfeldlab/n5/N5Exception.java | 2 +- .../janelia/saalfeldlab/n5/N5FSWriter.java | 8 ++--- .../saalfeldlab/n5/N5KeyValueReader.java | 4 +-- 10 files changed, 28 insertions(+), 45 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Reader.java index 9539a7ea..fd512020 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Reader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/CachedGsonKeyValueN5Reader.java @@ -29,6 +29,7 @@ import java.lang.reflect.Type; import com.google.gson.JsonSyntaxException; +import org.janelia.saalfeldlab.n5.N5Exception.N5IOException; import org.janelia.saalfeldlab.n5.cache.N5JsonCache; import org.janelia.saalfeldlab.n5.cache.N5JsonCacheableContainer; @@ -78,7 +79,7 @@ default DatasetAttributes getDatasetAttributes(final String pathName) { return createDatasetAttributes(attributes); } - default DatasetAttributes normalGetDatasetAttributes(final String pathName) throws N5Exception.N5IOException { + default DatasetAttributes normalGetDatasetAttributes(final String pathName) throws N5IOException { final String normalPath = N5URI.normalizeGroupPath(pathName); final JsonElement attributes = GsonKeyValueN5Reader.super.getAttributes(normalPath); @@ -172,7 +173,7 @@ default boolean isGroupFromAttributes(final String normalCacheKey, final JsonEle } @Override - default boolean datasetExists(final String pathName) throws N5Exception.N5IOException { + default boolean datasetExists(final String pathName) throws N5IOException { final String normalPathName = N5URI.normalizeGroupPath(pathName); if (cacheMeta()) { @@ -182,7 +183,7 @@ default boolean datasetExists(final String pathName) throws N5Exception.N5IOExce } @Override - default boolean isDatasetFromContainer(final String normalPathName) throws N5Exception.N5IOException { + default boolean isDatasetFromContainer(final String normalPathName) throws N5IOException { return normalGetDatasetAttributes(normalPathName) != null; } @@ -198,11 +199,11 @@ default boolean isDatasetFromAttributes(final String normalCacheKey, final JsonE * * @param pathName * group path - * @return - * @throws IOException + * @return the attribute + * @throws N5IOException if an IO error occurs while reading the attribute */ @Override - default JsonElement getAttributes(final String pathName) throws N5Exception.N5IOException { + default JsonElement getAttributes(final String pathName) throws N5IOException { final String groupPath = N5URI.normalizeGroupPath(pathName); @@ -215,7 +216,7 @@ default JsonElement getAttributes(final String pathName) throws N5Exception.N5IO } @Override - default String[] list(final String pathName) throws N5Exception.N5IOException { + default String[] list(final String pathName) throws N5IOException { final String normalPath = N5URI.normalizeGroupPath(pathName); if (cacheMeta()) { @@ -232,23 +233,6 @@ default String[] listFromContainer(final String normalPathName) { return GsonKeyValueN5Reader.super.list(normalPathName); } - /** - * Constructs the path for a data block in a dataset at a given grid - * position. - *

    - * The returned path is - * - * 

    -	 * $basePath/datasetPathName/$gridPosition[0]/$gridPosition[1]/.../$gridPosition[n]
-	 *
    - *

    - * This is the file into which the data block will be stored. - * - * @param normalPath - * normalized dataset path - * @param gridPosition - * @return - */ @Override default String absoluteDataBlockPath( final String normalPath, diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DataBlock.java b/src/main/java/org/janelia/saalfeldlab/n5/DataBlock.java index c79dcf26..3d9dc92a 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DataBlock.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DataBlock.java @@ -32,7 +32,7 @@ * grid, a size, and can read itself from and write itself into a * {@link ByteBuffer}. * - * @param + * @param type of the data contained in the DataBlock * * @author Stephan Saalfeld */ diff --git a/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockWriter.java b/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockWriter.java index 1f1aad39..c53aae2d 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockWriter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/DefaultBlockWriter.java @@ -55,7 +55,7 @@ public default void write( /** * Writes a {@link DataBlock} into an {@link OutputStream}. * - * + * @param the type of data * @param out * the output stream * @param datasetAttributes diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java index c32fcd3c..0ffb492f 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Reader.java @@ -66,7 +66,7 @@ default boolean datasetExists(final String pathName) throws N5Exception { * * @param pathName * group path - * @return + * @return the attribute * @throws N5Exception if the attributes cannot be read */ @Override @@ -129,8 +129,8 @@ default String[] list(final String pathName) throws N5Exception { * * @param normalPath * normalized dataset path - * @param gridPosition - * @return + * @param gridPosition to the target data block + * @return the absolute path to the data block ad gridPosition */ default String absoluteDataBlockPath( final String normalPath, diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java index 6f3ecd82..acd8cc0e 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonKeyValueN5Writer.java @@ -49,10 +49,9 @@ public interface GsonKeyValueN5Writer extends GsonN5Writer, GsonKeyValueN5Reader * if this is the desired behavior or if it is always overridden, e.g. as by * the caching version. If this is true, delete this implementation. * - * @param path - * @throws IOException + * @param path to the group to write the version into */ - default void setVersion(final String path) throws IOException { + default void setVersion(final String path) { if (!VERSION.equals(getVersion())) setAttribute("/", VERSION_KEY, VERSION.toString()); @@ -82,7 +81,7 @@ default void createGroup(final String path) throws N5Exception { * Helper method that writes an attributes tree into the store * * TODO This method is not part of the public API and should be protected - * in Java >8 + * in Java versions greater than 8 * * @param normalGroupPath * to write the attributes to @@ -120,7 +119,7 @@ default void setAttributes( * the attributes store. * * TODO This method is not part of the public API and should be protected - * in Java >8 + * in Java greater than 8 * * @param normalGroupPath * to write the attributes to diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonN5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonN5Reader.java index bfadb345..be16ed08 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonN5Reader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonN5Reader.java @@ -110,12 +110,12 @@ default T getAttribute(final String pathName, final String key, final Type t } /** - * Reads or creates the attributes map of a group or dataset. + * Reads or the attributes of a group or dataset. * * @param pathName * group path - * @return - * @throws N5Exception + * @return the attributes identified by pathName + * @throws N5Exception if the attribute cannot be returned */ JsonElement getAttributes(final String pathName) throws N5Exception; } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonN5Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonN5Writer.java index f0030441..66231425 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonN5Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonN5Writer.java @@ -42,7 +42,7 @@ public interface GsonN5Writer extends GsonN5Reader, N5Writer { * to write the attributes to * @param attributes * to write - * @throws N5Exception + * @throws N5Exception if the attributes cannot be set */ void setAttributes( final String groupPath, diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5Exception.java b/src/main/java/org/janelia/saalfeldlab/n5/N5Exception.java index 819e0ba5..35ae208c 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5Exception.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5Exception.java @@ -61,7 +61,7 @@ protected N5IOException( } /** - * This excpetion represents the situation when an attribute is requested by key as a specific Class, + * This excpetion represents the situation when an attribute is requested by key as a specific Class, * that attribute key does exist, but is not parseable as the desired Class */ public static class N5ClassCastException extends N5Exception { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java b/src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java index d69740f9..9873c095 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5FSWriter.java @@ -82,9 +82,9 @@ public N5FSWriter(final String basePath, final GsonBuilder gsonBuilder, final bo * compatible with this implementation, the N5 version of this container * will be set to the current N5 version of this implementation. * - * @param basePathn5 + * @param basePath * base path - * @param cacheAttributescache + * @param cacheAttributes * attributes and meta data * Setting this to true avoidsfrequent reading and parsing of * JSON encoded attributes andother meta data that requires @@ -114,9 +114,9 @@ public N5FSWriter(final String basePath, final boolean cacheAttributes) throws N * will be set to the current N5 version of this implementation. *

    * - * @param basePathn5 + * @param basePath * base path - * @param gsonBuilderthe + * @param gsonBuilder * gson builder * * @throws N5Exception diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java index 29308ac9..81dad8de 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java @@ -88,10 +88,10 @@ public N5KeyValueReader( * Opens an {@link N5KeyValueReader} at a given base path with a custom * {@link GsonBuilder} to support custom attributes. * - * @param checkVersiondo + * @param checkVersion * the version check * @param keyValueAccess - * @param basePathN5 + * @param basePath * base path * @param gsonBuilder * @param cacheMeta From bbfe0f37919af5c9c341e1e3c9636f4fa171530f Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Tue, 27 Jun 2023 11:04:20 -0400 Subject: [PATCH 242/243] doc: add and clarify javadoc * N5Reader * LockedChannel * KeyValueAccess * FileSystemKeyValueAccess --- .../n5/FileSystemKeyValueAccess.java | 19 ++++++++------- .../saalfeldlab/n5/KeyValueAccess.java | 23 +++++++++---------- .../janelia/saalfeldlab/n5/LockedChannel.java | 10 +++++--- .../org/janelia/saalfeldlab/n5/N5Reader.java | 9 ++++---- 4 files changed, 33 insertions(+), 28 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java index 19792b57..01dc75d1 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/FileSystemKeyValueAccess.java @@ -148,9 +148,9 @@ public void close() throws IOException { protected final FileSystem fileSystem; /** - * Opens a {@link FileSystemKeyValueAccess}. + * Opens a {@link FileSystemKeyValueAccess} with a {@link FileSystem}. * - * @param fileSystem + * @param fileSystem the file system */ public FileSystemKeyValueAccess(final FileSystem fileSystem) { @@ -267,7 +267,7 @@ public String relativize(final String path, final String base) { * otherwise {@code pathName} is treated as UNC path on Windows, and * {@code Paths.get(pathName, ...)} fails with {@code InvalidPathException}. * - * @param path + * @param path the path * @return the normalized path, without leading slash */ @Override @@ -470,14 +470,17 @@ protected static Path createDirectories(Path dir, final FileAttribute... attr } /** - * This is a copy of a previous - * Files#createAndCheckIsDirectory(Path, FileAttribute...) method that - * follows symlinks. + * This is a copy of a previous Files#createAndCheckIsDirectory(Path, + * FileAttribute...) method that follows symlinks. * * Workaround for https://bugs.openjdk.java.net/browse/JDK-8130464 * - * Used by createDirectories to attempt to create a directory. A no-op if - * the directory already exists. + * Used by createDirectories to attempt to create a directory. A no-op if the + * directory already exists. + * + * @param dir directory path + * @param attrs file attributes + * @throws IOException the exception */ protected static void createAndCheckIsDirectory( final Path dir, diff --git a/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccess.java index 77cf5297..ea09269b 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/KeyValueAccess.java @@ -98,15 +98,15 @@ public default String compose(final URI uri, final String... components) { public String normalize(final String path); /** - * Get the absolute (including scheme) URI of the given path + * Get the absolute (including scheme) {@link URI} of the given path * - * @param normalPath + * @param uriString * is expected to be in normalized form, no further * efforts are made to normalize it. * @return absolute URI + * @throws URISyntaxException if the given path is not a proper URI */ - public URI uri(final String normalPath) throws URISyntaxException; - + public URI uri(final String uriString) throws URISyntaxException; /** * Test whether the path exists. * @@ -151,7 +151,7 @@ public default String compose(final URI uri, final String... components) { * efforts are made to normalize it. * @return the locked channel * @throws IOException - * the exception + * if a locked channel could not be created */ public LockedChannel lockForReading(final String normalPath) throws IOException; @@ -171,7 +171,7 @@ public default String compose(final URI uri, final String... components) { * efforts are made to normalize it. * @return the locked channel * @throws IOException - * the exception + * if a locked channel could not be created */ public LockedChannel lockForWriting(final String normalPath) throws IOException; @@ -183,7 +183,7 @@ public default String compose(final URI uri, final String... components) { * efforts are made to normalize it. * @return the directories * @throws IOException - * the exception + * if an error occurs during listing */ public String[] listDirectories(final String normalPath) throws IOException; @@ -194,10 +194,7 @@ public default String compose(final URI uri, final String... components) { * is expected to be in normalized form, no further * efforts are made to normalize it. * @return the the child paths - * @throws IOException - * the exception - * - * + * @throws IOException if an error occurs during listing */ public String[] list(final String normalPath) throws IOException; @@ -211,7 +208,7 @@ public default String compose(final URI uri, final String... components) { * is expected to be in normalized form, no further * efforts are made to normalize it. * @throws IOException - * the exception + * if an error occurs during creation */ public void createDirectories(final String normalPath) throws IOException; @@ -221,6 +218,8 @@ public default String compose(final URI uri, final String... components) { * @param normalPath * is expected to be in normalized form, no further * efforts are made to normalize it. + * @throws IOException + * if an error occurs during deletion */ public void delete(final String normalPath) throws IOException; } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/LockedChannel.java b/src/main/java/org/janelia/saalfeldlab/n5/LockedChannel.java index 3e931eb9..bd34a59d 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/LockedChannel.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/LockedChannel.java @@ -45,16 +45,16 @@ public interface LockedChannel extends Closeable { * * @return the reader * @throws IOException - * the exception + * if the reader could not be created */ public Reader newReader() throws IOException; /** * Create a new {@link InputStream}. * - * @return the reader + * @return the input stream * @throws IOException - * the exception + * if an input stream could not be created */ public InputStream newInputStream() throws IOException; @@ -62,6 +62,8 @@ public interface LockedChannel extends Closeable { * Create a new UTF-8 {@link Writer}. * * @return the writer + * @throws IOException + * if a writer could not be created */ public Writer newWriter() throws IOException; @@ -69,6 +71,8 @@ public interface LockedChannel extends Closeable { * Create a new {@link OutputStream}. * * @return the output stream + * @throws IOException + * if an output stream could not be created */ public OutputStream newOutputStream() throws IOException; } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java index c5601129..6e708e66 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5Reader.java @@ -763,9 +763,8 @@ static void deepListHelper( * * @param pathName * group path - * @return the attribute map - * @throws N5Exception - * the exception + * @return a map of attribute keys to their inferred class + * @throws N5Exception if an error occurred during listing */ Map> listAttributes(final String pathName) throws N5Exception; @@ -784,8 +783,8 @@ default String getGroupSeparator() { * defined by {@link #getGroupSeparator()}. The string will not have a * leading or trailing node separator symbol. * - * @param nodes - * @return + * @param nodes a collection of child node names + * @return the full group path */ default String groupPath(final String... nodes) { From e81ce86abb744f8c38061ad199a05ff3faa31ad5 Mon Sep 17 00:00:00 2001 From: Caleb Hulbert Date: Tue, 27 Jun 2023 11:04:48 -0400 Subject: [PATCH 243/243] doc: more javadoc warnings --- .../org/janelia/saalfeldlab/n5/GsonUtils.java | 25 +++++++++++++++++-- .../janelia/saalfeldlab/n5/N5Exception.java | 2 +- .../saalfeldlab/n5/N5KeyValueReader.java | 4 +++ .../saalfeldlab/n5/N5KeyValueWriter.java | 2 ++ .../saalfeldlab/n5/cache/N5JsonCache.java | 2 ++ 5 files changed, 32 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/GsonUtils.java b/src/main/java/org/janelia/saalfeldlab/n5/GsonUtils.java index eda92a70..830d29d2 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/GsonUtils.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/GsonUtils.java @@ -63,9 +63,11 @@ static Gson registerGson(final GsonBuilder gsonBuilder) { * * @param reader * the reader + * @param gson + * to parse Json from the {@code reader} * @return the root {@link JsonObject} of the attributes * @throws IOException - * the exception + * if an error occurs reading json from the {@code reader} */ static JsonElement readAttributes(final Reader reader, final Gson gson) throws IOException { @@ -151,6 +153,8 @@ static T parseAttributeElement(final JsonElement attribute, final Gson gson, * to search for the {@link JsonElement} at location * {@code normalizedAttributePath} * + * @param root + * containing an attribute at normalizedAttributePath * @param normalizedAttributePath * to the attribute * @return the attribute as a {@link JsonElement}. @@ -376,12 +380,18 @@ else if (jsonPrimitive.isNumber()) { * to write the modified {@code root} to after removal of the attribute to * remove the attribute from * + * + * @param the type of the attribute to be removed + * @param writer to write the modified JsonElement, which no longer contains the removed attribute + * @param root element to remove the attribute from * @param normalizedAttributePath * to the attribute location of the attribute to remove to * deserialize the attribute with of the removed attribute + * @param cls of the type of the attribute to be removed + * @param gson used to deserialize the JsonElement as {@code T} * @return the removed attribute, or null if nothing removed * @throws IOException - * the exception + * if unable to write to the {@code writer} */ static T removeAttribute( final Writer writer, @@ -403,9 +413,13 @@ static T removeAttribute( * to write the modified {@code root} to after removal of the attribute to * remove the attribute from * + * @param writer to write the modified JsonElement, which no longer contains the removed attribute + * @param root to remove the attribute from * @param normalizedAttributePath * to the attribute location to deserialize the attribute with + * @param gson to write the attribute to the {@code writer} * @return if the attribute was removed or not + * @throws IOException if an error occurs while writing to the {@code writer} */ static boolean removeAttribute( final Writer writer, @@ -431,10 +445,17 @@ static boolean removeAttribute( * {@code T}, then it is not removed. * to remove the attribute from * + * @param the type of the attribute to be removed + * @param root element to remove the attribute from * @param normalizedAttributePath * to the attribute location of the attribute to remove to * deserialize the attribute with of the removed attribute + * @param cls of the type of the attribute to be removed + * @param gson used to deserialize the JsonElement as {@code T} * @return the removed attribute, or null if nothing removed + * @throws JsonSyntaxException if the attribute is not valid json + * @throws NumberFormatException if {@code T} is a {@link Number} but the attribute cannot be parsed as {@code T} + * @throws ClassCastException if an attribute exists at this path, but is not of type {@code T} */ static T removeAttribute( final JsonElement root, diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5Exception.java b/src/main/java/org/janelia/saalfeldlab/n5/N5Exception.java index 35ae208c..f26705c9 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5Exception.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5Exception.java @@ -62,7 +62,7 @@ protected N5IOException( /** * This excpetion represents the situation when an attribute is requested by key as a specific Class, - * that attribute key does exist, but is not parseable as the desired Class + * that attribute key does exist, but is not parseable as the desired Class */ public static class N5ClassCastException extends N5Exception { diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java index 81dad8de..690724d3 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueReader.java @@ -58,9 +58,11 @@ public class N5KeyValueReader implements CachedGsonKeyValueN5Reader { * {@link GsonBuilder} to support custom attributes. * * @param keyValueAccess + * the KeyValueAccess backend used * @param basePath * N5 base path * @param gsonBuilder + * the GsonBuilder * @param cacheMeta * cache attributes and meta data * Setting this to true avoidsfrequent reading and parsing of @@ -91,9 +93,11 @@ public N5KeyValueReader( * @param checkVersion * the version check * @param keyValueAccess + * the backend KeyValueAccess used * @param basePath * base path * @param gsonBuilder + * the GsonBuilder * @param cacheMeta * cache attributes and meta data * Setting this to true avoids frequent reading and parsing of diff --git a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriter.java b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriter.java index 93e14c68..88dfd6c4 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriter.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/N5KeyValueWriter.java @@ -46,9 +46,11 @@ public class N5KeyValueWriter extends N5KeyValueReader implements CachedGsonKeyV * will be set to the current N5 version of this implementation. * * @param keyValueAccess + * the backend key value access to use * @param basePath * n5 base path * @param gsonBuilder + * the gson builder * @param cacheAttributes * Setting this to true avoids frequent reading and parsing of * JSON encoded attributes, this is most interesting for high diff --git a/src/main/java/org/janelia/saalfeldlab/n5/cache/N5JsonCache.java b/src/main/java/org/janelia/saalfeldlab/n5/cache/N5JsonCache.java index 873d8b5b..e26b18dc 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/cache/N5JsonCache.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/cache/N5JsonCache.java @@ -208,6 +208,8 @@ protected N5CacheInfo getOrMakeCacheInfo(final String normalPathKey) { * * @param normalPathKey * the normalized path key + * @param normalCacheKey + * the normalize key to cache */ public void updateCacheInfo(final String normalPathKey, final String normalCacheKey) {