Skip to content

Commit

Permalink
Add --v3 option to write Zarr v3
Browse files Browse the repository at this point in the history 
Uses zarr-java. Right now, only writes v3 metadata; array writing will use v2 logic.
zarr-java is not deployed anywhere, so artifacts need to be downloaded from:

https://github.com/zarr-developers/zarr-java/suites/16044480122/artifacts/915858843
https://github.com/zarr-developers/zarr-java/raw/main/pom.xml

and installed manually:

$ mvn org.apache.maven.plugins:maven-install-plugin:2.5.2:install-file -Dfile=zarr-java-1.0-SNAPSHOT.jar
$ mvn org.apache.maven.plugins:maven-install-plugin:2.5.2:install-file -Dfile=pom.xml -DgroupId=com.scalableminds.zarrjava -DartifactId=zarr-java -Dversion=1.0-SNAPSHOT -Dpackaging=pom
  • Loading branch information
@melissalinkert
melissalinkert committed Sep 28, 2023
1 parent 609f092 commit 567fd3e
Show file tree
Hide file tree
Showing 2 changed files with 107 additions and 36 deletions.
2 changes: 1 addition & 1 deletion build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,14 +41,14 @@ dependencies {
implementation 'info.picocli:picocli:4.6.1'
implementation 'com.univocity:univocity-parsers:2.8.4'
implementation 'dev.zarr:jzarr:0.4.0'
implementation 'com.scalableminds.zarrjava:zarr-java:1.0-SNAPSHOT'
// implementation 'org.carlspring.cloud.aws:s3fs-nio:1.0-SNAPSHOT'
// implementation 'io.nextflow:nxf-s3fs:1.1.0'
implementation 'org.lasersonlab:s3fs:2.2.3'
implementation 'javax.xml.bind:jaxb-api:2.3.0'
implementation group: 'ch.qos.logback', name: 'logback-classic', version: '1.3.4'
implementation group: 'ch.qos.logback', name: 'logback-core', version: '1.3.4'


implementation 'org.openpnp:opencv:3.4.2-1'
implementation 'me.tongfei:progressbar:0.9.0'

Expand Down
141 changes: 106 additions & 35 deletions src/main/java/com/glencoesoftware/bioformats2raw/Converter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,10 @@
import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;
import com.google.common.math.DoubleMath;
import com.scalableminds.zarrjava.ZarrException;
import com.scalableminds.zarrjava.store.FilesystemStore;
import com.scalableminds.zarrjava.v3.Group;
import com.scalableminds.zarrjava.v3.GroupMetadata;
import com.univocity.parsers.csv.CsvParser;
import com.univocity.parsers.csv.CsvParserSettings;

Expand Down Expand Up @@ -174,6 +178,8 @@ public class Converter implements Callable<Integer> {
private volatile boolean reuseExistingResolutions = false;
private volatile int minSize = MIN_SIZE;

private volatile boolean useVersion3 = false;

/** Scaling implementation that will be used during downsampling. */
private volatile IImageScaler scaler = new SimpleImageScaler();

Expand Down Expand Up @@ -799,6 +805,20 @@ public void setDimensionOrder(DimensionOrder order) {
dimensionOrder = order;
}

/**
* Set whether or not to write Zarr v3. Defaults to v2.
*
* @param version3 true if Zarr v3 should be written
*/
@Option(
names = "--v3",
description = "Write Zarr v3",
defaultValue = "false"
)
public void setUseVersion3(boolean version3) {
useVersion3 = version3;
}

// Option getters

/**
Expand Down Expand Up @@ -1042,6 +1062,13 @@ public DimensionOrder getDimensionOrder() {
return dimensionOrder;
}

/**
* @return true if Zarr v3 should be written
*/
public boolean useVersion3() {
return useVersion3;
}

// Conversion methods

/**
Expand Down Expand Up @@ -1306,35 +1333,10 @@ public void convert()

// fileset level metadata
if (!noRootGroup) {
final ZarrGroup root = ZarrGroup.create(getRootPath());
Map<String, Object> attributes = new HashMap<String, Object>();
attributes.put("bioformats2raw.layout", LAYOUT);

root.writeAttributes(attributes);
createRootGroup();
}
if (!noOMEMeta) {
Path metadataPath = getRootPath().resolve("OME");
final ZarrGroup root = ZarrGroup.create(metadataPath);
Map<String, Object> attributes = new HashMap<String, Object>();

// record the path to each series (multiscales) and the corresponding
// series (OME-XML Image) index
// using the index as the key would mean that the index is stored
// as a string instead of an integer
List<String> groups = new ArrayList<String>();
for (Integer index : seriesList) {
String resolutionString = String.format(
scaleFormatString, getScaleFormatStringArgs(index, 0));
String seriesString = "";
if (resolutionString.indexOf('/') >= 0) {
seriesString = resolutionString.substring(0,
resolutionString.lastIndexOf('/'));
}
groups.add(seriesString);
}
attributes.put("series", groups);

root.writeAttributes(attributes);
writeOMEMetadata();
}

for (Integer index : seriesList) {
Expand Down Expand Up @@ -2052,7 +2054,7 @@ private void saveHCSMetadata(IMetadata meta) throws IOException {

// assumes only one plate defined
Path rootPath = getRootPath();
ZarrGroup root = ZarrGroup.open(rootPath);

int plate = 0;
Map<String, Object> plateMap = new HashMap<String, Object>();

Expand Down Expand Up @@ -2118,9 +2120,8 @@ private void saveHCSMetadata(IMetadata meta) throws IOException {
List<Map<String, Object>> imageList =
new ArrayList<Map<String, Object>>();
String rowPath = index.getRowPath();
ZarrGroup rowGroup = root.createSubGroup(rowPath);
createGroupWithAttributes(rowPath, null);
String columnPath = index.getColumnPath();
ZarrGroup columnGroup = rowGroup.createSubGroup(columnPath);
for (HCSIndex field : hcsIndexes) {
if (field.getPlateIndex() == index.getPlateIndex() &&
field.getWellRowIndex() == index.getWellRowIndex() &&
Expand All @@ -2138,9 +2139,12 @@ private void saveHCSMetadata(IMetadata meta) throws IOException {

Map<String, Object> wellMap = new HashMap<String, Object>();
wellMap.put("images", imageList);
Map<String, Object> attributes = columnGroup.getAttributes();
Map<String, Object> attributes = new HashMap<String, Object>();
attributes.put("well", wellMap);
columnGroup.writeAttributes(attributes);
// TODO: better way to set relative path here?
createGroupWithAttributes(
rowPath + File.separator + columnPath,
attributes);

// make sure the row/column indexes are added to the plate attributes
// this is necessary when Plate.Rows or Plate.Columns is not set
Expand Down Expand Up @@ -2190,9 +2194,24 @@ private void saveHCSMetadata(IMetadata meta) throws IOException {
plateMap.put("field_count", maxField + 1);
plateMap.put("version", NGFF_VERSION);

Map<String, Object> attributes = root.getAttributes();
attributes.put("plate", plateMap);
root.writeAttributes(attributes);
if (useVersion3()) {
try {
Group root = Group.open(
new FilesystemStore(getRootPath()).resolve("."));
Map<String, Object> attributes = root.metadata.attributes;
attributes.put("plate", plateMap);
root.setAttributes(attributes);
}
catch (ZarrException e) {
throw new IOException("Failed to set HCS metadata", e);
}
}
else {
ZarrGroup root = ZarrGroup.open(rootPath);
Map<String, Object> attributes = root.getAttributes();
attributes.put("plate", plateMap);
root.writeAttributes(attributes);
}
}

/**
Expand Down Expand Up @@ -2791,6 +2810,58 @@ private int calculateResolutions(int width, int height) {
return resolutions;
}

private void createGroupWithAttributes(String relativePath,
Map<String, Object> attrs)
throws IOException
{
if (useVersion3()) {
// group won't be created if Map is passed directly
try {
Group g = Group.create(
new FilesystemStore(getRootPath()).resolve(relativePath),
new GroupMetadata(attrs));
}
catch (ZarrException e) {
throw new IOException("Failed to create group " + relativePath, e);
}
}
else {
final ZarrGroup root =
ZarrGroup.create(getRootPath().resolve(relativePath));
root.writeAttributes(attrs);
}
}

private void createRootGroup() throws IOException {
Map<String, Object> attributes = new HashMap<String, Object>();
attributes.put("bioformats2raw.layout", LAYOUT);

createGroupWithAttributes(".", attributes);
}

private void writeOMEMetadata() throws IOException {
Map<String, Object> attributes = new HashMap<String, Object>();

// record the path to each series (multiscales) and the corresponding
// series (OME-XML Image) index
// using the index as the key would mean that the index is stored
// as a string instead of an integer
List<String> groups = new ArrayList<String>();
for (Integer index : seriesList) {
String resolutionString = String.format(
scaleFormatString, getScaleFormatStringArgs(index, 0));
String seriesString = "";
if (resolutionString.indexOf('/') >= 0) {
seriesString = resolutionString.substring(0,
resolutionString.lastIndexOf('/'));
}
groups.add(seriesString);
}
attributes.put("series", groups);

createGroupWithAttributes("OME", attributes);
}

private static Slf4JStopWatch stopWatch() {
return new Slf4JStopWatch(LOGGER, Slf4JStopWatch.DEBUG_LEVEL);
}
Expand Down

0 comments on commit 567fd3e

Please sign in to comment.