Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

ZarrReader: HCS validation fixes #24

Merged
merged 7 commits into from
Apr 28, 2022
Merged

ZarrReader: HCS validation fixes #24

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
ed0a7c6
ZarrReader: HCS validation fixes
dgault Mar 29, 2022
7d318ea
ZarrReader: HCS Multi-Res fixes
dgault Mar 29, 2022
155f3ba
Merge branch 'main' into 0.4-HCS-validation
dgault Apr 8, 2022
b02bfbb
ZarrReader: Fixes for 8bit data and default dimensionOrder
dgault Apr 14, 2022
f8bca98
Merge 'origin/main' into 0.4-HCS-validation
joshmoore Apr 20, 2022
574fddb
Update ZarrReader.java
dgault Apr 21, 2022
c4d2d7a
ZarrReader: Fix for multiRes HCS
dgault Apr 27, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
199 changes: 103 additions & 96 deletions src/loci/formats/in/ZarrReader.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,13 +9,13 @@
* %%
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
Expand Down Expand Up @@ -78,7 +78,9 @@ public class ZarrReader extends FormatReader {
private HashMap<Integer, ArrayList<String>> resSeries = new HashMap<Integer, ArrayList<String>>();
private HashMap<String, Integer> resCounts = new HashMap<String, Integer>();
private HashMap<String, Integer> resIndexes = new HashMap<String, Integer>();
private String dimensionOrder = "XYCZT";
private String dimensionOrder = "XYZCT";
private int wellCount = 0;
private int wellSamplesCount = 0;
private HashMap<String, ArrayList<String>> pathArrayDimensions = new HashMap<String, ArrayList<String>>();

private boolean hasSPW = false;
Expand Down Expand Up @@ -208,7 +210,7 @@ protected void initFile(String id) throws FormatException, IOException {
throw new FormatException("Image dimensions not found");
}

Boolean endian = null;
Boolean endian = zarrService.isLittleEndian();;
String pixType = omexmlMeta.getPixelsType(i).toString();
ms.dimensionOrder = omexmlMeta.getPixelsDimensionOrder(i).toString();
ms.sizeX = w.intValue();
Expand Down Expand Up @@ -242,7 +244,7 @@ protected void initFile(String id) throws FormatException, IOException {
try {
jsonAttr = ZarrUtils.toJson(attr, true);
store.setXMLAnnotationValue(jsonAttr, attrIndex);
String xml_id = MetadataTools.createLSID("AttributesAnnotation:", attrIndex);
String xml_id = MetadataTools.createLSID("Annotation", attrIndex);
store.setXMLAnnotationID(xml_id, attrIndex);
} catch (JZarrException e) {
LOGGER.warn("Failed to convert attributes to JSON");
Expand All @@ -262,7 +264,7 @@ protected void initFile(String id) throws FormatException, IOException {
try {
jsonAttr = ZarrUtils.toJson(attributes, true);
store.setXMLAnnotationValue(jsonAttr, attrIndex);
String xml_id = MetadataTools.createLSID("AttributesAnnotation:"+key, attrIndex);
String xml_id = MetadataTools.createLSID("Annotation", attrIndex);
store.setXMLAnnotationID(xml_id, attrIndex);
} catch (JZarrException e) {
LOGGER.warn("Failed to convert attributes to JSON");
Expand All @@ -280,7 +282,7 @@ protected void initFile(String id) throws FormatException, IOException {
try {
jsonAttr = ZarrUtils.toJson(attributes, true);
store.setXMLAnnotationValue(jsonAttr, attrIndex);
String xml_id = MetadataTools.createLSID("AttributesAnnotation:"+key, attrIndex);
String xml_id = MetadataTools.createLSID("Annotation", attrIndex);
store.setXMLAnnotationID(xml_id, attrIndex);
} catch (JZarrException e) {
LOGGER.warn("Failed to convert attributes to JSON");
Expand Down Expand Up @@ -331,7 +333,7 @@ protected void initFile(String id) throws FormatException, IOException {
ms.sizeZ = shape[2];
ms.sizeC = shape[1];
ArrayList<String> pathDimensions = pathArrayDimensions.get(arrayPaths.get(i));
if (!pathDimensions.isEmpty()) {
if (pathDimensions != null && !pathDimensions.isEmpty()) {
ms.sizeX = shape[pathDimensions.indexOf("x")];
ms.sizeY = shape[pathDimensions.indexOf("y")];
ms.sizeT = shape[pathDimensions.indexOf("t")];
Expand All @@ -351,12 +353,14 @@ protected void initFile(String id) throws FormatException, IOException {
ms.resolutionCount = resolutionCount;
}
}

MetadataTools.populatePixels( store, this, true );
for (int i = 0; i < getSeriesCount(); i++) {
store.setImageName(arrayPaths.get(i), i);
store.setImageID(arrayPaths.get(i), i);
store.setImageName(arrayPaths.get(seriesToCoreIndex(i)), i);
store.setImageID(MetadataTools.createLSID("Image", i), i);
}
parsePlate(zarrRootPath, "", store);

setSeries(0);
}

Expand Down Expand Up @@ -433,7 +437,10 @@ public byte[] openBytes(int no, byte[] buf, int x, int y, int w, int h) throws F
boolean little = zarrService.isLittleEndian();
int bpp = FormatTools.getBytesPerPixel(zarrService.getPixelType());
if (image instanceof byte[]) {
buf = (byte []) image;
byte [] data = (byte []) image;
for (int i = 0; i < data.length; i++) {
DataTools.unpackBytes(data[i], buf, i, 1, little);
}
}
else if (image instanceof short[]) {
short[] data = (short[]) image;
Expand Down Expand Up @@ -481,7 +488,7 @@ public void setSeries(int no) {
super.setSeries(no);
openZarr();
}

@Override
public void setResolution(int no) {
super.setResolution(no);
Expand Down Expand Up @@ -521,12 +528,12 @@ private void orderArrayPaths(String root) {
private void parseResolutionCount(String root, String key) throws IOException, FormatException {
String path = key.isEmpty() ? root : root + File.separator + key;
Map<String, Object> attr = zarrService.getGroupAttr(path);
ArrayList<String> pathDimensions = new ArrayList<String> ();
ArrayList<Object> multiscales = (ArrayList<Object>) attr.get("multiscales");
if (multiscales != null) {
for (int x = 0; x < multiscales.size(); x++) {
Map<String, Object> datasets = (Map<String, Object>) multiscales.get(x);
List<Object> multiscaleAxes = (List<Object>)datasets.get("axes");
ArrayList<String> pathDimensions = new ArrayList<String> ();
if (multiscaleAxes != null) {
for (int i = 0; i < multiscaleAxes.size(); i++) {
if (multiscaleAxes.get(i) instanceof String) {
Expand Down Expand Up @@ -561,7 +568,7 @@ else if (multiscaleAxes.get(i) instanceof HashMap) {
String scalePath = (String) multiScale.get("path");
int numRes = multiscalePaths.size();
if (i == 0) {
resCounts.put(scalePath, numRes);
resCounts.put(key.isEmpty() ? scalePath : key + File.separator + scalePath, numRes);
}
resIndexes.put(scalePath, i);
ArrayList<String> list = resSeries.get(resCounts.size() - 1);
Expand Down Expand Up @@ -590,100 +597,100 @@ private void parsePlate(String root, String key, MetadataStore store) throws IOE
Map<String, Object> attr = zarrService.getGroupAttr(path);
Map<Object, Object> plates = (Map<Object, Object>) attr.get("plate");
if (plates != null) {
for (int p=0; p < plates.size(); p++) {

Map<String, Object> plate = (Map<String, Object>) plates.get(p);
ArrayList<Object> columns = (ArrayList<Object>)plates.get("columns");
ArrayList<Object> rows = (ArrayList<Object>)plates.get("rows");
ArrayList<Object> wells = (ArrayList<Object>)plates.get("wells");
ArrayList<Object> acquisitions = (ArrayList<Object> )plates.get("acquisitions");
String plateName = (String) plates.get("name");
String fieldCount = (String) plates.get("filed_count");

String plate_id = MetadataTools.createLSID("Plate", p);
store.setPlateID(plate_id, p);
store.setPlateName(plateName, p);
int wellSamplesCount = 0;
HashMap<Integer, Integer> acqIdsIndexMap = new HashMap<Integer, Integer>();
if (acquisitions != null) {
for (int a = 0; a < acquisitions.size(); a++) {
Map<String, Object> acquistion = (Map<String, Object>) acquisitions.get(a);
Integer acqId = (Integer) acquistion.get("id");
String acqName = (String) acquistion.get("name");
String acqStartTime = (String) acquistion.get("starttime");
Integer maximumfieldcount = (Integer) acquistion.get("maximumfieldcount");
acqIdsIndexMap.put(acqId, a);
store.setPlateAcquisitionID(
MetadataTools.createLSID("PlateAcquisition", p, acqId), p, a);
}
}
for (int c = 0; c < columns.size(); c++) {
Map<String, Object> column = (Map<String, Object>) columns.get(c);
String colName = (String) column.get("name");
}
for (int r = 0; r < rows.size(); r++) {
Map<String, Object> row = (Map<String, Object>) rows.get(r);
String rowName = (String) row.get("name");
}
for (int w = 0; w < wells.size(); w++) {
Map<String, Object> well = (Map<String, Object>) wells.get(w);
String wellPath = (String) well.get("path");
String wellCol = (String) well.get("column_index");
String wellRow = (String) well.get("row_index");
String well_id = MetadataTools.createLSID("Well", w);
store.setWellID(well_id, p, w);
String[] parts = wellPath.split("/");
if (wellRow == null || wellRow.length() == 0) {
wellRow = parts[parts.length - 2];
}
if (wellCol == null || wellCol.length() == 0) {
wellCol = parts[parts.length - 1];
}
int rowIndex = "ABCDEFGHIJKLMNOPQRSTUVWXYZ".indexOf(wellRow.toUpperCase());
if (rowIndex == -1) {
rowIndex = Integer.parseInt(wellRow);
}
int colIndex = "ABCDEFGHIJKLMNOPQRSTUVWXYZ".indexOf(wellCol.toUpperCase());
if (colIndex == -1) {
colIndex = Integer.parseInt(wellCol);
}
store.setWellRow(new NonNegativeInteger(rowIndex), p, w);
store.setWellColumn(new NonNegativeInteger(colIndex), p, w);
store.setWellExternalIdentifier(wellPath, p, w);
wellSamplesCount = parseWells(root, wellPath, store, p, w, wellSamplesCount, acqIdsIndexMap);
ArrayList<Object> columns = (ArrayList<Object>)plates.get("columns");
ArrayList<Object> rows = (ArrayList<Object>)plates.get("rows");
ArrayList<Object> wells = (ArrayList<Object>)plates.get("wells");
ArrayList<Object> acquisitions = (ArrayList<Object> )plates.get("acquisitions");
String plateName = (String) plates.get("name");
String fieldCount = (String) plates.get("filed_count");

String plate_id = MetadataTools.createLSID("Plate", 0);
store.setPlateID(plate_id, 0);
store.setPlateName(plateName, 0);
HashMap<Integer, Integer> acqIdsIndexMap = new HashMap<Integer, Integer>();
if (acquisitions != null) {
for (int a = 0; a < acquisitions.size(); a++) {
Map<String, Object> acquistion = (Map<String, Object>) acquisitions.get(a);
Integer acqId = (Integer) acquistion.get("id");
String acqName = (String) acquistion.get("name");
String acqStartTime = (String) acquistion.get("starttime");
Integer maximumfieldcount = (Integer) acquistion.get("maximumfieldcount");
acqIdsIndexMap.put(acqId, a);
store.setPlateAcquisitionID(
MetadataTools.createLSID("PlateAcquisition", 0, acqId), 0, a);
}
}
for (int c = 0; c < columns.size(); c++) {
Map<String, Object> column = (Map<String, Object>) columns.get(c);
String colName = (String) column.get("name");
}
for (int r = 0; r < rows.size(); r++) {
Map<String, Object> row = (Map<String, Object>) rows.get(r);
String rowName = (String) row.get("name");
}
for (int w = 0; w < wells.size(); w++) {
Map<String, Object> well = (Map<String, Object>) wells.get(w);
String wellPath = (String) well.get("path");
String wellCol = (String) well.get("column_index");
String wellRow = (String) well.get("row_index");
String well_id = MetadataTools.createLSID("Well", wellCount);
store.setWellID(well_id, 0, w);
String[] parts = wellPath.split("/");
if (wellRow == null || wellRow.isEmpty()) {
wellRow = parts[parts.length - 2];
}
if (wellCol == null || wellCol.isEmpty()) {
wellCol = parts[parts.length - 1];
}
int rowIndex = "ABCDEFGHIJKLMNOPQRSTUVWXYZ".indexOf(wellRow.toUpperCase());
if (rowIndex == -1) {
rowIndex = Integer.parseInt(wellRow);
}
int colIndex = "ABCDEFGHIJKLMNOPQRSTUVWXYZ".indexOf(wellCol.toUpperCase());
if (colIndex == -1) {
colIndex = Integer.parseInt(wellCol);
}
store.setWellRow(new NonNegativeInteger(rowIndex), 0, w);
store.setWellColumn(new NonNegativeInteger(colIndex), 0, w);
store.setWellExternalIdentifier(wellPath, 0, w);
parseWells(root, wellPath, store, 0, w, acqIdsIndexMap);
wellCount++;
}
}
}

private int parseWells(String root, String key, MetadataStore store, int plateIndex, int wellIndex, int wellSamplesCount,
private void parseWells(String root, String key, MetadataStore store, int plateIndex, int wellIndex,
HashMap<Integer, Integer> acqIdsIndexMap) throws IOException, FormatException {
String path = key.isEmpty() ? root : root + File.separator + key;
Map<String, Object> attr = zarrService.getGroupAttr(path);
Map<Object, Object> wells = (Map<Object, Object>) attr.get("well");
if (wells != null) {
for (int w=0; w < wells.size(); w++) {
Map<String, Object> well = (Map<String, Object>) wells.get(w);
ArrayList<Object> images = (ArrayList<Object>)wells.get("images");
for (int i = 0; i < images.size(); i++) {
Map<String, Object> image = (Map<String, Object>) images.get(i);
String imagePath = (String) image.get("path");
Integer acquisition = (Integer) image.get("acquisition");
if (acqIdsIndexMap.containsKey(acquisition)) {
acquisition = acqIdsIndexMap.get(acquisition);
}
String site_id = MetadataTools.createLSID("WellSample", wellSamplesCount);
store.setWellSampleID(site_id, plateIndex, wellIndex, i);
store.setWellSampleIndex(new NonNegativeInteger(i), plateIndex, wellIndex, i);
store.setWellSampleImageRef(arrayPaths.get(wellSamplesCount), plateIndex, wellIndex, i);
if (acquisition != null) {
store.setPlateAcquisitionWellSampleRef(site_id, plateIndex, (int) acquisition, i);
}
wellSamplesCount++;
}
ArrayList<Object> images = (ArrayList<Object>)wells.get("images");
for (int i = 0; i < images.size(); i++) {
Map<String, Object> image = (Map<String, Object>) images.get(i);
String imagePath = (String) image.get("path");
Integer acquisition = (Integer) image.get("acquisition");
if (acqIdsIndexMap.containsKey(acquisition)) {
acquisition = acqIdsIndexMap.get(acquisition);
}
String site_id = MetadataTools.createLSID("WellSample", wellSamplesCount);
store.setWellSampleID(site_id, plateIndex, wellIndex, i);
store.setWellSampleIndex(new NonNegativeInteger(i), plateIndex, wellIndex, i);
String imageRefPath = "" + i;
if (key != null && !key.isEmpty()) {
imageRefPath = key + File.separator + i;
}
if (resCounts.containsKey(imageRefPath + File.separator + "0")) {
imageRefPath += File.separator + "0";
}
String imageID = MetadataTools.createLSID("Image", coreIndexToSeries(arrayPaths.indexOf(imageRefPath)));
store.setWellSampleImageRef(imageID, plateIndex, wellIndex, i);
if (acquisition != null) {
store.setPlateAcquisitionWellSampleRef(site_id, plateIndex, (int) acquisition, i);
}
wellSamplesCount++;
}
}
return wellSamplesCount;
}

private void parseLabels(String root, String key) throws IOException, FormatException {
Expand Down
2 changes: 1 addition & 1 deletion src/test/loci/formats/utests/ZarrReaderTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -161,7 +161,7 @@ public void testOpenBytes() {

when(zarrService.readBytes(readerShape, readerOffset)).thenReturn(expectedBuf);
when(zarrService.getPixelType()).thenReturn(0);
buf = new byte[1024*512];
buf = new byte[1024*512*4];
buf = reader.openBytes(0, buf);
assertEquals(expectedBuf, buf);
} catch (FormatException | IOException e) {
Expand Down