Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Reorder group keys #53

Closed
wants to merge 4 commits into from
Closed

Reorder group keys #53

Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
ef1a013
Group keys will now be reordered before processing
dgault May 1, 2023
bec6f0f
Make group key comparator static
dgault May 5, 2023
306c0d0
Reorder group keys: Use original series ordering
dgault Jul 13, 2023
0fd6c3f
Merge branch 'main' into reorderGroupKeys
dgault Jul 13, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
110 changes: 106 additions & 4 deletions src/loci/formats/in/ZarrReader.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,10 +37,13 @@
import java.nio.file.Paths;
import java.nio.file.FileVisitOption;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.Hashtable;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Stream;

import javax.xml.parsers.ParserConfigurationException;
Expand Down Expand Up @@ -161,8 +164,9 @@ protected void initFile(String id) throws FormatException, IOException {

initializeZarrService(canonicalPath);

ArrayList<String> omeSeriesOrder = new ArrayList<String>();
if(omeMetaFile.exists()) {
parseOMEXML(omeMetaFile, store);
parseOMEXML(omeMetaFile, store, omeSeriesOrder);
}
// Parse base level attributes
Map<String, Object> attr = zarrService.getGroupAttr(canonicalPath);
Expand All @@ -183,7 +187,9 @@ protected void initFile(String id) throws FormatException, IOException {
}

// Parse group attributes
for (String key: zarrService.getGroupKeys(canonicalPath)) {
Set<String> groupKeys = zarrService.getGroupKeys(canonicalPath);
List<String> orderedGroupKeys = reorderGroupKeys(groupKeys, omeSeriesOrder);
for (String key: orderedGroupKeys) {
Map<String, Object> attributes = zarrService.getGroupAttr(canonicalPath+File.separator+key);
if (attributes != null && !attributes.isEmpty()) {
parseResolutionCount(zarrRootPath, key);
Expand Down Expand Up @@ -293,6 +299,68 @@ protected void initFile(String id) throws FormatException, IOException {
setSeries(0);
}

private List<String> reorderGroupKeys(Set<String> groupKeys, List<String> originalKeys) {
// Reorder group keys to maintain the original order from the OME-XML provided by bioformats2raw
if (originalKeys.isEmpty() || !groupKeys.containsAll(originalKeys)) {
LOGGER.warn("Mismatch with group key paths and original OME-XML metadata, original ordering wont be maintained");
return reorderGroupKeys(groupKeys);
}
List<String> groupKeysList = new ArrayList<String>();
groupKeys.removeAll(originalKeys);
groupKeysList.addAll(originalKeys);
groupKeysList.addAll(groupKeys);
return groupKeysList;
}

private List<String> reorderGroupKeys(Set<String> groupKeys) {
// Reorder group keys to avoid order such A/1, A/10, A/11, A/12, A/2, A/20, A/3, A/4
List<String> groupKeysList = new ArrayList<String>();
groupKeysList.addAll(groupKeys);
Collections.sort(groupKeysList, keyComparator);
return groupKeysList;
}

private static Comparator<String> keyComparator = (a,b)->{
String[] aParts = a.split("/");
String[] bParts = b.split("/");

int numParts = aParts.length - bParts.length;
if (numParts != 0) return numParts;

for (int i = 0; i < aParts.length; i++) {
String aPart = aParts[i];
String bPart = bParts[i];

boolean isAInt = isInteger(aPart);
boolean isBInt = isInteger(bPart);
if (isAInt && !isBInt) return -1;
if (!isAInt && isBInt) return 1;

if (isAInt) {
int numResult = Integer.compare(Integer.valueOf(aPart), Integer.valueOf(bPart));
if (numResult != 0) return numResult;
}
else {
int stringResult = aPart.compareTo(bPart);
if (stringResult != 0) return stringResult;
}
}

return 0;
};

private static boolean isInteger(String s) {
if(s.isEmpty()) return false;
for(int i = 0; i < s.length(); i++) {
if(i == 0 && s.charAt(i) == '-') {
if(s.length() == 1) return false;
else continue;
}
if(Character.digit(s.charAt(i), 10) < 0) return false;
}
return true;
}

/**
* In the event that .zarray does not contain a 5d shape
* The dimensions of the original shape will be assumed based on tczyx
Expand Down Expand Up @@ -733,7 +801,7 @@ private void parseOmeroMetadata(String root, String key) throws IOException, For
}
}

private void parseOMEXML(Location omeMetaFile, MetadataStore store) throws IOException, FormatException {
private void parseOMEXML(Location omeMetaFile, MetadataStore store, ArrayList<String> origSeries) throws IOException, FormatException {
Document omeDocument = null;
try (RandomAccessInputStream measurement =
new RandomAccessInputStream(omeMetaFile.getAbsolutePath())) {
Expand Down Expand Up @@ -775,6 +843,24 @@ private void parseOMEXML(Location omeMetaFile, MetadataStore store) throws IOExc

int numDatasets = omexmlMeta.getImageCount();

// Map of the well location for each imageReference
// Later we will map the series index to the imageReference
// This allows us to maintain the series order when parsing the Zarr groups
Map<String, String> imageRefPaths = new HashMap<String, String>();
for (int plateIndex = 0; plateIndex < omexmlMeta.getPlateCount(); plateIndex++) {
for (int wellIndex = 0; wellIndex < omexmlMeta.getWellCount(plateIndex); wellIndex++) {
NonNegativeInteger col = omexmlMeta.getWellColumn(plateIndex, wellIndex);
NonNegativeInteger row = omexmlMeta.getWellRow(plateIndex, wellIndex);

String rowLetter = getRowString(row.getValue());
String expectedPath = rowLetter + File.separator + (col.getValue() + 1) + File.separator + "0";
for (int wellSampleIndex = 0; wellSampleIndex < omexmlMeta.getWellSampleCount(plateIndex, wellIndex); wellSampleIndex++) {
String imageRef = omexmlMeta.getWellSampleImageRef(plateIndex, wellIndex, wellSampleIndex);
imageRefPaths.put(imageRef, expectedPath);
}
}
}

int oldSeries = getSeries();
core.clear();
for (int i=0; i<numDatasets; i++) {
Expand All @@ -791,7 +877,13 @@ private void parseOMEXML(Location omeMetaFile, MetadataStore store) throws IOExc
throw new FormatException("Image dimensions not found");
}

Boolean endian = zarrService.isLittleEndian();;
String imageId = omexmlMeta.getImageID(i);
if (!imageRefPaths.isEmpty() && imageRefPaths.containsKey(imageId)) {
String expectedZarrPath = imageRefPaths.get(imageId);
origSeries.add(expectedZarrPath);
}

Boolean endian = zarrService.isLittleEndian();
String pixType = omexmlMeta.getPixelsType(i).toString();
ms.dimensionOrder = omexmlMeta.getPixelsDimensionOrder(i).toString();
ms.sizeX = w.intValue();
Expand Down Expand Up @@ -849,6 +941,16 @@ private void parseOMEXML(Location omeMetaFile, MetadataStore store) throws IOExc
MetadataConverter.convertMetadata( omexmlMeta, store );
}

public static String getRowString(int rowIndex) {
StringBuilder sb = new StringBuilder();
if (rowIndex == 0) sb.append('A');
while (rowIndex > 0) {
sb.append((char)('A' + (rowIndex % 26)));
rowIndex /= 26;
}
return sb.reverse().toString();
}

private Double getDouble(Map<String, Object> src, String key) {
Number val = (Number) src.get(key);
if (val == null) {
Expand Down