Skip to content

Commit

Permalink
Merge pull request #3229 from melissalinkert/micromanager-backport
Browse files Browse the repository at this point in the history 
Micromanager: better handling of very large *metadata.txt files
  • Loading branch information
@dgault
dgault authored Oct 4, 2018
2 parents 6d54312 + 2f401f2 commit b1163d6
Showing 1 changed file with 91 additions and 21 deletions.
112 changes: 91 additions & 21 deletions components/formats-bsd/src/loci/formats/in/MicromanagerReader.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,11 +34,12 @@

import java.io.File;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.util.Arrays;
import java.util.HashMap;
import java.util.StringTokenizer;
import java.util.Vector;

import loci.common.Constants;
import loci.common.DataTools;
import loci.common.DateTools;
import loci.common.Location;
Expand Down Expand Up @@ -100,6 +101,9 @@ public class MicromanagerReader extends FormatReader {
private MinimalTiffReader tiffReader;

private Vector<Position> positions;
private int start = 0;

private boolean spim = false;

// -- Constructor --

Expand Down Expand Up @@ -227,6 +231,8 @@ public void close(boolean fileOnly) throws IOException {
if (tiffReader != null) tiffReader.close(fileOnly);
if (!fileOnly) {
positions = null;
start = 0;
spim = false;
}
}

Expand Down Expand Up @@ -304,8 +310,30 @@ public void initFile(String id) throws FormatException, IOException {
setSeries(i);
parsePosition(i);
}

setSeries(0);

// collapse original metadata so that keys with the same
// per-plane value are stored only once

for (int i=0; i<seriesCount; i++) {
for (String key : core.get(i).seriesMetadata.keySet()) {
if (core.get(i).seriesMetadata.get(key) instanceof Vector) {
Vector v = (Vector) core.get(i).seriesMetadata.get(key);
boolean collapse = true;
for (Object o : v) {
if (!o.equals(v.get(0))) {
collapse = false;
break;
}
}
if (collapse) {
core.get(i).seriesMetadata.put(key, v.get(0));
}
}
}
}

populateMetadata();
}

Expand Down Expand Up @@ -437,16 +465,19 @@ public void populateMetadataStore(String[] jsonData)

private void parsePosition(int posIndex) throws IOException, FormatException {
Position p = positions.get(posIndex);
String s = DataTools.readFile(p.metadataFile);
parsePosition(s, posIndex);

parsePosition(p.metadataFile, posIndex);

buildTIFFList(posIndex);

// parse original metadata from each TIFF's JSON
p.positions = new Double[p.tiffs.size()][3];
int digits = String.valueOf(p.tiffs.size() - 1).length();

boolean parseMMJSONTag = true;
// safe to assume that no extra data (i.e. not already in _metadata.txt)
// will be included in the MM_JSON tags for SPIM data
boolean parseMMJSONTag = !spim;

for (int plane=0; plane<p.tiffs.size(); ) {
String path = p.tiffs.get(plane);
// use getFile(...) lookup if possible, to make sure that
Expand Down Expand Up @@ -510,6 +541,7 @@ private void parsePosition(int posIndex) throws IOException, FormatException {
String[] tokens = json.split("[\\{\\}:,\"]");
String key = null, value = null, propType = null;
int nEmptyTokens = 0;

for (int q=0; q<tokens.length; q++) {
String token = tokens[q];
if (token.length() == 0) {
Expand Down Expand Up @@ -571,26 +603,28 @@ else if (((propType != null && propType.equals("PropType")) || token.equals("]")
}
}

private void parseKeyAndValue(String key, String value, int digits, int plane, int nPlanes) {
Position p = positions.get(getCoreIndex());

private void parseKeyAndValue(String key, String value, int digits, int plane, int nPlanes) {
// using key alone will result in conflicts with metadata.txt values
for (int i=plane; i<plane+nPlanes; i++) {
addSeriesMeta(String.format("Plane #%0" + digits + "d %s", i, key), value);
if (key.equals("XPositionUm")) {
try {
Position p = positions.get(getCoreIndex());
p.positions[i][0] = new Double(value);
}
catch (NumberFormatException e) { }
}
else if (key.equals("YPositionUm")) {
try {
Position p = positions.get(getCoreIndex());
p.positions[i][1] = new Double(value);
}
catch (NumberFormatException e) { }
}
else if (key.equals("ZPositionUm")) {
try {
Position p = positions.get(getCoreIndex());
p.positions[i][2] = new Double(value);
}
catch (NumberFormatException e) { }
Expand Down Expand Up @@ -675,10 +709,24 @@ private void parsePosition(String jsonData, int posIndex)
Vector<Double> stamps = new Vector<Double>();
p.voltage = new Vector<Double>();

StringTokenizer st = new StringTokenizer(jsonData, "\n");
RandomAccessInputStream s = new RandomAccessInputStream(jsonData);

if (s.length() > Integer.MAX_VALUE) {
LOGGER.warn(jsonData + " exceeds 2GB; metadata parsing is likely to fail");
}
else if (s.length() > 100 * 1024 * 1024) {
LOGGER.warn(jsonData + " is larger than 100MB and may require additional memory to parse. " +
"A minimum of 1024MB is suggested.");
}

byte[] b = new byte[(int) s.length()];
s.readFully(b);
s.close();

int[] slice = new int[3];
while (st.hasMoreTokens()) {
String token = st.nextToken().trim();
start = 0;
while (start < b.length) {
String token = getNextLine(b).trim();
boolean open = token.indexOf('[') != -1;
boolean closed = token.indexOf(']') != -1;
if (open || (!open && !closed && !token.equals("{") &&
Expand All @@ -694,7 +742,7 @@ private void parsePosition(String jsonData, int posIndex)
else if (!closed) {
final StringBuilder valueBuffer = new StringBuilder();
while (!closed) {
token = st.nextToken();
token = getNextLine(b);
closed = token.indexOf(']') != -1;
valueBuffer.append(token);
}
Expand Down Expand Up @@ -775,6 +823,9 @@ else if (key.equals("IJType")) {
throw new FormatException("Unknown type: " + type);
}
}
else if (key.equals("SPIMmode")) {
spim = true;
}
}

if (token.startsWith("\"FrameKey")) {
Expand All @@ -788,7 +839,7 @@ else if (key.equals("IJType")) {
slice[0] = Integer.parseInt(token.substring(dash,
token.indexOf("\"", dash)));

token = st.nextToken().trim();
token = getNextLine(b).trim();
String key = "";
StringBuilder valueBuffer = new StringBuilder();
boolean valueArray = false;
Expand All @@ -798,12 +849,12 @@ else if (key.equals("IJType")) {

if (token.trim().endsWith("{")) {
nestedCount++;
token = st.nextToken().trim();
token = getNextLine(b).trim();
continue;
}
else if (token.trim().startsWith("}")) {
nestedCount--;
token = st.nextToken().trim();
token = getNextLine(b).trim();
continue;
}

Expand All @@ -813,12 +864,12 @@ else if (token.trim().startsWith("}")) {
}
else {
valueBuffer.append(token.trim().replaceAll("\"", ""));
token = st.nextToken().trim();
token = getNextLine(b).trim();
continue;
}
}
else {
int colon = token.indexOf(':');
int colon = token.indexOf(":");
key = token.substring(1, colon).trim();
valueBuffer.setLength(0);
valueBuffer.append(token.substring(colon + 1, token.length() - 1).trim().replaceAll("\"", ""));
Expand All @@ -827,7 +878,7 @@ else if (token.trim().startsWith("}")) {

if (token.trim().endsWith("[")) {
valueArray = true;
token = st.nextToken().trim();
token = getNextLine(b).trim();
continue;
}
}
Expand Down Expand Up @@ -882,7 +933,7 @@ else if (key.equals("FileName")) {
}
}

token = st.nextToken().trim();
token = getNextLine(b).trim();
}
}
else if (token.startsWith("\"Coords-")) {
Expand Down Expand Up @@ -912,7 +963,7 @@ else if (key.equals("channel")) {
}
}

token = st.nextToken().trim();
token = getNextLine(b).trim();
}
Index idx = new Index(zct);
idx.position = position;
Expand Down Expand Up @@ -1070,12 +1121,18 @@ private void parseXMLFile() throws IOException {
XMLTools.parseXML(xmlData, handler);
}

/** Initialize the TIFF reader with the first file in the current series. */
/** Initialize the TIFF reader with the first non-null file in the current series. */
private void setupReader() {
try {
int plane = 0;
String file = positions.get(getSeries()).getFile(
getDimensionOrder(), getSizeZ(), getSizeC(), getSizeT(),
getImageCount(), 0);
getImageCount(), plane++);
while (file == null && plane < getImageCount()) {
file = positions.get(getSeries()).getFile(
getDimensionOrder(), getSizeZ(), getSizeC(), getSizeT(),
getImageCount(), plane++);
}
tiffReader.setId(file);
}
catch (Exception e) {
Expand All @@ -1101,7 +1158,7 @@ private void handleKeyValue(String key, String value) {
if (key == null || value == null) {
return;
}
addSeriesMeta(key, value);
addSeriesMetaList(key, value);

if (key.equals("MicroManagerVersion")) {
String[] version = value.split("\\.");
Expand All @@ -1121,6 +1178,19 @@ private void handleKeyValue(String key, String value) {
}
}

private String getNextLine(byte[] buf) throws UnsupportedEncodingException {
for (int i=start; i<buf.length; i++) {
if (buf[i] == '\n') {
String line = new String(buf, start, (i - start) + 1, Constants.ENCODING);
start = i + 1;
return line;
}
}
String line = new String(buf, start, buf.length - start, Constants.ENCODING);
start = buf.length;
return line;
}

// -- Helper classes --

/** SAX handler for parsing Acqusition.xml. */
Expand Down

0 comments on commit b1163d6

Please sign in to comment.