From 87da5c2766dca481bb66b0aea6f24f4996ca39aa Mon Sep 17 00:00:00 2001
From: Melissa Linkert <melissa@glencoesoftware.com>
Date: Tue, 12 Jul 2016 18:26:25 -0500
Subject: [PATCH 1/3] Micro-manager: reduce memory required for SPIM data

Conflicts:
	components/formats-bsd/src/loci/formats/in/MicromanagerReader.java

Conflicts:
	components/formats-bsd/src/loci/formats/in/MicromanagerReader.java
---
 .../loci/formats/in/MicromanagerReader.java   | 91 +++++++++++++++----
 1 file changed, 72 insertions(+), 19 deletions(-)

diff --git a/components/formats-bsd/src/loci/formats/in/MicromanagerReader.java b/components/formats-bsd/src/loci/formats/in/MicromanagerReader.java
index ed43b858bbe..dc10c2192b4 100644
--- a/components/formats-bsd/src/loci/formats/in/MicromanagerReader.java
+++ b/components/formats-bsd/src/loci/formats/in/MicromanagerReader.java
@@ -36,7 +36,6 @@
 import java.io.IOException;
 import java.util.Arrays;
 import java.util.HashMap;
-import java.util.StringTokenizer;
 import java.util.Vector;
 
 import loci.common.DataTools;
@@ -100,6 +99,9 @@ public class MicromanagerReader extends FormatReader {
   private MinimalTiffReader tiffReader;
 
   private Vector<Position> positions;
+  private int start = 0;
+
+  private boolean spim = false;
 
   // -- Constructor --
 
@@ -227,6 +229,8 @@ public void close(boolean fileOnly) throws IOException {
     if (tiffReader != null) tiffReader.close(fileOnly);
     if (!fileOnly) {
       positions = null;
+      start = 0;
+      spim = false;
     }
   }
 
@@ -304,8 +308,30 @@ public void initFile(String id) throws FormatException, IOException {
       setSeries(i);
       parsePosition(i);
     }
+
     setSeries(0);
 
+    // collapse original metadata so that keys with the same
+    // per-plane value are stored only once
+
+    for (int i=0; i<seriesCount; i++) {
+      for (String key : core.get(i).seriesMetadata.keySet()) {
+        if (core.get(i).seriesMetadata.get(key) instanceof Vector) {
+          Vector v = (Vector) core.get(i).seriesMetadata.get(key);
+          boolean collapse = true;
+          for (Object o : v) {
+            if (!o.equals(v.get(0))) {
+              collapse = false;
+              break;
+            }
+          }
+          if (collapse) {
+            core.get(i).seriesMetadata.put(key, v.get(0));
+          }
+        }
+      }
+    }
+
     populateMetadata();
   }
 
@@ -437,8 +463,8 @@ public void populateMetadataStore(String[] jsonData)
 
   private void parsePosition(int posIndex) throws IOException, FormatException {
     Position p = positions.get(posIndex);
-    String s = DataTools.readFile(p.metadataFile);
-    parsePosition(s, posIndex);
+
+    parsePosition(p.metadataFile, posIndex);
 
     buildTIFFList(posIndex);
 
@@ -446,7 +472,10 @@ private void parsePosition(int posIndex) throws IOException, FormatException {
     p.positions = new Double[p.tiffs.size()][3];
     int digits = String.valueOf(p.tiffs.size() - 1).length();
 
-    boolean parseMMJSONTag = true;
+    // safe to assume that no extra data (i.e. not already in _metadata.txt)
+    // will be included in the MM_JSON tags for SPIM data
+    boolean parseMMJSONTag = !spim;
+
     for (int plane=0; plane<p.tiffs.size(); ) {
       String path = p.tiffs.get(plane);
       // use getFile(...) lookup if possible, to make sure that
@@ -510,6 +539,7 @@ private void parsePosition(int posIndex) throws IOException, FormatException {
           String[] tokens = json.split("[\\{\\}:,\"]");
           String key = null, value = null, propType = null;
           int nEmptyTokens = 0;
+
           for (int q=0; q<tokens.length; q++) {
             String token = tokens[q];
             if (token.length() == 0) {
@@ -571,26 +601,28 @@ else if (((propType != null && propType.equals("PropType")) || token.equals("]")
     }
   }
 
-  private void parseKeyAndValue(String key, String value, int digits, int plane, int nPlanes) {
-    Position p = positions.get(getCoreIndex());
 
+  private void parseKeyAndValue(String key, String value, int digits, int plane, int nPlanes) {
     // using key alone will result in conflicts with metadata.txt values
     for (int i=plane; i<plane+nPlanes; i++) {
       addSeriesMeta(String.format("Plane #%0" + digits + "d %s", i, key), value);
       if (key.equals("XPositionUm")) {
         try {
+          Position p = positions.get(getCoreIndex());
           p.positions[i][0] = new Double(value);
         }
         catch (NumberFormatException e) { }
       }
       else if (key.equals("YPositionUm")) {
         try {
+          Position p = positions.get(getCoreIndex());
           p.positions[i][1] = new Double(value);
         }
         catch (NumberFormatException e) { }
       }
       else if (key.equals("ZPositionUm")) {
         try {
+          Position p = positions.get(getCoreIndex());
           p.positions[i][2] = new Double(value);
         }
         catch (NumberFormatException e) { }
@@ -675,10 +707,14 @@ private void parsePosition(String jsonData, int posIndex)
     Vector<Double> stamps = new Vector<Double>();
     p.voltage = new Vector<Double>();
 
-    StringTokenizer st = new StringTokenizer(jsonData, "\n");
+    RandomAccessInputStream s = new RandomAccessInputStream(jsonData);
+    byte[] b = new byte[(int) s.length()];
+    s.readFully(b);
+    s.close();
+
     int[] slice = new int[3];
-    while (st.hasMoreTokens()) {
-      String token = st.nextToken().trim();
+    while (start < b.length) {
+      String token = getNextLine(b).trim();
       boolean open = token.indexOf('[') != -1;
       boolean closed = token.indexOf(']') != -1;
       if (open || (!open && !closed && !token.equals("{") &&
@@ -694,7 +730,7 @@ private void parsePosition(String jsonData, int posIndex)
         else if (!closed) {
           final StringBuilder valueBuffer = new StringBuilder();
           while (!closed) {
-            token = st.nextToken();
+            token = getNextLine(b);
             closed = token.indexOf(']') != -1;
             valueBuffer.append(token);
           }
@@ -775,6 +811,9 @@ else if (key.equals("IJType")) {
               throw new FormatException("Unknown type: " + type);
           }
         }
+        else if (key.equals("SPIMmode")) {
+          spim = true;
+        }
       }
 
       if (token.startsWith("\"FrameKey")) {
@@ -788,7 +827,7 @@ else if (key.equals("IJType")) {
         slice[0] = Integer.parseInt(token.substring(dash,
           token.indexOf("\"", dash)));
 
-        token = st.nextToken().trim();
+        token = getNextLine(b).trim();
         String key = "";
         StringBuilder valueBuffer = new StringBuilder();
         boolean valueArray = false;
@@ -798,12 +837,12 @@ else if (key.equals("IJType")) {
 
           if (token.trim().endsWith("{")) {
             nestedCount++;
-            token = st.nextToken().trim();
+            token = getNextLine(b).trim();
             continue;
           }
           else if (token.trim().startsWith("}")) {
             nestedCount--;
-            token = st.nextToken().trim();
+            token = getNextLine(b).trim();
             continue;
           }
 
@@ -813,12 +852,15 @@ else if (token.trim().startsWith("}")) {
             }
             else {
               valueBuffer.append(token.trim().replaceAll("\"", ""));
-              token = st.nextToken().trim();
+              token = getNextLine(b).trim();
               continue;
             }
           }
           else {
-            int colon = token.indexOf(':');
+            int colon = token.indexOf("\":") + 1;
+            if (colon == 0) {
+              colon = token.indexOf(':');
+            }
             key = token.substring(1, colon).trim();
             valueBuffer.setLength(0);
             valueBuffer.append(token.substring(colon + 1, token.length() - 1).trim().replaceAll("\"", ""));
@@ -827,7 +869,7 @@ else if (token.trim().startsWith("}")) {
 
             if (token.trim().endsWith("[")) {
               valueArray = true;
-              token = st.nextToken().trim();
+              token = getNextLine(b).trim();
               continue;
             }
           }
@@ -882,7 +924,7 @@ else if (key.equals("FileName")) {
             }
           }
 
-          token = st.nextToken().trim();
+          token = getNextLine(b).trim();
         }
       }
       else if (token.startsWith("\"Coords-")) {
@@ -912,7 +954,7 @@ else if (key.equals("channel")) {
             }
           }
 
-          token = st.nextToken().trim();
+          token = getNextLine(b).trim();
         }
         Index idx = new Index(zct);
         idx.position = position;
@@ -1101,7 +1143,7 @@ private void handleKeyValue(String key, String value) {
     if (key == null || value == null) {
       return;
     }
-    addSeriesMeta(key, value);
+    addSeriesMetaList(key, value);
 
     if (key.equals("MicroManagerVersion")) {
       String[] version = value.split("\\.");
@@ -1121,6 +1163,17 @@ private void handleKeyValue(String key, String value) {
     }
   }
 
+  private String getNextLine(byte[] buf) {
+    for (int i=start; i<buf.length; i++) {
+      if (buf[i] == '\n') {
+        String line = new String(buf, start, (i - start) + 1);
+        start = i + 1;
+        return line;
+      }
+    }
+    return null;
+  }
+
   // -- Helper classes --
 
   /** SAX handler for parsing Acqusition.xml. */

From d9ae8ec675846acd900f699b2017013200e2b17a Mon Sep 17 00:00:00 2001
From: Melissa Linkert <melissa@glencoesoftware.com>
Date: Mon, 18 Jul 2016 08:32:17 -0500
Subject: [PATCH 2/3] Add logging when metadata.txt file exceeds 100MB

---
 .../src/loci/formats/in/MicromanagerReader.java          | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/components/formats-bsd/src/loci/formats/in/MicromanagerReader.java b/components/formats-bsd/src/loci/formats/in/MicromanagerReader.java
index dc10c2192b4..42754c2c3b0 100644
--- a/components/formats-bsd/src/loci/formats/in/MicromanagerReader.java
+++ b/components/formats-bsd/src/loci/formats/in/MicromanagerReader.java
@@ -708,6 +708,15 @@ private void parsePosition(String jsonData, int posIndex)
     p.voltage = new Vector<Double>();
 
     RandomAccessInputStream s = new RandomAccessInputStream(jsonData);
+
+    if (s.length() > Integer.MAX_VALUE) {
+      LOGGER.warn(jsonData + " exceeds 2GB; metadata parsing is likely to fail");
+    }
+    else if (s.length() > 100 * 1024 * 1024) {
+      LOGGER.warn(jsonData + " is larger than 100MB and may require additional memory to parse. " +
+        "A minimum of 1024MB is suggested.");
+    }
+
     byte[] b = new byte[(int) s.length()];
     s.readFully(b);
     s.close();

From 2f401f24d1fbceedb76cbaa443636ff818937930 Mon Sep 17 00:00:00 2001
From: Melissa Linkert <melissa@glencoesoftware.com>
Date: Thu, 20 Sep 2018 12:54:13 -0500
Subject: [PATCH 3/3] Fix test and Findbugs failures

---
 .../loci/formats/in/MicromanagerReader.java   | 26 ++++++++++++-------
 1 file changed, 17 insertions(+), 9 deletions(-)

diff --git a/components/formats-bsd/src/loci/formats/in/MicromanagerReader.java b/components/formats-bsd/src/loci/formats/in/MicromanagerReader.java
index 42754c2c3b0..9dc0fe39d98 100644
--- a/components/formats-bsd/src/loci/formats/in/MicromanagerReader.java
+++ b/components/formats-bsd/src/loci/formats/in/MicromanagerReader.java
@@ -34,10 +34,12 @@
 
 import java.io.File;
 import java.io.IOException;
+import java.io.UnsupportedEncodingException;
 import java.util.Arrays;
 import java.util.HashMap;
 import java.util.Vector;
 
+import loci.common.Constants;
 import loci.common.DataTools;
 import loci.common.DateTools;
 import loci.common.Location;
@@ -722,6 +724,7 @@ else if (s.length() > 100 * 1024 * 1024) {
     s.close();
 
     int[] slice = new int[3];
+    start = 0;
     while (start < b.length) {
       String token = getNextLine(b).trim();
       boolean open = token.indexOf('[') != -1;
@@ -866,10 +869,7 @@ else if (token.trim().startsWith("}")) {
             }
           }
           else {
-            int colon = token.indexOf("\":") + 1;
-            if (colon == 0) {
-              colon = token.indexOf(':');
-            }
+            int colon = token.indexOf(":");
             key = token.substring(1, colon).trim();
             valueBuffer.setLength(0);
             valueBuffer.append(token.substring(colon + 1, token.length() - 1).trim().replaceAll("\"", ""));
@@ -1121,12 +1121,18 @@ private void parseXMLFile() throws IOException {
     XMLTools.parseXML(xmlData, handler);
   }
 
-  /** Initialize the TIFF reader with the first file in the current series. */
+  /** Initialize the TIFF reader with the first non-null file in the current series. */
   private void setupReader() {
     try {
+      int plane = 0;
       String file = positions.get(getSeries()).getFile(
         getDimensionOrder(), getSizeZ(), getSizeC(), getSizeT(),
-        getImageCount(), 0);
+        getImageCount(), plane++);
+      while (file == null && plane < getImageCount()) {
+        file = positions.get(getSeries()).getFile(
+          getDimensionOrder(), getSizeZ(), getSizeC(), getSizeT(),
+          getImageCount(), plane++);
+      }
       tiffReader.setId(file);
     }
     catch (Exception e) {
@@ -1172,15 +1178,17 @@ private void handleKeyValue(String key, String value) {
     }
   }
 
-  private String getNextLine(byte[] buf) {
+  private String getNextLine(byte[] buf) throws UnsupportedEncodingException {
     for (int i=start; i<buf.length; i++) {
       if (buf[i] == '\n') {
-        String line = new String(buf, start, (i - start) + 1);
+        String line = new String(buf, start, (i - start) + 1, Constants.ENCODING);
         start = i + 1;
         return line;
       }
     }
-    return null;
+    String line = new String(buf, start, buf.length - start, Constants.ENCODING);
+    start = buf.length;
+    return line;
   }
 
   // -- Helper classes --