From 42eecdd8e4d85a7cd4b135e9603ecce3aa55472b Mon Sep 17 00:00:00 2001 From: Melissa Linkert Date: Mon, 30 Sep 2019 16:57:24 -0500 Subject: [PATCH 01/14] Initial version of Tecan Spark Cyto reader Adds a dependency upon sqlite-jdbc (https://github.com/xerial/sqlite-jdbc) --- .../formats-api/src/loci/formats/readers.txt | 1 + components/formats-gpl/pom.xml | 5 + .../src/loci/formats/in/TecanReader.java | 429 ++++++++++++++++++ pom.xml | 1 + 4 files changed, 436 insertions(+) create mode 100644 components/formats-gpl/src/loci/formats/in/TecanReader.java diff --git a/components/formats-api/src/loci/formats/readers.txt b/components/formats-api/src/loci/formats/readers.txt index 04f6258cdce..efc93cad364 100644 --- a/components/formats-api/src/loci/formats/readers.txt +++ b/components/formats-api/src/loci/formats/readers.txt @@ -137,6 +137,7 @@ loci.formats.in.ImarisHDFReader # ims [NetCDF] loci.formats.in.CellH5Reader # ch5 [JHDF] loci.formats.in.WlzReader # wlz [JWlz] loci.formats.in.VeecoReader # hdf [NetCDF] +loci.formats.in.TecanReader # db [SQLite JDBC] # TIFF-based readers with unique file extensions loci.formats.in.ZeissLSMReader # lsm, mdb [MDB Tools] diff --git a/components/formats-gpl/pom.xml b/components/formats-gpl/pom.xml index d425cddd964..45b168633db 100644 --- a/components/formats-gpl/pom.xml +++ b/components/formats-gpl/pom.xml @@ -170,6 +170,11 @@ json 20090211 + + org.xerial + sqlite-jdbc + ${sqlite.version} + diff --git a/components/formats-gpl/src/loci/formats/in/TecanReader.java b/components/formats-gpl/src/loci/formats/in/TecanReader.java new file mode 100644 index 00000000000..f180f74f8a2 --- /dev/null +++ b/components/formats-gpl/src/loci/formats/in/TecanReader.java @@ -0,0 +1,429 @@ +/* + * #%L + * OME Bio-Formats package for reading and converting biological file formats. + * %% + * Copyright (C) 2019 Open Microscopy Environment: + * - Board of Regents of the University of Wisconsin-Madison + * - Glencoe Software, Inc. + * - University of Dundee + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as + * published by the Free Software Foundation, either version 2 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public + * License along with this program. If not, see + * . + * #L% + */ + +package loci.formats.in; + +import java.io.IOException; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; + +import loci.common.DataTools; +import loci.common.Location; +import loci.formats.CoreMetadata; +import loci.formats.FormatException; +import loci.formats.FormatReader; +import loci.formats.FormatTools; +import loci.formats.MetadataTools; +import loci.formats.meta.MetadataStore; + +import ome.units.UNITS; +import ome.units.quantity.Length; +import ome.units.quantity.Power; +import ome.units.quantity.Time; +import ome.xml.model.primitives.Color; +import ome.xml.model.primitives.NonNegativeInteger; +import ome.xml.model.primitives.PositiveFloat; +import ome.xml.model.primitives.PositiveInteger; +import ome.xml.model.primitives.Timestamp; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.sqlite.SQLiteConfig; + +/** + * + */ +public class TecanReader extends FormatReader { + + // -- Constants -- + + private static final Logger LOGGER = LoggerFactory.getLogger(TecanReader.class); + + // -- Fields -- + + private int plateRows = 0; + private int plateColumns = 0; + private transient String plateName = null; + private ArrayList images = new ArrayList(); + private transient MinimalTiffReader helperReader = null; + private transient ArrayList channels = new ArrayList(); + private String imageDirectory = null; + + // -- Constructor -- + + /** Constructs a new Tecan Spark Cyto reader. */ + public TecanReader() { + super("Tecan Spark Cyto", new String[] {"db"}); + hasCompanionFiles = true; + domains = new String[] {FormatTools.HCS_DOMAIN}; + datasetDescription = "SQLite database, TIFF files, optional analysis output"; + } + + // -- IFormatReader API methods -- + + /* @see loci.formats.IFormatReader#getRequiredDirectories(String[]) */ + @Override + public int getRequiredDirectories(String[] files) + throws FormatException, IOException + { + return 3; + } + + /* @see loci.formats.IFormatReader#isSingleFile(String) */ + @Override + public boolean isSingleFile(String id) throws FormatException, IOException { + return false; + } + + /* @see loci.formats.IFormatReader#fileGroupOption(String) */ + @Override + public int fileGroupOption(String id) throws FormatException, IOException { + return FormatTools.MUST_GROUP; + } + + /* @see loci.formats.IFormatReader#getSeriesUsedFiles(boolean) */ + @Override + public String[] getSeriesUsedFiles(boolean noPixels) { + FormatTools.assertId(currentId, true, 1); + + ArrayList files = new ArrayList(); + files.add(getCurrentFile()); + for (Image img : images) { + if (img.series == getSeries()) { + if (img.result || img.overlay || !noPixels) { + files.add(getImageFile(img.file)); + } + } + } + String[] rtn = files.toArray(new String[files.size()]); + Arrays.sort(rtn); + return rtn; + } + + /* @see loci.formats.IFormatReader#close(boolean) */ + @Override + public void close(boolean fileOnly) throws IOException { + super.close(fileOnly); + if (helperReader != null) { + helperReader.close(fileOnly); + } + if (!fileOnly) { + images.clear(); + channels.clear(); + imageDirectory = null; + plateRows = 0; + plateColumns = 0; + plateName = null; + helperReader = null; + } + } + + /** + * @see loci.formats.IFormatReader#openBytes(int, byte[], int, int, int, int) + */ + @Override + public byte[] openBytes(int no, byte[] buf, int x, int y, int w, int h) + throws FormatException, IOException + { + FormatTools.checkPlaneParameters(this, no, buf.length, x, y, w, h); + + Arrays.fill(buf, (byte) 0); + Image img = lookupImage(getSeries(), no, false, false); + if (img != null && img.file != null) { + if (helperReader == null) { + helperReader = new MinimalTiffReader(); + } + helperReader.setId(getImageFile(img.file)); + helperReader.openBytes(0, buf, x, y, w, h); + } + return buf; + } + + // -- Internal FormatReader API methods -- + + /* @see loci.formats.FormatReader#initFile(String) */ + @Override + protected void initFile(String id) throws FormatException, IOException { + super.initFile(id); + + // find the parent directory for image data + Location parent = new Location(currentId).getAbsoluteFile().getParentFile(); + parent = parent.getParentFile(); + Location images = new Location(parent, "Images"); + if (!images.exists() || !images.isDirectory()) { + throw new IOException("Cannot find expected 'Images' directory"); + } + imageDirectory = images.getAbsolutePath(); + + Connection conn = openConnection(); + HashMap wellLabels = null; + + try { + findPlateDimensions(conn); + + wellLabels = getWellLabels(conn); + findImages(conn, wellLabels); + } + catch (SQLException e) { + throw new IOException("Could not assemble plate", e); + } + finally { + if (conn != null) { + try { + conn.close(); + } + catch (SQLException e) { + LOGGER.warn("Could not close database connection", e); + } + } + } + + core.clear(); + helperReader = new MinimalTiffReader(); + for (int i=0; i getWellLabels(Connection conn) throws SQLException { + HashMap labels = new HashMap(); + + PreparedStatement statement = conn.prepareStatement( + "SELECT Id, AlphanumericCoordinate FROM SelectedWell"); + ResultSet wells = statement.executeQuery(); + + while (wells.next()) { + labels.put(wells.getInt(1), wells.getString(2)); + } + return labels; + } + + private void findImages(Connection conn, HashMap wells) throws SQLException { + PreparedStatement imageQuery = conn.prepareStatement( + "SELECT ImageTypeId, ImagingResultId, RelativePath, PixelSizeInNm " + + "FROM Image ORDER BY Id"); + ResultSet allImages = imageQuery.executeQuery(); + while (allImages.next()) { + Image img = new Image(); + img.file = allImages.getString(3); + img.pixelSize = FormatTools.getPhysicalSize(allImages.getDouble(4), "nm"); + + String imageTypeId = allImages.getString(1); + String resultId = allImages.getString(2); + + PreparedStatement typeQuery = conn.prepareStatement( + "SELECT ChannelTypeId, IsResult, IsOverlay " + + "FROM ImageType WHERE Id = ?"); + typeQuery.setInt(1, Integer.parseInt(imageTypeId)); + ResultSet imageType = typeQuery.executeQuery(); + imageType.next(); + + img.result = imageType.getBoolean(2); + img.overlay = imageType.getBoolean(3); + + int channelTypeId = imageType.getInt(1); + PreparedStatement channelQuery = conn.prepareStatement( + "SELECT Name FROM ChannelType WHERE Id=?"); + // TODO: join on AcquisitionChannelSetting (ChannelTypeId) + channelQuery.setInt(1, channelTypeId); + ResultSet channel = channelQuery.executeQuery(); + // might return 0 rows for overlay/result images + if (channel.next()) { + img.channelName = channel.getString(1); + + if (!channels.contains(img.channelName)) { + channels.add(img.channelName); + } + img.plane = channels.indexOf(img.channelName); + } + + // now map the image to a well + + PreparedStatement linkQuery = conn.prepareStatement( + "SELECT SelectedWellId FROM ImagingResult " + + "INNER JOIN ResultContext ON " + + "ImagingResult.ResultContextId=ResultContext.Id WHERE " + + "ImagingResult.Id=?"); + linkQuery.setInt(1, Integer.parseInt(resultId)); + ResultSet wellLink = linkQuery.executeQuery(); + wellLink.next(); + + Integer wellId = wellLink.getInt(1); + String wellLabel = wells.get(wellId); + img.wellRow = wellLabel.charAt(0) - 'A'; + img.wellColumn = Integer.parseInt(wellLabel.substring(1)) - 1; + img.series = wellId - 1; + + images.add(img); + } + } + + private Image lookupImage(int series, int plane, + boolean overlay, boolean result) + { + for (Image img : images) { + if (img.series == series && img.plane == plane && + img.overlay == overlay && img.result == result) + { + return img; + } + } + return null; + } + + /** + * Turn a relative image file path into an absolute path. + * Storing relative paths and one copy of the parent directory + * saves some space in the memo file, especially for large plates. + * + * @param file relative path to an image file + * @return absolute path to the image file + */ + private String getImageFile(String file) { + return new Location(imageDirectory, file).getAbsolutePath(); + } + + class Image { + public String file; + public int wellRow = -1; + public int wellColumn = -1; + public int field = -1; + public int series = -1; + public int plane = -1; + public Length pixelSize; + public String channelName; + public boolean overlay = false; + public boolean result = false; + } + +} diff --git a/pom.xml b/pom.xml index cc31d4f997f..94d0b29f5b4 100644 --- a/pom.xml +++ b/pom.xml @@ -60,6 +60,7 @@ 0.3.0 0.2.4 2.7.2 + 3.28.0 3.0.1 From 7b2cf27b3145ab8a97bd1a440a29b580b065b521 Mon Sep 17 00:00:00 2001 From: Melissa Linkert Date: Wed, 2 Oct 2019 15:45:14 -0500 Subject: [PATCH 02/14] Add extra checks ensure that openBytes only reads from raw TIFFs --- .../src/loci/formats/in/TecanReader.java | 55 +++++++++++++------ 1 file changed, 39 insertions(+), 16 deletions(-) diff --git a/components/formats-gpl/src/loci/formats/in/TecanReader.java b/components/formats-gpl/src/loci/formats/in/TecanReader.java index f180f74f8a2..78b3037f251 100644 --- a/components/formats-gpl/src/loci/formats/in/TecanReader.java +++ b/components/formats-gpl/src/loci/formats/in/TecanReader.java @@ -163,7 +163,14 @@ public byte[] openBytes(int no, byte[] buf, int x, int y, int w, int h) if (helperReader == null) { helperReader = new MinimalTiffReader(); } - helperReader.setId(getImageFile(img.file)); + String filePath = getImageFile(img.file); + LOGGER.debug("Reading plane {} in series {} from {}", + no, getSeries(), filePath); + helperReader.setId(filePath); + if (helperReader.getImageCount() > 1) { + LOGGER.warn("File {} has {} planes", + img.file, helperReader.getImageCount()); + } helperReader.openBytes(0, buf, x, y, w, h); } return buf; @@ -342,7 +349,7 @@ private void findImages(Connection conn, HashMap wells) throws String resultId = allImages.getString(2); PreparedStatement typeQuery = conn.prepareStatement( - "SELECT ChannelTypeId, IsResult, IsOverlay " + + "SELECT ChannelTypeId, IsResult, IsOverlay, Name " + "FROM ImageType WHERE Id = ?"); typeQuery.setInt(1, Integer.parseInt(imageTypeId)); ResultSet imageType = typeQuery.executeQuery(); @@ -350,21 +357,37 @@ private void findImages(Connection conn, HashMap wells) throws img.result = imageType.getBoolean(2); img.overlay = imageType.getBoolean(3); - - int channelTypeId = imageType.getInt(1); - PreparedStatement channelQuery = conn.prepareStatement( - "SELECT Name FROM ChannelType WHERE Id=?"); - // TODO: join on AcquisitionChannelSetting (ChannelTypeId) - channelQuery.setInt(1, channelTypeId); - ResultSet channel = channelQuery.executeQuery(); - // might return 0 rows for overlay/result images - if (channel.next()) { - img.channelName = channel.getString(1); - - if (!channels.contains(img.channelName)) { - channels.add(img.channelName); + img.channelName = imageType.getString(4); + + // make sure the image is "Raw" and not "Processed" + // without this check, the channel and image counts will be wrong + // as processed files will be included + PreparedStatement acquisitionType = conn.prepareStatement( + "SELECT Name FROM ImagingResultType INNER JOIN ImagingResult " + + "ON ImagingResultType.Id=ImagingResult.ImagingResultTypeId " + + "WHERE ImagingResult.Id=?"); + acquisitionType.setInt(1, Integer.parseInt(resultId)); + ResultSet acqType = acquisitionType.executeQuery(); + acqType.next(); + + if ("Raw".equals(acqType.getString(1)) && !img.result && !img.overlay) { + int channelTypeId = imageType.getInt(1); + PreparedStatement channelQuery = conn.prepareStatement( + "SELECT Name FROM ChannelType WHERE Id=?"); + // TODO: join on AcquisitionChannelSetting (ChannelTypeId) + channelQuery.setInt(1, channelTypeId); + ResultSet channel = channelQuery.executeQuery(); + // might return 0 rows for overlay/result images + if (channel.next()) { + // a single ChannelType (e.g. brightfield) can map + // to multiple ImageTypes in the same well + img.channelName += " " + channel.getString(1); + + if (!channels.contains(img.channelName)) { + channels.add(img.channelName); + } + img.plane = channels.indexOf(img.channelName); } - img.plane = channels.indexOf(img.channelName); } // now map the image to a well From 726b3b49f46ca7b56f44821cdac160ba1ab7e9ee Mon Sep 17 00:00:00 2001 From: Melissa Linkert Date: Wed, 2 Oct 2019 15:56:58 -0500 Subject: [PATCH 03/14] Add exported PDF and Excel files to used files list --- .../src/loci/formats/in/TecanReader.java | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/components/formats-gpl/src/loci/formats/in/TecanReader.java b/components/formats-gpl/src/loci/formats/in/TecanReader.java index 78b3037f251..6131e8e639d 100644 --- a/components/formats-gpl/src/loci/formats/in/TecanReader.java +++ b/components/formats-gpl/src/loci/formats/in/TecanReader.java @@ -78,6 +78,7 @@ public class TecanReader extends FormatReader { private transient MinimalTiffReader helperReader = null; private transient ArrayList channels = new ArrayList(); private String imageDirectory = null; + private ArrayList extraFiles = new ArrayList(); // -- Constructor -- @@ -111,6 +112,24 @@ public int fileGroupOption(String id) throws FormatException, IOException { return FormatTools.MUST_GROUP; } + /* @see loci.formats.IFormatReader#getUsedFiles(boolean) */ + @Override + public String[] getUsedFiles(boolean noPixels) { + FormatTools.assertId(currentId, true, 1); + + ArrayList files = new ArrayList(); + files.add(getCurrentFile()); + files.addAll(extraFiles); + for (Image img : images) { + if (img.result || img.overlay || !noPixels) { + files.add(getImageFile(img.file)); + } + } + String[] rtn = files.toArray(new String[files.size()]); + Arrays.sort(rtn); + return rtn; + } + /* @see loci.formats.IFormatReader#getSeriesUsedFiles(boolean) */ @Override public String[] getSeriesUsedFiles(boolean noPixels) { @@ -118,6 +137,7 @@ public String[] getSeriesUsedFiles(boolean noPixels) { ArrayList files = new ArrayList(); files.add(getCurrentFile()); + files.addAll(extraFiles); for (Image img : images) { if (img.series == getSeries()) { if (img.result || img.overlay || !noPixels) { @@ -145,6 +165,7 @@ public void close(boolean fileOnly) throws IOException { plateColumns = 0; plateName = null; helperReader = null; + extraFiles.clear(); } } @@ -192,6 +213,12 @@ protected void initFile(String id) throws FormatException, IOException { } imageDirectory = images.getAbsolutePath(); + // look for extra files in the "Export" directory + Location export = new Location(parent, "Export"); + if (export.exists() && export.isDirectory()) { + findAllFiles(export, extraFiles); + } + Connection conn = openConnection(); HashMap wellLabels = null; @@ -436,6 +463,19 @@ private String getImageFile(String file) { return new Location(imageDirectory, file).getAbsolutePath(); } + private void findAllFiles(Location root, ArrayList files) { + if (root.isDirectory()) { + String[] list = root.list(true); + for (String file : list) { + Location path = new Location(root, file); + findAllFiles(path, files); + } + } + else { + files.add(root.getAbsolutePath()); + } + } + class Image { public String file; public int wellRow = -1; From bb476b7e4e1e15a877bf609b3568e8c29a014bf1 Mon Sep 17 00:00:00 2001 From: Melissa Linkert Date: Wed, 2 Oct 2019 19:59:19 -0500 Subject: [PATCH 04/14] Parse some additional metadata and attach .svgz files --- .../src/loci/formats/in/TecanReader.java | 82 +++++++++++++++---- 1 file changed, 65 insertions(+), 17 deletions(-) diff --git a/components/formats-gpl/src/loci/formats/in/TecanReader.java b/components/formats-gpl/src/loci/formats/in/TecanReader.java index 6131e8e639d..2d56ec10830 100644 --- a/components/formats-gpl/src/loci/formats/in/TecanReader.java +++ b/components/formats-gpl/src/loci/formats/in/TecanReader.java @@ -37,7 +37,7 @@ import java.util.HashMap; import java.util.List; -import loci.common.DataTools; +import loci.common.DateTools; import loci.common.Location; import loci.formats.CoreMetadata; import loci.formats.FormatException; @@ -48,11 +48,8 @@ import ome.units.UNITS; import ome.units.quantity.Length; -import ome.units.quantity.Power; import ome.units.quantity.Time; -import ome.xml.model.primitives.Color; import ome.xml.model.primitives.NonNegativeInteger; -import ome.xml.model.primitives.PositiveFloat; import ome.xml.model.primitives.PositiveInteger; import ome.xml.model.primitives.Timestamp; @@ -312,6 +309,18 @@ protected void initFile(String id) throws FormatException, IOException { store.setPixelsPhysicalSizeX(firstImage.pixelSize, i); store.setPixelsPhysicalSizeY(firstImage.pixelSize, i); } + if (firstImage.timestamp != null) { + store.setImageAcquisitionDate(new Timestamp( + DateTools.formatDate(firstImage.timestamp, + "yyyy-MM-dd HH:mm:ss.SSSSSS")), i); + } + + for (int p=0; p wells) throws // without this check, the channel and image counts will be wrong // as processed files will be included PreparedStatement acquisitionType = conn.prepareStatement( - "SELECT Name FROM ImagingResultType INNER JOIN ImagingResult " + + "SELECT Name, CreatedAt FROM ImagingResultType INNER JOIN ImagingResult " + "ON ImagingResultType.Id=ImagingResult.ImagingResultTypeId " + "WHERE ImagingResult.Id=?"); acquisitionType.setInt(1, Integer.parseInt(resultId)); @@ -400,8 +409,10 @@ private void findImages(Connection conn, HashMap wells) throws if ("Raw".equals(acqType.getString(1)) && !img.result && !img.overlay) { int channelTypeId = imageType.getInt(1); PreparedStatement channelQuery = conn.prepareStatement( - "SELECT Name FROM ChannelType WHERE Id=?"); - // TODO: join on AcquisitionChannelSetting (ChannelTypeId) + "SELECT Name, ExposureTimeInUs FROM ChannelType " + + "INNER JOIN AcquisitionChannelSetting " + + "ON AcquisitionChannelSetting.ChannelTypeId=ChannelType.Id " + + "WHERE ChannelType.Id=?"); channelQuery.setInt(1, channelTypeId); ResultSet channel = channelQuery.executeQuery(); // might return 0 rows for overlay/result images @@ -414,21 +425,14 @@ private void findImages(Connection conn, HashMap wells) throws channels.add(img.channelName); } img.plane = channels.indexOf(img.channelName); + img.exposureTime = FormatTools.getTime(channel.getDouble(2), "µs"); + img.timestamp = acqType.getString(2); } } // now map the image to a well - PreparedStatement linkQuery = conn.prepareStatement( - "SELECT SelectedWellId FROM ImagingResult " + - "INNER JOIN ResultContext ON " + - "ImagingResult.ResultContextId=ResultContext.Id WHERE " + - "ImagingResult.Id=?"); - linkQuery.setInt(1, Integer.parseInt(resultId)); - ResultSet wellLink = linkQuery.executeQuery(); - wellLink.next(); - - Integer wellId = wellLink.getInt(1); + Integer wellId = getWellID(conn, resultId); String wellLabel = wells.get(wellId); img.wellRow = wellLabel.charAt(0) - 'A'; img.wellColumn = Integer.parseInt(wellLabel.substring(1)) - 1; @@ -436,6 +440,48 @@ private void findImages(Connection conn, HashMap wells) throws images.add(img); } + + // list of SVG objects is stored separately + // each row in ObjectList has a file which needs to be attached to a well + PreparedStatement objectQuery = conn.prepareStatement( + "SELECT ImagingResultId, Path FROM ObjectList ORDER BY Id"); + ResultSet objects = objectQuery.executeQuery(); + while (objects.next()) { + String resultId = objects.getString(1); + String path = objects.getString(2); + + Integer well = getWellID(conn, resultId); + if (well != null) { + String wellLabel = wells.get(well); + Image img = new Image(); + img.wellRow = wellLabel.charAt(0) - 'A'; + img.wellColumn = Integer.parseInt(wellLabel.substring(1)) - 1; + img.series = well - 1; + img.result = true; + img.file = path; + images.add(img); + } + } + } + + /** + * Get the well identifier for the given Id in the ImagingResult table. + * @param imagingResultID + * @return well ID + */ + private Integer getWellID(Connection conn, String imagingResultID) + throws SQLException + { + PreparedStatement linkQuery = conn.prepareStatement( + "SELECT SelectedWellId FROM ImagingResult " + + "INNER JOIN ResultContext ON " + + "ImagingResult.ResultContextId=ResultContext.Id WHERE " + + "ImagingResult.Id=?"); + linkQuery.setInt(1, Integer.parseInt(imagingResultID)); + ResultSet wellLink = linkQuery.executeQuery(); + wellLink.next(); + + return wellLink.getInt(1); } private Image lookupImage(int series, int plane, @@ -484,9 +530,11 @@ class Image { public int series = -1; public int plane = -1; public Length pixelSize; + public Time exposureTime; public String channelName; public boolean overlay = false; public boolean result = false; + public String timestamp; } } From 4de916615f63eb7a8f239ffa33d03ad4b7fb7e19 Mon Sep 17 00:00:00 2001 From: Melissa Linkert Date: Thu, 3 Oct 2019 16:45:41 -0500 Subject: [PATCH 05/14] Support multiple fields and improve initialization time --- .../src/loci/formats/in/TecanReader.java | 162 ++++++++++++------ 1 file changed, 107 insertions(+), 55 deletions(-) diff --git a/components/formats-gpl/src/loci/formats/in/TecanReader.java b/components/formats-gpl/src/loci/formats/in/TecanReader.java index 2d56ec10830..b2f90bd4788 100644 --- a/components/formats-gpl/src/loci/formats/in/TecanReader.java +++ b/components/formats-gpl/src/loci/formats/in/TecanReader.java @@ -25,12 +25,14 @@ package loci.formats.in; +import java.io.File; import java.io.IOException; import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; +import java.sql.Statement; import java.util.ArrayList; import java.util.Arrays; @@ -64,7 +66,8 @@ public class TecanReader extends FormatReader { // -- Constants -- - private static final Logger LOGGER = LoggerFactory.getLogger(TecanReader.class); + private static final Logger LOGGER = + LoggerFactory.getLogger(TecanReader.class); // -- Fields -- @@ -76,6 +79,7 @@ public class TecanReader extends FormatReader { private transient ArrayList channels = new ArrayList(); private String imageDirectory = null; private ArrayList extraFiles = new ArrayList(); + private Integer maxField = 1; // -- Constructor -- @@ -84,7 +88,8 @@ public TecanReader() { super("Tecan Spark Cyto", new String[] {"db"}); hasCompanionFiles = true; domains = new String[] {FormatTools.HCS_DOMAIN}; - datasetDescription = "SQLite database, TIFF files, optional analysis output"; + datasetDescription = + "SQLite database, TIFF files, optional analysis output"; } // -- IFormatReader API methods -- @@ -163,6 +168,7 @@ public void close(boolean fileOnly) throws IOException { plateName = null; helperReader = null; extraFiles.clear(); + maxField = 1; } } @@ -204,11 +210,11 @@ protected void initFile(String id) throws FormatException, IOException { // find the parent directory for image data Location parent = new Location(currentId).getAbsoluteFile().getParentFile(); parent = parent.getParentFile(); - Location images = new Location(parent, "Images"); - if (!images.exists() || !images.isDirectory()) { + Location imageDir = new Location(parent, "Images"); + if (!imageDir.exists() || !imageDir.isDirectory()) { throw new IOException("Cannot find expected 'Images' directory"); } - imageDirectory = images.getAbsolutePath(); + imageDirectory = imageDir.getAbsolutePath(); // look for extra files in the "Export" directory Location export = new Location(parent, "Export"); @@ -239,13 +245,23 @@ protected void initFile(String id) throws FormatException, IOException { } } + // update series index for each Image to account for fields + for (Image img : images) { + img.series *= maxField; + img.series += (img.field - 1); + } + core.clear(); helperReader = new MinimalTiffReader(); - for (int i=0; i getWellLabels(Connection conn) throws SQLException { + private HashMap getWellLabels(Connection conn) + throws SQLException + { HashMap labels = new HashMap(); PreparedStatement statement = conn.prepareStatement( @@ -368,25 +397,45 @@ private HashMap getWellLabels(Connection conn) throws SQLExcept while (wells.next()) { labels.put(wells.getInt(1), wells.getString(2)); } + wells.close(); return labels; } - private void findImages(Connection conn, HashMap wells) throws SQLException { + private void findImages(Connection conn, HashMap wells) + throws SQLException + { PreparedStatement imageQuery = conn.prepareStatement( "SELECT ImageTypeId, ImagingResultId, RelativePath, PixelSizeInNm " + "FROM Image ORDER BY Id"); + PreparedStatement wellLinkQuery = conn.prepareStatement( + "SELECT SelectedWellId, CycleIndex FROM ImagingResult " + + "INNER JOIN ResultContext ON " + + "ImagingResult.ResultContextId=ResultContext.Id WHERE " + + "ImagingResult.Id=?"); + PreparedStatement typeQuery = conn.prepareStatement( + "SELECT ChannelTypeId, IsResult, IsOverlay, Name " + + "FROM ImageType WHERE Id = ?"); + PreparedStatement acquisitionType = conn.prepareStatement( + "SELECT Name, CreatedAt FROM ImagingResultType " + + "INNER JOIN ImagingResult " + + "ON ImagingResultType.Id=ImagingResult.ImagingResultTypeId " + + "WHERE ImagingResult.Id=?"); + PreparedStatement channelQuery = conn.prepareStatement( + "SELECT Name, ExposureTimeInUs FROM ChannelType " + + "INNER JOIN AcquisitionChannelSetting " + + "ON AcquisitionChannelSetting.ChannelTypeId=ChannelType.Id " + + "WHERE ChannelType.Id=?"); + ResultSet allImages = imageQuery.executeQuery(); while (allImages.next()) { Image img = new Image(); img.file = allImages.getString(3); img.pixelSize = FormatTools.getPhysicalSize(allImages.getDouble(4), "nm"); + LOGGER.debug("processing image file = {}", img.file); String imageTypeId = allImages.getString(1); String resultId = allImages.getString(2); - PreparedStatement typeQuery = conn.prepareStatement( - "SELECT ChannelTypeId, IsResult, IsOverlay, Name " + - "FROM ImageType WHERE Id = ?"); typeQuery.setInt(1, Integer.parseInt(imageTypeId)); ResultSet imageType = typeQuery.executeQuery(); imageType.next(); @@ -398,21 +447,12 @@ private void findImages(Connection conn, HashMap wells) throws // make sure the image is "Raw" and not "Processed" // without this check, the channel and image counts will be wrong // as processed files will be included - PreparedStatement acquisitionType = conn.prepareStatement( - "SELECT Name, CreatedAt FROM ImagingResultType INNER JOIN ImagingResult " + - "ON ImagingResultType.Id=ImagingResult.ImagingResultTypeId " + - "WHERE ImagingResult.Id=?"); acquisitionType.setInt(1, Integer.parseInt(resultId)); ResultSet acqType = acquisitionType.executeQuery(); acqType.next(); if ("Raw".equals(acqType.getString(1)) && !img.result && !img.overlay) { int channelTypeId = imageType.getInt(1); - PreparedStatement channelQuery = conn.prepareStatement( - "SELECT Name, ExposureTimeInUs FROM ChannelType " + - "INNER JOIN AcquisitionChannelSetting " + - "ON AcquisitionChannelSetting.ChannelTypeId=ChannelType.Id " + - "WHERE ChannelType.Id=?"); channelQuery.setInt(1, channelTypeId); ResultSet channel = channelQuery.executeQuery(); // might return 0 rows for overlay/result images @@ -428,18 +468,24 @@ private void findImages(Connection conn, HashMap wells) throws img.exposureTime = FormatTools.getTime(channel.getDouble(2), "µs"); img.timestamp = acqType.getString(2); } + channel.close(); } + imageType.close(); + acqType.close(); // now map the image to a well - Integer wellId = getWellID(conn, resultId); - String wellLabel = wells.get(wellId); + Integer[] ids = getWellAndField(wellLinkQuery, resultId); + String wellLabel = wells.get(ids[0]); img.wellRow = wellLabel.charAt(0) - 'A'; img.wellColumn = Integer.parseInt(wellLabel.substring(1)) - 1; - img.series = wellId - 1; + img.series = ids[0] - 1; + img.field = ids[1]; + maxField = (int) Math.max(img.field, maxField); images.add(img); } + allImages.close(); // list of SVG objects is stored separately // each row in ObjectList has a file which needs to be attached to a well @@ -449,39 +495,41 @@ private void findImages(Connection conn, HashMap wells) throws while (objects.next()) { String resultId = objects.getString(1); String path = objects.getString(2); + LOGGER.debug("processing object {}", path); - Integer well = getWellID(conn, resultId); - if (well != null) { - String wellLabel = wells.get(well); + Integer[] ids = getWellAndField(wellLinkQuery, resultId); + if (ids != null) { + String wellLabel = wells.get(ids[0]); Image img = new Image(); img.wellRow = wellLabel.charAt(0) - 'A'; img.wellColumn = Integer.parseInt(wellLabel.substring(1)) - 1; - img.series = well - 1; + img.series = ids[0] - 1; + img.field = ids[1]; img.result = true; img.file = path; images.add(img); } } + objects.close(); } /** - * Get the well identifier for the given Id in the ImagingResult table. + * Get the well identifier and field indexfor the given Id in the + * ImagingResult table. * @param imagingResultID * @return well ID */ - private Integer getWellID(Connection conn, String imagingResultID) + private Integer[] getWellAndField(PreparedStatement linkQuery, + String imagingResultID) throws SQLException { - PreparedStatement linkQuery = conn.prepareStatement( - "SELECT SelectedWellId FROM ImagingResult " + - "INNER JOIN ResultContext ON " + - "ImagingResult.ResultContextId=ResultContext.Id WHERE " + - "ImagingResult.Id=?"); linkQuery.setInt(1, Integer.parseInt(imagingResultID)); ResultSet wellLink = linkQuery.executeQuery(); wellLink.next(); - return wellLink.getInt(1); + Integer[] rtn = new Integer[] {wellLink.getInt(1), wellLink.getInt(2)}; + wellLink.close(); + return rtn; } private Image lookupImage(int series, int plane, @@ -506,7 +554,11 @@ private Image lookupImage(int series, int plane, * @return absolute path to the image file */ private String getImageFile(String file) { - return new Location(imageDirectory, file).getAbsolutePath(); + // sanitize the relative path first + // files might be stored in a subdirectory of "Images", + // in which case the database may store a "\" to separate the path + String sanitized = File.separator + file.replaceAll("\\\\", File.separator); + return new Location(imageDirectory + sanitized).getAbsolutePath(); } private void findAllFiles(Location root, ArrayList files) { From 25e4898d8e3ae73a1c369a1526cdac800c15c1d7 Mon Sep 17 00:00:00 2001 From: Melissa Linkert Date: Thu, 3 Oct 2019 17:14:24 -0500 Subject: [PATCH 06/14] Fix field support for single-field cases --- .../src/loci/formats/in/TecanReader.java | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/components/formats-gpl/src/loci/formats/in/TecanReader.java b/components/formats-gpl/src/loci/formats/in/TecanReader.java index b2f90bd4788..fa94e53d1b8 100644 --- a/components/formats-gpl/src/loci/formats/in/TecanReader.java +++ b/components/formats-gpl/src/loci/formats/in/TecanReader.java @@ -351,12 +351,8 @@ private Connection openConnection() throws IOException { try { // see https://github.com/xerial/sqlite-jdbc/issues/247 SQLiteConfig config = new SQLiteConfig(); - // restore the DB in memory instead of reading directly - // this is much faster, especially as the DB file size grows - // a read-only connection isn't allowed though, so be careful - conn = config.createConnection("jdbc:sqlite::memory:"); - Statement restore = conn.createStatement(); - restore.execute("restore from " + + config.setReadOnly(true); + conn = config.createConnection("jdbc:sqlite:" + new Location(getCurrentFile()).getAbsolutePath()); } catch (SQLException e) { @@ -407,6 +403,10 @@ private void findImages(Connection conn, HashMap wells) PreparedStatement imageQuery = conn.prepareStatement( "SELECT ImageTypeId, ImagingResultId, RelativePath, PixelSizeInNm " + "FROM Image ORDER BY Id"); + + // not clear if CycleIndex is a field or timepoint index + // treating as a field for now (since that's more difficult), + // but easy to switch to timepoint if needed PreparedStatement wellLinkQuery = conn.prepareStatement( "SELECT SelectedWellId, CycleIndex FROM ImagingResult " + "INNER JOIN ResultContext ON " + @@ -480,7 +480,8 @@ private void findImages(Connection conn, HashMap wells) img.wellRow = wellLabel.charAt(0) - 'A'; img.wellColumn = Integer.parseInt(wellLabel.substring(1)) - 1; img.series = ids[0] - 1; - img.field = ids[1]; + // always set a 1-based index + img.field = (int) Math.max(1, ids[1]); maxField = (int) Math.max(img.field, maxField); images.add(img); @@ -504,7 +505,8 @@ private void findImages(Connection conn, HashMap wells) img.wellRow = wellLabel.charAt(0) - 'A'; img.wellColumn = Integer.parseInt(wellLabel.substring(1)) - 1; img.series = ids[0] - 1; - img.field = ids[1]; + // always set a 1-based index + img.field = (int) Math.max(1, ids[1]); img.result = true; img.file = path; images.add(img); From 51ca1b5ec8ee7408a4408796a0e99a533fae7aa4 Mon Sep 17 00:00:00 2001 From: Melissa Linkert Date: Thu, 3 Oct 2019 17:26:24 -0500 Subject: [PATCH 07/14] Use try-with-resources to close ResultSets See https://github.com/xerial/sqlite-jdbc/issues/9 --- .../src/loci/formats/in/TecanReader.java | 193 +++++++++--------- 1 file changed, 99 insertions(+), 94 deletions(-) diff --git a/components/formats-gpl/src/loci/formats/in/TecanReader.java b/components/formats-gpl/src/loci/formats/in/TecanReader.java index fa94e53d1b8..ddd275be442 100644 --- a/components/formats-gpl/src/loci/formats/in/TecanReader.java +++ b/components/formats-gpl/src/loci/formats/in/TecanReader.java @@ -367,18 +367,17 @@ private void findPlateDimensions(Connection conn) throws SQLException { // expect only one plate to be defined PreparedStatement statement = conn.prepareStatement( "SELECT Name, Rows, Columns FROM PlateDefinition ORDER BY Id"); - ResultSet plates = statement.executeQuery(); - - if (plates.next()) { - plateName = plates.getString(1); - plateRows = plates.getInt(2); - plateColumns = plates.getInt(3); - } + try (ResultSet plates = statement.executeQuery()) { + if (plates.next()) { + plateName = plates.getString(1); + plateRows = plates.getInt(2); + plateColumns = plates.getInt(3); + } - if (plates.next()) { - LOGGER.warn("Found more than one plate; only using the first one"); + if (plates.next()) { + LOGGER.warn("Found more than one plate; only using the first one"); + } } - plates.close(); } private HashMap getWellLabels(Connection conn) @@ -388,12 +387,11 @@ private HashMap getWellLabels(Connection conn) PreparedStatement statement = conn.prepareStatement( "SELECT Id, AlphanumericCoordinate FROM SelectedWell"); - ResultSet wells = statement.executeQuery(); - - while (wells.next()) { - labels.put(wells.getInt(1), wells.getString(2)); + try (ResultSet wells = statement.executeQuery()) { + while (wells.next()) { + labels.put(wells.getInt(1), wells.getString(2)); + } } - wells.close(); return labels; } @@ -426,93 +424,102 @@ private void findImages(Connection conn, HashMap wells) "ON AcquisitionChannelSetting.ChannelTypeId=ChannelType.Id " + "WHERE ChannelType.Id=?"); - ResultSet allImages = imageQuery.executeQuery(); - while (allImages.next()) { - Image img = new Image(); - img.file = allImages.getString(3); - img.pixelSize = FormatTools.getPhysicalSize(allImages.getDouble(4), "nm"); - LOGGER.debug("processing image file = {}", img.file); - - String imageTypeId = allImages.getString(1); - String resultId = allImages.getString(2); - - typeQuery.setInt(1, Integer.parseInt(imageTypeId)); - ResultSet imageType = typeQuery.executeQuery(); - imageType.next(); - - img.result = imageType.getBoolean(2); - img.overlay = imageType.getBoolean(3); - img.channelName = imageType.getString(4); - - // make sure the image is "Raw" and not "Processed" - // without this check, the channel and image counts will be wrong - // as processed files will be included - acquisitionType.setInt(1, Integer.parseInt(resultId)); - ResultSet acqType = acquisitionType.executeQuery(); - acqType.next(); - - if ("Raw".equals(acqType.getString(1)) && !img.result && !img.overlay) { - int channelTypeId = imageType.getInt(1); - channelQuery.setInt(1, channelTypeId); - ResultSet channel = channelQuery.executeQuery(); - // might return 0 rows for overlay/result images - if (channel.next()) { - // a single ChannelType (e.g. brightfield) can map - // to multiple ImageTypes in the same well - img.channelName += " " + channel.getString(1); - - if (!channels.contains(img.channelName)) { - channels.add(img.channelName); - } - img.plane = channels.indexOf(img.channelName); - img.exposureTime = FormatTools.getTime(channel.getDouble(2), "µs"); - img.timestamp = acqType.getString(2); + try (ResultSet allImages = imageQuery.executeQuery()) { + while (allImages.next()) { + Image img = new Image(); + img.file = allImages.getString(3); + img.pixelSize = + FormatTools.getPhysicalSize(allImages.getDouble(4), "nm"); + LOGGER.debug("processing image file = {}", img.file); + + String imageTypeId = allImages.getString(1); + String resultId = allImages.getString(2); + + typeQuery.setInt(1, Integer.parseInt(imageTypeId)); + int channelTypeId = 0; + try (ResultSet imageType = typeQuery.executeQuery()) { + imageType.next(); + + channelTypeId = imageType.getInt(1); + img.result = imageType.getBoolean(2); + img.overlay = imageType.getBoolean(3); + img.channelName = imageType.getString(4); } - channel.close(); - } - imageType.close(); - acqType.close(); - // now map the image to a well + // make sure the image is "Raw" and not "Processed" + // without this check, the channel and image counts will be wrong + // as processed files will be included + acquisitionType.setInt(1, Integer.parseInt(resultId)); - Integer[] ids = getWellAndField(wellLinkQuery, resultId); - String wellLabel = wells.get(ids[0]); - img.wellRow = wellLabel.charAt(0) - 'A'; - img.wellColumn = Integer.parseInt(wellLabel.substring(1)) - 1; - img.series = ids[0] - 1; - // always set a 1-based index - img.field = (int) Math.max(1, ids[1]); - maxField = (int) Math.max(img.field, maxField); + String acqTypeName = null; + String acqTimestamp = null; - images.add(img); - } - allImages.close(); + try (ResultSet acqType = acquisitionType.executeQuery()) { + acqType.next(); + acqTypeName = acqType.getString(1); + acqTimestamp = acqType.getString(2); + } - // list of SVG objects is stored separately - // each row in ObjectList has a file which needs to be attached to a well - PreparedStatement objectQuery = conn.prepareStatement( - "SELECT ImagingResultId, Path FROM ObjectList ORDER BY Id"); - ResultSet objects = objectQuery.executeQuery(); - while (objects.next()) { - String resultId = objects.getString(1); - String path = objects.getString(2); - LOGGER.debug("processing object {}", path); - - Integer[] ids = getWellAndField(wellLinkQuery, resultId); - if (ids != null) { + if ("Raw".equals(acqTypeName) && !img.result && !img.overlay) { + channelQuery.setInt(1, channelTypeId); + try (ResultSet channel = channelQuery.executeQuery()) { + // might return 0 rows for overlay/result images + if (channel.next()) { + // a single ChannelType (e.g. brightfield) can map + // to multiple ImageTypes in the same well + img.channelName += " " + channel.getString(1); + + if (!channels.contains(img.channelName)) { + channels.add(img.channelName); + } + img.plane = channels.indexOf(img.channelName); + img.exposureTime = + FormatTools.getTime(channel.getDouble(2), "µs"); + img.timestamp = acqTimestamp; + } + } + } + + // now map the image to a well + + Integer[] ids = getWellAndField(wellLinkQuery, resultId); String wellLabel = wells.get(ids[0]); - Image img = new Image(); img.wellRow = wellLabel.charAt(0) - 'A'; img.wellColumn = Integer.parseInt(wellLabel.substring(1)) - 1; img.series = ids[0] - 1; // always set a 1-based index img.field = (int) Math.max(1, ids[1]); - img.result = true; - img.file = path; + maxField = (int) Math.max(img.field, maxField); + images.add(img); } } - objects.close(); + + // list of SVG objects is stored separately + // each row in ObjectList has a file which needs to be attached to a well + PreparedStatement objectQuery = conn.prepareStatement( + "SELECT ImagingResultId, Path FROM ObjectList ORDER BY Id"); + try (ResultSet objects = objectQuery.executeQuery()) { + while (objects.next()) { + String resultId = objects.getString(1); + String path = objects.getString(2); + LOGGER.debug("processing object {}", path); + + Integer[] ids = getWellAndField(wellLinkQuery, resultId); + if (ids != null) { + String wellLabel = wells.get(ids[0]); + Image img = new Image(); + img.wellRow = wellLabel.charAt(0) - 'A'; + img.wellColumn = Integer.parseInt(wellLabel.substring(1)) - 1; + img.series = ids[0] - 1; + // always set a 1-based index + img.field = (int) Math.max(1, ids[1]); + img.result = true; + img.file = path; + images.add(img); + } + } + } } /** @@ -526,12 +533,10 @@ private Integer[] getWellAndField(PreparedStatement linkQuery, throws SQLException { linkQuery.setInt(1, Integer.parseInt(imagingResultID)); - ResultSet wellLink = linkQuery.executeQuery(); - wellLink.next(); - - Integer[] rtn = new Integer[] {wellLink.getInt(1), wellLink.getInt(2)}; - wellLink.close(); - return rtn; + try (ResultSet wellLink = linkQuery.executeQuery()) { + wellLink.next(); + return new Integer[] {wellLink.getInt(1), wellLink.getInt(2)}; + } } private Image lookupImage(int series, int plane, From f81f1755b1aa2de9a526815a1388acfa2b1220b4 Mon Sep 17 00:00:00 2001 From: Melissa Linkert Date: Fri, 17 Apr 2020 17:02:49 -0500 Subject: [PATCH 08/14] Treat cycles as timepoints, not fields --- .../src/loci/formats/in/TecanReader.java | 23 ++++++++++++------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/components/formats-gpl/src/loci/formats/in/TecanReader.java b/components/formats-gpl/src/loci/formats/in/TecanReader.java index ddd275be442..68c34f27db5 100644 --- a/components/formats-gpl/src/loci/formats/in/TecanReader.java +++ b/components/formats-gpl/src/loci/formats/in/TecanReader.java @@ -80,6 +80,7 @@ public class TecanReader extends FormatReader { private String imageDirectory = null; private ArrayList extraFiles = new ArrayList(); private Integer maxField = 1; + private Integer maxCycle = 1; // -- Constructor -- @@ -169,6 +170,7 @@ public void close(boolean fileOnly) throws IOException { helperReader = null; extraFiles.clear(); maxField = 1; + maxCycle = 1; } } @@ -245,10 +247,11 @@ protected void initFile(String id) throws FormatException, IOException { } } - // update series index for each Image to account for fields + // update series and plane index for each Image to account for fields/timepoints for (Image img : images) { img.series *= maxField; img.series += (img.field - 1); + img.plane += (img.cycle - 1) * channels.size(); } core.clear(); @@ -264,7 +267,8 @@ protected void initFile(String id) throws FormatException, IOException { for (int f=0; f wells) // now map the image to a well - Integer[] ids = getWellAndField(wellLinkQuery, resultId); + Integer[] ids = getWellLink(wellLinkQuery, resultId); String wellLabel = wells.get(ids[0]); img.wellRow = wellLabel.charAt(0) - 'A'; img.wellColumn = Integer.parseInt(wellLabel.substring(1)) - 1; img.series = ids[0] - 1; // always set a 1-based index - img.field = (int) Math.max(1, ids[1]); + img.cycle = (int) Math.max(1, ids[1]); + + maxCycle = (int) Math.max(img.cycle, maxCycle); maxField = (int) Math.max(img.field, maxField); images.add(img); @@ -505,7 +511,7 @@ private void findImages(Connection conn, HashMap wells) String path = objects.getString(2); LOGGER.debug("processing object {}", path); - Integer[] ids = getWellAndField(wellLinkQuery, resultId); + Integer[] ids = getWellLink(wellLinkQuery, resultId); if (ids != null) { String wellLabel = wells.get(ids[0]); Image img = new Image(); @@ -513,7 +519,7 @@ private void findImages(Connection conn, HashMap wells) img.wellColumn = Integer.parseInt(wellLabel.substring(1)) - 1; img.series = ids[0] - 1; // always set a 1-based index - img.field = (int) Math.max(1, ids[1]); + img.cycle = (int) Math.max(1, ids[1]); img.result = true; img.file = path; images.add(img); @@ -528,7 +534,7 @@ private void findImages(Connection conn, HashMap wells) * @param imagingResultID * @return well ID */ - private Integer[] getWellAndField(PreparedStatement linkQuery, + private Integer[] getWellLink(PreparedStatement linkQuery, String imagingResultID) throws SQLException { @@ -585,7 +591,8 @@ class Image { public String file; public int wellRow = -1; public int wellColumn = -1; - public int field = -1; + public int field = 1; + public int cycle = -1; public int series = -1; public int plane = -1; public Length pixelSize; From 887f7eca6f20ea77a31bf4ba59d587464eb58a6c Mon Sep 17 00:00:00 2001 From: Melissa Linkert Date: Tue, 11 Aug 2020 21:17:27 -0500 Subject: [PATCH 09/14] Add a bunch of extra original metadata keys --- .../src/loci/formats/in/TecanReader.java | 267 +++++++++++++++++- 1 file changed, 258 insertions(+), 9 deletions(-) diff --git a/components/formats-gpl/src/loci/formats/in/TecanReader.java b/components/formats-gpl/src/loci/formats/in/TecanReader.java index 68c34f27db5..08059b687df 100644 --- a/components/formats-gpl/src/loci/formats/in/TecanReader.java +++ b/components/formats-gpl/src/loci/formats/in/TecanReader.java @@ -38,9 +38,11 @@ import java.util.Arrays; import java.util.HashMap; import java.util.List; +import javax.xml.parsers.ParserConfigurationException; import loci.common.DateTools; import loci.common.Location; +import loci.common.xml.XMLTools; import loci.formats.CoreMetadata; import loci.formats.FormatException; import loci.formats.FormatReader; @@ -59,6 +61,10 @@ import org.slf4j.LoggerFactory; import org.sqlite.SQLiteConfig; +import org.w3c.dom.Element; +import org.w3c.dom.NodeList; +import org.xml.sax.SAXException; + /** * */ @@ -76,7 +82,7 @@ public class TecanReader extends FormatReader { private transient String plateName = null; private ArrayList images = new ArrayList(); private transient MinimalTiffReader helperReader = null; - private transient ArrayList channels = new ArrayList(); + private transient ArrayList channels = new ArrayList(); private String imageDirectory = null; private ArrayList extraFiles = new ArrayList(); private Integer maxField = 1; @@ -272,6 +278,8 @@ protected void initFile(String id) throws FormatException, IOException { } } + findExtraMetadata(); + // populate the MetadataStore MetadataStore store = makeFilterMetadata(); @@ -296,7 +304,7 @@ protected void initFile(String id) throws FormatException, IOException { store.setWellRow(new NonNegativeInteger(row), 0, nextWell); store.setWellColumn(new NonNegativeInteger(col), 0, nextWell); - String label = (char) (row + 'A') + String.valueOf(col + 1); + String label = makeWellLabel(row, col); if (!wellLabels.containsValue(label)) { nextWell++; continue; @@ -325,7 +333,7 @@ protected void initFile(String id) throws FormatException, IOException { for (int i=0; i 0) { + Element plateStrip = (Element) plateStrips.item(0); + addGlobalMeta("Lid lifter", plateStrip.getAttribute("LidType")); + addGlobalMeta("Humidity Cassette", plateStrip.getAttribute("HumidityCassetteType")); + } + + if (plateDefinitions.getLength() > 0) { + Element plateDef = (Element) plateDefinitions.item(0); + addGlobalMeta("Plate", + plateDef.getAttribute("DisplayName") + " " + plateDef.getAttribute("Comment")); + } + + StringBuffer channelNames = new StringBuffer(); + for (int i=0; i 0) { + channelNames.append(", "); + } + channelNames.append(name); + + int index = lookupChannelIndex(name); + if (index >= 0) { + NodeList crosstalkCorrections = channel.getElementsByTagName("tadodssdf:CrosstalkSettings.CrosstalkCorrectionDict"); + if (crosstalkCorrections.getLength() > 0) { + Element crosstalkCorrection = (Element) crosstalkCorrections.item(0); + NodeList dict = crosstalkCorrection.getElementsByTagName("x:Int16"); + StringBuffer crosstalk = new StringBuffer(); + + for (int d=0; d 0) { + crosstalk.append(", "); + } + Element component = (Element) dict.item(d); + crosstalk.append(component.getAttribute("x:Key")); + crosstalk.append(": "); + crosstalk.append(component.getTextContent()); + crosstalk.append("%"); + } + + addGlobalMeta("Channel #" + (index + 1) + " Cross-talk settings", + crosstalk.toString()); + } + } + } + } + addGlobalMeta("Channels", channelNames.toString()); + } + catch (ParserConfigurationException|SAXException e) { + LOGGER.debug("Could not parse XML", e); + } + } + } + } + catch (IOException|SQLException e) { + LOGGER.warn("Could not read all extra metadata", e); + } + finally { + if (conn != null) { + try { + conn.close(); + } + catch (SQLException ex) { + LOGGER.warn("Could not close database connection", ex); + } + } + } + + for (int c=0; c wells) "ON ImagingResultType.Id=ImagingResult.ImagingResultTypeId " + "WHERE ImagingResult.Id=?"); PreparedStatement channelQuery = conn.prepareStatement( - "SELECT Name, ExposureTimeInUs FROM ChannelType " + + "SELECT Name, ExposureTimeInUs, FocusOffsetInUm, LedIntensityInPercent FROM ChannelType " + "INNER JOIN AcquisitionChannelSetting " + "ON AcquisitionChannelSetting.ChannelTypeId=ChannelType.Id " + "WHERE ChannelType.Id=?"); @@ -473,12 +694,16 @@ private void findImages(Connection conn, HashMap wells) // to multiple ImageTypes in the same well img.channelName += " " + channel.getString(1); - if (!channels.contains(img.channelName)) { - channels.add(img.channelName); + if (lookupChannelIndex(img.channelName) < 0) { + Channel c = new Channel(img.channelName); + c.originalName = channel.getString(1); + c.exposureTime = channel.getDouble(2); + c.focusOffset = channel.getDouble(3); + c.intensity = channel.getDouble(4); + channels.add(c); } - img.plane = channels.indexOf(img.channelName); - img.exposureTime = - FormatTools.getTime(channel.getDouble(2), "µs"); + img.plane = lookupChannelIndex(img.channelName); + img.exposureTime = FormatTools.getTime(channel.getDouble(2), "µs"); img.timestamp = acqTimestamp; } } @@ -558,6 +783,17 @@ private Image lookupImage(int series, int plane, return null; } + private int lookupChannelIndex(String name) { + for (int i=0; i Date: Wed, 12 Aug 2020 20:23:35 -0500 Subject: [PATCH 10/14] Add remaining original metadata keys Includes basic xlsx spreadsheet parsing, since some of the metadata is not actually in the database. --- .../src/loci/formats/in/TecanReader.java | 124 ++++++++++++++++-- 1 file changed, 112 insertions(+), 12 deletions(-) diff --git a/components/formats-gpl/src/loci/formats/in/TecanReader.java b/components/formats-gpl/src/loci/formats/in/TecanReader.java index 08059b687df..698cabc2607 100644 --- a/components/formats-gpl/src/loci/formats/in/TecanReader.java +++ b/components/formats-gpl/src/loci/formats/in/TecanReader.java @@ -26,6 +26,7 @@ package loci.formats.in; import java.io.File; +import java.io.FileInputStream; import java.io.IOException; import java.sql.Connection; @@ -38,10 +39,15 @@ import java.util.Arrays; import java.util.HashMap; import java.util.List; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; import javax.xml.parsers.ParserConfigurationException; +import loci.common.Constants; +import loci.common.DataTools; import loci.common.DateTools; import loci.common.Location; +import loci.common.ZipHandle; import loci.common.xml.XMLTools; import loci.formats.CoreMetadata; import loci.formats.FormatException; @@ -75,6 +81,11 @@ public class TecanReader extends FormatReader { private static final Logger LOGGER = LoggerFactory.getLogger(TecanReader.class); + private static final String[] SPREADSHEET_KEYS = new String[] { + "Application:", "Device", "Firmware", + "System", "User", "Smooth mode", "Part of Plate" + }; + // -- Fields -- private int plateRows = 0; @@ -439,7 +450,7 @@ private void findExtraMetadata() { ); try (ResultSet applicationType = application.executeQuery()) { if (applicationType.next()) { - addGlobalMeta("Application", applicationType.getString(1)); + addGlobalMeta("Application type", applicationType.getString(1)); } } @@ -552,10 +563,6 @@ private void findExtraMetadata() { addGlobalMeta(prefix + "Exposure time [µs]", channel.exposureTime); } - int minRow = Integer.MAX_VALUE; - int maxRow = Integer.MIN_VALUE; - int minCol = Integer.MAX_VALUE; - int maxCol = Integer.MIN_VALUE; for (int s=0; s= strings.getLength()) { + continue; + } + String cellValue = strings.item(cellValueIndex).getTextContent(); + + if (DataTools.indexOf(SPREADSHEET_KEYS, cellValue) >= 0) { + key = cellValue; + } + else if (key != null) { + addGlobalMeta(key, value + " " + cellValue); + key = null; + value = ""; + } + else { + for (String knownKey : SPREADSHEET_KEYS) { + if (cellValue.startsWith(knownKey) || + cellValue.startsWith(knownKey + ":")) + { + int index = cellValue.indexOf(":"); + if (index > 0) { + key = cellValue.substring(0, index); + value = cellValue.substring(index + 1).trim(); + } + } + } + } + } + } + } + } + catch (IOException|ParserConfigurationException|SAXException e) { + LOGGER.debug("Could not parse spreadsheet", e); + } } private String makeWellLabel(int row, int col) { From 94a70a2d3bf4b3781372f729e4b2b243adb11af8 Mon Sep 17 00:00:00 2001 From: Melissa Linkert Date: Wed, 12 Aug 2020 20:24:23 -0500 Subject: [PATCH 11/14] Fix file path sanitizing bug on Windows --- components/formats-gpl/src/loci/formats/in/TecanReader.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/components/formats-gpl/src/loci/formats/in/TecanReader.java b/components/formats-gpl/src/loci/formats/in/TecanReader.java index 698cabc2607..a9d5450426d 100644 --- a/components/formats-gpl/src/loci/formats/in/TecanReader.java +++ b/components/formats-gpl/src/loci/formats/in/TecanReader.java @@ -906,7 +906,11 @@ private String getImageFile(String file) { // sanitize the relative path first // files might be stored in a subdirectory of "Images", // in which case the database may store a "\" to separate the path - String sanitized = File.separator + file.replaceAll("\\\\", File.separator); + String sanitized = File.separator + file; + try { + sanitized = File.separator + file.replaceAll("\\\\", File.separator); + } + catch (IllegalArgumentException e) { } return new Location(imageDirectory + sanitized).getAbsolutePath(); } From d8eb193a5bd5286e00f1b25d4a4c2273e42e8c58 Mon Sep 17 00:00:00 2001 From: Melissa Linkert Date: Wed, 12 Aug 2020 22:34:15 -0500 Subject: [PATCH 12/14] Catch NumberFormatException when reading spreadsheet Fixes remaining issue with kinetics data. --- components/formats-gpl/src/loci/formats/in/TecanReader.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/components/formats-gpl/src/loci/formats/in/TecanReader.java b/components/formats-gpl/src/loci/formats/in/TecanReader.java index a9d5450426d..86315509897 100644 --- a/components/formats-gpl/src/loci/formats/in/TecanReader.java +++ b/components/formats-gpl/src/loci/formats/in/TecanReader.java @@ -681,6 +681,9 @@ else if (key != null) { catch (IOException|ParserConfigurationException|SAXException e) { LOGGER.debug("Could not parse spreadsheet", e); } + catch (NumberFormatException n) { + LOGGER.debug("Unexpected spreadsheet contents", n); + } } private String makeWellLabel(int row, int col) { From a730fe71744b1d67f4adcec892f5bbd8a5231a71 Mon Sep 17 00:00:00 2001 From: Melissa Linkert Date: Fri, 20 Nov 2020 12:38:04 -0600 Subject: [PATCH 13/14] Fix isThisType so that invalid *.db files are not picked up --- .../src/loci/formats/in/TecanReader.java | 26 ++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/components/formats-gpl/src/loci/formats/in/TecanReader.java b/components/formats-gpl/src/loci/formats/in/TecanReader.java index 86315509897..b03cdc43b3d 100644 --- a/components/formats-gpl/src/loci/formats/in/TecanReader.java +++ b/components/formats-gpl/src/loci/formats/in/TecanReader.java @@ -108,10 +108,30 @@ public TecanReader() { domains = new String[] {FormatTools.HCS_DOMAIN}; datasetDescription = "SQLite database, TIFF files, optional analysis output"; + suffixSufficient = false; } // -- IFormatReader API methods -- + /* @see loci.formats.IFormatReader#isThisType(String, boolean) */ + @Override + public boolean isThisType(String name, boolean open) { + if (!checkSuffix(name, "db")) { + return false; + } + if (!open) { + return super.isThisType(name, open); + } + try { + Connection conn = openConnection(name); + findPlateDimensions(conn); + return true; + } + catch (Exception e) { + } + return false; + } + /* @see loci.formats.IFormatReader#getRequiredDirectories(String[]) */ @Override public int getRequiredDirectories(String[] files) @@ -370,13 +390,17 @@ protected void initFile(String id) throws FormatException, IOException { } private Connection openConnection() throws IOException { + return openConnection(getCurrentFile()); + } + + private Connection openConnection(String file) throws IOException { Connection conn = null; try { // see https://github.com/xerial/sqlite-jdbc/issues/247 SQLiteConfig config = new SQLiteConfig(); config.setReadOnly(true); conn = config.createConnection("jdbc:sqlite:" + - new Location(getCurrentFile()).getAbsolutePath()); + new Location(file).getAbsolutePath()); } catch (SQLException e) { LOGGER.warn("Could not read from database"); From 4c75576511f3a18104a9deb843c4cf6756a7dd33 Mon Sep 17 00:00:00 2001 From: Melissa Linkert Date: Tue, 24 Nov 2020 09:40:00 -0600 Subject: [PATCH 14/14] Add isThisType/used files test exclusions for TecanReader --- .../loci/tests/testng/FormatReaderTest.java | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/components/test-suite/src/loci/tests/testng/FormatReaderTest.java b/components/test-suite/src/loci/tests/testng/FormatReaderTest.java index 00ed56d565b..53d760a5a90 100644 --- a/components/test-suite/src/loci/tests/testng/FormatReaderTest.java +++ b/components/test-suite/src/loci/tests/testng/FormatReaderTest.java @@ -1986,6 +1986,13 @@ else if (success) { continue; } + // Tecan datasets can only be detected with the .db file + if (reader.getFormat().equals("Tecan Spark Cyto") && + !base[i].toLowerCase().endsWith(".db")) + { + continue; + } + r.setId(base[i]); String[] comp = r.getUsedFiles(); @@ -2739,6 +2746,20 @@ public void testIsThisType() { continue; } + // Tecan data can only be detected with the .db file + if (!result && readers[j] instanceof TecanReader && + !used[i].toLowerCase().endsWith(".db")) + { + continue; + } + + // OK for other readers to flag Tecan files other than .db + if (result && r instanceof TecanReader && + !used[i].toLowerCase().endsWith(".db")) + { + continue; + } + boolean expected = r == readers[j]; if (result != expected) { success = false;