From cb55d7fe749d962fd7504b04b6ef39b2b71cd4d7 Mon Sep 17 00:00:00 2001 From: Ed Scanlon Date: Tue, 21 Nov 2023 12:10:26 -0500 Subject: [PATCH 1/4] Work-around for breaking change in CellSens 4.1: missing metadata in .vsi file was causing exception when parsing .ets file. Added logic to dynamically associate .efs files with .vsi file metadata. --- .../src/loci/formats/in/CellSensReader.java | 72 +++++++++++++++---- 1 file changed, 60 insertions(+), 12 deletions(-) diff --git a/components/formats-gpl/src/loci/formats/in/CellSensReader.java b/components/formats-gpl/src/loci/formats/in/CellSensReader.java index 04cf9190152..2cbb4504193 100644 --- a/components/formats-gpl/src/loci/formats/in/CellSensReader.java +++ b/components/formats-gpl/src/loci/formats/in/CellSensReader.java @@ -771,8 +771,15 @@ else if (zCount > 0) { if (s < files.size() - 1) { setCoreIndex(index); String ff = files.get(s); + /** + * If there are fewer metadata 'pyramids' defined in the vsi than there + * are frame_*.ets files, we are missing the metadata associated with + * one or more frame_*.ets files. In this case we need to dynamically + * match the metadata we do have with the appropriate frame_*.ets file. + */ + boolean missingMetadata = pyramids.size() < (files.size() - 1); try (RandomAccessInputStream stream = new RandomAccessInputStream(ff)) { - parseETSFile(stream, ff, s); + parseETSFile(stream, ff, s, missingMetadata); } ms.littleEndian = compressionType.get(index) == RAW; @@ -797,10 +804,6 @@ else if (zCount > 0) { } index += ms.resolutionCount; - if (s < pyramids.size()) { - ms.seriesMetadata = pyramids.get(s).originalMetadata; - } - setCoreIndex(0); ms.dimensionOrder = "XYCZT"; } @@ -1188,7 +1191,8 @@ else if (dim.equals("T")) { return buf; } - private void parseETSFile(RandomAccessInputStream etsFile, String file, int s) + private void parseETSFile(RandomAccessInputStream etsFile, String file, int s, + boolean missingMetadata) throws FormatException, IOException { fileMap.put(core.size() - 1, file); @@ -1286,7 +1290,40 @@ private void parseETSFile(RandomAccessInputStream etsFile, String file, int s) int[] maxC = new int[maxResolution]; int[] maxT = new int[maxResolution]; - HashMap dimOrder = pyramids.get(s).dimensionOrdering; + HashMap dimOrder = new HashMap(); + Pyramid pyramid = null; + /** + * If there is metadata missing from the vsi file, we need to see if we can match up + * this frame_*.ets file with its correct metadata block if it exists. To do that we + * search the metadata blocks (ie 'pyramids') for the one whose width and height are + * within the correct range for this frame_*.ets file. + */ + if (missingMetadata) { + int maxXAtRes0 = 0; + int maxYAtRes0 = 0; + for (TileCoordinate t : tmpTiles) { + if (!usePyramid || t.coordinate[t.coordinate.length - 1] == 0) { + maxXAtRes0 = Math.max(maxXAtRes0, t.coordinate[0]); + maxYAtRes0 = Math.max(maxYAtRes0, t.coordinate[1]); + } + } + int maxPixelWidth = (maxXAtRes0 + 1) * tileX.get(tileX.size()-1); + int maxPixelHeight = (maxYAtRes0 + 1) * tileY.get(tileY.size()-1); + for (Pyramid p : pyramids) { + if ( (p.width <= maxPixelWidth ) && (p.width >= maxPixelWidth - tileX.get(tileX.size()-1)) + && (p.height <= maxPixelHeight) && (p.height >= maxPixelHeight - tileY.get(tileY.size()-1)) ) { + pyramid = p; + break; + } + } + } + else { + pyramid = pyramids.get(s); + } + + if (pyramid != null) { + dimOrder = pyramid.dimensionOrdering; + } for (TileCoordinate t : tmpTiles) { int resolution = usePyramid ? t.coordinate[t.coordinate.length - 1] : 0; @@ -1381,12 +1418,23 @@ private void parseETSFile(RandomAccessInputStream etsFile, String file, int s) maxC[resolution] = t.coordinate[cIndex]; } } - - if (pyramids.get(s).width != null) { - ms.sizeX = pyramids.get(s).width; + /** + * If this ets file has an associated metadata block, grab the correct width and height, + * and link it to the full metadata block. + */ + if (pyramid != null) { + ms.sizeX = pyramid.width; + ms.sizeY = pyramid.height; + ms.seriesMetadata = pyramid.originalMetadata; } - if (pyramids.get(s).height != null) { - ms.sizeY = pyramids.get(s).height; + else { + /** + * Otherwise, compute the closest approximation, since the real info couldn't be found. + */ + ms.sizeX = (maxX[0] + 1) * tileX.get(tileX.size()-1); + ms.sizeY = (maxY[0] + 1) * tileY.get(tileY.size()-1); + ms.seriesMetadata = new Hashtable(); + ms.seriesMetadata.put("Metadata", "Not Found"); // leave a trail that the metadata was missing for this ets. } ms.sizeZ = maxZ[0] + 1; if (maxC[0] > 0) { From 4557c98af3d19e872f35977573007988c7b9e683 Mon Sep 17 00:00:00 2001 From: Ed Scanlon Date: Thu, 7 Dec 2023 17:18:05 -0500 Subject: [PATCH 2/4] Altered the original work-around logic so that when a .ets file has no associated metadata block in the .vsi file, its contents are omitted from the series list, and the file is treated the same way as the blob_*.meta files, which is to say, their names are included in the 'extraFiles' list, and they are otherwise ignored. --- .../src/loci/formats/in/CellSensReader.java | 83 +++++++++++-------- 1 file changed, 47 insertions(+), 36 deletions(-) diff --git a/components/formats-gpl/src/loci/formats/in/CellSensReader.java b/components/formats-gpl/src/loci/formats/in/CellSensReader.java index 2cbb4504193..8827ab8de69 100644 --- a/components/formats-gpl/src/loci/formats/in/CellSensReader.java +++ b/components/formats-gpl/src/loci/formats/in/CellSensReader.java @@ -772,14 +772,28 @@ else if (zCount > 0) { setCoreIndex(index); String ff = files.get(s); /** - * If there are fewer metadata 'pyramids' defined in the vsi than there - * are frame_*.ets files, we are missing the metadata associated with - * one or more frame_*.ets files. In this case we need to dynamically - * match the metadata we do have with the appropriate frame_*.ets file. + * If there are more frame_*.ets files than there are metadata 'pyramids' + * defined in the .vsi, then we have orphaned frame files. In this case + * we need to determine which are the valid frame files and which are + * the orphans. The valid ones need to be matched with the appropriate + * metadata blocks in the vsi, and the orphaned ones need to be ignored. */ - boolean missingMetadata = pyramids.size() < (files.size() - 1); + boolean hasOrphanEtsFiles = pyramids.size() < (files.size() - 1); try (RandomAccessInputStream stream = new RandomAccessInputStream(ff)) { - parseETSFile(stream, ff, s, missingMetadata); + boolean validFrameFile = parseETSFile(stream, ff, s, hasOrphanEtsFiles); + /* + * If this frame file is an orphan, "undo" the work that was done for + * it and change its status to being an "extra" file. + */ + if (!validFrameFile) { + core.remove(core.size()-1); + extraFiles.add(files.get(s)); + files.remove(s); + usedFiles = files.toArray(new String[files.size()]); + s--; + seriesCount--; + continue; + } } ms.littleEndian = compressionType.get(index) == RAW; @@ -1191,8 +1205,8 @@ else if (dim.equals("T")) { return buf; } - private void parseETSFile(RandomAccessInputStream etsFile, String file, int s, - boolean missingMetadata) + private boolean parseETSFile(RandomAccessInputStream etsFile, String file, int s, + boolean hasOrphanEtsFiles) throws FormatException, IOException { fileMap.put(core.size() - 1, file); @@ -1293,12 +1307,13 @@ private void parseETSFile(RandomAccessInputStream etsFile, String file, int s, HashMap dimOrder = new HashMap(); Pyramid pyramid = null; /** - * If there is metadata missing from the vsi file, we need to see if we can match up - * this frame_*.ets file with its correct metadata block if it exists. To do that we - * search the metadata blocks (ie 'pyramids') for the one whose width and height are - * within the correct range for this frame_*.ets file. - */ - if (missingMetadata) { + * If there are orphaned .ets files with this vsi file, we need to determine whether + * the current one is an orphan or a legit file. The logic to determine this is to + * see of there is a metadata block (ie 'pyramid') whose width and height are + * within the correct range for this .ets file. If there is no matching metadata + * block, then we have to assume this is an orphan + **/ + if (hasOrphanEtsFiles) { int maxXAtRes0 = 0; int maxYAtRes0 = 0; for (TileCoordinate t : tmpTiles) { @@ -1316,14 +1331,24 @@ private void parseETSFile(RandomAccessInputStream etsFile, String file, int s, break; } } + /** + * No matching metadata block. This is an orphan ets file. Undo and erase + * all the data elements that have been gathered up for this .ets file. + **/ + if (pyramid == null) { + nDimensions.remove(nDimensions.size() - 1); + compressionType.remove(compressionType.size() - 1); + tileX.remove(tileX.size() - 1); + tileY.remove(tileY.size() - 1); + backgroundColor.remove(getCoreIndex()); + tileOffsets.remove(tileOffsets.size()-1); + return(false); + } } else { pyramid = pyramids.get(s); } - - if (pyramid != null) { - dimOrder = pyramid.dimensionOrdering; - } + dimOrder = pyramid.dimensionOrdering; for (TileCoordinate t : tmpTiles) { int resolution = usePyramid ? t.coordinate[t.coordinate.length - 1] : 0; @@ -1418,24 +1443,9 @@ private void parseETSFile(RandomAccessInputStream etsFile, String file, int s, maxC[resolution] = t.coordinate[cIndex]; } } - /** - * If this ets file has an associated metadata block, grab the correct width and height, - * and link it to the full metadata block. - */ - if (pyramid != null) { - ms.sizeX = pyramid.width; - ms.sizeY = pyramid.height; - ms.seriesMetadata = pyramid.originalMetadata; - } - else { - /** - * Otherwise, compute the closest approximation, since the real info couldn't be found. - */ - ms.sizeX = (maxX[0] + 1) * tileX.get(tileX.size()-1); - ms.sizeY = (maxY[0] + 1) * tileY.get(tileY.size()-1); - ms.seriesMetadata = new Hashtable(); - ms.seriesMetadata.put("Metadata", "Not Found"); // leave a trail that the metadata was missing for this ets. - } + ms.sizeX = pyramid.width; + ms.sizeY = pyramid.height; + ms.seriesMetadata = pyramid.originalMetadata; ms.sizeZ = maxZ[0] + 1; if (maxC[0] > 0) { ms.sizeC *= (maxC[0] + 1); @@ -1529,6 +1539,7 @@ else if (newResolution.sizeY > maxSizeY) { ms.resolutionCount = finalResolution; } + return(true); } private int convertPixelType(int pixelType) throws FormatException { From 5493c1ea8df70308a6e4309b1493426e0748e6fa Mon Sep 17 00:00:00 2001 From: Ed Scanlon Date: Tue, 27 Feb 2024 17:51:28 -0500 Subject: [PATCH 3/4] Improved the method of detecting orphan ETS file by ensuring that any given 'pyramid' metadata struct is not associated with more than one ETS file. This change is to handle the case where the image sizes of multiple ETS files are the same or nearly the same as a single 'pyramid' metadata struct, resulting in an erroneous associations that cause the program to crash. --- .../formats-gpl/src/loci/formats/in/CellSensReader.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/components/formats-gpl/src/loci/formats/in/CellSensReader.java b/components/formats-gpl/src/loci/formats/in/CellSensReader.java index 8827ab8de69..86c699bdecf 100644 --- a/components/formats-gpl/src/loci/formats/in/CellSensReader.java +++ b/components/formats-gpl/src/loci/formats/in/CellSensReader.java @@ -1325,9 +1325,12 @@ private boolean parseETSFile(RandomAccessInputStream etsFile, String file, int s int maxPixelWidth = (maxXAtRes0 + 1) * tileX.get(tileX.size()-1); int maxPixelHeight = (maxYAtRes0 + 1) * tileY.get(tileY.size()-1); for (Pyramid p : pyramids) { + if (p.HasAssociatedEtsFile) // If this pyramid has already been linked to an ETS + continue; // then don't allow it to be linked to another. if ( (p.width <= maxPixelWidth ) && (p.width >= maxPixelWidth - tileX.get(tileX.size()-1)) && (p.height <= maxPixelHeight) && (p.height >= maxPixelHeight - tileY.get(tileY.size()-1)) ) { pyramid = p; + p.HasAssociatedEtsFile = true; // Rememeber that this pyramid is now taken by an Ets. break; } } @@ -2664,6 +2667,7 @@ class Pyramid { public transient Double zStart; public transient Double zIncrement; public transient ArrayList zValues = new ArrayList(); + public boolean HasAssociatedEtsFile = false; } } From ec129c92fd40a3d58700f7381b340ea86edcdde7 Mon Sep 17 00:00:00 2001 From: Ed Scanlon Date: Wed, 6 Mar 2024 13:49:44 -0500 Subject: [PATCH 4/4] When an ETS file is determined to be an orphan, also remove it from the filemap. --- components/formats-gpl/src/loci/formats/in/CellSensReader.java | 1 + 1 file changed, 1 insertion(+) diff --git a/components/formats-gpl/src/loci/formats/in/CellSensReader.java b/components/formats-gpl/src/loci/formats/in/CellSensReader.java index 86c699bdecf..0884c9b4767 100644 --- a/components/formats-gpl/src/loci/formats/in/CellSensReader.java +++ b/components/formats-gpl/src/loci/formats/in/CellSensReader.java @@ -1339,6 +1339,7 @@ private boolean parseETSFile(RandomAccessInputStream etsFile, String file, int s * all the data elements that have been gathered up for this .ets file. **/ if (pyramid == null) { + fileMap.remove(core.size() - 1); nDimensions.remove(nDimensions.size() - 1); compressionType.remove(compressionType.size() - 1); tileX.remove(tileX.size() - 1);