Chunk API: Add new API and functionality for reading and writing chunks

components/formats-api/src/loci/formats/FormatReader.java

@@ -924,6 +924,29 @@ public byte[] openBytes(int no, int x, int y, int w, int h)
   public abstract byte[] openBytes(int no, byte[] buf, int x, int y,
     int w, int h) throws FormatException, IOException;
 
+  /* @see IFormatReader#openBytes(int) */
+  @Override
+  public byte[] openBytes(byte[] buf, int [] shape, int [] offsets) throws FormatException, IOException {
+    FormatTools.checkParameters(this, buf.length, shape, offsets);
+    String order = getDimensionOrder();
+    int[] XYZTCshape = FormatTools.getXYZCTIndexes(order, shape);
+    int[] XYZTCoffsets = FormatTools.getXYZCTIndexes(order, offsets);
+    int num = getSizeZ() * getSizeC() * getSizeT();
+    int bufOffset = 0;
+    for (int z = 0; z < XYZTCshape[2]; z++) {
+      for (int c = 0; c < XYZTCshape[3]; c++) {
+        for (int t = 0; t < XYZTCshape[4]; t++) {
+          int no = FormatTools.getIndex(order, getSizeZ(), getSizeC(), getSizeT(), num, 
+              XYZTCoffsets[2] + z, XYZTCoffsets[3] + c, XYZTCoffsets[4] + t);
+          byte[] plane = openBytes(no, XYZTCoffsets[0], XYZTCoffsets[1], XYZTCshape[0], XYZTCshape[1]);
+          System.arraycopy(plane, 0, buf, bufOffset, plane.length);
+          bufOffset += plane.length;
+        }
+      }
+    }
+    return buf;
+  }
+
   /* @see IFormatReader#openPlane(int, int, int, int, int int) */
   @Override
   public Object openPlane(int no, int x, int y, int w, int h)
@@ -1287,6 +1310,13 @@ public int getOptimalTileHeight() {
      return (int) Math.min(maxHeight, getSizeY());
   }
 
+  /* @see IFormatReader#getOptimalChunkSize() */
+  @Override
+  public int[] getOptimalChunkSize() {
+    int[] chunkSize = {getOptimalTileWidth(), getOptimalTileHeight(), 1, 1, 1};
+    return chunkSize;
+  }
+
   // -- Sub-resolution API methods --
 
   @Override

components/formats-api/src/loci/formats/FormatTools.java

@@ -697,6 +697,21 @@ public static int[] getZCTCoords(String order,
     };
   }
 
+  public static int[] getXYZCTIndexes(String order, int[] values) {
+    int xIndex = order.indexOf("X");
+    int yIndex = order.indexOf("Y");
+    int zIndex = order.indexOf("Z");
+    int cIndex = order.indexOf("C");
+    int tIndex = order.indexOf("T");
+      return new int[] {
+          values[xIndex],
+          values[yIndex],
+          values[zIndex],
+          values[cIndex],
+          values[tIndex],
+      };
+  }
+
   /**
    * Converts index from the given dimension order to the reader's native one.
    * This method is useful for shuffling the planar order around
@@ -1003,6 +1018,20 @@ public static void checkPlaneParameters(IFormatReader r, int no,
     if (bufLength >= 0) checkBufferSize(r, bufLength, w, h);
   }
 
+  /**
+   * Convenience method for checking that the chunk shape, chunk offsets and
+   * buffer sizes are all valid for the given reader.
+   * If 'bufLength' is less than 0, then the buffer length check is not
+   * performed.
+   */
+  public static void checkParameters(IFormatReader r, int bufLength,
+    int[] shape, int[] offsets) throws FormatException
+  {
+    assertId(r.getCurrentFile(), true, 2);
+    checkChunkSize(r, shape, offsets);
+    if (bufLength >= 0) checkBufferSize(r, bufLength, shape);
+  }
+
   /** Checks that the given plane number is valid for the given reader. */
   public static void checkPlaneNumber(IFormatReader r, int no)
     throws FormatException
@@ -1028,6 +1057,22 @@ public static void checkTileSize(IFormatReader r, int x, int y, int w, int h)
     }
   }
 
+  /** Checks that the given chunk size is valid for the given reader. */
+  public static void checkChunkSize(IFormatReader r, int[] shape, int[] offsets)
+    throws FormatException
+  {
+    int[] dimensionSizes = {r.getSizeX(), r.getSizeY(), r.getSizeZ(), r.getSizeC(), r.getSizeT()};
+    int[] XYZTCshape = getXYZCTIndexes(r.getDimensionOrder(), shape);
+    int[] XYZTCoffsets = getXYZCTIndexes(r.getDimensionOrder(), offsets);
+    for (int i = 0; i < XYZTCoffsets.length; i++) {
+      if (XYZTCoffsets[i] < 0 || (XYZTCoffsets[i] + XYZTCshape[i]) > dimensionSizes[i]) {
+        char dim = DimensionOrder.XYZCT.toString().charAt(i);
+        throw new FormatException("Invalid chunk size: " + dim + " shape = " + XYZTCshape[i] + " " + dim +
+            " offset = " + XYZTCoffsets[i] + " " + dim + " maxSize = " + dimensionSizes[i]);
+      }
+    }
+  }
+
   public static void checkBufferSize(IFormatReader r, int len)
     throws FormatException
   {
@@ -1049,6 +1094,21 @@ public static void checkBufferSize(IFormatReader r, int len, int w, int h)
     }
   }
 
+  /**
+   * Checks that the given buffer size is large enough to hold a 5D chunk
+   * image as returned by the given reader.
+   * @throws FormatException if the buffer is too small
+   */
+  public static void checkBufferSize(IFormatReader r, int len, int[] shape)
+    throws FormatException
+  {
+    int size = getChunkSize(r, shape);
+    if (size > len) {
+      throw new FormatException("Buffer too small (got " + len +
+        ", expected " + size + ").");
+    }
+  }
+
   /**
    * Returns true if the given RandomAccessInputStream conatins at least
    * 'len' bytes.
@@ -1071,6 +1131,15 @@ public static int getPlaneSize(IFormatReader r, int w, int h) {
     return w * h * r.getRGBChannelCount() * getBytesPerPixel(r.getPixelType());
   }
 
+  /** Returns the size in bytes of a w * h tile. */
+  public static int getChunkSize(IFormatReader r, int[] shape) {
+    int size = r.getRGBChannelCount() * getBytesPerPixel(r.getPixelType());
+    for (int i = 0; i < shape.length; i++) {
+      size *= shape[i];
+    }
+    return size;
+  }
+
   // -- Utility methods -- export
 
   public static String getTileFilename(int tileX, int tileY,

components/formats-api/src/loci/formats/FormatWriter.java

@@ -35,8 +35,10 @@
 import java.awt.image.ColorModel;
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.List;
 
+import ome.xml.model.enums.DimensionOrder;
 import ome.xml.model.primitives.PositiveInteger;
 
 import loci.common.DataTools;
@@ -162,6 +164,36 @@ public void savePlane(int no, Object plane, int x, int y, int w, int h)
     saveBytes(no, (byte[]) plane, x, y, w, h);
   }
 
+  /* @see IFormatWriter#savePlane(byte[], int[], int[]) */
+  @Override
+  public void saveBytes(byte[] buf, int[] shape, int[] offsets)
+    throws FormatException, IOException
+  {
+    checkParams(buf, shape, offsets);
+    MetadataRetrieve r = getMetadataRetrieve();
+    String order = r.getPixelsDimensionOrder(series).toString();
+    int[] XYZTCshape = FormatTools.getXYZCTIndexes(order, shape);
+    int[] XYZTCoffsets = FormatTools.getXYZCTIndexes(order, offsets);
+    int num = r.getPixelsSizeZ(series).getValue() * r.getPixelsSizeC(series).getValue() * r.getPixelsSizeT(series).getValue();
+    int bufOffset = 0;
+    for (int z = 0; z < XYZTCshape[2]; z++) {
+      for (int c = 0; c < XYZTCshape[3]; c++) {
+        for (int t = 0; t < XYZTCshape[4]; t++) {
+          int no = FormatTools.getIndex(order, r.getPixelsSizeZ(series).getValue(), r.getPixelsSizeC(series).getValue(), 
+              r.getPixelsSizeT(series).getValue(), num, XYZTCoffsets[2] + z, XYZTCoffsets[3] + c, XYZTCoffsets[4] + t);
+          int pixelType = FormatTools.pixelTypeFromString(r.getPixelsType(series).toString());
+          int bpp = FormatTools.getBytesPerPixel(pixelType);
+          PositiveInteger samples = r.getChannelSamplesPerPixel(series, 0);
+          if (samples == null) samples = new PositiveInteger(1);
+          int planeSize =  bpp * XYZTCshape[0] * XYZTCshape[1] * samples.getValue();
+          byte[] plane = Arrays.copyOfRange(buf, bufOffset, planeSize);
+          saveBytes(no, plane, XYZTCoffsets[0], XYZTCoffsets[1], XYZTCshape[0], XYZTCshape[1]);
+          bufOffset += planeSize;
+        }
+      }
+    }
+  }
+
   /* @see IFormatWriter#savePlane(int, Object, Region) */
   @Override
   public void savePlane(int no, Object plane, Region tile)
@@ -330,6 +362,20 @@ public int setTileSizeY(int tileSize) throws FormatException {
     return height;
   }
 
+  /* @see IFormatWriter#getChunkSize() */
+  @Override
+  public int[] getChunkSize() throws FormatException {
+    int[] defaultChunkSize = {getTileSizeX(), getTileSizeY(), 1, 1, 1};
+    return defaultChunkSize;
+  }
+
+  /* @see IFormatWriter#setChunkSize(int[]) */
+  @Override
+  public int[] setChunkSize(int[] chunkSize) throws FormatException {
+    int[] returnChunkSize = {setTileSizeX(chunkSize[0]), setTileSizeY(chunkSize[1]), 1, 1, 1};
+    return returnChunkSize;
+  }  
+
   /* @see IFormatWriter#setResolutions(List<Resolution>) */
   @Override
   public void setResolutions(List<Resolution> resolutions) {
@@ -483,6 +529,51 @@ protected void checkParams(int no, byte[] buf, int x, int y, int w, int h)
     }
   }
 
+  /**
+   * Ensure that the arguments that are being passed to saveBytes(...) are
+   * valid.
+   * @throws FormatException if any of the arguments is invalid.
+   */
+  protected void checkParams(byte[] buf, int[] shape, int[] offsets)
+    throws FormatException
+  {
+    MetadataRetrieve r = getMetadataRetrieve();
+    MetadataTools.verifyMinimumPopulated(r, series);
+
+    int pixelType =
+        FormatTools.pixelTypeFromString(r.getPixelsType(series).toString());
+    PositiveInteger samples = r.getChannelSamplesPerPixel(series, 0);
+    if (samples == null) samples = new PositiveInteger(1);
+    int minSize = samples.getValue() * FormatTools.getBytesPerPixel(pixelType);
+    for (int i = 0; i < shape.length; i++) {
+      minSize *= shape[i];
+    }
+    if (buf.length < minSize) {
+      throw new FormatException("Buffer is too small; expected " + minSize +
+        " bytes, got " + buf.length + " bytes.");
+    }
+
+    String order = r.getPixelsDimensionOrder(series).toString();
+    int sizeZ = r.getPixelsSizeZ(series).getValue().intValue();
+    int sizeT = r.getPixelsSizeT(series).getValue().intValue();
+    int sizeC = r.getChannelCount(series);
+    int[] dimensionSizes = {getSizeX(), getSizeY(), sizeZ, sizeC, sizeT};
+    int[] XYZTCshape = FormatTools.getXYZCTIndexes(order, shape);
+    int[] XYZTCoffsets = FormatTools.getXYZCTIndexes(order, offsets);
+    for (int i = 0; i < XYZTCoffsets.length; i++) {
+      if (XYZTCoffsets[i] < 0 || (XYZTCoffsets[i] + XYZTCshape[i]) > dimensionSizes[i]) {
+        char dim = DimensionOrder.XYZCT.toString().charAt(i);
+        throw new FormatException("Invalid chunk size: " + dim + " shape = " + XYZTCshape[i] + " " + dim +
+            " offset = " + XYZTCoffsets[i] + " " + dim + " maxSize = " + dimensionSizes[i]);
+      }
+    }
+
+    if (!DataTools.containsValue(getPixelTypes(compression), pixelType)) {
+      throw new FormatException("Unsupported image type '" +
+        FormatTools.getPixelTypeString(pixelType) + "'.");
+    }
+  }
+
   /**
    * Seek to the given (x, y) coordinate of the image that starts at
    * the given offset.

components/formats-api/src/loci/formats/IFormatReader.java

@@ -283,6 +283,19 @@ byte[] openBytes(int no, byte[] buf)
    */
   byte[] openBytes(int no, byte[] buf, int x, int y, int w, int h)
     throws FormatException, IOException;
+
+  /**
+   * Obtains a chunk of the specified image into a pre-allocated byte array.
+   * 
+   * @param buf a pre-allocated buffer.
+   * @param shape the shape of the chunk consisting of the size of each dimension
+   * @param offsets the offset of each dimension for the chunk
+   * @return the pre-allocated buffer <code>buf</code> for convenience.
+   * @throws FormatException if there was a problem parsing the metadata of the
+   *   file.
+   * @throws IOException if there was a problem reading the file.
+   */
+  byte[] openBytes(byte[] buf, int[] shape, int[] offsets) throws FormatException, IOException;
 
   /**
    * Obtains the specified image plane (or sub-image thereof) in the reader's
@@ -591,6 +604,9 @@ Object openPlane(int no, int x, int y, int w, int h)
   /** Returns the optimal sub-image height for use with openBytes. */
   int getOptimalTileHeight();
 
+  /** Returns the optimal chunk size for use with openBytes. */
+  int[] getOptimalChunkSize();
+
   // -- Sub-resolution API methods --
 
   /** Returns the first core index corresponding to the specified series.

components/formats-api/src/loci/formats/IFormatWriter.java

@@ -82,6 +82,17 @@ void saveBytes(int no, byte[] buf, int x, int y, int w, int h)
   void saveBytes(int no, byte[] buf, Region tile)
     throws FormatException, IOException;
 
+  /**
+   * Saves the given chunk to the current series in the current file.
+   *
+   * @param buf the byte array that represents the image chunk.
+   * @param shape the size of the chunk for every dimension.
+   * @param offsets the offset of the chunk for every dimension.
+   * @throws FormatException if one of the parameters is invalid.
+   * @throws IOException if there was a problem writing to the file.
+   */
+  void saveBytes(byte[] buf, int[] shape, int[] offsets) throws FormatException, IOException;
+
   /**
    * Saves the given image plane to the current series in the current file.
    *
@@ -248,4 +259,8 @@ void savePlane(int no, Object plane, Region tile)
    */
   List<Resolution> getResolutions();
 
+  int[] setChunkSize(int[] chunkSize) throws FormatException;
+
+  int[] getChunkSize() throws FormatException;
+
 }

components/formats-api/src/loci/formats/ImageReader.java

@@ -465,6 +465,12 @@ public byte[] openBytes(int no, byte[] buf, int x, int y, int w, int h)
     return getReader().openBytes(no, buf, x, y, w, h);
   }
 
+  /* @see IFormatReader#openBytes(byte[] buf, int[] shape, int[] offsets) */
+  @Override
+  public byte[] openBytes(byte[] buf, int[] shape, int[] offsets) throws FormatException, IOException {
+    return getReader().openBytes(buf, shape, offsets);
+  }
+
   /* @see IFormatReader#openPlane(int, int, int, int, int) */
   @Override
   public Object openPlane(int no, int x, int y, int w, int h)
@@ -747,6 +753,12 @@ public int getOptimalTileHeight() {
     return getReader().getOptimalTileHeight();
   }
 
+  /* @see IFormatReader#getOptimalChunkSize() */
+  @Override
+  public int[] getOptimalChunkSize() {
+    return getReader().getOptimalChunkSize();
+  }
+
   /* @see IFormatReader#getCoreIndex() */
   @Override
   public int getCoreIndex() {

components/formats-api/src/loci/formats/ImageWriter.java

@@ -201,6 +201,16 @@ public int setTileSizeY(int tileSize) throws FormatException {
     return getWriter().setTileSizeY(tileSize);
   }
 
+  @Override
+  public int[] getChunkSize() throws FormatException {
+    return getWriter().getChunkSize();
+  }
+
+  @Override
+  public int[] setChunkSize(int[] chunkSize) throws FormatException {
+    return getWriter().setChunkSize(chunkSize);
+  }
+
   @Override
   public void setResolutions(List<Resolution> resolutions) {
     for (IFormatWriter w : writers) {
@@ -268,6 +278,12 @@ public void saveBytes(int no, byte[] buf, Region tile)
     getWriter().saveBytes(no, buf, tile);
   }
 
+  /* @see IFormatWriter#saveBytes(byte[] buf, int[] shape, int[] offsets) */
+  @Override
+  public void saveBytes(byte[] buf, int[] shape, int[] offsets) throws FormatException, IOException {
+    getWriter().saveBytes(buf, shape, offsets);
+  }
+
   /* @see IFormatWriter#savePlane(int, Object) */
   @Override
   public void savePlane(int no, Object plane)