Implement fixes for opening OME-Zarr with credentials from S3

src/main/java/org/embl/mobie/MoBIE.java

@@ -112,6 +112,8 @@ public class MoBIE
 
 	public MoBIE( String uri, MoBIESettings settings ) throws IOException
 	{
+		initImageJAndMoBIE();
+
 		this.settings = settings;
 		this.projectLocation = uri;
 
@@ -134,7 +136,6 @@ public MoBIE( String uri, MoBIESettings settings ) throws IOException
 							.columnTypesPartial( nameToType );
 			Table table = Table.read().usingOptions( builder );
 
-			initImageJAndMoBIE();
 			initProject( IOHelper.getFileName( uri ) );
 
 			CollectionTableDataSetter dataSetter = new CollectionTableDataSetter( table, settings.values.getDataRoot() );
@@ -146,7 +147,6 @@ public MoBIE( String uri, MoBIESettings settings ) throws IOException
 		else if ( settings.values.getProjectType().equals( ProjectType.MoBIEJSON ) )
 		{
 			initTableSaw();
-			initImageJAndMoBIE();
 
 			IJ.log( "\n# MoBIE" );
 			IJ.log( "Opening: " + uri );
@@ -163,11 +163,11 @@ public static MoBIE getInstance()
 
 	public MoBIE( String hcsDataLocation, MoBIESettings settings, double relativeWellMargin, double relativeSiteMargin, @Nullable VoxelDimensions voxelDimensions ) throws IOException
 	{
+		initImageJAndMoBIE();
+
 		this.settings = settings;
 		this.projectLocation = hcsDataLocation;
 
-		initImageJAndMoBIE();
-
 		IJ.log("\n# MoBIE" );
 		IJ.log("Opening: " + hcsDataLocation );
 
@@ -176,6 +176,8 @@ public MoBIE( String hcsDataLocation, MoBIESettings settings, double relativeWel
 
 	public MoBIE( List< String > imagePaths, List< String > labelPaths, List< String > labelTablePaths, String root, GridType grid, MoBIESettings settings ) throws IOException
 	{
+		initImageJAndMoBIE();
+
 		IJ.log("\n# MoBIE" );
 		IJ.log("Opening images: " + Arrays.toString( imagePaths.toArray() ) );
 		IJ.log("Opening labels: " + Arrays.toString( labelPaths.toArray() ) );
@@ -195,6 +197,8 @@ public MoBIE( List< String > imagePaths, List< String > labelPaths, List< String
 	// open an image or object table
 	public MoBIE( String tablePath, List< String > imageColumns, List< String > labelColumns, String root, String pathMapping, GridType grid, MoBIESettings settings ) throws IOException
 	{
+		initImageJAndMoBIE();
+
 		IJ.log("\n# MoBIE" );
 		IJ.log("Opening data from table: " + tablePath );
 
@@ -226,8 +230,6 @@ private void initTableSaw()
 
 	private void openMoBIEProject() throws IOException
 	{
-		S3Utils.setS3AccessAndSecretKey( settings.values.getS3AccessAndSecretKey() );
-
 		setProjectImageAndTableRootLocations();
 		registerProjectPlugins( projectLocation );
 		project = new ProjectJsonParser().parseProject( combinePath( projectRoot, "project.json" ) );
@@ -239,8 +241,6 @@ private void openMoBIEProject() throws IOException
 	// TODO 2D or 3D?
 	private void openImageAndLabelGrids( List< ImageGridSources > images, List< LabelGridSources > labels, Table regionTable )
 	{
-		initImageJAndMoBIE();
-
 		initProject( "" );
 
 		new GridSourcesDataSetter( images, labels, regionTable ).addDataAndDisplaysAndViews( dataset );
@@ -250,10 +250,10 @@ private void openImageAndLabelGrids( List< ImageGridSources > images, List< Labe
 
 	public MoBIE( String projectName, ImageData< ? > imageData, ImageData< ? > labelData, @Nullable StorageLocation tableStorageLocation, @Nullable TableDataFormat tableDataFormat )
 	{
-		settings = new MoBIESettings();
-
 		initImageJAndMoBIE();
 
+		settings = new MoBIESettings();
+
 		initProject( projectName );
 
 		new ImageDataAdder(
@@ -481,7 +481,6 @@ public void close()
 		{
 			IJ.log( "Closing MoBIE..." );
 			IJ.log( "Closing I/O threads..." );
-			S3Utils.setS3AccessAndSecretKey( null );
 			ThreadHelper.resetIOThreads();
 			viewManager.close();
 			IJ.log( "MoBIE closed." );

src/main/java/org/embl/mobie/command/open/OpenCollectionTableCommand.java

@@ -71,6 +71,18 @@ public static enum DataRoot{
 			required = false )
 	public BdvViewingMode bdvViewingMode = BdvViewingMode.ThreeDimensional;
 
+	@Parameter ( label = "( S3 Access Key )",
+			description = "Optional. Access key for a protected S3 bucket.",
+			persist = false,
+			required = false )
+	public String s3AccessKey;
+
+	@Parameter ( label = "( S3 Secret Key )",
+			description = "Optional. Secret key for a protected S3 bucket.",
+			persist = false,
+			required = false )
+	public String s3SecretKey;
+
 
 	@Override
 	public void run()
@@ -97,6 +109,9 @@ public void run()
 				.dataRoot( dataRootString )
 				.bdvViewingMode( bdvViewingMode );
 
+		if ( MoBIEHelper.notNullOrEmpty( s3AccessKey ) )
+			settings.s3AccessAndSecretKey( new String[]{ s3AccessKey, s3SecretKey } );
+
 		try
 		{
 			new MoBIE( MoBIEHelper.toURI( table ), settings );

src/main/java/org/embl/mobie/lib/ImageDataAdder.java

@@ -115,7 +115,7 @@ private void addData( ImageData< ? > imageData, boolean isSegmentation )
 
 	private String getImageName( String setupName, int numImages )
 	{
-		String imageName = FilenameUtils.removeExtension( new File( setupName ).getName() );
+		String imageName = MoBIEHelper.removeExtension( new File( setupName ).getName() );
 		if ( numImages == 1)
 		{
 			imageName = imageName.replaceAll( " channel.*", "" );

src/main/java/org/embl/mobie/lib/SourcesFromTableCreator.java

@@ -29,6 +29,8 @@
 package org.embl.mobie.lib;
 
 import ij.IJ;
+import org.embl.mobie.DataStore;
+import org.embl.mobie.io.ImageDataFormat;
 import org.embl.mobie.io.ImageDataOpener;
 import org.embl.mobie.lib.data.ImageGridSources;
 import org.embl.mobie.lib.data.LabelGridSources;
@@ -74,7 +76,8 @@ public SourcesFromTableCreator( String tablePath, List< String > imageColumns, L
 				String relativeFolder = table.getString( 0, imageColumn.replace(  "FileName_", "PathName_" ) );
 				String referenceImagePath = MoBIEHelper.createAbsolutePath( root, fileName, relativeFolder );
 				IJ.log( "Determining number of channels of \"" + imageColumn + "\" from " + referenceImagePath + "..." );
-				int numChannels = ImageDataOpener.open( referenceImagePath, ThreadHelper.sharedQueue ).getNumDatasets();
+				ImageDataFormat imageDataFormat = ImageDataFormat.fromPath( referenceImagePath );
+				int numChannels = DataStore.fetchImageData( referenceImagePath, imageDataFormat, ThreadHelper.sharedQueue ).getNumDatasets();
 				IJ.log( "Number of channels: " + numChannels );
 				for ( int channelIndex = 0; channelIndex < numChannels; channelIndex++ )
 				{

src/main/java/org/embl/mobie/lib/bdv/view/SliceViewer.java

@@ -217,7 +217,7 @@ public static BdvHandle createBdv( boolean is2D, String frameTitle )
 		sOptions.is2D = is2D;
 		sOptions.frameTitle = frameTitle;
 		sOptions.interpolate = false;
-		IJ.log("BigDataViewer (BDV) initialised.");
+		IJ.log("BigDataViewer (BDV) initialised");
 		IJ.log("BDV navigation mode: " + ( is2D ? "2D" : "3D" ));
 		IJ.log("BDV interpolation: Nearest neighbour");
 		IJ.log("Use [I] keyboard shortcut in BDV window to change the interpolation mode" );

src/main/java/org/embl/mobie/lib/create/ImagesCreator.java

@@ -43,12 +43,15 @@
 import net.imglib2.type.numeric.integer.IntType;
 import net.imglib2.type.numeric.real.FloatType;
 import org.apache.commons.io.FileUtils;
+import org.embl.mobie.MoBIE;
+import org.embl.mobie.io.ImageDataFormat;
 import org.embl.mobie.io.ImageDataOpener;
 import org.embl.mobie.io.OMEZarrWriter;
 import org.embl.mobie.io.imagedata.ImageData;
 import org.embl.mobie.io.util.IOHelper;
 import org.embl.mobie.lib.color.ColorHelper;
 import org.embl.mobie.lib.source.SourceHelper;
+import org.embl.mobie.lib.util.ThreadHelper;
 
 import javax.swing.*;
 import java.io.File;
@@ -286,7 +289,12 @@ public void addOMEZarrImage( String uri,
     {
         File imagesDirectory = new File( getDefaultLocalImageDirPath( datasetName ) );
 
-        ImageData< ? > imageData = ImageDataOpener.open( uri );
+        ImageDataFormat imageDataFormat = ImageDataFormat.fromPath( uri );
+        imageDataFormat.setS3SecretAndAccessKey( MoBIE.getInstance().getSettings().values.getS3AccessAndSecretKey() );
+        ImageData< ? > imageData = ImageDataOpener.open(
+                uri,
+                imageDataFormat,
+                ThreadHelper.sharedQueue );
 
         if ( ! isImageValid( imageData, projectCreator.getVoxelUnit() ) ) {
             return;
@@ -363,7 +371,12 @@ private void addDefaultTableForImage ( String imageName, String datasetName ) {
 
             // xml file or zarr file, depending on imageDataFormat
             String filePath = getDefaultLocalImagePath( datasetName, imageName );
-            ImageData< ? > imageData = ImageDataOpener.open( filePath );
+            ImageDataFormat imageDataFormat = ImageDataFormat.fromPath( filePath );
+            imageDataFormat.setS3SecretAndAccessKey( MoBIE.getInstance().getSettings().values.getS3AccessAndSecretKey() );
+            ImageData< ? > imageData = ImageDataOpener.open(
+                    filePath,
+                    imageDataFormat,
+                    ThreadHelper.sharedQueue );
             final Source< ? > labelsSource = imageData.getSourcePair( 0 ).getA();
 
             boolean hasMultipleTimepoints = labelsSource.isPresent( 1 ); // 0,1,...

src/main/java/org/embl/mobie/lib/create/ProjectCreatorHelper.java

@@ -33,11 +33,13 @@
 import ij.gui.GenericDialog;
 import mpicbg.spim.data.SpimData;
 import net.imglib2.realtransform.AffineTransform3D;
+import org.embl.mobie.io.ImageDataFormat;
 import org.embl.mobie.io.ImageDataOpener;
 import org.embl.mobie.io.imagedata.ImageData;
 import org.embl.mobie.io.util.IOHelper;
 import org.embl.mobie.lib.serialize.Dataset;
 import org.embl.mobie.lib.serialize.View;
+import org.embl.mobie.lib.util.ThreadHelper;
 import org.embl.mobie.lib.view.AdditionalViews;
 import ucar.units.*;
 
@@ -135,7 +137,7 @@ public static File getDataLocation( File projectLocation ) {
 
     public static boolean is2D( String uri )
     {
-        return is2D( ImageDataOpener.open( uri ) );
+        return is2D( ImageDataOpener.open( uri, ImageDataFormat.fromPath( uri ), ThreadHelper.sharedQueue ) );
     }
 
     public static boolean is2D( ImageData< ? > imageData ) {
@@ -289,7 +291,7 @@ public static boolean unitsEqual( String unit1, String unit2 ) throws PrefixDBEx
     //       see also: https://imagesc.zulipchat.com/#narrow/stream/327326-BigDataViewer/topic/N5.20viewer.20support.20for.20different.20voxel.20units.3F
     public static boolean isImageValid( String uri, String projectVoxelUnit )
     {
-        return isImageValid( ImageDataOpener.open( uri ), projectVoxelUnit );
+        return isImageValid( ImageDataOpener.open( uri, ImageDataFormat.fromPath( uri ), ThreadHelper.sharedQueue ), projectVoxelUnit );
     }
 
     public static boolean isImageValid(  ImageData< ? > imageData, String projectVoxelUnit )

src/main/java/org/embl/mobie/lib/create/ui/ProjectsCreatorUI.java

@@ -42,6 +42,8 @@
 import org.embl.mobie.lib.create.ProjectCreatorHelper;
 import org.embl.mobie.lib.serialize.Dataset;
 import org.embl.mobie.lib.serialize.Project;
+import org.embl.mobie.lib.util.MoBIEHelper;
+import org.embl.mobie.lib.util.ThreadHelper;
 import org.embl.mobie.ui.SwingHelper;
 import org.embl.mobie.ui.UserInterfaceHelper;
 
@@ -474,7 +476,7 @@ private void addOMEZarr( String uri, String datasetName )
             return;
         }
 
-        if ( ! is2D( ImageDataOpener.open( uri ) ) && projectCreator.getDataset( datasetName ).is2D() ) {
+        if ( ! is2D( ImageDataOpener.open( uri, ImageDataFormat.fromPath( uri ), ThreadHelper.sharedQueue ) ) && projectCreator.getDataset( datasetName ).is2D() ) {
             if ( ! changeDatasetDimensionDialog (datasetName ) ) {
                 return;
             }
@@ -632,7 +634,7 @@ public void addCurrentImageDialog() {
 
             final GenericDialog gd = new GenericDialog( "Add Current Image To MoBIE Project..." );
             gd.addMessage( "Make sure your voxel size, and unit,\n are set properly under Image > Properties...");
-            gd.addStringField( "Image Name", FilenameUtils.removeExtension(currentImage.getTitle()), 35 );
+            gd.addStringField( "Image Name", MoBIEHelper.removeExtension(currentImage.getTitle()), 35 );
             gd.addChoice( "Image Type", imageTypes, imageType.toString() );
             gd.addCheckbox("Make view exclusive", exclusive );
 

src/main/java/org/embl/mobie/lib/data/CollectionTableDataSetter.java

@@ -19,6 +19,7 @@
 import org.embl.mobie.lib.table.TableDataFormat;
 import org.embl.mobie.lib.table.TableSource;
 import org.embl.mobie.lib.table.columns.CollectionTableConstants;
+import org.embl.mobie.lib.util.MoBIEHelper;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 import tech.tablesaw.api.Row;
@@ -275,7 +276,7 @@ private static String getName( Row row )
     private static String getNameFromURI( Row row )
     {
         String uri = getUri( row );
-        return FilenameUtils.removeExtension( IOHelper.getFileName( uri ) );
+        return MoBIEHelper.removeExtension( IOHelper.getFileName( uri ) );
     }
 
     private static String getPixelType( Row row )

src/main/java/org/embl/mobie/lib/data/ImageGridSources.java

@@ -34,17 +34,19 @@
 import net.imglib2.realtransform.AffineTransform3D;
 import net.imglib2.util.Intervals;
 import org.apache.commons.io.FilenameUtils;
-import org.embl.mobie.io.ImageDataOpener;
+import org.embl.mobie.DataStore;
+import org.embl.mobie.MoBIE;
+import org.embl.mobie.io.ImageDataFormat;
 import org.embl.mobie.io.imagedata.ImageData;
 import org.embl.mobie.io.util.IOHelper;
-import org.embl.mobie.lib.util.MoBIEHelper;
-import org.embl.mobie.lib.util.ThreadHelper;
 import org.embl.mobie.lib.source.Metadata;
 import org.embl.mobie.lib.source.SourceHelper;
-import org.embl.mobie.lib.table.columns.ColumnNames;
 import org.embl.mobie.lib.table.TableDataFormat;
+import org.embl.mobie.lib.table.columns.ColumnNames;
 import org.embl.mobie.lib.table.columns.SegmentColumnNames;
 import org.embl.mobie.lib.transform.GridType;
+import org.embl.mobie.lib.util.MoBIEHelper;
+import org.embl.mobie.lib.util.ThreadHelper;
 import org.janelia.saalfeldlab.n5.universe.metadata.canonical.CanonicalDatasetMetadata;
 import tech.tablesaw.api.NumberColumn;
 import tech.tablesaw.api.StringColumn;
@@ -181,10 +183,16 @@ private void setMetadata( Integer channelIndex )
 		// Take the first source to fetch metadata
 		metadataSource = nameToFullPath.keySet().iterator().next();
 		long start = System.currentTimeMillis();
-		IJ.log( "Fetching metadata for " + name + ", channel " + channelIndex );
-		IJ.log( "Source: " + nameToFullPath.get( metadataSource ) );
-		// FIXME: Cache the image data, because for multiple channels it is now reloaded!
-		ImageData< ? > imageData = ImageDataOpener.open( nameToFullPath.get( metadataSource ), ThreadHelper.sharedQueue );
+		IJ.log( "Fetching metadata for \"" + name + "\", channel " + channelIndex );
+		String uri = nameToFullPath.get( metadataSource );
+		IJ.log( "Source: " + uri );
+		ImageDataFormat imageDataFormat = ImageDataFormat.fromPath( uri );
+		String[] s3AccessAndSecretKey = MoBIE.getInstance().getSettings().values.getS3AccessAndSecretKey();
+		imageDataFormat.setS3SecretAndAccessKey( s3AccessAndSecretKey );
+		ImageData< ? > imageData = DataStore.fetchImageData(
+				uri,
+				imageDataFormat,
+				ThreadHelper.sharedQueue );
 		CanonicalDatasetMetadata canonicalDatasetMetadata = imageData.getMetadata( channelIndex );
 		metadata = new Metadata( canonicalDatasetMetadata );
 		Source< ? > source = imageData.getSourcePair( channelIndex ).getA();
@@ -201,7 +209,7 @@ private void setMetadata( Integer channelIndex )
 			metadata.contrastLimits = SourceHelper.estimateMinMax( ( RandomAccessibleInterval ) lowResRAI );
 
 		IJ.log( "Contrast limits: " + Arrays.toString( metadata.contrastLimits ) );
-		IJ.log( "Fetched metadata in " + ( System.currentTimeMillis() - start ) + " ms." );
+		IJ.log( "Fetched metadata in " + ( System.currentTimeMillis() - start ) + " ms" );
 
 	}
 
@@ -247,7 +255,7 @@ private void dealWithTimepointsInObjectTableIfNeeded( String name, Table table,
 
 	private String createImageName( Integer channelIndex, String fileName )
 	{
-		String imageName = FilenameUtils.removeExtension( fileName );
+		String imageName = MoBIEHelper.removeExtension( fileName );
 
 		if ( channelIndex != null )
 		{

src/main/java/org/embl/mobie/lib/data/LabelGridSources.java

@@ -83,7 +83,6 @@ public LabelGridSources( String name, String labelsPath, Integer channelIndex, S
 		super( name, labelsPath, channelIndex, root, grid );
 	}
 
-
 	public LabelGridSources( String name, String path, Integer channelIndex, String labelTablePath, String root, GridType grid )
 	{
 		super( name, path, channelIndex, root, grid );

src/main/java/org/embl/mobie/lib/hcs/Plate.java

@@ -215,7 +215,7 @@ private void buildPlateMap( List< String > imagePaths )
 					{
 						int datasetIndex = channel.getIndex();
 
-						IJ.log( "Fetching metadata for " + channelName + " from " + imagePath );
+						IJ.log( "Fetching metadata for \"" + channelName + "\" from " + imagePath );
 						Source< ? > source = imageData.getSourcePair( datasetIndex ).getA();
 						int numMipmapLevels = source.getNumMipmapLevels();
 						numSlices = ( int ) source.getSource( 0, 0 ).dimension( 2 );

src/main/java/org/embl/mobie/lib/io/FileImageSource.java

@@ -29,6 +29,7 @@
 package org.embl.mobie.lib.io;
 
 import org.apache.commons.io.FilenameUtils;
+import org.embl.mobie.lib.util.MoBIEHelper;
 
 import java.io.File;
 
@@ -63,7 +64,7 @@ public FileImageSource( String string )
 		}
 		else
 		{
-			name = FilenameUtils.removeExtension( new File( split[ 0 ] ).getName() );
+			name = MoBIEHelper.removeExtension( new File( split[ 0 ] ).getName() );
 			path = split[ 0 ];
 		}
 	}