Merge pull request #6150 from johnhaddon/onnx

GafferML

.github/workflows/main.yml

@@ -66,7 +66,7 @@ jobs:
             publish: true
             containerImage:
             testRunner: Invoke-Expression
-            testArguments: -excludedCategories performance GafferTest GafferVDBTest GafferUSDTest GafferSceneTest GafferDispatchTest GafferOSLTest GafferImageTest GafferUITest GafferImageUITest GafferSceneUITest GafferDispatchUITest GafferOSLUITest GafferUSDUITest GafferVDBUITest GafferDelightUITest GafferTractorTest GafferTractorUITest
+            testArguments: -excludedCategories performance GafferTest GafferVDBTest GafferUSDTest GafferSceneTest GafferDispatchTest GafferOSLTest GafferImageTest GafferUITest GafferImageUITest GafferSceneUITest GafferDispatchUITest GafferOSLUITest GafferUSDUITest GafferVDBUITest GafferDelightUITest GafferTractorTest GafferTractorUITest GafferMLTest GafferMLUITest
             sconsCacheMegabytes: 800
             jobs: 4
 
@@ -139,6 +139,8 @@ jobs:
         echo GAFFER_DEPENDENCIES_HASH=`python .github/workflows/main/installDependencies.py ${{ matrix.dependenciesURL != '' && format( '--archiveURL {0}', matrix.dependenciesURL ) || '' }} --dependenciesDir ${{ env.GAFFER_BUILD_DIR }} --outputFormat "{archiveDigest}"` >> $GITHUB_ENV
         ./.github/workflows/main/installDelight.py
         echo DELIGHT=$GITHUB_WORKSPACE/3delight >> $GITHUB_ENV
+        ./.github/workflows/main/installONNX.py
+        echo ONNX_ROOT=$GITHUB_WORKSPACE/onnxruntime >> $GITHUB_ENV
       shell: bash
 
     - name: Install Mesa (Windows)

.github/workflows/main/installONNX.py

@@ -0,0 +1,63 @@
+#!/usr/bin/env python3
+##########################################################################
+#
+#  Copyright (c) 2024, Cinesite VFX Ltd. All rights reserved.
+#
+#  Redistribution and use in source and binary forms, with or without
+#  modification, are permitted provided that the following conditions are
+#  met:
+#
+#	 * Redistributions of source code must retain the above copyright
+#	   notice, this list of conditions and the following disclaimer.
+#
+#	 * Redistributions in binary form must reproduce the above copyright
+#	   notice, this list of conditions and the following disclaimer in the
+#	   documentation and/or other materials provided with the distribution.
+#
+#	 * Neither the name of Image Engine Design nor the names of any
+#	   other contributors to this software may be used to endorse or
+#	   promote products derived from this software without specific prior
+#	   written permission.
+#
+#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+#  IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+#  THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+#  PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+#  CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+#  EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+#  PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+#  PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+#  LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+#  NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+#  SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+##########################################################################
+
+import pathlib
+import sys
+import shutil
+import subprocess
+from urllib.request import urlretrieve
+
+version = "1.19.2"
+
+if sys.platform == "linux" :
+	url = f"https://github.com/microsoft/onnxruntime/releases/download/v{version}/onnxruntime-linux-x64-{version}.tgz"
+elif sys.platform == "darwin" :
+	url = f"https://github.com/microsoft/onnxruntime/releases/download/v{version}/onnxruntime-osx-arm64-{version}.tgz"
+elif sys.platform == "win32" :
+	url = f"https://github.com/microsoft/onnxruntime/releases/download/v{version}/onnxruntime-win-x64-{version}.zip"
+
+print( "Downloading ONNX \"{}\"".format( url ) )
+archiveFileName, headers = urlretrieve( url )
+
+if sys.platform in ( "linux", "darwin" ) :
+	subprocess.check_call(
+		[ "tar", "-xf", archiveFileName ]
+	)
+else :
+	subprocess.check_call(
+		[ "7z", "x", archiveFileName, "-o./", "-y" ]
+	)
+
+shutil.move( pathlib.Path( url ).stem, "onnxruntime" )

.github/workflows/main/sconsOptions

@@ -45,6 +45,7 @@ BUILD_CACHEDIR = os.environ["GAFFER_CACHE_DIR"]
 
 ARNOLD_ROOT = os.environ.get( "ARNOLD_ROOT", "" )
 DELIGHT_ROOT = os.environ["DELIGHT"]
+ONNX_ROOT = os.environ["ONNX_ROOT"]
 
 BUILD_DIR = os.environ["GAFFER_BUILD_DIR"]
 INSTALL_DIR = os.path.join( "install", os.environ["GAFFER_BUILD_NAME"] )

Changes.md

@@ -1,7 +1,21 @@
 1.5.x.x (relative to 1.5.1.0)
 =======
 
+> Caution : The GafferML features introduced in this release are considered experimental, and are not subject to the usual backwards compatibility guarantees that apply to the rest of Gaffer.
 
+Features
+--------
+
+- GafferML : Added a new module with the following nodes for running maching learning models via ONNX Runtime :
+  - DataToTensor : Converts Gaffer data to tensors.
+  - Inference : Loads ONNX models and performance inference using an array of input tensors.
+  - ImageToTensor : Converts images to tensors for use with the Inference node.
+  - TensorToImage : Converts tensors back to images following inference.
+
+API
+---
+
+- PlugLayout : Activations may now depend on the presence of certain plugs, as they are now reevaluated when child plugs are added and removed.
 
 1.5.1.0 (relative to 1.5.0.1)
 =======

SConstruct

@@ -307,6 +307,12 @@ options.Add(
 	)
 )
 
+options.Add(
+	"ONNX_ROOT",
+	"The directory in which the ONNX runtime is installed. Used to build GafferML",
+	"",
+)
+
 # general variables
 
 options.Add(
@@ -771,7 +777,7 @@ commandEnv["ENV"]["PYTHONPATH"] = commandEnv.subst( os.path.pathsep.join( [ "$BU
 # SIP on MacOS prevents DYLD_LIBRARY_PATH being passed down so we make sure
 # we also pass through to gaffer the other base vars it uses to populate paths
 # for third-party support.
-for v in ( 'ARNOLD_ROOT', 'DELIGHT_ROOT' ) :
+for v in ( 'ARNOLD_ROOT', 'DELIGHT_ROOT', 'ONNX_ROOT' ) :
 	commandEnv["ENV"][ v ] = commandEnv[ v ]
 
 def runCommand( command ) :
@@ -1109,6 +1115,33 @@ libraries = {
 		},
 	},
 
+	"GafferML" : {
+		"envAppends" : {
+			"CPPPATH" : [ "$ONNX_ROOT/include" ],
+			"LIBPATH" : [ "$ONNX_ROOT/lib" ],
+			"LIBS" : [ "Gaffer", "GafferImage", "onnxruntime" ],
+		},
+		"pythonEnvAppends" : {
+			"CPPPATH" : [ "$ONNX_ROOT/include" ],
+			"LIBPATH" : [ "$ONNX_ROOT/lib" ],
+			"LIBS" : [ "GafferBindings", "GafferImage", "GafferML", "onnxruntime" ],
+		},
+		"requiredOptions" : [ "ONNX_ROOT" ],
+	},
+
+	"GafferMLTest" : {
+		"requiredOptions" : [ "ONNX_ROOT" ],
+		"additionalFiles" : glob.glob( "python/GafferMLTest/models/*" )
+	},
+
+	"GafferMLUI" : {
+		"requiredOptions" : [ "ONNX_ROOT" ],
+	},
+
+	"GafferMLUITest" : {
+		"requiredOptions" : [ "ONNX_ROOT" ],
+	},
+
 	"IECoreArnold" : {
 		"envAppends" : {
 			"LIBPATH" : [ "$ARNOLD_ROOT/bin" ] if env["PLATFORM"] != "win32" else [ "$ARNOLD_ROOT/bin", "$ARNOLD_ROOT/lib" ],

bin/gaffer

@@ -305,6 +305,19 @@ if [[ -n $DELIGHT ]] ; then
 
 fi
 
+# Set up ONNX
+##########################################################################
+
+if [[ -n $ONNX_ROOT ]] ; then
+
+	if [[ `uname` = "Linux" ]] ; then
+		appendToPath "$ONNX_ROOT/lib" LD_LIBRARY_PATH
+	else
+		appendToPath "$ONNX_ROOT/lib" DYLD_LIBRARY_PATH
+	fi
+
+fi
+
 # Set up 3rd Party extensions
 ##########################################################################
 

bin/gaffer.cmd

@@ -140,6 +140,12 @@ if "%CYCLES_ROOT%" NEQ "" (
 	call :prependToPath "%CYCLES_ROOT%\bin" PATH
 )
 
+rem ONNX
+rem ====
+
+if "%ONNX_ROOT%" NEQ "" (
+	call :appendToPath "%ONNX_ROOT%\lib" PATH
+)
 
 rem Set up 3rd party extensions
 rem Batch files are awkward at `for` loops. The default `for`, without `/f`

config/jh/options

@@ -50,3 +50,5 @@ DELIGHT_ROOT = os.environ["DELIGHT"]
 ARNOLD_ROOT = os.environ["ARNOLD_ROOT"]
 VTUNE_ROOT = "/disk1/apps/intel/system_studio_2018/vtune_amplifier_2018.1.0.535340"
 GAFFERCORTEX=1
+
+ONNX_ROOT = os.environ["ONNX_ROOT"]

include/GafferML/DataToTensor.h

@@ -0,0 +1,102 @@
+//////////////////////////////////////////////////////////////////////////
+//
+//  Copyright (c) 2024, Cinesite VFX Ltd. All rights reserved.
+//
+//  Redistribution and use in source and binary forms, with or without
+//  modification, are permitted provided that the following conditions are
+//  met:
+//
+//      * Redistributions of source code must retain the above
+//        copyright notice, this list of conditions and the following
+//        disclaimer.
+//
+//      * Redistributions in binary form must reproduce the above
+//        copyright notice, this list of conditions and the following
+//        disclaimer in the documentation and/or other materials provided with
+//        the distribution.
+//
+//      * Neither the name of John Haddon nor the names of
+//        any other contributors to this software may be used to endorse or
+//        promote products derived from this software without specific prior
+//        written permission.
+//
+//  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+//  IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+//  THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+//  PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+//  CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+//  EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+//  PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+//  PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+//  LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+//  NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+//  SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+//
+//////////////////////////////////////////////////////////////////////////
+
+#pragma once
+
+#include "GafferML/Export.h"
+#include "GafferML/TensorPlug.h"
+
+#include "Gaffer/ComputeNode.h"
+#include "Gaffer/NumericPlug.h"
+#include "Gaffer/TypedObjectPlug.h"
+
+namespace GafferML
+{
+
+class GAFFERML_API DataToTensor : public Gaffer::ComputeNode
+{
+
+	public :
+
+		explicit DataToTensor( const std::string &name=defaultName<DataToTensor>() );
+		~DataToTensor() override;
+
+		GAFFER_NODE_DECLARE_TYPE( GafferML::DataToTensor, DataToTensorTypeId, Gaffer::ComputeNode );
+
+		enum class ShapeMode
+		{
+			Automatic,
+			Custom
+		};
+
+		bool canSetup( const Gaffer::ValuePlug *prototypeDataPlug );
+		void setup( const Gaffer::ValuePlug *prototypeDataPlug );
+
+		template<typename T = Gaffer::ValuePlug>
+		T *dataPlug();
+		template<typename T = Gaffer::ValuePlug>
+		const T *dataPlug() const;
+
+		Gaffer::IntPlug *shapeModePlug();
+		const Gaffer::IntPlug *shapeModePlug() const;
+
+		Gaffer::Int64VectorDataPlug *shapePlug();
+		const Gaffer::Int64VectorDataPlug *shapePlug() const;
+
+		TensorPlug *tensorPlug();
+		const TensorPlug *tensorPlug() const;
+
+		void affects( const Gaffer::Plug *input, AffectedPlugsContainer &outputs ) const override;
+
+	protected :
+
+		void hash( const Gaffer::ValuePlug *output, const Gaffer::Context *context, IECore::MurmurHash &h ) const override;
+		void compute( Gaffer::ValuePlug *output, const Gaffer::Context *context ) const override;
+
+		Gaffer::ValuePlug::CachePolicy computeCachePolicy( const Gaffer::ValuePlug *output ) const override;
+
+	private :
+
+		static size_t g_firstPlugIndex;
+		static const IECore::InternedString g_dataPlugName;
+
+};
+
+IE_CORE_DECLAREPTR( DataToTensor )
+
+} // namespace GafferML
+
+#include "GafferML/DataToTensor.inl"

include/GafferML/DataToTensor.inl

@@ -0,0 +1,54 @@
+//////////////////////////////////////////////////////////////////////////
+//
+//  Copyright (c) 2024, Cinesite VFX Ltd. All rights reserved.
+//
+//  Redistribution and use in source and binary forms, with or without
+//  modification, are permitted provided that the following conditions are
+//  met:
+//
+//      * Redistributions of source code must retain the above
+//        copyright notice, this list of conditions and the following
+//        disclaimer.
+//
+//      * Redistributions in binary form must reproduce the above
+//        copyright notice, this list of conditions and the following
+//        disclaimer in the documentation and/or other materials provided with
+//        the distribution.
+//
+//      * Neither the name of John Haddon nor the names of
+//        any other contributors to this software may be used to endorse or
+//        promote products derived from this software without specific prior
+//        written permission.
+//
+//  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+//  IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+//  THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+//  PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+//  CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+//  EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+//  PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+//  PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+//  LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+//  NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+//  SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+//
+//////////////////////////////////////////////////////////////////////////
+
+#pragma once
+
+namespace GafferML
+{
+
+template<typename T>
+T *DataToTensor::dataPlug()
+{
+	return getChild<T>( g_dataPlugName );
+}
+
+template<typename T>
+const T *DataToTensor::dataPlug() const
+{
+	return getChild<T>( g_dataPlugName );
+}
+
+} // namespace GafferML