diff --git a/configure.ac b/configure.ac index 8655c153..0d1b1498 100644 --- a/configure.ac +++ b/configure.ac @@ -1,4 +1,4 @@ -# Copyright (C) 2018 RidgeRun, LLC (http://www.ridgerun.com) +# Copyright (C) 2018-2020 RidgeRun, LLC (http://www.ridgerun.com) # All Rights Reserved. # # The contents of this software are proprietary and confidential to RidgeRun, @@ -10,7 +10,7 @@ # Initialize autoconf. AC_PREREQ([2.69]) -AC_INIT([RidgeRun inference library],[0.7.0],[https://github.com/RidgeRun/r2inference/issues],[r2inference]) +AC_INIT([RidgeRun inference library],[0.8.0],[https://github.com/RidgeRun/r2inference/issues],[r2inference]) # Initialize our build utils RR_INIT @@ -63,6 +63,9 @@ AM_COND_IF([HAVE_EDGETPU], [AM_COND_IF([HAVE_TFLITE], [], [AC_MSG_ERROR(The Edge RR_CHECK_FEATURE_LIB(TENSORRT, TensorRT Installation, nvinfer, createInferBuilder_INTERNAL, NvInfer.h, no) + +RR_CHECK_FEATURE_LIB(ONNXRT, ONNX Runtime Installation, + onnxruntime, OrtGetApiBase, onnxruntime/core/session/onnxruntime_cxx_api.h, no) AC_LANG_POP([C++]) AM_CONDITIONAL([PLATFORM_IS_GPU], [false]) @@ -81,12 +84,14 @@ examples/Makefile examples/r2i/Makefile examples/r2i/edgetpu/Makefile examples/r2i/ncsdk/Makefile +examples/r2i/onnxrt/Makefile examples/r2i/tensorflow/Makefile examples/r2i/tensorrt/Makefile examples/r2i/tflite/Makefile r2i/Makefile r2i/edgetpu/Makefile r2i/ncsdk/Makefile +r2i/onnxrt/Makefile r2i/tensorflow/Makefile r2i/tensorrt/Makefile r2i/tflite/Makefile @@ -96,6 +101,7 @@ tests/acceptance/Makefile tests/unit/Makefile tests/unit/r2i/Makefile tests/unit/r2i/ncsdk/Makefile +tests/unit/r2i/onnxrt/Makefile tests/unit/r2i/tensorflow/Makefile tests/unit/r2i/tensorrt/Makefile tests/unit/r2i/tflite/Makefile diff --git a/examples/external/list_backends.cc b/examples/external/list_backends.cc index c45ea693..0babcc78 100644 --- a/examples/external/list_backends.cc +++ b/examples/external/list_backends.cc @@ -16,6 +16,7 @@ void PrintFramework (r2i::FrameworkMeta &meta) { std::cout << "Name : " << meta.name << std::endl; std::cout << "Description : " << meta.description << std::endl; std::cout << "Version : " << meta.version << std::endl; + std::cout << "Label : " << meta.label << std::endl; std::cout << "---" << std::endl; } diff --git a/examples/r2i/Makefile.am b/examples/r2i/Makefile.am index 6d645ae9..bc4de473 100644 --- a/examples/r2i/Makefile.am +++ b/examples/r2i/Makefile.am @@ -27,6 +27,10 @@ if HAVE_NCSDK SUBDIRS += ncsdk endif +if HAVE_ONNXRT +SUBDIRS += onnxrt +endif + if HAVE_TENSORFLOW SUBDIRS += tensorflow endif diff --git a/examples/r2i/meson.build b/examples/r2i/meson.build index ef662b4b..1101b285 100644 --- a/examples/r2i/meson.build +++ b/examples/r2i/meson.build @@ -17,6 +17,10 @@ if get_option('enable-edgetpu') subdir('edgetpu') endif +if get_option('enable-onnxrt') + subdir('onnxrt') +endif + if get_option('enable-tensorflow') subdir('tensorflow') endif diff --git a/examples/r2i/onnxrt/Makefile.am b/examples/r2i/onnxrt/Makefile.am new file mode 100644 index 00000000..b1a18661 --- /dev/null +++ b/examples/r2i/onnxrt/Makefile.am @@ -0,0 +1,36 @@ +# Copyright (C) 2020 RidgeRun, LLC (http://www.ridgerun.com) +# All Rights Reserved. +# +# The contents of this software are proprietary and confidential to RidgeRun, +# LLC. No part of this program may be photocopied, reproduced or translated +# into another programming language without prior written consent of +# RidgeRun, LLC. The user is free to modify the source code after obtaining +# a software license from RidgeRun. All source code changes must be provided +# back to RidgeRun without any encumbrance. + +AM_DEFAULT_SOURCE_EXT = .cc + +if ENABLE_EXAMPLES + +noinst_PROGRAMS = \ + inception + +AM_CXXFLAGS = \ + $(RR_CXXFLAGS) \ + $(CODE_COVERAGE_CXXFLAGS) \ + -I../common/ + +AM_CFLAGS = \ + $(RR_CFLAGS) \ + $(CODE_COVERAGE_CFLAGS) + +AM_CPPFLAGS = \ + $(RR_CPPFLAGS) \ + $(CODE_COVERAGE_CPPFLAGS) + +AM_LDFLAGS = \ + $(RR_LIBS) \ + $(CODE_COVERAGE_LIBS) \ + $(top_builddir)/r2i/libr2inference-@RR_PACKAGE_VERSION@.la + +endif # ENABLE_EXAMPLES diff --git a/examples/r2i/onnxrt/inception.cc b/examples/r2i/onnxrt/inception.cc new file mode 100644 index 00000000..5eeda9fc --- /dev/null +++ b/examples/r2i/onnxrt/inception.cc @@ -0,0 +1,215 @@ +/* Copyright (C) 2018-2020 RidgeRun, LLC (http://www.ridgerun.com) + * All Rights Reserved. + * + * The contents of this software are proprietary and confidential to RidgeRun, + * LLC. No part of this program may be photocopied, reproduced or translated + * into another programming language without prior written consent of + * RidgeRun, LLC. The user is free to modify the source code after obtaining + * a software license from RidgeRun. All source code changes must be provided + * back to RidgeRun without any encumbrance. +*/ + +#include + +#include +#include +#include +#include + +#define STB_IMAGE_IMPLEMENTATION +#include "stb_image.h" + +#define STB_IMAGE_RESIZE_IMPLEMENTATION +#include "stb_image_resize.h" + +void PrintTopPrediction (std::shared_ptr prediction) { + r2i::RuntimeError error; + int index = 0; + double max = -1; + int num_labels = prediction->GetResultSize(); + + for (int i = 0; i < num_labels; ++i) { + double current = prediction->At(i, error); + if (current > max) { + max = current; + index = i; + } + } + + std::cout << "Highest probability is label " + << index << " (" << max << ")" << std::endl; +} + +void PrintUsage() { + std::cerr << "Required arguments: " + << "-i [JPG input_image] " + << "-m [Inception ONNX Model] " + << "-s [Model Input Size] " + << "-I [Input Node] " + << "-O [Output Node] \n" + << " Example: " + << " ./inception -i cat.jpg -m graph_inceptionv2_tensorflow.pb " + << "-s 224" + << std::endl; +} + +std::unique_ptr PreProcessImage (const unsigned char *input, + int width, int height, int reqwidth, int reqheight) { + + const int channels = 3; + const int scaled_size = channels * reqwidth * reqheight; + std::unique_ptr scaled (new unsigned char[scaled_size]); + std::unique_ptr adjusted (new float[scaled_size]); + + stbir_resize_uint8(input, width, height, 0, scaled.get(), reqwidth, + reqheight, 0, channels); + + for (int i = 0; i < scaled_size; i += channels) { + /* RGB = (RGB - Mean)*StdDev */ + adjusted[i + 0] = (static_cast(scaled[i + 0]) - 128) / 128.0; + adjusted[i + 1] = (static_cast(scaled[i + 1]) - 128) / 128.0; + adjusted[i + 2] = (static_cast(scaled[i + 2]) - 128) / 128.0; + } + + return adjusted; +} + +std::unique_ptr LoadImage(const std::string &path, int reqwidth, + int reqheight) { + int channels = 3; + int width, height, cp; + + unsigned char *img = stbi_load(path.c_str(), &width, &height, &cp, channels); + if (!img) { + std::cerr << "The picture " << path << " could not be loaded"; + return nullptr; + } + + auto ret = PreProcessImage(img, width, height, reqwidth, reqheight); + free (img); + + return ret; +} + +bool ParseArgs (int &argc, char *argv[], std::string &image_path, + std::string &model_path, int &index, int &size, + std::string &in_node, std::string &out_node) { + int option = 0; + while ((option = getopt(argc, argv, "i:m:p:s:I:O:")) != -1) { + switch (option) { + case 'i' : + image_path = optarg; + break; + case 'm' : + model_path = optarg; + break; + case 'p' : + index = std::stoi (optarg); + break; + case 's' : + size = std::stoi (optarg); + break; + case 'I' : + in_node = optarg; + break; + case 'O' : + out_node = optarg; + break; + default: + return false; + } + } + return true; +} + +int main (int argc, char *argv[]) { + r2i::RuntimeError error; + std::string model_path; + std::string image_path; + std::string in_node; + std::string out_node; + int Index = 0; + int size = 0; + + if (false == ParseArgs (argc, argv, image_path, model_path, Index, + size, in_node, out_node)) { + PrintUsage (); + exit (EXIT_FAILURE); + } + + if (image_path.empty() || model_path.empty ()) { + PrintUsage (); + exit (EXIT_FAILURE); + } + + auto factory = r2i::IFrameworkFactory::MakeFactory( + r2i::FrameworkCode::ONNXRT, + error); + + if (nullptr == factory) { + std::cerr << "TensorFlow backend is not built: " << error << std::endl; + exit(EXIT_FAILURE); + } + + std::cout << "Loading Model: " << model_path << std::endl; + auto loader = factory->MakeLoader (error); + std::shared_ptr model = loader->Load (model_path, error); + if (error.IsError ()) { + std::cerr << "Loader error: " << error << std::endl; + exit(EXIT_FAILURE); + } + + std::cout << "Setting model to engine" << std::endl; + std::shared_ptr engine = factory->MakeEngine (error); + error = engine->SetModel (model); + + std::cout << "Configuring ONNXRT session parameters" << std::endl; + auto params = factory->MakeParameters (error); + error = params->Configure(engine, model); + /* Set OrtLoggingLevel::ORT_LOGGING_LEVEL_WARNING */ + error = params->Set ("logging-level", 2); + error = params->Set ("log-id", "onnxrt_example"); + error = params->Set ("intra-num-threads", 1); + /* Set GraphOptimizationLevel::ORT_ENABLE_EXTENDED */ + error = params->Set ("graph-optimization-level", 2); + + if (error.IsError ()) { + std::cerr << "Parameters error: " << error << std::endl; + exit(EXIT_FAILURE); + } + + std::cout << "Loading image: " << image_path << std::endl; + std::unique_ptr image_data = LoadImage (image_path, size, + size); + + std::cout << "Configuring frame" << std::endl; + std::shared_ptr frame = factory->MakeFrame (error); + + error = frame->Configure (image_data.get(), size, size, + r2i::ImageFormat::Id::RGB); + + std::cout << "Starting engine" << std::endl; + error = engine->Start (); + if (error.IsError ()) { + std::cerr << "Engine start error: " << error << std::endl; + exit(EXIT_FAILURE); + } + + std::cout << "Predicting..." << std::endl; + auto prediction = engine->Predict (frame, error); + if (error.IsError ()) { + std::cerr << "Engine prediction error: " << error << std::endl; + exit(EXIT_FAILURE); + } + + PrintTopPrediction (prediction); + + std::cout << "Stopping engine" << std::endl; + error = engine->Stop (); + if (error.IsError ()) { + std::cerr << "Engine stop error: " << error << std::endl; + exit(EXIT_FAILURE); + } + + return EXIT_SUCCESS; +} diff --git a/examples/r2i/onnxrt/meson.build b/examples/r2i/onnxrt/meson.build new file mode 100644 index 00000000..b568b5f2 --- /dev/null +++ b/examples/r2i/onnxrt/meson.build @@ -0,0 +1,11 @@ +# Compile examples +app_examples = [ + 'inception', +] + +foreach app : app_examples + executable(app, '@0@.cc'.format(app), + include_directories: [configinc, common_inc_dir], + dependencies : [r2inference_lib_dep], + install: false) +endforeach diff --git a/meson.build b/meson.build index 4ee72e87..69a4ca6d 100644 --- a/meson.build +++ b/meson.build @@ -1,5 +1,5 @@ project('r2inference', ['cpp'], default_options : ['cpp_std=c++11'], - version : '0.7.0', + version : '0.8.0', meson_version : '>= 0.50',) # Set project information @@ -88,8 +88,17 @@ if get_option('enable-tensorrt') cdata.set('HAVE_TENSORRT', 1) endif +# Define library dependencies for ONNX support +if get_option('enable-onnxrt') + onnxrt = cpp.find_library('onnxruntime', required: true) + onnxrt_dep = declare_dependency(dependencies: onnxrt) + lib_onnxrt_dep = [onnxrt_dep] + cdata.set('HAVE_ONNXRT', 1) +endif + # Check if at least one backend is enabled -if not (cdata.has('HAVE_TENSORFLOW') or cdata.has('HAVE_TFLITE') or cdata.has('HAVE_TENSORRT')) +if not (cdata.has('HAVE_TENSORFLOW') or cdata.has('HAVE_TFLITE') or + cdata.has('HAVE_TENSORRT') or cdata.has('HAVE_ONNXRT')) error ('No backend selected, you must choose at least one') endif diff --git a/meson_options.txt b/meson_options.txt index c9fd6c80..8f42e424 100644 --- a/meson_options.txt +++ b/meson_options.txt @@ -14,3 +14,5 @@ option('enable-tflite', type : 'boolean', value: false, description : 'Enable Tensorflow Lite backend support') option('enable-tensorrt', type : 'boolean', value: false, description : 'Enable TensorRT backend support') +option('enable-onnxrt', type : 'boolean', value: false, + description : 'Enable ONNX Runtime backend support') diff --git a/r2i/Makefile.am b/r2i/Makefile.am index d275106e..25787f02 100644 --- a/r2i/Makefile.am +++ b/r2i/Makefile.am @@ -1,4 +1,4 @@ -# Copyright (C) 2018 RidgeRun, LLC (http://www.ridgerun.com) +# Copyright (C) 2018-2020 RidgeRun, LLC (http://www.ridgerun.com) # All Rights Reserved. # # The contents of this software are proprietary and confidential to RidgeRun, @@ -18,6 +18,10 @@ if HAVE_NCSDK SUBDIRS += ncsdk endif +if HAVE_ONNXRT +SUBDIRS += onnxrt +endif + if HAVE_TENSORFLOW SUBDIRS += tensorflow endif @@ -33,6 +37,7 @@ endif DIST_SUBDIRS = \ edgetpu \ ncsdk \ + onnxrt \ tensorflow \ tensorrt \ tflite @@ -91,6 +96,11 @@ libr2inference_@RR_PACKAGE_VERSION@_la_LIBADD += \ $(top_builddir)/r2i/ncsdk/libncsdk.la endif +if HAVE_ONNXRT +libr2inference_@RR_PACKAGE_VERSION@_la_LIBADD += \ + $(top_builddir)/r2i/onnxrt/libonnxrt.la +endif + if HAVE_TENSORFLOW libr2inference_@RR_PACKAGE_VERSION@_la_LIBADD += \ $(top_builddir)/r2i/tensorflow/libtf.la diff --git a/r2i/edgetpu/frameworkfactory.cc b/r2i/edgetpu/frameworkfactory.cc index c08c4f68..90220401 100644 --- a/r2i/edgetpu/frameworkfactory.cc +++ b/r2i/edgetpu/frameworkfactory.cc @@ -28,7 +28,8 @@ r2i::FrameworkMeta FrameworkFactory::GetDescription ( RuntimeError &error) { const FrameworkMeta meta { .code = r2i::FrameworkCode::EDGETPU, - .name = "edgetpu", + .name = "EdgeTPU", + .label = "edgetpu", .description = "Google's TensorFlow Lite with EdgeTPU support", .version = ::edgetpu::EdgeTpuManager::GetSingleton()->Version() }; diff --git a/r2i/frameworkmeta.h b/r2i/frameworkmeta.h index 02d41442..542a0abe 100644 --- a/r2i/frameworkmeta.h +++ b/r2i/frameworkmeta.h @@ -35,6 +35,11 @@ struct FrameworkMeta { */ const std::string name; + /** + * A short string to identify the framework + */ + const std::string label; + /** * A string with a description for the framework */ diff --git a/r2i/frameworks.h b/r2i/frameworks.h index 3000e275..3ff627b5 100644 --- a/r2i/frameworks.h +++ b/r2i/frameworks.h @@ -14,11 +14,11 @@ namespace r2i { /** - * Numerical codes identifying supported frameworks. Not that not + * Numerical codes identifying supported frameworks. Note that not * all frameworks will be available at runtime. For example, some of * them may be disabled at configure by the user, automatically if * no development where found or if the system doesn't seem to have - * to appropriate hardware. + * the appropriate hardware. */ enum FrameworkCode { @@ -32,6 +32,11 @@ enum FrameworkCode { */ NCSDK, + /** + * Open Neural Network Exchange Runtime + */ + ONNXRT, + /** * Google's TensorFlow */ diff --git a/r2i/iframeworkfactory.cc b/r2i/iframeworkfactory.cc index 7f330b3a..10c47545 100644 --- a/r2i/iframeworkfactory.cc +++ b/r2i/iframeworkfactory.cc @@ -17,6 +17,7 @@ #include "config.h" #include "edgetpu/frameworkfactory.h" #include "ncsdk/frameworkfactory.h" +#include "onnxrt/frameworkfactory.h" #include "tensorflow/frameworkfactory.h" #include "tensorrt/frameworkfactory.h" #include "tflite/frameworkfactory.h" @@ -38,6 +39,15 @@ MakeNcsdkFactory (RuntimeError &error) { } #endif // HAVE_NCSDK +#ifdef HAVE_ONNXRT +static std::unique_ptr +MakeOnnxrtFactory(RuntimeError &error) { + return std::unique_ptr(new + onnxrt::FrameworkFactory); +} +#endif // HAVE_ONNXRT + + #ifdef HAVE_TENSORFLOW static std::unique_ptr MakeTensorflowFactory (RuntimeError &error) { @@ -74,6 +84,10 @@ const std::unordered_map frameworks ({ {FrameworkCode::NCSDK, MakeNcsdkFactory}, #endif //HAVE_NCSDK +#ifdef HAVE_ONNXRT + {FrameworkCode::ONNXRT, MakeOnnxrtFactory}, +#endif //HAVE_ONNXRT + #ifdef HAVE_TENSORFLOW {FrameworkCode::TENSORFLOW, MakeTensorflowFactory}, #endif //HAVE_TENSORFLOW diff --git a/r2i/meson.build b/r2i/meson.build index ac559e42..6de3384d 100644 --- a/r2i/meson.build +++ b/r2i/meson.build @@ -20,6 +20,11 @@ if get_option('enable-tensorrt') r2inference_internal_dep += [internal_trt_dep] endif +if get_option('enable-onnxrt') + subdir('onnxrt') + r2inference_internal_dep += [internal_onnxrt_dep] +endif + # Define source code r2inference_sources = [ 'datatype.cc', diff --git a/r2i/ncsdk/frameworkfactory.cc b/r2i/ncsdk/frameworkfactory.cc index 93e0e847..b944767d 100644 --- a/r2i/ncsdk/frameworkfactory.cc +++ b/r2i/ncsdk/frameworkfactory.cc @@ -51,6 +51,7 @@ r2i::FrameworkMeta FrameworkFactory::GetDescription (RuntimeError &error) { const FrameworkMeta meta { .code = r2i::FrameworkCode::NCSDK, .name = "NCSDK", + .label = "ncsdk", .description = "Intel Movidius Neural Compute software developer kit", .version = "2" }; diff --git a/r2i/onnxrt/Makefile.am b/r2i/onnxrt/Makefile.am new file mode 100644 index 00000000..6523f202 --- /dev/null +++ b/r2i/onnxrt/Makefile.am @@ -0,0 +1,55 @@ +# Copyright (C) 2020 RidgeRun, LLC (http://www.ridgerun.com) +# All Rights Reserved. +# +# The contents of this software are proprietary and confidential to RidgeRun, +# LLC. No part of this program may be photocopied, reproduced or translated +# into another programming language without prior written consent of +# RidgeRun, LLC. The user is free to modify the source code after obtaining +# a software license from RidgeRun. All source code changes must be provided +# back to RidgeRun without any encumbrance. + +if HAVE_ONNXRT + +noinst_LTLIBRARIES = libonnxrt.la + +# Define a custom area for our headers +onnxrtincludedir = @R2IINCLUDEDIR@/r2i/onnxrt + +libonnxrt_la_SOURCES = \ + engine.cc \ + frame.cc \ + frameworkfactory.cc \ + loader.cc \ + model.cc \ + parameters.cc \ + prediction.cc + +libonnxrt_la_CPPFLAGS = \ + $(RR_CPPFLAGS) \ + $(CODE_COVERAGE_CPPFLAGS) + +libonnxrt_la_CFLAGS = \ + $(RR_CFLAGS) \ + $(ONNXRT_CFLAGS) \ + $(CODE_COVERAGE_CFLAGS) + +libonnxrt_la_CXXFLAGS = \ + $(RR_CXXFLAGS) \ + $(ONNXRT_CFLAGS) \ + $(CODE_COVERAGE_CXXFLAGS) + +libonnxrt_la_LIBADD = \ + $(RR_LIBS) \ + $(ONNXRT_LIBS) \ + $(CODE_COVERAGE_LIBS) + +onnxrtinclude_HEADERS = \ + engine.h \ + frame.h \ + frameworkfactory.h \ + loader.h \ + model.h \ + parameters.h \ + prediction.h + +endif # HAVE_ONNXRT diff --git a/r2i/onnxrt/engine.cc b/r2i/onnxrt/engine.cc new file mode 100644 index 00000000..b876a8c8 --- /dev/null +++ b/r2i/onnxrt/engine.cc @@ -0,0 +1,515 @@ +/* Copyright (C) 2020 RidgeRun, LLC (http://www.ridgerun.com) + * All Rights Reserved. + * + * The contents of this software are proprietary and confidential to RidgeRun, + * LLC. No part of this program may be photocopied, reproduced or translated + * into another programming language without prior written consent of + * RidgeRun, LLC. The user is free to modify the source code after obtaining + * a software license from RidgeRun. All source code changes must be provided + * back to RidgeRun without any encumbrance. + */ + +#include "r2i/onnxrt/engine.h" + +#include +#include + +#include +#include +#include + +#include "r2i/onnxrt/frame.h" +#include "r2i/onnxrt/model.h" +#include "r2i/onnxrt/prediction.h" + +namespace r2i { +namespace onnxrt { + +Engine::Engine () { + /* Initialize all variable members */ + logging_level = OrtLoggingLevel::ORT_LOGGING_LEVEL_WARNING; + intra_num_threads = 1; + graph_opt_level = GraphOptimizationLevel::ORT_DISABLE_ALL; + log_id = ""; + state = State::STOPPED; + model = nullptr; + output_size = 0; + num_input_nodes = 0; + num_output_nodes = 0; +} + +Engine::~Engine () { + this->Stop(); +} + +RuntimeError Engine::SetModel (std::shared_ptr in_model) { + RuntimeError error; + + if (State::STOPPED != this->state) { + error.Set (RuntimeError::Code::WRONG_ENGINE_STATE, + "Attempting to set model in stopped state"); + return error; + } + + if (nullptr == in_model) { + error.Set (RuntimeError::Code:: NULL_PARAMETER, + "Received null model pointer"); + return error; + } + auto model = std::dynamic_pointer_cast + (in_model); + + if (nullptr == model) { + error.Set (RuntimeError::Code::FRAMEWORK_ERROR, + "The provided model is not an ONNXRT model"); + return error; + } + + this->model = model; + + return error; +} + +void Engine::CreateEnv() { + env = Ort::Env(this->logging_level, this->log_id.c_str()); +} + +void Engine::CreateSessionOptions() { + session_options = Ort::SessionOptions(); + session_options.SetIntraOpNumThreads(this->intra_num_threads); + session_options.SetGraphOptimizationLevel( + this->graph_opt_level); +} + +void Engine::CreateSession(const void *model_data, + size_t model_data_size, + RuntimeError &error) { + + if (nullptr == this->env) { + error.Set (RuntimeError::Code:: NULL_PARAMETER, + "Ort::Env not initialized"); + } + + if (nullptr == this->session_options) { + error.Set (RuntimeError::Code:: NULL_PARAMETER, + "Ort::SessionOptions not initialized"); + } + + if (nullptr == model_data) { + error.Set (RuntimeError::Code:: NULL_PARAMETER, + "Received null model data pointer"); + } + + this->session = std::make_shared(this->env, model_data, + model_data_size, this->session_options); +} + +RuntimeError Engine::Start () { + RuntimeError error; + + if (State::STARTED == this->state) { + error.Set (RuntimeError::Code::WRONG_ENGINE_STATE, + "Engine already started"); + return error; + } + + if (nullptr == this->model) { + error.Set (RuntimeError::Code:: NULL_PARAMETER, + "Model not set yet"); + return error; + } + + try { + this->CreateEnv(); + this->CreateSessionOptions(); + this->CreateSession((void *) this->model->GetOnnxrtModel().get(), + this->model->GetOnnxrtModelSize(), error); + this->num_input_nodes = this->GetSessionInputCount(this->session, error); + this->num_output_nodes = this->GetSessionOutputCount(this->session, error); + } + + catch (std::exception &excep) { + error.Set(RuntimeError::Code::FRAMEWORK_ERROR, excep.what()); + return error; + } + + /* In case it fails but not by an exception */ + if (error.IsError ()) { + return error; + } + + if (this->num_input_nodes > 1) { + error.Set(RuntimeError::Code::INCOMPATIBLE_MODEL, + "Number of inputs in the model is greater than 1, this is not supported"); + return error; + } + + if (this->num_output_nodes > 1) { + error.Set(RuntimeError::Code::INCOMPATIBLE_MODEL, + "Number of outputs in the model is greater than 1, this is not supported"); + return error; + } + + this->input_node_names.resize(num_input_nodes); + this->output_node_names.resize(num_output_nodes); + + /* Warning: only 1 input and output supported */ + error = GetSessionInfo(this->session, 0); + if (error.IsError ()) { + return error; + } + + this->state = State::STARTED; + + return error; +} + +RuntimeError Engine::Stop () { + RuntimeError error; + + if (State::STOPPED == this->state) { + error.Set (RuntimeError::Code::WRONG_ENGINE_STATE, + "Engine already stopped"); + } + + this->state = State::STOPPED; + + return error; +} + +std::shared_ptr Engine::Predict (std::shared_ptr + in_frame, r2i::RuntimeError &error) { + ImageFormat frame_format; + int frame_width = 0; + int frame_height = 0; + int frame_channels = 0; + size_t input_image_size = 0; + + auto prediction = std::make_shared(); + + error.Clean (); + + if (State::STARTED != this->state) { + error.Set (RuntimeError::Code::WRONG_ENGINE_STATE, + "Engine not started"); + return nullptr; + } + + auto frame = std::dynamic_pointer_cast (in_frame); + if (!frame) { + error.Set (RuntimeError::Code::FRAMEWORK_ERROR, + "The provided frame is not an onnxrt frame"); + return nullptr; + } + + frame_format = frame->GetFormat(); + frame_channels = frame_format.GetNumPlanes(); + frame_height = frame->GetHeight(); + frame_width = frame->GetWidth(); + input_image_size = frame_height * frame_width * frame_channels; + + error = this->ValidateInputTensorShape(frame_channels, frame_height, + frame_width, this->input_node_dims); + if (error.IsError ()) { + return nullptr; + } + + /* Score model with input tensor, get back Prediction set with pointer + * of the output tensor result. + * Note that this implementation only supports 1 input and 1 output models. + */ + error = this->ScoreModel(this->session, frame, + input_image_size, + this->output_size, + this->input_node_dims, + prediction); + + if (error.IsError ()) { + return nullptr; + } + + return prediction; +} + +size_t Engine::GetSessionInputCount(std::shared_ptr session, + RuntimeError &error) { + if (nullptr == session) { + error.Set (RuntimeError::Code:: NULL_PARAMETER, + "Received null session pointer"); + return 0; + } + return session->GetInputCount(); +} + +size_t Engine::GetSessionOutputCount(std::shared_ptr session, + RuntimeError &error) { + if (nullptr == session) { + error.Set (RuntimeError::Code:: NULL_PARAMETER, + "Received null session pointer"); + return 0; + } + return session->GetOutputCount(); +} + +std::vector Engine::GetSessionInputNodeDims( + std::shared_ptr session, size_t index, RuntimeError &error) { + std::vector dims; + if (nullptr == session) { + error.Set (RuntimeError::Code:: NULL_PARAMETER, + "Received null session pointer"); + return dims; + } + dims = session->GetInputTypeInfo( + index).GetTensorTypeAndShapeInfo().GetShape(); + return dims; +} + +size_t Engine::GetSessionOutputSize(std::shared_ptr session, + size_t index, RuntimeError &error) { + if (nullptr == session) { + error.Set (RuntimeError::Code:: NULL_PARAMETER, + "Received null session pointer"); + return 0; + } + return session->GetOutputTypeInfo( + index).GetTensorTypeAndShapeInfo().GetElementCount(); +} + +const char *Engine::GetSessionInputName(std::shared_ptr session, + size_t index, OrtAllocator *allocator, + RuntimeError &error) { + const char *name = ""; + + if (nullptr == session) { + error.Set (RuntimeError::Code:: NULL_PARAMETER, + "Received null session pointer"); + return name; + } + + if (nullptr == allocator) { + error.Set (RuntimeError::Code:: NULL_PARAMETER, + "Received null allocator pointer"); + return name; + } + + name = session->GetInputName(index, allocator); + return name; +} + +const char *Engine::GetSessionOutputName(std::shared_ptr session, + size_t index, OrtAllocator *allocator, + RuntimeError &error) { + const char *name = ""; + + if (nullptr == session) { + error.Set (RuntimeError::Code:: NULL_PARAMETER, + "Received null session pointer"); + return name; + } + + if (nullptr == allocator) { + error.Set (RuntimeError::Code:: NULL_PARAMETER, + "Received null allocator pointer"); + return name; + } + + name = session->GetOutputName(index, allocator); + return name; +} + +RuntimeError Engine::GetSessionInfo(std::shared_ptr session, + size_t index) { + RuntimeError error; + + Ort::AllocatorWithDefaultOptions input_allocator; + Ort::AllocatorWithDefaultOptions output_allocator; + + try { + this->input_node_dims = this->GetSessionInputNodeDims(session, index, error); + this->output_size = this->GetSessionOutputSize(session, index, error); + this->input_node_names[index] = this->GetSessionInputName(session, index, + input_allocator, error); + this->output_node_names[index] = this->GetSessionOutputName(session, index, + output_allocator, error); + } + + catch (std::exception &excep) { + error.Set(RuntimeError::Code::FRAMEWORK_ERROR, excep.what()); + return error; + } + + return error; +} + +RuntimeError Engine::ValidateInputTensorShape (int channels, int height, + int width, std::vector input_dims) { + RuntimeError error; + + /* We only support 1 batch */ + if (1 != input_dims.at(0)) { + error.Set (RuntimeError::Code::INVALID_FRAMEWORK_PARAMETER, + "We only support a batch of 1 image(s) in our frames"); + return error; + } + + /* Check that channels match */ + if (channels != input_dims.at(3)) { + std::string error_msg; + error_msg = "Channels per image:" + std::to_string(channels) + + ", needs to be equal to model input channels:" + + std::to_string(input_dims.at(1)); + error.Set (RuntimeError::Code::INVALID_FRAMEWORK_PARAMETER, error_msg); + return error; + } + + /* Check that heights match */ + if (height != input_dims.at(2)) { + error.Set (RuntimeError::Code::INVALID_FRAMEWORK_PARAMETER, + "Unsupported image height"); + return error; + } + + /* Check that widths match */ + if (width != input_dims.at(1)) { + error.Set (RuntimeError::Code::INVALID_FRAMEWORK_PARAMETER, + "Unsupported image width"); + return error; + } + + return error; +} + +float *Engine::SessionRun (std::shared_ptr session, + std::shared_ptr frame, + size_t input_image_size, + std::vector input_node_dims, + std::vector &output_tensor, + RuntimeError &error) { + Ort::Value input_tensor{nullptr}; + Ort::MemoryInfo memory_info = Ort::MemoryInfo::CreateCpu(OrtArenaAllocator, + OrtMemTypeDefault); + input_tensor = Ort::Value::CreateTensor(memory_info, + reinterpret_cast(frame->GetData()), + input_image_size, + input_node_dims.data(), + input_node_dims.size()); + output_tensor = + session->Run(Ort::RunOptions{nullptr}, input_node_names.data(), + &input_tensor, num_input_nodes, output_node_names.data(), + num_output_nodes); + return output_tensor.at(0).GetTensorMutableData(); +} + +RuntimeError Engine::ScoreModel (std::shared_ptr session, + std::shared_ptr frame, + size_t input_size, + size_t output_size, + std::vector input_node_dims, + std::shared_ptr prediction) { + RuntimeError error; + float *result; + std::vector output_tensor; + + if (!frame->GetData()) { + error.Set (RuntimeError::Code::NULL_PARAMETER, + "The provided frame does not contain valid data"); + return error; + } + + try { + result = this->SessionRun(session, frame, input_size, input_node_dims, + output_tensor, error); + } + + catch (std::exception &excep) { + error.Set(RuntimeError::Code::FRAMEWORK_ERROR, excep.what()); + return error; + } + + error = prediction->SetTensorValues(result, output_size); + if (error.IsError ()) { + return error; + } + + return error; +} + +RuntimeError Engine::SetLoggingLevel (int logging_level) { + RuntimeError error; + + if (State::STARTED == this->state) { + error.Set (RuntimeError::Code::WRONG_ENGINE_STATE, + "Parameter can't be set, engine already started"); + return error; + } + + /* We need to convert int to OrtLoggingLevel enum */ + this->logging_level = static_cast(logging_level); + + return error; +} + +RuntimeError Engine::SetIntraNumThreads (int intra_num_threads) { + RuntimeError error; + + if (State::STARTED == this->state) { + error.Set (RuntimeError::Code::WRONG_ENGINE_STATE, + "Parameter can't be set, engine already started"); + return error; + } + + this->intra_num_threads = intra_num_threads; + + return error; +} + +RuntimeError Engine::SetGraphOptLevel (int graph_opt_level) { + RuntimeError error; + + if (State::STARTED == this->state) { + error.Set (RuntimeError::Code::WRONG_ENGINE_STATE, + "Parameter can't be set, engine already started"); + return error; + } + + /* We need to convert int to GraphOptimizationLevel enum */ + this->graph_opt_level = static_cast(graph_opt_level); + + return error; +} + +RuntimeError Engine::SetLogId (const std::string &log_id) { + RuntimeError error; + + if (State::STARTED == this->state) { + error.Set (RuntimeError::Code::WRONG_ENGINE_STATE, + "Parameter can't be set, engine already started"); + return error; + } + + this->log_id = log_id; + + return error; +} + +int Engine::GetLoggingLevel () { + /* We need to convert OrtLoggingLevel enum to int */ + return static_cast(this->logging_level); +} + +int Engine::GetIntraNumThreads () { + + return this->intra_num_threads; +} + +int Engine::GetGraphOptLevel () { + /* We need to convert GraphOptimizationLevel enum to int */ + return static_cast(this->graph_opt_level); +} + +const std::string Engine::GetLogId () { + return this->log_id; +} + +} // namespace onnxrt +} // namespace r2i diff --git a/r2i/onnxrt/engine.h b/r2i/onnxrt/engine.h new file mode 100644 index 00000000..ed73d6cf --- /dev/null +++ b/r2i/onnxrt/engine.h @@ -0,0 +1,109 @@ +/* Copyright (C) 2020 RidgeRun, LLC (http://www.ridgerun.com) + * All Rights Reserved. + * + * The contents of this software are proprietary and confidential to RidgeRun, + * LLC. No part of this program may be photocopied, reproduced or translated + * into another programming language without prior written consent of + * RidgeRun, LLC. The user is free to modify the source code after obtaining + * a software license from RidgeRun. All source code changes must be provided + * back to RidgeRun without any encumbrance. +*/ + +#ifndef R2I_ONNXRT_ENGINE_H +#define R2I_ONNXRT_ENGINE_H + +#include + +#include + +#include +#include + +#include +#include +#include + +namespace r2i { +namespace onnxrt { + +class Engine : public IEngine { + public: + Engine (); + ~Engine (); + r2i::RuntimeError SetModel (std::shared_ptr in_model) override; + r2i::RuntimeError Start () override; + r2i::RuntimeError Stop () override; + std::shared_ptr Predict (std::shared_ptr + in_frame, r2i::RuntimeError &error) override; + r2i::RuntimeError SetLoggingLevel (int logging_level); + r2i::RuntimeError SetLogId (const std::string &log_id); + r2i::RuntimeError SetIntraNumThreads (int intra_num_threads); + r2i::RuntimeError SetGraphOptLevel (int graph_opt_level); + int GetLoggingLevel (); + int GetIntraNumThreads (); + int GetGraphOptLevel (); + const std::string GetLogId (); + + private: + enum State { + STARTED, + STOPPED + }; + + /* ONNXRT parameters must be initialized in case user does not set any */ + OrtLoggingLevel logging_level; + int intra_num_threads; + GraphOptimizationLevel graph_opt_level; + std::string log_id; + State state; + std::shared_ptr model; + std::vector input_node_dims; + std::vector input_node_names; + std::vector output_node_names; + size_t output_size; + size_t num_input_nodes; + size_t num_output_nodes; + Ort::Env env = Ort::Env(nullptr); + Ort::SessionOptions session_options = Ort::SessionOptions(nullptr); + std::shared_ptr session; + + void CreateEnv(); + void CreateSessionOptions(); + void CreateSession(const void *model_data, size_t model_data_size, + RuntimeError &error); + size_t GetSessionInputCount(std::shared_ptr session, + RuntimeError &error); + size_t GetSessionOutputCount(std::shared_ptr session, + RuntimeError &error); + std::vector GetSessionInputNodeDims(std::shared_ptr + session, size_t index, RuntimeError &error); + size_t GetSessionOutputSize(std::shared_ptr session, + size_t index, RuntimeError &error); + const char *GetSessionInputName(std::shared_ptr session, + size_t index, + OrtAllocator *allocator, RuntimeError &error); + const char *GetSessionOutputName(std::shared_ptr session, + size_t index, + OrtAllocator *allocator, RuntimeError &error); + float *SessionRun (std::shared_ptr session, + std::shared_ptr frame, + size_t input_image_size, + std::vector input_node_dims, + std::vector &output_tensor, + RuntimeError &error); + r2i::RuntimeError GetSessionInfo(std::shared_ptr session, + size_t index); + r2i::RuntimeError ValidateInputTensorShape (int channels, int height, int width, + std::vector input_dims); + r2i::RuntimeError ScoreModel (std::shared_ptr session, + std::shared_ptr frame, + size_t input_size, + size_t output_size, + std::vector input_node_dims, + std::shared_ptr prediction); +}; + +} // namespace onnxrt +} // namespace r2i + +#endif //R2I_ONNXRT_ENGINE_H diff --git a/r2i/onnxrt/frame.cc b/r2i/onnxrt/frame.cc new file mode 100644 index 00000000..b1c7ca47 --- /dev/null +++ b/r2i/onnxrt/frame.cc @@ -0,0 +1,71 @@ +/* Copyright (C) 2020 RidgeRun, LLC (http://www.ridgerun.com) + * All Rights Reserved. + * + * The contents of this software are proprietary and confidential to RidgeRun, + * LLC. No part of this program may be photocopied, reproduced or translated + * into another programming language without prior written consent of + * RidgeRun, LLC. The user is free to modify the source code after obtaining + * a software license from RidgeRun. All source code changes must be provided + * back to RidgeRun without any encumbrance. +*/ + +#include "r2i/onnxrt/frame.h" + +namespace r2i { +namespace onnxrt { + +Frame::Frame () : + frame_data(nullptr), frame_width(0), frame_height(0), + frame_format(ImageFormat::Id::UNKNOWN_FORMAT) { +} + +RuntimeError Frame::Configure (void *in_data, int width, + int height, r2i::ImageFormat::Id format) { + RuntimeError error; + ImageFormat imageformat (format); + + if (nullptr == in_data) { + error.Set (RuntimeError::Code::NULL_PARAMETER, "Received a NULL data pointer"); + return error; + } + if (width <= 0) { + error.Set (RuntimeError::Code::WRONG_API_USAGE, + "Received an invalid image width"); + return error; + } + if (height <= 0) { + error.Set (RuntimeError::Code::WRONG_API_USAGE, + "Received an invalid image height"); + return error; + } + + this->frame_data = static_cast(in_data); + this->frame_width = width; + this->frame_height = height; + this->frame_format = imageformat; + + return error; +} + +void *Frame::GetData () { + return this->frame_data; +} + +int Frame::GetWidth () { + return this->frame_width; +} + +int Frame::GetHeight () { + return this->frame_height; +} + +ImageFormat Frame::GetFormat () { + return this->frame_format; +} + +DataType Frame::GetDataType () { + return r2i::DataType::Id::FLOAT; +} + +} // namespace onnxrt +} // namespace r2i diff --git a/r2i/onnxrt/frame.h b/r2i/onnxrt/frame.h new file mode 100644 index 00000000..eab3cc7d --- /dev/null +++ b/r2i/onnxrt/frame.h @@ -0,0 +1,48 @@ +/* Copyright (C) 2020 RidgeRun, LLC (http://www.ridgerun.com) + * All Rights Reserved. + * + * The contents of this software are proprietary and confidential to RidgeRun, + * LLC. No part of this program may be photocopied, reproduced or translated + * into another programming language without prior written consent of + * RidgeRun, LLC. The user is free to modify the source code after obtaining + * a software license from RidgeRun. All source code changes must be provided + * back to RidgeRun without any encumbrance. +*/ + +#ifndef R2I_ONNXRT_FRAME_H +#define R2I_ONNXRT_FRAME_H + +#include + +namespace r2i { +namespace onnxrt { + +class Frame : public IFrame { + public: + Frame (); + + RuntimeError Configure (void *in_data, int width, + int height, r2i::ImageFormat::Id format) override; + + void *GetData () override; + + int GetWidth () override; + + int GetHeight () override; + + ImageFormat GetFormat () override; + + virtual DataType GetDataType () override; + + private: + float *frame_data; + int frame_width; + int frame_height; + ImageFormat frame_format; + +}; + +} // namespace onnxrt +} // namespace r2i + +#endif //R2I_ONNXRT_FRAME_H diff --git a/r2i/onnxrt/frameworkfactory.cc b/r2i/onnxrt/frameworkfactory.cc new file mode 100644 index 00000000..7ae7e81b --- /dev/null +++ b/r2i/onnxrt/frameworkfactory.cc @@ -0,0 +1,67 @@ +/* Copyright (C) 2020 RidgeRun, LLC (http://www.ridgerun.com) + * All Rights Reserved. + * + * The contents of this software are proprietary and confidential to RidgeRun, + * LLC. No part of this program may be photocopied, reproduced or translated + * into another programming language without prior written consent of + * RidgeRun, LLC. The user is free to modify the source code after obtaining + * a software license from RidgeRun. All source code changes must be provided + * back to RidgeRun without any encumbrance. +*/ + +#include "frameworkfactory.h" + +#include + +#include "engine.h" +#include "frame.h" +#include "loader.h" +#include "parameters.h" + +namespace r2i { +namespace onnxrt { + +std::unique_ptr FrameworkFactory::MakeLoader ( + RuntimeError &error) { + error.Clean (); + + return std::unique_ptr (new Loader); +} + +std::unique_ptr FrameworkFactory::MakeEngine ( + RuntimeError &error) { + error.Clean (); + + return std::unique_ptr (new Engine); +} + +std::unique_ptr FrameworkFactory::MakeParameters ( + RuntimeError &error) { + error.Clean (); + + return std::unique_ptr (new Parameters); +} + +std::unique_ptr FrameworkFactory::MakeFrame ( + RuntimeError &error) { + error.Clean (); + + return std::unique_ptr