diff --git a/configure.ac b/configure.ac index 97506b33..f8b6febe 100644 --- a/configure.ac +++ b/configure.ac @@ -10,7 +10,7 @@ # Initialize autoconf. AC_PREREQ([2.69]) -AC_INIT([RidgeRun inference library],[0.4.2],[https://github.com/RidgeRun/r2inference/issues],[r2inference]) +AC_INIT([RidgeRun inference library],[0.5.0],[https://github.com/RidgeRun/r2inference/issues],[r2inference]) # Initialize our build utils RR_INIT diff --git a/r2i/iparameters.h b/r2i/iparameters.h index 32205943..32a9fa23 100644 --- a/r2i/iparameters.h +++ b/r2i/iparameters.h @@ -63,6 +63,14 @@ class IParameters { */ virtual RuntimeError Get (const std::string &in_parameter, int &value) = 0; + /** + * \brief Queries a double parameter. + * \param in_parameter Name of the parameter to get a value + * \param value [out] Return value of the parameter to query + * \return RuntimeError with a description of an error. + */ + virtual RuntimeError Get (const std::string &in_parameter, double &value) = 0; + /** * \brief Queries a string parameter. * \param in_parameter Name of the parameter to get a value @@ -89,6 +97,14 @@ class IParameters { */ virtual RuntimeError Set (const std::string &in_parameter, int in_value) = 0; + /** + * \brief Sets an double parameter. + * \param in_parameter Name of the parameter to set a value + * \param in_value New value to set for in_parameter + * \return RuntimeError with a description of an error. + */ + virtual RuntimeError Set (const std::string &in_parameter, double in_value) = 0; + /** * \brief Lists the available parameters for this framework * \param metas [out] A vector of ParameterMeta with the description of diff --git a/r2i/parametermeta.h b/r2i/parametermeta.h index 6b278989..a3ae14a7 100644 --- a/r2i/parametermeta.h +++ b/r2i/parametermeta.h @@ -62,7 +62,12 @@ struct ParameterMeta { /** * A standard string */ - STRING + STRING, + + /** + * System dependent double + */ + DOUBLE }; /** diff --git a/r2i/tensorflow/engine.cc b/r2i/tensorflow/engine.cc index 91252f67..0f31ef2a 100644 --- a/r2i/tensorflow/engine.cc +++ b/r2i/tensorflow/engine.cc @@ -10,9 +10,7 @@ */ #include "r2i/tensorflow/engine.h" - #include - #include "r2i/tensorflow/prediction.h" #include "r2i/tensorflow/frame.h" @@ -55,6 +53,18 @@ RuntimeError Engine::SetModel (std::shared_ptr in_model) { return error; } +RuntimeError Engine::SetMemoryUsage (double memory_usage) { + RuntimeError error; + + if (memory_usage > 1.0 || memory_usage < 0.1) { + error.Set (RuntimeError::Code::WRONG_API_USAGE, "Invalid memory usage value"); + return error; + } + + this->session_memory_usage_index = (static_cast(memory_usage * 10) - 1); + return error; +} + static RuntimeError FreeSession (TF_Session *session) { RuntimeError error; std::shared_ptr pstatus(TF_NewStatus (), TF_DeleteStatus); @@ -97,6 +107,8 @@ RuntimeError Engine::Start () { TF_Graph *graph = pgraph.get(); TF_Status *status = pstatus.get (); TF_SessionOptions *opt = popt.get (); + TF_SetConfig(opt, this->config[this->session_memory_usage_index], + RAM_ARRAY_SIZE, status); std::shared_ptr session (TF_NewSession(graph, opt, status), FreeSession); diff --git a/r2i/tensorflow/engine.h b/r2i/tensorflow/engine.h index a8f4b78b..a2283c0e 100644 --- a/r2i/tensorflow/engine.h +++ b/r2i/tensorflow/engine.h @@ -18,6 +18,8 @@ #include +#define RAM_ARRAY_SIZE 11 + namespace r2i { namespace tensorflow { @@ -27,6 +29,8 @@ class Engine : public IEngine { r2i::RuntimeError SetModel (std::shared_ptr in_model) override; + r2i::RuntimeError SetMemoryUsage (double memory_usage); + r2i::RuntimeError Start () override; r2i::RuntimeError Stop () override; @@ -41,10 +45,25 @@ class Engine : public IEngine { STARTED, STOPPED }; + State state; + int session_memory_usage_index; std::shared_ptr session; std::shared_ptr model; + + const uint8_t config[10][RAM_ARRAY_SIZE] = { + {0x32, 0x9, 0x9, 0x9a, 0x99, 0x99, 0x99, 0x99, 0x99, 0xb9, 0x3f}, + {0x32, 0x9, 0x9, 0x9a, 0x99, 0x99, 0x99, 0x99, 0x99, 0xc9, 0x3f}, + {0x32, 0x9, 0x9, 0x33, 0x33, 0x33, 0x33, 0x33, 0x33, 0xd3, 0x3f}, + {0x32, 0x9, 0x9, 0x9a, 0x99, 0x99, 0x99, 0x99, 0x99, 0xd9, 0x3f}, + {0x32, 0x9, 0x9, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xe0, 0x3f}, + {0x32, 0x9, 0x9, 0x33, 0x33, 0x33, 0x33, 0x33, 0x33, 0xe3, 0x3f}, + {0x32, 0x9, 0x9, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0xe6, 0x3f}, + {0x32, 0x9, 0x9, 0x9a, 0x99, 0x99, 0x99, 0x99, 0x99, 0xe9, 0x3f}, + {0x32, 0x9, 0x9, 0xcd, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xec, 0x3f}, + {0x32, 0x9, 0x9, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xf0, 0x3f} + }; }; } diff --git a/r2i/tensorflow/parameters.cc b/r2i/tensorflow/parameters.cc index 311c7b6d..e6b2cae1 100644 --- a/r2i/tensorflow/parameters.cc +++ b/r2i/tensorflow/parameters.cc @@ -38,6 +38,10 @@ Parameters::Parameters () : r2i::ParameterMeta::Flags::READ, r2i::ParameterMeta::Type::STRING, std::make_shared(this)), + PARAM("gpu-memory-usage", "Per process GPU memory usage fraction", + r2i::ParameterMeta::Flags::READWRITE | r2i::ParameterMeta::Flags::WRITE_BEFORE_START, + r2i::ParameterMeta::Type::DOUBLE, + std::make_shared(this)), /* Model parameters */ PARAM("input-layer", "Name of the input layer in the graph", @@ -147,6 +151,25 @@ RuntimeError Parameters::Get (const std::string &in_parameter, int &value) { return error; } +RuntimeError Parameters::Get (const std::string &in_parameter, double &value) { + RuntimeError error; + + ParamDesc param = this->Validate (in_parameter, + r2i::ParameterMeta::Type::DOUBLE, "double", error); + if (error.IsError ()) { + return error; + } + + auto accessor = std::dynamic_pointer_cast(param.accessor); + + error = accessor->Get (); + if (error.IsError ()) { + return error; + } + + value = accessor->value; + return error; +} RuntimeError Parameters::Get (const std::string &in_parameter, std::string &value) { @@ -195,6 +218,22 @@ RuntimeError Parameters::Set (const std::string &in_parameter, return accessor->Set (); } +RuntimeError Parameters::Set (const std::string &in_parameter, + double in_value) { + RuntimeError error; + + ParamDesc param = this->Validate (in_parameter, + r2i::ParameterMeta::Type::DOUBLE, "double", error); + if (error.IsError ()) { + return error; + } + + auto accessor = std::dynamic_pointer_cast(param.accessor); + + accessor->value = in_value; + return accessor->Set (); +} + RuntimeError Parameters::Set (const std::string &in_parameter, int in_value) { RuntimeError error; diff --git a/r2i/tensorflow/parameters.h b/r2i/tensorflow/parameters.h index 079149f3..1fabafb9 100644 --- a/r2i/tensorflow/parameters.h +++ b/r2i/tensorflow/parameters.h @@ -37,6 +37,8 @@ class Parameters : public IParameters { RuntimeError Get (const std::string &in_parameter, int &value) override; + RuntimeError Get (const std::string &in_parameter, double &value) override; + RuntimeError Get (const std::string &in_parameter, std::string &value) override; @@ -45,6 +47,8 @@ class Parameters : public IParameters { RuntimeError Set (const std::string &in_parameter, int in_value) override; + RuntimeError Set (const std::string &in_parameter, double in_value) override; + RuntimeError List (std::vector &metas) override; private: @@ -75,6 +79,12 @@ class Parameters : public IParameters { int value; }; + class DoubleAccessor : public Accessor { + public: + DoubleAccessor (Parameters *target) : Accessor(target) {} + double value; + }; + class VersionAccessor : public StringAccessor { public: VersionAccessor (Parameters *target) : StringAccessor(target) {} @@ -89,6 +99,18 @@ class Parameters : public IParameters { } }; + class MemoryUsageAccessor : public DoubleAccessor { + public: + MemoryUsageAccessor (Parameters *target) : DoubleAccessor(target) {} + RuntimeError Set () { + return target->engine->SetMemoryUsage(this->value); + } + + RuntimeError Get () { + return RuntimeError (); + } + }; + class InputLayerAccessor : public StringAccessor { public: InputLayerAccessor (Parameters *target) : StringAccessor(target) {} diff --git a/r2i/tflite/engine.cc b/r2i/tflite/engine.cc index 65cce27c..9bc726ec 100644 --- a/r2i/tflite/engine.cc +++ b/r2i/tflite/engine.cc @@ -193,6 +193,13 @@ std::shared_ptr Engine::Predict (std::shared_ptr return nullptr; } + // Check the model quantization, only 32 bits allowed + if (kTfLiteFloat32 != interpreter->tensor(input)->type) { + error.Set (RuntimeError::Code::FRAMEWORK_ERROR, + "The provided model quantization is not allowed, only float32 is supported"); + return nullptr; + } + auto input_tensor = this->interpreter->typed_tensor(input); auto input_data = (float *)frame->GetData();