From 31916110201caa522c6295a2df0927ceb7cc084a Mon Sep 17 00:00:00 2001 From: Marcos Bento Date: Wed, 2 Oct 2024 13:27:40 +0100 Subject: [PATCH 01/15] Fix ecFlow Python check_job_creation related crash Re ECFLOW-1976 --- libs/node/src/ecflow/node/AvisoAttr.cpp | 2 +- libs/node/src/ecflow/node/MirrorAttr.cpp | 12 +++++++++--- libs/pyext/test/py_u_TestAviso.py | 19 +++++++++++++++++++ libs/pyext/test/py_u_TestMirror.py | 18 ++++++++++++++++++ 4 files changed, 47 insertions(+), 4 deletions(-) diff --git a/libs/node/src/ecflow/node/AvisoAttr.cpp b/libs/node/src/ecflow/node/AvisoAttr.cpp index 31b66f7b2..013c6873a 100644 --- a/libs/node/src/ecflow/node/AvisoAttr.cpp +++ b/libs/node/src/ecflow/node/AvisoAttr.cpp @@ -88,7 +88,7 @@ bool AvisoAttr::why(std::string& theReasonWhy) const { void AvisoAttr::reset() { state_change_no_ = Ecf::incr_state_change_no(); - if (parent_->state() == NState::QUEUED) { + if (parent_ && (parent_->state() == NState::QUEUED)) { start(); } } diff --git a/libs/node/src/ecflow/node/MirrorAttr.cpp b/libs/node/src/ecflow/node/MirrorAttr.cpp index 25db0edde..bce3abbcd 100644 --- a/libs/node/src/ecflow/node/MirrorAttr.cpp +++ b/libs/node/src/ecflow/node/MirrorAttr.cpp @@ -160,6 +160,10 @@ void MirrorAttr::mirror() { std::optional MirrorAttr::resolve_cfg(const std::string& value, std::string_view default_value) const { // Substitude variable in local value std::string local = value; + if(!parent_) { + return std::nullopt; + } + parent_->variableSubstitution(local); // Ensure substituted value is not default @@ -199,9 +203,11 @@ void MirrorAttr::start_controller() { state_change_no_ = Ecf::incr_state_change_no(); reason_ = Message("Unable to start mirror. Failed to resolve mirror remote host: ", remote_host_).str(); - parent_->flag().set(Flag::REMOTE_ERROR); - parent_->flag().set_state_change_no(state_change_no_); - parent_->setStateOnly(NState::UNKNOWN, true); + if (parent_) { + parent_->flag().set(Flag::REMOTE_ERROR); + parent_->flag().set_state_change_no(state_change_no_); + parent_->setStateOnly(NState::UNKNOWN, true); + } return; } // For the remaining configuration, we use fallback values if the resolution fails diff --git a/libs/pyext/test/py_u_TestAviso.py b/libs/pyext/test/py_u_TestAviso.py index eb7041559..c78a8acba 100644 --- a/libs/pyext/test/py_u_TestAviso.py +++ b/libs/pyext/test/py_u_TestAviso.py @@ -79,6 +79,24 @@ def cannot_have_multiple_avisos_in_single_task(): assert True +def can_check_job_creation_with_aviso(): + defs = ecf.Defs() + + suite = ecf.Suite("s") + defs.add_suite(suite) + + family = ecf.Family("f") + suite.add_family(family) + + task = ecf.Task("t") + family.add_task(task) + + aviso = ecf.AvisoAttr("name", "listener", "url", "schema", "polling", "auth") + task.add_aviso(aviso) + + defs.check_job_creation() + + if __name__ == "__main__": Test.print_test_start(os.path.basename(__file__)) @@ -86,5 +104,6 @@ def cannot_have_multiple_avisos_in_single_task(): can_add_aviso_to_task() can_embed_aviso_into_task() cannot_have_multiple_avisos_in_single_task() + can_check_job_creation_with_aviso() print("All tests pass") diff --git a/libs/pyext/test/py_u_TestMirror.py b/libs/pyext/test/py_u_TestMirror.py index fd3f9afed..02bdc1535 100644 --- a/libs/pyext/test/py_u_TestMirror.py +++ b/libs/pyext/test/py_u_TestMirror.py @@ -79,6 +79,23 @@ def cannot_have_multiple_mirrors_in_single_task(): except RuntimeError as e: assert True +def can_check_job_creation_with_mirror(): + defs = ecf.Defs() + + suite = ecf.Suite("s") + defs.add_suite(suite) + + family = ecf.Family("f") + suite.add_family(family) + + task = ecf.Task("t"); + family.add_task(task) + + mirror = ecf.MirrorAttr("name", "r_path", "r_host", "r_port", "polling", True, "auth") + task.add_mirror(mirror) + + defs.check_job_creation() + if __name__ == "__main__": Test.print_test_start(os.path.basename(__file__)) @@ -87,5 +104,6 @@ def cannot_have_multiple_mirrors_in_single_task(): can_add_mirror_to_task() can_embed_mirror_into_task() cannot_have_multiple_mirrors_in_single_task() + can_check_job_creation_with_mirror() print("All tests pass") From fb8b609abfe5e7e83c1aab44f2a63d8837353666 Mon Sep 17 00:00:00 2001 From: Marcos Bento Date: Tue, 8 Oct 2024 09:44:01 +0100 Subject: [PATCH 02/15] Add generated variables to .../tree endpoint response - When using ?content=full&gen_vars=true Re ECFLOW-1977 --- libs/node/src/ecflow/node/Node.cpp | 6 ++++ libs/node/src/ecflow/node/Node.hpp | 3 +- libs/rest/src/ecflow/http/ApiV1.cpp | 16 +++++++---- libs/rest/src/ecflow/http/ApiV1Impl.cpp | 4 +-- libs/rest/src/ecflow/http/ApiV1Impl.hpp | 2 +- libs/rest/src/ecflow/http/TreeGeneration.hpp | 29 ++++++++++++++------ 6 files changed, 43 insertions(+), 17 deletions(-) diff --git a/libs/node/src/ecflow/node/Node.cpp b/libs/node/src/ecflow/node/Node.cpp index 712866388..1e24dfd69 100644 --- a/libs/node/src/ecflow/node/Node.cpp +++ b/libs/node/src/ecflow/node/Node.cpp @@ -2668,6 +2668,12 @@ void Node::gen_variables(std::vector& vec) const { repeat_.gen_variables(vec); // if repeat_ is empty vec is unchanged } +std::vector Node::gen_variables() const { + std::vector generated; + gen_variables(generated); + return generated; +} + const Variable& Node::findGenVariable(const std::string& name) const { return repeat_.find_gen_variable(name); // if repeat_ is empty find returns empty variable by ref } diff --git a/libs/node/src/ecflow/node/Node.hpp b/libs/node/src/ecflow/node/Node.hpp index 73a91fe16..4f6b44376 100644 --- a/libs/node/src/ecflow/node/Node.hpp +++ b/libs/node/src/ecflow/node/Node.hpp @@ -420,7 +420,8 @@ class Node : public std::enable_shared_from_this { ecf::Flag& flag() { return flag_; } const ecf::Flag& get_flag() const { return flag_; } - virtual void gen_variables(std::vector&) const; + [[deprecated]] virtual void gen_variables(std::vector&) const; + std::vector gen_variables() const; bool getLabelValue(const std::string& name, std::string& value) const; bool getLabelNewValue(const std::string& name, std::string& value) const; diff --git a/libs/rest/src/ecflow/http/ApiV1.cpp b/libs/rest/src/ecflow/http/ApiV1.cpp index 45e689757..b93feb764 100644 --- a/libs/rest/src/ecflow/http/ApiV1.cpp +++ b/libs/rest/src/ecflow/http/ApiV1.cpp @@ -209,13 +209,18 @@ ojson filter_json(const ojson& j, const httplib::Request& r) { } static std::string get_tree_content_kind(const httplib::Request& request) { - constexpr const char* content = "content"; - return request.has_param(content) ? request.get_param_value(content) : "basic"; + constexpr const char* parameter = "content"; + return request.has_param(parameter) ? request.get_param_value(parameter) : "basic"; } static bool get_tree_content_id_flag(const httplib::Request& request) { - constexpr const char* content = "with_id"; - return request.has_param(content) ? (request.get_param_value(content) == "true" ? true : false) : false; + constexpr const char* parameter = "with_id"; + return request.has_param(parameter) ? (request.get_param_value(parameter) == "true" ? true : false) : false; +} + +static bool get_tree_content_gen_vars(const httplib::Request& request) { + constexpr const char* parameter = "gen_vars"; + return request.has_param(parameter) ? (request.get_param_value(parameter) == "true" ? true : false) : false; } } // namespace @@ -269,7 +274,8 @@ void node_tree_read(const httplib::Request& request, httplib::Response& response const std::string path = request.matches[1]; std::string tree_kind = get_tree_content_kind(request); bool with_id = get_tree_content_id_flag(request); - ojson tree_content = (tree_kind == "full") ? get_full_node_tree(path, with_id) : get_basic_node_tree(path); + bool gen_vars = get_tree_content_gen_vars(request); + ojson tree_content = (tree_kind == "full") ? get_full_node_tree(path, with_id, gen_vars) : get_basic_node_tree(path); ojson j = filter_json(tree_content, request); response.status = HttpStatusCode::success_ok; response.set_content(j.dump(), "application/json"); diff --git a/libs/rest/src/ecflow/http/ApiV1Impl.cpp b/libs/rest/src/ecflow/http/ApiV1Impl.cpp index 9a4bcfbbb..0e641c933 100644 --- a/libs/rest/src/ecflow/http/ApiV1Impl.cpp +++ b/libs/rest/src/ecflow/http/ApiV1Impl.cpp @@ -129,8 +129,8 @@ ojson get_basic_node_tree(const std::string& path) { return tree.content(); } -ojson get_full_node_tree(const std::string& path, bool with_id) { - FullTree tree{with_id}; +ojson get_full_node_tree(const std::string& path, bool with_id, bool with_gen_vars) { + FullTree tree{with_id, with_gen_vars}; DefsTreeVisitor(get_defs(), tree).visit_at(path); return tree.content(); } diff --git a/libs/rest/src/ecflow/http/ApiV1Impl.hpp b/libs/rest/src/ecflow/http/ApiV1Impl.hpp index 97c5d1885..adedec3ed 100644 --- a/libs/rest/src/ecflow/http/ApiV1Impl.hpp +++ b/libs/rest/src/ecflow/http/ApiV1Impl.hpp @@ -18,7 +18,7 @@ namespace ecf::http { ojson get_basic_node_tree(const std::string& path); -ojson get_full_node_tree(const std::string& path, bool with_id); +ojson get_full_node_tree(const std::string& path, bool with_id, bool with_gen_vars); ojson get_sparser_node_tree(const std::string& path); void add_suite(const httplib::Request& request, httplib::Response& response); diff --git a/libs/rest/src/ecflow/http/TreeGeneration.hpp b/libs/rest/src/ecflow/http/TreeGeneration.hpp index 8d224b54f..ba33eb3c1 100644 --- a/libs/rest/src/ecflow/http/TreeGeneration.hpp +++ b/libs/rest/src/ecflow/http/TreeGeneration.hpp @@ -63,7 +63,11 @@ struct BasicTree struct FullTree { - FullTree(bool with_id) : root_(ojson::object({})), stack_{&root_}, with_id_{with_id} {} + FullTree(bool with_id, bool with_gen_vars) + : root_(ojson::object({})), + stack_{&root_}, + with_id_{with_id}, + with_gen_vars_{with_gen_vars} {} void begin_visit(const Suite& suite) { ojson& parent_ = *stack_.back(); @@ -74,7 +78,7 @@ struct FullTree current["id"] = suite.absNodePath(); } publish_state(suite, current); - publish_attributes(suite, current); + publish_attributes(suite, current, with_gen_vars_); ojson& children = current["children"] = ojson::object({}); stack_.push_back(&children); @@ -91,7 +95,7 @@ struct FullTree current["id"] = family.absNodePath(); } publish_state(family, current); - publish_attributes(family, current); + publish_attributes(family, current, with_gen_vars_); ojson& children = current["children"] = ojson::object({}); stack_.push_back(&children); @@ -108,7 +112,7 @@ struct FullTree current["id"] = task.absNodePath(); } publish_state(task, current); - publish_attributes(task, current); + publish_attributes(task, current, with_gen_vars_); ojson& children = current["aliases"] = ojson::object({}); stack_.push_back(&children); @@ -126,7 +130,7 @@ struct FullTree current["id"] = alias.absNodePath(); } publish_state(alias, current); - publish_attributes(alias, current); + publish_attributes(alias, current, with_gen_vars_); } void end_visit(const Alias& alias [[maybe_unused]]) { stack_.pop_back(); } @@ -141,15 +145,18 @@ struct FullTree state["default"] = DState::toString(node.dstate()); } - template - static ojson publish_atribute(const T& attr, std::string_view type) { + template + static ojson publish_atribute(const T& attr, std::string_view type, P... parameters) { auto j = ojson::object({}); to_json(j, attr); j["type"] = type; + if constexpr (sizeof...(P) > 0) { + ((j[parameters.first] = parameters.second), ...); + } return j; } - static void publish_attributes(const Node& node, ojson& parent) { + static void publish_attributes(const Node& node, ojson& parent, bool with_gen_vars) { ojson& array = parent["attributes"] = ojson::array(); for (const auto& attr : node.labels()) { @@ -164,6 +171,11 @@ struct FullTree for (const auto& attr : node.variables()) { array.emplace_back(publish_atribute(attr, "variable")); } + if (with_gen_vars) { + for (const auto& attr : node.gen_variables()) { + array.emplace_back(publish_atribute(attr, "variable", std::make_pair("generated", true))); + } + } for (const auto& attr : node.limits()) { array.emplace_back(publish_atribute(*attr, "limit")); } @@ -252,6 +264,7 @@ struct FullTree ojson root_; std::vector stack_; bool with_id_; + bool with_gen_vars_; }; } // namespace ecf::http From 5801cb7442f19f0b64fa6e2abb83405d7d5acc53 Mon Sep 17 00:00:00 2001 From: Marcos Bento Date: Tue, 8 Oct 2024 13:29:34 +0100 Subject: [PATCH 03/15] Enable test for generated variables Re ECFLOW-1977 --- libs/rest/test/TestApiV1.cpp | 65 ++++++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/libs/rest/test/TestApiV1.cpp b/libs/rest/test/TestApiV1.cpp index 8d06e0eb1..e8318b50f 100644 --- a/libs/rest/test/TestApiV1.cpp +++ b/libs/rest/test/TestApiV1.cpp @@ -475,6 +475,71 @@ BOOST_AUTO_TEST_CASE(test_node_full_tree) { wait_until([] { return false == check_for_path("/v1/suites/full_suite/definition"); }); } +BOOST_AUTO_TEST_CASE(test_node_full_tree_with_generated_variables) { + std::cout << "======== " << boost::unit_test::framework::current_test_case().p_name << " =========" << std::endl; + + // (0) Clean up -- in case there is any left-over from passed/failed tests + request("delete", "/v1/suites/full_suite/definition", "", API_KEY); + wait_until([] { return false == check_for_path("/v1/suites/full_suite/definition"); }); + + // (1) Publish 'full_tree' suite + + std::string suite_definition = + R"({"definition" : "suite full_suite\n family f\n task t\n label l \"value\"\n meter m 0 100 50\n event e\n endfamily\nendsuite\n# comment"})"; + handle_response(request("post", "/v1/suites", suite_definition, API_KEY), HttpStatusCode::success_created); + wait_until([] { return check_for_path("/v1/suites/full_suite/definition"); }); + + // (2) Retrieve 'full_suite' suite tree, explicitly requesting generated variables + { + auto result = handle_response(request("get", "/v1/suites/full_suite/f/tree?content=full&gen_vars=true")); + auto content = ojson::parse(result.body); + + BOOST_REQUIRE(content.contains("f")); + // Check family attributes + BOOST_REQUIRE(content["f"].contains("attributes")); + BOOST_REQUIRE(content["f"]["attributes"].size() == 2); + { + size_t count = 0; + for (const auto& attr : content["f"]["attributes"]) { + BOOST_REQUIRE(attr.contains("type")); + if (attr["type"] == "variable") { + BOOST_REQUIRE(attr.contains("name")); + BOOST_REQUIRE(attr.contains("value")); + if (attr.contains("generated")) { + BOOST_REQUIRE(attr["generated"] == true); + ++count; + } + } + } + BOOST_REQUIRE(count >= 2); + } + BOOST_REQUIRE(content["f"].contains("children")); + BOOST_REQUIRE(content["f"]["children"].contains("t")); + // Check task attributes + BOOST_REQUIRE(content["f"]["children"]["t"].contains("attributes")); + BOOST_REQUIRE(content["f"]["children"]["t"]["attributes"].size() == 11); + { + size_t count = 0; + for (const auto& attr : content["f"]["children"]["t"]["attributes"]) { + BOOST_REQUIRE(attr.contains("type")); + if (attr["type"] == "variable") { + BOOST_REQUIRE(attr.contains("name")); + BOOST_REQUIRE(attr.contains("value")); + if (attr.contains("generated")) { + BOOST_REQUIRE(attr["generated"] == true); + ++count; + } + } + } + BOOST_REQUIRE(count >= 2); + } + } + + // (4) Clean up + request("delete", "/v1/suites/full_suite/definition", "", API_KEY); + wait_until([] { return false == check_for_path("/v1/suites/full_suite/definition"); }); +} + BOOST_AUTO_TEST_CASE(test_token_authentication) { std::cout << "======== " << boost::unit_test::framework::current_test_case().p_name << " =========" << std::endl; From 72e3ed3ab10daa844c4b0ee01c34fe128e0b19e1 Mon Sep 17 00:00:00 2001 From: Marcos Bento Date: Tue, 8 Oct 2024 13:58:21 +0100 Subject: [PATCH 04/15] Add gen_vars description to documentation Re ECFLOW-1977 --- docs/rest_api.rst | 142 +++++++++++++++++++++++++--------------------- 1 file changed, 78 insertions(+), 64 deletions(-) diff --git a/docs/rest_api.rst b/docs/rest_api.rst index 385c23199..6299dfa65 100644 --- a/docs/rest_api.rst +++ b/docs/rest_api.rst @@ -492,7 +492,12 @@ Obtain the tree of all Suites * - Description - **Read** a tree with all suites * - Parameters - - :code:`content`, (optional), possible values: :code:`basic`, :code:`full`; :code:`with_id`, (optional), possible values: :code:`true`, :code:`false` + - :code:`content`, (optional), possible values: :code:`basic`, :code:`full` + + :code:`with_id`, (optional), possible values: :code:`true`, :code:`false` + + :code:`gen_vars`, (optional), possible values: :code:`true` + * - Payload - *empty* * - Response @@ -513,7 +518,69 @@ When query parameter :code:`content` is not provided, or query parameter is :cod } } -When query parameters :code:`content=full&with_id=true`, the full Suite tree is provided: +When query parameter :code:`content=full` is used, the full Suite tree is provided: + +.. code:: json + + { + "some_suite": { + "type": "suite", + "state": { + "node": "active", + "default": "complete" + }, + "attributes": [ + { + "name": "YMD", + "value": "20100114", + "const": false, + "type": "variable" + } + ], + "children": { + "some_family": { + "type": "family", + "state": { + "node": "active", + "default": "unknown" + }, + "attributes": [], + "children": { + "some_task": { + "type": "task", + "state": { + "node": "active", + "default": "unknown" + }, + "attributes": [ + { + "name": "some_label", + "value": "value", + "type": "label" + }, + { + "name": "some_meter", + "min": 0, + "max": 30, + "value": 0, + "type": "meter" + }, + { + "name": "some_event", + "value": false, + "initial_value": false, + "type": "event" + } + ], + "aliases": {} + } + } + } + } + } + } + +When query parameters :code:`content=full&with_id=true` are used, the full Suite tree with ids is provided: .. code:: json @@ -578,6 +645,9 @@ When query parameters :code:`content=full&with_id=true`, the full Suite tree is } } +When using the query parameter :code:`gen_vars=true` the generated variables are included in the :code:`attributes` section. +Generated variables can be identified by a :code:`generate` attribute set to :code:`true`. + Endpoint :code:`/v1/suites/{path}` ---------------------------------- @@ -621,71 +691,15 @@ Obtain the tree of a Node - **Read** a tree view of the node at :code:`/{path}` * - Parameters - :code:`content`, (optional), possible values: :code:`basic`, :code:`full` - * - Payload - - *empty* - * - Response - - :code:`{"b":{"c":""}}`, when parameter :code:`content` is not provided, or parameter :code:`content=basic` - - :code:`{"b":{"state":{"node":"...","default":"..."},"children":{...},"attributes":[...]}` - when parameter :code:`content=full` - -Response with Node tree -""""""""""""""""""""""" - -When query parameter :code:`content` is not provided, or query parameter is :code:`content=basic`, the basic Node tree is provided: -.. code:: json - - { - "family": { - "task": "" - } - } - -When query parameter :code:`content=full`, the full Node tree is provided: + :code:`with_id`, (optional), possible values: :code:`true`, :code:`false` -.. code:: json + :code:`gen_vars`, (optional), possible values: :code:`true` - { - "some_family": { - "type": "family", - "state": { - "node": "active", - "default": "unknown" - }, - "attributes": [], - "children": { - "some_task": { - "type": "task", - "state": { - "node": "active", - "default": "unknown" - }, - "attributes": [ - { - "name": "some_label", - "value": "value", - "type": "label" - }, - { - "name": "some_meter", - "min": 0, - "max": 30, - "value": 0, - "type": "meter" - }, - { - "name": "some_event", - "value": false, - "initial_value": false, - "type": "event" - } - ], - "aliases": {} - } - } - } - } + * - Payload + - *empty* + * - Response + - Same as `Suite tree `_ Endpoint :code:`/v1/suites/{path}/definition` --------------------------------------------- From f8cb542a1c234fe02dec226a8fff02df71d1ee4b Mon Sep 17 00:00:00 2001 From: Marcos Bento Date: Tue, 8 Oct 2024 14:53:38 +0100 Subject: [PATCH 05/15] Store Node inherited variables in array Re ECFLOW-1978 --- libs/rest/src/ecflow/http/ApiV1Impl.cpp | 53 +++++++++++++++---------- 1 file changed, 33 insertions(+), 20 deletions(-) diff --git a/libs/rest/src/ecflow/http/ApiV1Impl.cpp b/libs/rest/src/ecflow/http/ApiV1Impl.cpp index 0e641c933..1ae0cb453 100644 --- a/libs/rest/src/ecflow/http/ApiV1Impl.cpp +++ b/libs/rest/src/ecflow/http/ApiV1Impl.cpp @@ -228,6 +228,24 @@ ojson get_server_attributes() { return j; } +static ojson get_node_variables_array(const Node& node) { + auto container = ojson::array(); + // Collect 'normal' variables + for (const auto& variable : node.variables()) { + ojson object; + to_json(object, variable); + container.push_back(object); + } + // ... and generated variables + for (const auto& variable : node.gen_variables()) { + ojson object; + to_json(object, variable); + object["generated"] = true; + container.push_back(object); + } + return container; +} + ojson get_node_attributes(const std::string& path) { ojson j; @@ -237,7 +255,6 @@ ojson get_node_attributes(const std::string& path) { j["limits"] = node->limits(); j["inlimits"] = node->inlimits(); j["events"] = node->events(); - j["variables"] = node->variables(); j["labels"] = node->labels(); j["dates"] = node->dates(); j["days"] = node->days(); @@ -259,30 +276,26 @@ ojson get_node_attributes(const std::string& path) { j["avisos"] = node->avisos(); j["mirrors"] = node->mirrors(); - { - // Collect 'normal' variables - auto vars = node->variables(); - // ... and generated variables - node->gen_variables(vars); + j["variables"] = get_node_variables_array(*node); - j["variables"] = vars; - } - - apply_to_parents(node->parent(), [&j](const Node* n) { - // Collect 'normal' variables - auto vars = n->variables(); - // ... and generated variables - n->gen_variables(vars); + { + auto inherited = ojson::array(); - j["inherited_variables"][n->name()] = vars; - }); + // Collect 'inherited' variables from parents + apply_to_parents(node->parent(), [&inherited](const Node* n) { + inherited.push_back(ojson::object( + {{"name", n->name()}, {"path", n->absNodePath()}, {"variables", get_node_variables_array(*n)}})); + }); - auto server_variables = get_defs()->server().server_variables(); - auto user_variables = get_defs()->server().user_variables(); + // ... and from server + auto server_variables = get_defs()->server().server_variables(); + auto user_variables = get_defs()->server().user_variables(); + server_variables.insert(server_variables.end(), user_variables.begin(), user_variables.end()); - server_variables.insert(server_variables.end(), user_variables.begin(), user_variables.end()); + inherited.push_back(ojson::object({{"name", "server"}, {"path", "/"}, {"variables", server_variables}})); - j["inherited_variables"]["server"] = server_variables; + j["inherited_variables"] = inherited; + } return j; } From 2399ed3a8dfe4fa2153b70b803a3e30c6284c497 Mon Sep 17 00:00:00 2001 From: Marcos Bento Date: Wed, 9 Oct 2024 08:08:39 +0100 Subject: [PATCH 06/15] Add attributes response to documentation Re ECFLOW-1978 --- docs/rest_api.rst | 88 ++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 83 insertions(+), 5 deletions(-) diff --git a/docs/rest_api.rst b/docs/rest_api.rst index 6299dfa65..055788bed 100644 --- a/docs/rest_api.rst +++ b/docs/rest_api.rst @@ -501,7 +501,9 @@ Obtain the tree of all Suites * - Payload - *empty* * - Response - - See below for details. + - See :ref:`below ` for details. + +.. _response with suite tree: Response with Suite tree """""""""""""""""""""""" @@ -699,7 +701,7 @@ Obtain the tree of a Node * - Payload - *empty* * - Response - - Same as `Suite tree `_ + - Same as :ref:`Suite tree `. Endpoint :code:`/v1/suites/{path}/definition` --------------------------------------------- @@ -943,10 +945,86 @@ Obtain all Node attributes * - Payload - *empty* * - Response - - :code:`{"meters":[...],"variables":[...]}` + - See :ref:`below ` for details * - Example - :code:`curl https://localhost:8080/v1/suites/path/to/node/attributes` +.. _response with node attributes: + +Response with Node attributes +""""""""""""""""""""""""""""" + +The response to a request for node attributes has the following JSON format. +Notice how the node variables are separated from inherited variables. +The inherited variables grouped by ancestor node, each identified by the node name and absolute path. + +.. code-block:: text + + { + "path": "..." + + "meters": [ { ... }, ... ], + "limits": [ { ... }, ... ], + "inlimits": [ { ... }, ... ], + "events": [ { ... }, ... ], + "labels": [ { ... }, ... ], + "dates": [ { ... }, ... ], + "days": [ { ... }, ... ], + "crons": [ { ... }, ... ], + "times": [ { ... }, ... ], + "todays": [ { ... }, ... ], + "repeat": { ... }, + "trigger": { ... }, + "complete": { ... }, + "flag": { ... }, + "late": { ... }, + "zombies": [ { ... }, ... ], + "generics": [ { ... }, ... ], + "queues": [ { ... }, ... ], + "autocancel": { ... }, + "autoarchive": { ... }, + "autorestore": { ... }, + "avisos": [ { ... }, ... ], + "mirrors" : [ { ... }, ... ], + "variables": [ + { + "name": "..." + "value": "...", + "const": true|false + }, + { + "name": "..." + "value": "...", + "generated": true # only present if the variable is generated + "const": true|false + }, + ... + ] + "inherited_variables": [ + { + "name": "......" + "path": "......", + "variables: [ + { + "name": "..." + "value": "...", + "type": "variable", + "const": true|false + }, + { + "name": "..." + "value": "...", + "type": "variable", + "generated": true # only present if the variable is generated + "const": true|false + }, + ... + ] + }, + ... + } + } + Update a Node attribute ^^^^^^^^^^^^^^^^^^^^^^^ @@ -1276,8 +1354,8 @@ Create a new Server attribute * - Response - :code:`{"message":"Attribute added successfully"}` -Obtain a Server attribute -^^^^^^^^^^^^^^^^^^^^^^^^^ +Obtain all Server attributes +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .. list-table:: :stub-columns: 1 From ad4fa4e6154f5d82f3f618b7cd3a5ce921531678 Mon Sep 17 00:00:00 2001 From: Marcos Bento Date: Fri, 25 Oct 2024 15:35:19 +0100 Subject: [PATCH 07/15] Update list of Server and Suite generated variables --- .../ecflow_variables/generated_variables.rst | 74 +++++++++++++++---- 1 file changed, 61 insertions(+), 13 deletions(-) diff --git a/docs/ug/user_manual/ecflow_variables/generated_variables.rst b/docs/ug/user_manual/ecflow_variables/generated_variables.rst index 3ccb8c561..f8e48228c 100644 --- a/docs/ug/user_manual/ecflow_variables/generated_variables.rst +++ b/docs/ug/user_manual/ecflow_variables/generated_variables.rst @@ -25,6 +25,10 @@ The table below shows a list of generated variables. - Variable name - Explanation - Example + * - server + - ECF_MICRO + - The default preprocessor character used during variable substitution + - % * - server - ECF_TRIES - The default number of tries for each task @@ -45,18 +49,46 @@ The table below shows a list of generated variables. - ECF_JOB_CMD - Command to be executed to send a job - %ECF_JOB% 1> %ECF_JOBOUT% 2>&1 + * - server + - ECF_KILL_CMD + - Command to be executed to kill a job + - kill -15 %ECF_RID% + * - server + - ECF_STATUS_CMD + - Command to be executed to retrieve the job status + - ps --pid %ECF_RID% -f > %ECF_JOB%.stat 2>&1 + * - server + - ECF_URL_CMD + - Command to be executed to open job files in a browser + - ${BROWSER:=firefox} -new-tab %ECF_URL_BASE%/%ECF_URL% + * - server + - ECF_URL_BASE + - The base prefix for the job URL + - https://confluence.ecmwf.int + * - server + - ECF_URL + - The stem suffix for the job URL + - display/ECFLOW/ecflow+home * - server - ECF_LISTS - Name of the ecFlow white-list file - ecf.lists * - server - - ECF_PASS + - ECF_PASSWD - Default password string to replace the real password, when communicating with the server. - DJP + * - server + - ECF_CUSTOM_PASSWD + - Default custom password string to replace the real password, when communicating with the server. + - DJP * - server - ECF_LOG - Name of the ecFlow history, or log file - ecf.log + * - server + - ECF_INTERVAL + - The interval between check of time dependencies and job submission + - 60 * - server - ECF_CHECK - Name of the checkpoint file @@ -65,54 +97,70 @@ The table below shows a list of generated variables. - ECF_CHECKOLD - Name of the backup of the checkpoint file - ecf.check.b - * - Suite + * - server + - ECF_CHECKINTERVAL + - The interval, in seconds, between saving current state into the checkpoint file + - 120 + * - server + - ECF_CHECKMODE + - The mode to perform checkpoint file saving + - CHECK_NEVER, CHECK_ON_TIME, CHECK_ALWAYS + * - suite - SUITE - The name of the suite - Backarc - * - server + * - suite - DATE - Date of the suite in format DD.MM.YYYY - 21.02.2012 - * - server + * - suite - DAY - Full name of the weekday - Sunday - * - server + * - suite - DD - Day of the month, with two digits - 07 - * - server + * - suite - DOW - Day Of the Week - 0 - * - server + * - suite - DOY - Day Of the Year - 52 - * - server + * - suite - MM - The month of the year, with two digits - 02 - * - server + * - suite - MONTH - Full name of the month - February - * - server + * - suite - YYYY - A year with four digits - 2012 - * - server + * - suite - ECF_DATE - Single date in format YYYYMMDD - 20120221 - * - server + * - suite + - TIME + - Time of the suites clock, HHMM + - 2032 + * - suite - ECF_TIME - Time of the suites clock, HH:MM - 20:32 - * - server + * - suite - ECF_CLOCK - Composite weekday, month, Day Of the Week, Day Of Year - sunday:february:0:52 + * - suite + - ECF_JULIAN + - Julian day + - 2455979 * - family - FAMILY - The name of the family, (avoid using) From 690e64cbb560f07a029ce397a96715fc5c9de358 Mon Sep 17 00:00:00 2001 From: Marcos Bento Date: Fri, 25 Oct 2024 16:19:09 +0100 Subject: [PATCH 08/15] Add description of Auth file for Aviso and Mirror --- docs/glossary.rst | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/docs/glossary.rst b/docs/glossary.rst index c2870816a..57f92e111 100644 --- a/docs/glossary.rst +++ b/docs/glossary.rst @@ -1,4 +1,3 @@ - .. index:: single: Glossary @@ -87,6 +86,18 @@ responsible for polling the Aviso server, and periodically processing the latest notifications. + The authentication credentials file is expected to be in JSON format, following the `ECMWF Web API `_: + + .. code-block:: json + + { + "url" : "https://api.ecmwf.int/v1", + "key" : "", + "email" : "" + } + + Only the fields :code:`url`, :code:`key`, and :code:`email` are required; any additional fields are ignored. + check point The check point file is like the :term:`suite definition`, but includes all the state information. @@ -1417,6 +1428,17 @@ responsible for polling the remote ecFlow server, and periodically synchronise node status. + The authentication credentials file is expected to be in JSON, according to the following format: + + .. code-block:: json + + { + "username" : "", + "password" : "", + } + + Only the fields :code:`username`, and :code:`password` are required; any additional fields are ignored. + node :term:`suite`, :term:`family` and :term:`task` form a hierarchy. Where a :term:`suite` serves as the root of the hierarchy. From dff9526bd8b414ee0a58fa662a4d4c529fa811ae Mon Sep 17 00:00:00 2001 From: Marcos Bento Date: Fri, 25 Oct 2024 16:47:41 +0100 Subject: [PATCH 09/15] Allow default parameters for Aviso attribute Re ECFLOW-1981 --- .../src/ecflow/python/ExportNodeAttr.cpp | 36 ++++++++- libs/pyext/test/py_u_TestAviso.py | 76 +++++++++++++++++++ 2 files changed, 108 insertions(+), 4 deletions(-) diff --git a/libs/pyext/src/ecflow/python/ExportNodeAttr.cpp b/libs/pyext/src/ecflow/python/ExportNodeAttr.cpp index 384f6ac7d..f59a7824f 100644 --- a/libs/pyext/src/ecflow/python/ExportNodeAttr.cpp +++ b/libs/pyext/src/ecflow/python/ExportNodeAttr.cpp @@ -285,14 +285,38 @@ static job_creation_ctrl_ptr makeJobCreationCtrl() { static std::shared_ptr aviso_init(const std::string& name, const std::string& listener, - const std::string& url, - const std::string& schema, - const std::string& polling, - const std::string& auth) { + const std::string& url = AvisoAttr::default_url, + const std::string& schema = AvisoAttr::default_schema, + const std::string& polling = AvisoAttr::default_polling, + const std::string& auth = AvisoAttr::default_auth) { auto attr = std::make_shared(nullptr, name, listener, url, schema, polling, 0, auth, ""); return attr; } +static std::shared_ptr aviso_init_defaults_0(const std::string& name, const std::string& listener) { + return aviso_init(name, listener); +} + +static std::shared_ptr +aviso_init_defaults_1(const std::string& name, const std::string& listener, const std::string& url) { + return aviso_init(name, listener, url); +} + +static std::shared_ptr aviso_init_defaults_2(const std::string& name, + const std::string& listener, + const std::string& url, + const std::string& schema) { + return aviso_init(name, listener, url, schema); +} + +static std::shared_ptr aviso_init_defaults_3(const std::string& name, + const std::string& listener, + const std::string& url, + const std::string& schema, + const std::string& polling) { + return aviso_init(name, listener, url, schema, polling); +} + static std::shared_ptr mirror_init(const std::string& name, const std::string& path, const std::string& host, @@ -1018,6 +1042,10 @@ void export_NodeAttr() { class_("AvisoAttr", NodeAttrDoc::aviso_doc()) .def("__init__", make_constructor(&aviso_init)) + .def("__init__", make_constructor(&aviso_init_defaults_0)) + .def("__init__", make_constructor(&aviso_init_defaults_1)) + .def("__init__", make_constructor(&aviso_init_defaults_2)) + .def("__init__", make_constructor(&aviso_init_defaults_3)) .def(self == self) // __eq__ .def("__str__", &ecf::to_python_string) // __str__ .def("__copy__", copyObject) // __copy__ uses copy constructor diff --git a/libs/pyext/test/py_u_TestAviso.py b/libs/pyext/test/py_u_TestAviso.py index c78a8acba..99d7a516e 100644 --- a/libs/pyext/test/py_u_TestAviso.py +++ b/libs/pyext/test/py_u_TestAviso.py @@ -25,6 +25,78 @@ def can_create_aviso_from_parameters(): assert aviso.auth() == "auth" +def can_create_aviso_from_default_parameters_0(): + suite = ecf.Suite("s1") + + family = ecf.Family("f1") + suite.add_family(family) + + task = ecf.Task("f1", ecf.AvisoAttr("name", "listener")) + assert len(list(task.avisos)) == 1 + + actual = list(task.avisos)[0] + assert actual.name() == "name" + assert actual.listener() == "listener" + assert actual.url() == "%ECF_AVISO_URL%" + assert actual.schema() == "%ECF_AVISO_SCHEMA%" + assert actual.polling() == "%ECF_AVISO_POLLING%" + assert actual.auth() == "%ECF_AVISO_AUTH%" + + +def can_create_aviso_from_default_parameters_1(): + suite = ecf.Suite("s1") + + family = ecf.Family("f1") + suite.add_family(family) + + task = ecf.Task("f1", ecf.AvisoAttr("name", "listener", "url")) + assert len(list(task.avisos)) == 1 + + actual = list(task.avisos)[0] + assert actual.name() == "name" + assert actual.listener() == "listener" + assert actual.url() == "url" + assert actual.schema() == "%ECF_AVISO_SCHEMA%" + assert actual.polling() == "%ECF_AVISO_POLLING%" + assert actual.auth() == "%ECF_AVISO_AUTH%" + + +def can_create_aviso_from_default_parameters_2(): + suite = ecf.Suite("s1") + + family = ecf.Family("f1") + suite.add_family(family) + + task = ecf.Task("f1", ecf.AvisoAttr("name", "listener", "url", "schema")) + assert len(list(task.avisos)) == 1 + + actual = list(task.avisos)[0] + assert actual.name() == "name" + assert actual.listener() == "listener" + assert actual.url() == "url" + assert actual.schema() == "schema" + assert actual.polling() == "%ECF_AVISO_POLLING%" + assert actual.auth() == "%ECF_AVISO_AUTH%" + + +def can_create_aviso_from_default_parameters_3(): + suite = ecf.Suite("s1") + + family = ecf.Family("f1") + suite.add_family(family) + + task = ecf.Task("f1", ecf.AvisoAttr("name", "listener", "url", "schema", "polling")) + assert len(list(task.avisos)) == 1 + + actual = list(task.avisos)[0] + assert actual.name() == "name" + assert actual.listener() == "listener" + assert actual.url() == "url" + assert actual.schema() == "schema" + assert actual.polling() == "polling" + assert actual.auth() == "%ECF_AVISO_AUTH%" + + def can_add_aviso_to_task(): suite = ecf.Suite("s1") @@ -101,6 +173,10 @@ def can_check_job_creation_with_aviso(): Test.print_test_start(os.path.basename(__file__)) can_create_aviso_from_parameters() + can_create_aviso_from_default_parameters_0() + can_create_aviso_from_default_parameters_1() + can_create_aviso_from_default_parameters_2() + can_create_aviso_from_default_parameters_3() can_add_aviso_to_task() can_embed_aviso_into_task() cannot_have_multiple_avisos_in_single_task() From b69a0bf64f4fe8264b607bb94ed9c2c841e44af4 Mon Sep 17 00:00:00 2001 From: Marcos Bento Date: Fri, 25 Oct 2024 16:47:52 +0100 Subject: [PATCH 10/15] Allow default parameters for Mirror attribute Re ECFLOW-1981 --- .../src/ecflow/python/ExportNodeAttr.cpp | 52 +++++++++++++--- libs/pyext/test/py_u_TestMirror.py | 59 +++++++++++++++++++ 2 files changed, 104 insertions(+), 7 deletions(-) diff --git a/libs/pyext/src/ecflow/python/ExportNodeAttr.cpp b/libs/pyext/src/ecflow/python/ExportNodeAttr.cpp index f59a7824f..627dd40a7 100644 --- a/libs/pyext/src/ecflow/python/ExportNodeAttr.cpp +++ b/libs/pyext/src/ecflow/python/ExportNodeAttr.cpp @@ -319,15 +319,48 @@ static std::shared_ptr aviso_init_defaults_3(const std::string& name, static std::shared_ptr mirror_init(const std::string& name, const std::string& path, - const std::string& host, - const std::string& port, - const std::string& polling, - bool ssl, - const std::string& auth) { + const std::string& host = ecf::MirrorAttr::default_remote_host, + const std::string& port = ecf::MirrorAttr::default_remote_port, + const std::string& polling = ecf::MirrorAttr::default_polling, + bool ssl = false, + const std::string& auth = ecf::MirrorAttr::default_remote_auth) { auto attr = std::make_shared(nullptr, name, path, host, port, polling, ssl, auth, ""); return attr; } +static std::shared_ptr mirror_init_defaults_0(const std::string& name, const std::string& path) { + return mirror_init(name, path); +} + +static std::shared_ptr +mirror_init_defaults_1(const std::string& name, const std::string& path, const std::string& host) { + return mirror_init(name, path, host); +} + +static std::shared_ptr mirror_init_defaults_2(const std::string& name, + const std::string& path, + const std::string& host, + const std::string& port) { + return mirror_init(name, path, host, port); +} + +static std::shared_ptr mirror_init_defaults_3(const std::string& name, + const std::string& path, + const std::string& host, + const std::string& port, + const std::string& polling) { + return mirror_init(name, path, host, port, polling); +} + +static std::shared_ptr mirror_init_defaults_4(const std::string& name, + const std::string& path, + const std::string& host, + const std::string& port, + const std::string& polling, + bool ssl) { + return mirror_init(name, path, host, port, polling, ssl); +} + void export_NodeAttr() { // Trigger & Complete thin wrapper over Expression, allows us to call: Task("a").add(Trigger("a=1"),Complete("b=1")) class_>("Trigger", DefsDoc::trigger(), init()) @@ -969,8 +1002,8 @@ void export_NodeAttr() { .def("step", &Repeat::step, "The increment for the repeat, as an integer") .def("value", &Repeat::last_valid_value, "The current value of the repeat as an integer"); - void (CronAttr::*add_time_series)(const TimeSeries&) = &CronAttr::addTimeSeries; - void (CronAttr::*add_time_series_2)(const TimeSlot& s, const TimeSlot& f, const TimeSlot& i) = + void (CronAttr::* add_time_series)(const TimeSeries&) = &CronAttr::addTimeSeries; + void (CronAttr::* add_time_series_2)(const TimeSlot& s, const TimeSlot& f, const TimeSlot& i) = &CronAttr::addTimeSeries; class_>("Cron", NodeAttrDoc::cron_doc()) .def("__init__", raw_function(&cron_raw_constructor, 1)) // will call -> cron_init or cron_init1 @@ -1073,6 +1106,11 @@ void export_NodeAttr() { class_("MirrorAttr", NodeAttrDoc::mirror_doc()) .def("__init__", make_constructor(&mirror_init)) + .def("__init__", make_constructor(&mirror_init_defaults_0)) + .def("__init__", make_constructor(&mirror_init_defaults_1)) + .def("__init__", make_constructor(&mirror_init_defaults_2)) + .def("__init__", make_constructor(&mirror_init_defaults_3)) + .def("__init__", make_constructor(&mirror_init_defaults_4)) .def(self == self) // __eq__ .def("__str__", &ecf::to_python_string) // __str__ .def("__copy__", copyObject) // __copy__ uses copy constructor diff --git a/libs/pyext/test/py_u_TestMirror.py b/libs/pyext/test/py_u_TestMirror.py index 02bdc1535..c3822dabd 100644 --- a/libs/pyext/test/py_u_TestMirror.py +++ b/libs/pyext/test/py_u_TestMirror.py @@ -26,6 +26,61 @@ def can_create_mirror_from_parameters(): assert mirror.auth() == "auth" +def can_create_mirror_from_default_parameters_0(): + mirror = ecf.MirrorAttr("name", "r_path") + assert mirror.name() == "name" + assert mirror.remote_path() == "r_path" + assert mirror.remote_host() == "%ECF_MIRROR_REMOTE_HOST%" + assert mirror.remote_port() == "%ECF_MIRROR_REMOTE_PORT%" + assert mirror.polling() == "%ECF_MIRROR_REMOTE_POLLING%" + assert mirror.ssl() == False + assert mirror.auth() == "%ECF_MIRROR_REMOTE_AUTH%" + + +def can_create_mirror_from_default_parameters_1(): + mirror = ecf.MirrorAttr("name", "r_path", "r_host") + assert mirror.name() == "name" + assert mirror.remote_path() == "r_path" + assert mirror.remote_host() == "r_host" + assert mirror.remote_port() == "%ECF_MIRROR_REMOTE_PORT%" + assert mirror.polling() == "%ECF_MIRROR_REMOTE_POLLING%" + assert mirror.ssl() == False + assert mirror.auth() == "%ECF_MIRROR_REMOTE_AUTH%" + + +def can_create_mirror_from_default_parameters_2(): + mirror = ecf.MirrorAttr("name", "r_path", "r_host", "r_port") + assert mirror.name() == "name" + assert mirror.remote_path() == "r_path" + assert mirror.remote_host() == "r_host" + assert mirror.remote_port() == "r_port" + assert mirror.polling() == "%ECF_MIRROR_REMOTE_POLLING%" + assert mirror.ssl() == False + assert mirror.auth() == "%ECF_MIRROR_REMOTE_AUTH%" + + +def can_create_mirror_from_default_parameters_3(): + mirror = ecf.MirrorAttr("name", "r_path", "r_host", "r_port", "polling") + assert mirror.name() == "name" + assert mirror.remote_path() == "r_path" + assert mirror.remote_host() == "r_host" + assert mirror.remote_port() == "r_port" + assert mirror.polling() == "polling" + assert mirror.ssl() == False + assert mirror.auth() == "%ECF_MIRROR_REMOTE_AUTH%" + + +def can_create_mirror_from_default_parameters_4(): + mirror = ecf.MirrorAttr("name", "r_path", "r_host", "r_port", "polling", True) + assert mirror.name() == "name" + assert mirror.remote_path() == "r_path" + assert mirror.remote_host() == "r_host" + assert mirror.remote_port() == "r_port" + assert mirror.polling() == "polling" + assert mirror.ssl() == True + assert mirror.auth() == "%ECF_MIRROR_AUTH%" + + def can_add_mirror_to_task(): suite = ecf.Suite("s1") @@ -101,6 +156,10 @@ def can_check_job_creation_with_mirror(): Test.print_test_start(os.path.basename(__file__)) can_create_mirror_from_parameters() + can_create_mirror_from_default_parameters_0() + can_create_mirror_from_default_parameters_1() + can_create_mirror_from_default_parameters_2() + can_create_mirror_from_default_parameters_3() can_add_mirror_to_task() can_embed_mirror_into_task() cannot_have_multiple_mirrors_in_single_task() From 2c51e848d0eee576cbb5f6dcc290a1bbff2aff7d Mon Sep 17 00:00:00 2001 From: Marcos Bento Date: Fri, 25 Oct 2024 16:54:55 +0100 Subject: [PATCH 11/15] Update Aviso/Mirror Python documentation Re ECFLOW-1981 --- docs/python_api/AvisoAttr.rst | 2 ++ docs/python_api/MirrorAttr.rst | 2 ++ libs/pyext/src/ecflow/python/NodeAttrDoc.cpp | 8 ++++++-- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/docs/python_api/AvisoAttr.rst b/docs/python_api/AvisoAttr.rst index 0aa7b6473..d106dff92 100644 --- a/docs/python_api/AvisoAttr.rst +++ b/docs/python_api/AvisoAttr.rst @@ -33,6 +33,8 @@ Usage: t2 = Task('t2') t2.add_aviso('name', '{...}', 'http://aviso.com', '60', '/path/to/auth') +The parameters `url`, `schema`, `polling`, and `auth` are optional + .. py:method:: AvisoAttr.auth( (AvisoAttr)arg1) -> str : :module: ecflow diff --git a/docs/python_api/MirrorAttr.rst b/docs/python_api/MirrorAttr.rst index 0e613d7b8..a69c0d434 100644 --- a/docs/python_api/MirrorAttr.rst +++ b/docs/python_api/MirrorAttr.rst @@ -34,6 +34,8 @@ Usage: t2 = Task('t2') t2.add_aviso('name', '/remote/task', 'remote-ecflow', '3141', '60', True, '/path/to/auth') +The parameters `remote_host`, `remote_port`, `polling`, `ssl`, and `auth` are optional + .. py:method:: MirrorAttr.auth( (MirrorAttr)arg1) -> str : :module: ecflow diff --git a/libs/pyext/src/ecflow/python/NodeAttrDoc.cpp b/libs/pyext/src/ecflow/python/NodeAttrDoc.cpp index 8388b7ee7..5954496c1 100644 --- a/libs/pyext/src/ecflow/python/NodeAttrDoc.cpp +++ b/libs/pyext/src/ecflow/python/NodeAttrDoc.cpp @@ -728,7 +728,9 @@ const char* NodeAttrDoc::aviso_doc() { " AvisoAttr('name', '{...}', 'http://aviso.com', '60', '/path/to/auth'))\n" "\n" " t2 = Task('t2')\n" - " t2.add_aviso('name', '{...}', 'http://aviso.com', '60', '/path/to/auth')\n"; + " t2.add_aviso('name', '{...}', 'http://aviso.com', '60', '/path/to/auth')\n" + "\n" + "The parameters `url`, `schema`, `polling`, and `auth` are optional\n"; } const char* NodeAttrDoc::mirror_doc() { @@ -754,5 +756,7 @@ const char* NodeAttrDoc::mirror_doc() { " MirrorAttr('name', '/remote/task', 'remote-ecflow', '3141', '60', True, '/path/to/auth'))\n" "\n" " t2 = Task('t2')\n" - " t2.add_aviso('name', '/remote/task', 'remote-ecflow', '3141', '60', True, '/path/to/auth')\n"; + " t2.add_aviso('name', '/remote/task', 'remote-ecflow', '3141', '60', True, '/path/to/auth')\n" + "\n" + "The parameters `remote_host`, `remote_port`, `polling`, `ssl`, and `auth` are optional\n"; } From b861b3c31c2f352e59d0a0609451f6a017afd1e9 Mon Sep 17 00:00:00 2001 From: Marcos Bento Date: Fri, 25 Oct 2024 12:52:34 +0100 Subject: [PATCH 12/15] Correct Aviso Auth token format Re ECFLOW-1982 --- libs/node/src/ecflow/node/AvisoAttr.cpp | 4 +++- libs/rest/src/ecflow/http/HttpLibrary.hpp | 10 ++++++++-- libs/service/src/ecflow/service/auth/Credentials.cpp | 4 +++- libs/service/src/ecflow/service/auth/Credentials.hpp | 1 + libs/service/src/ecflow/service/aviso/Aviso.cpp | 9 +++++++++ libs/service/src/ecflow/service/aviso/AvisoService.cpp | 4 +++- libs/service/src/ecflow/service/aviso/etcd/Client.cpp | 2 +- 7 files changed, 28 insertions(+), 6 deletions(-) diff --git a/libs/node/src/ecflow/node/AvisoAttr.cpp b/libs/node/src/ecflow/node/AvisoAttr.cpp index 013c6873a..a5c5eb369 100644 --- a/libs/node/src/ecflow/node/AvisoAttr.cpp +++ b/libs/node/src/ecflow/node/AvisoAttr.cpp @@ -94,7 +94,6 @@ void AvisoAttr::reset() { } bool AvisoAttr::isFree() const { - SLOG(D, "AvisoAttr: check Aviso attribute (name: " << name_ << ", listener: " << listener_ << ") is free"); if (controller_ == nullptr) { return false; @@ -105,6 +104,7 @@ bool AvisoAttr::isFree() const { if (notifications.empty()) { // No notifications, nothing to do -- task continues to wait + SLOG(D, "AvisoAttr: (path: " << this->path() << ", name: " << name_ << ", listener: " << listener_ << "): no notifications found"); return false; } @@ -143,6 +143,8 @@ bool AvisoAttr::isFree() const { ecf::visit_parents(*parent_, [n = this->state_change_no_](Node& node) { node.set_state_change_no(n); }); + SLOG(D, "AvisoAttr: (path: " << this->path() << ", name: " << name_ << ", listener: " << listener_ << ") " << std::string{(is_free ? "" : "no ")} + "notifications found"); + return is_free; } diff --git a/libs/rest/src/ecflow/http/HttpLibrary.hpp b/libs/rest/src/ecflow/http/HttpLibrary.hpp index 0afdacee9..8ee15690a 100644 --- a/libs/rest/src/ecflow/http/HttpLibrary.hpp +++ b/libs/rest/src/ecflow/http/HttpLibrary.hpp @@ -23,9 +23,15 @@ #define CPPHTTPLIB_RECV_FLAGS MSG_NOSIGNAL #endif -#ifdef ECF_OPENSSL - #define CPPHTTPLIB_OPENSSL_SUPPORT +#if defined(ECF_OPENSSL) + #include + #if OPENSSL_VERSION_NUMBER < 0x1010100fL + #warning OpenSSL versions prior to 1.1.1 detected. Aviso ETCD HTTP client will be build without OpenSSL support! + #else + #define CPPHTTPLIB_OPENSSL_SUPPORT + #endif #endif + #ifdef ECF_HTTP_COMPRESSION #define CPPHTTPLIB_ZLIB_SUPPORT #endif diff --git a/libs/service/src/ecflow/service/auth/Credentials.cpp b/libs/service/src/ecflow/service/auth/Credentials.cpp index e1825b20b..20e0b442a 100644 --- a/libs/service/src/ecflow/service/auth/Credentials.cpp +++ b/libs/service/src/ecflow/service/auth/Credentials.cpp @@ -42,7 +42,9 @@ std::optional Credentials::user() const { std::optional Credentials::key() const { if (auto key = value("key"); key) { - return KeyCredentials{std::move(*key)}; + if (auto email = value("email"); email) { + return KeyCredentials{std::move(*email), std::move(*key)}; + } } return std::nullopt; } diff --git a/libs/service/src/ecflow/service/auth/Credentials.hpp b/libs/service/src/ecflow/service/auth/Credentials.hpp index bcb2132a2..a9fae62a5 100644 --- a/libs/service/src/ecflow/service/auth/Credentials.hpp +++ b/libs/service/src/ecflow/service/auth/Credentials.hpp @@ -27,6 +27,7 @@ class Credentials { struct KeyCredentials { + std::string email; std::string key; }; diff --git a/libs/service/src/ecflow/service/aviso/Aviso.cpp b/libs/service/src/ecflow/service/aviso/Aviso.cpp index c0f487b1e..30c1b5d19 100644 --- a/libs/service/src/ecflow/service/aviso/Aviso.cpp +++ b/libs/service/src/ecflow/service/aviso/Aviso.cpp @@ -10,6 +10,15 @@ #include "ecflow/service/aviso/Aviso.hpp" +#if defined(ECF_OPENSSL) + #include + #if OPENSSL_VERSION_NUMBER < 0x1010100fL + #warning OpenSSL versions prior to 1.1.1 detected. Aviso ETCD HTTP client will be build without OpenSSL support! + #else + #define CPPHTTPLIB_OPENSSL_SUPPORT + #endif +#endif + #include #include #include diff --git a/libs/service/src/ecflow/service/aviso/AvisoService.cpp b/libs/service/src/ecflow/service/aviso/AvisoService.cpp index d6849504d..ad7099d3e 100644 --- a/libs/service/src/ecflow/service/aviso/AvisoService.cpp +++ b/libs/service/src/ecflow/service/aviso/AvisoService.cpp @@ -129,7 +129,9 @@ void AvisoService::register_listener(const AvisoSubscribe& listen) { if (auto auth = listen.auth(); !auth.empty()) { auto credentials = ecf::service::auth::Credentials::load(auth); if (auto key_credentials = credentials.key(); key_credentials) { - inserted.auth_token = key_credentials->key; + auto email = key_credentials->email; + auto key = key_credentials->key; + inserted.auth_token = email + ":" + key; } else { SLOG(I, "AvisoService: no key found in auth token for listener {" << listener.path() << "}"); diff --git a/libs/service/src/ecflow/service/aviso/etcd/Client.cpp b/libs/service/src/ecflow/service/aviso/etcd/Client.cpp index 130795aba..c054a4008 100644 --- a/libs/service/src/ecflow/service/aviso/etcd/Client.cpp +++ b/libs/service/src/ecflow/service/aviso/etcd/Client.cpp @@ -42,7 +42,7 @@ std::vector> Client::poll(std::string_view k if (!auth_token_.empty()) { SLOG(D, "EtcdClient: using authorization token"); - headers.emplace("Authorization", "Bearer " + auth_token_); + headers.emplace("Authorization", "EmailKey " + auth_token_); } auto range = Range(key_prefix); From 43d4f5f684bf2363a89b685af3b03073c2df27f5 Mon Sep 17 00:00:00 2001 From: Marcos Bento Date: Tue, 29 Oct 2024 16:07:28 +0000 Subject: [PATCH 13/15] Correct enabling of Session dialog buttons - Remove unnecessary Label in Clone/Update Session dialog Re ECFLOW-1980 --- Viewer/ecflowUI/src/SessionDialog.cpp | 68 +++------------------- Viewer/ecflowUI/src/SessionDialog.hpp | 5 +- Viewer/ecflowUI/src/SessionRenameDialog.ui | 4 +- 3 files changed, 12 insertions(+), 65 deletions(-) diff --git a/Viewer/ecflowUI/src/SessionDialog.cpp b/Viewer/ecflowUI/src/SessionDialog.cpp index ee97e606b..156ad15b1 100644 --- a/Viewer/ecflowUI/src/SessionDialog.cpp +++ b/Viewer/ecflowUI/src/SessionDialog.cpp @@ -30,18 +30,20 @@ SessionDialog::SessionDialog(QWidget* parent) : QDialog(parent) { // what was saved last time? std::string lastSessionName = SessionHandler::instance()->lastSessionName(); int index = SessionHandler::instance()->indexFromName(lastSessionName); - if (index != -1) + if (index != -1) { savedSessionsList_->setCurrentItem(savedSessionsList_->topLevelItem(index)); // select this one in the table + } - if (SessionHandler::instance()->loadLastSessionAtStartup()) + if (SessionHandler::instance()->loadLastSessionAtStartup()) { restoreLastSessionCb_->setCheckState(Qt::Checked); - else + } + else { restoreLastSessionCb_->setCheckState(Qt::Unchecked); + } newButton_->setVisible(false); // XXX TODO: enable New Session functionality // ensure the correct state of the Save button - on_sessionNameEdit__textChanged(); setButtonsEnabledStatus(); } @@ -101,39 +103,7 @@ void SessionDialog::setButtonsEnabledStatus() { cloneButton_->setEnabled(true); } } - -bool SessionDialog::validSaveName(const std::string& /*name*/) { - /* - QString boxName(QObject::tr("Save session")); - // name empty? - if (name.empty()) - { - QMessageBox::critical(0,boxName, tr("Please enter a name for the session")); - return false; - } - - - // is there already a session with this name? - bool sessionWithThisName = (SessionHandler::instance()->find(name) != NULL); - - if (sessionWithThisName) - { - QMessageBox::critical(0,boxName, tr("A session with that name already exists - please choose another - name")); return false; - } - else - { - return true; - }*/ - return true; -} - -void SessionDialog::on_sessionNameEdit__textChanged() { - // only allow to save a non-empty session name - // saveButton_->setEnabled(!sessionNameEdit_->text().isEmpty()); -} - -void SessionDialog::on_savedSessionsList__currentRowChanged(int /*currentRow*/) { +void SessionDialog::on_savedSessionsList__currentItemChanged(QTreeWidgetItem* current, QTreeWidgetItem* previous) { setButtonsEnabledStatus(); } @@ -142,6 +112,7 @@ void SessionDialog::on_cloneButton__clicked() { assert(!sessionName.empty()); // it should not be possible for the name to be empty SessionRenameDialog renameDialog; + renameDialog.setWindowTitle("Clone Session"); renameDialog.exec(); int result = renameDialog.result(); @@ -176,6 +147,7 @@ void SessionDialog::on_renameButton__clicked() { assert(item); // it should not be possible for the name to be empty SessionRenameDialog renameDialog; + renameDialog.setWindowTitle("Rename Session"); renameDialog.exec(); int result = renameDialog.result(); @@ -202,25 +174,3 @@ void SessionDialog::on_switchToButton__clicked() { accept(); // close the dialogue and continue loading the main user interface } - -void SessionDialog::on_saveButton__clicked() { - /* - std::string name = sessionNameEdit_->text().toStdString(); - - if (validSaveName(name)) - { - SessionItem* newSession = - SessionHandler::instance()->copySession(SessionHandler::instance()->current(), name); if (newSession) - { - //SessionHandler::instance()->add(name); - refreshListOfSavedSessions(); - SessionHandler::instance()->current(newSession); - QMessageBox::information(0,tr("Session"), tr("Session saved")); - } - else - { - QMessageBox::critical(0,tr("Session"), tr("Failed to save session")); - } - close(); - }*/ -} diff --git a/Viewer/ecflowUI/src/SessionDialog.hpp b/Viewer/ecflowUI/src/SessionDialog.hpp index 7eb0266e6..d5772bfcd 100644 --- a/Viewer/ecflowUI/src/SessionDialog.hpp +++ b/Viewer/ecflowUI/src/SessionDialog.hpp @@ -28,9 +28,7 @@ class SessionDialog : public QDialog, protected Ui::SessionDialog { ~SessionDialog() override; public Q_SLOTS: - void on_saveButton__clicked(); - void on_sessionNameEdit__textChanged(); - void on_savedSessionsList__currentRowChanged(int currentRow); + void on_savedSessionsList__currentItemChanged(QTreeWidgetItem *current, QTreeWidgetItem *previous); void on_cloneButton__clicked(); void on_deleteButton__clicked(); void on_renameButton__clicked(); @@ -39,7 +37,6 @@ public Q_SLOTS: private: // Ui::SaveSessionAsDialog *ui; void addSessionToTable(SessionItem* s); - bool validSaveName(const std::string& name); void refreshListOfSavedSessions(); void setButtonsEnabledStatus(); std::string selectedSessionName(); diff --git a/Viewer/ecflowUI/src/SessionRenameDialog.ui b/Viewer/ecflowUI/src/SessionRenameDialog.ui index a82d3c899..b30228fd4 100644 --- a/Viewer/ecflowUI/src/SessionRenameDialog.ui +++ b/Viewer/ecflowUI/src/SessionRenameDialog.ui @@ -11,7 +11,7 @@ - Add variable + true @@ -26,7 +26,7 @@ - New name: + From ed1acadedada8893e6e4c0fc1bfa8d4884165814 Mon Sep 17 00:00:00 2001 From: Marcos Bento Date: Tue, 29 Oct 2024 16:08:22 +0000 Subject: [PATCH 14/15] Avoid incorrect error message related to renamed session configuration file Re ECFLOW-1980 --- Viewer/ecflowUI/src/SessionHandler.cpp | 7 +++++++ Viewer/ecflowUI/src/SessionHandler.hpp | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/Viewer/ecflowUI/src/SessionHandler.cpp b/Viewer/ecflowUI/src/SessionHandler.cpp index cd46cec57..d96498efe 100644 --- a/Viewer/ecflowUI/src/SessionHandler.cpp +++ b/Viewer/ecflowUI/src/SessionHandler.cpp @@ -36,6 +36,13 @@ SessionItem::~SessionItem() { } } +void SessionItem::name(const std::string& name) { + name_ = name; + // Since the name changes, we need to update the related directory paths + dirPath_ = SessionHandler::sessionDirName(name_); + qtPath_ = SessionHandler::sessionQtDirName(name_); +} + void SessionItem::checkDir() { dirPath_ = SessionHandler::sessionDirName(name_); DirectoryHandler::createDir(dirPath_); diff --git a/Viewer/ecflowUI/src/SessionHandler.hpp b/Viewer/ecflowUI/src/SessionHandler.hpp index 47fa68c48..e114c21ad 100644 --- a/Viewer/ecflowUI/src/SessionHandler.hpp +++ b/Viewer/ecflowUI/src/SessionHandler.hpp @@ -21,7 +21,7 @@ class SessionItem { explicit SessionItem(const std::string&); virtual ~SessionItem(); - void name(const std::string& name) { name_ = name; } + void name(const std::string& name); const std::string& name() const { return name_; } std::string sessionFile() const; From e40d0a12f3ed8f1323ccd9b3b191da0cff7f57af Mon Sep 17 00:00:00 2001 From: Marcos Bento Date: Thu, 31 Oct 2024 15:38:19 +0000 Subject: [PATCH 15/15] Prepare ecFlow 5.13.5 release --- CMakeLists.txt | 2 +- docs/conf.py | 2 +- docs/release_notes/version_5.13.rst | 27 +++++++++++++++++++++++++++ libs/pyext/ecflow/__init__.py | 2 +- 4 files changed, 30 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index d4107e8fa..53e8b98a4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -24,7 +24,7 @@ endif() find_package( ecbuild 3.4 REQUIRED HINTS ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/../ecbuild /workspace/ecbuild) # Before project() -project( ecflow LANGUAGES CXX VERSION 5.13.4 ) +project( ecflow LANGUAGES CXX VERSION 5.13.5 ) # Important: the project version is used, as generated CMake variables, to filter .../ecflow/core/ecflow_version.h.in diff --git a/docs/conf.py b/docs/conf.py index 0dbfbf679..9f1dc5d68 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -93,7 +93,7 @@ def get_ecflow_version(): - version = "5.13.4" + version = "5.13.5" ecflow_version = version.split(".") print("Extracted ecflow version '" + str(ecflow_version)) return ecflow_version diff --git a/docs/release_notes/version_5.13.rst b/docs/release_notes/version_5.13.rst index ee8845ffc..ee6ba59ca 100644 --- a/docs/release_notes/version_5.13.rst +++ b/docs/release_notes/version_5.13.rst @@ -6,6 +6,33 @@ Version 5.13 updates .. role:: jiraissue :class: hidden +Version 5.13.5 +============== + +* `Released `__\ on 2024-10-31 + +General +------- + +- **Fix** correct the token used for Aviso authentication :jiraissue:`ECFLOW-1982` + +ecFlow UI +--------- + +- **Fix** correct the disabled/enabled status of Session dialog buttons :jiraissue:`ECFLOW-1980` + +ecFlow HTTP +----------- + +- **Fix** reorganise :code:`inherited_variables` to allow repeated names of Node ancestors :jiraissue:`ECFLOW-1978` +- **Fix** add missing inherited variables to response of endppoint :code:`.../tree` :jiraissue:`ECFLOW-1977` + +Python +------ + +- **Fix** correct :code:`check_job_creation()` crash on suite with Mirror/Aviso attribute :jiraissue:`ECFLOW-1976` +- **Fix** allow default parameters in Aviso/Mirror attribute constructors :jiraissue:`ECFLOW-1981` + Version 5.13.4 ============== diff --git a/libs/pyext/ecflow/__init__.py b/libs/pyext/ecflow/__init__.py index afc50e0da..48334cbdf 100644 --- a/libs/pyext/ecflow/__init__.py +++ b/libs/pyext/ecflow/__init__.py @@ -15,6 +15,6 @@ The ecFlow python module """ -__version__ = '5.13.4' +__version__ = '5.13.5' # http://stackoverflow.com/questions/13040646/how-do-i-create-documentation-with-pydoc