Merge pull request #973 from nedimtokic/management-api-interfaces-det…

…ails List all interface definitions
astarte-platform · Jul 18, 2024 · 11e9bc6 · 11e9bc6
2 parents c706a72 + bcdc5d3
commit 11e9bc6
Show file tree

Hide file tree

Showing 14 changed files with 248 additions and 11 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -34,6 +34,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 - [astarte_realm_management_api] Allow to read realm's maximum datastream
   storage retention period with the `/config/datastream_maximum_storage_retention`
   endpoint.
+- [astarte_realm_management_api] Allow to list all interfaces definitions using 
+  the `detailed=true` parameter
 
 ### Changed
 - Forward port changes from release 1.1.

diff --git a/apps/astarte_realm_management/lib/astarte_realm_management/engine.ex b/apps/astarte_realm_management/lib/astarte_realm_management/engine.ex
@@ -471,6 +471,14 @@ defmodule Astarte.RealmManagement.Engine do
     end
   end
 
+  def get_detailed_interfaces_list(realm_name) do
+    _ = Logger.debug("Get detailed interfaces list.")
+
+    with {:ok, client} <- Database.connect(realm: realm_name) do
+      Queries.get_detailed_interfaces_list(client)
+    end
+  end
+
   def get_jwt_public_key_pem(realm_name) do
     _ = Logger.debug("Get JWT public key PEM.")
 

diff --git a/apps/astarte_realm_management/lib/astarte_realm_management/queries.ex b/apps/astarte_realm_management/lib/astarte_realm_management/queries.ex
@@ -1027,6 +1027,94 @@ defmodule Astarte.RealmManagement.Queries do
     end
   end
 
+  def get_detailed_interfaces_list(client) do
+    with {:ok, interfaces} <- fetch_interfaces_without_mappings(client),
+         {:ok, mappings} <- fetch_mappings(client) do
+      # Convert list to a map grouped by interface_id
+      mappings_map = Enum.group_by(mappings, & &1.interface_id)
+
+      # Merge mappings into parent interfaces
+      interfaces_details =
+        Enum.map(interfaces, fn interface ->
+          interface_mappings = Map.get(mappings_map, interface.interface_id, [])
+          Map.put(interface, :mappings, interface_mappings)
+        end)
+
+      # Encode interfaces to JSON
+      interfaces_jsons =
+        Enum.map(interfaces_details, fn interface ->
+          %InterfaceDocument{
+            name: interface.name,
+            major_version: interface.major_version,
+            minor_version: interface.minor_version,
+            interface_id: interface.interface_id,
+            type: interface.type,
+            ownership: interface.ownership,
+            aggregation: interface.aggregation,
+            mappings: interface.mappings
+          }
+          |> Jason.encode!()
+        end)
+
+      {:ok, interfaces_jsons}
+    end
+  end
+
+  defp fetch_interfaces_without_mappings(client) do
+    interfaces_details_query = """
+    SELECT * FROM interfaces
+    """
+
+    query =
+      DatabaseQuery.new()
+      |> DatabaseQuery.statement(interfaces_details_query)
+      |> DatabaseQuery.consistency(:quorum)
+
+    with {:ok, interfaces_result} <- DatabaseQuery.call(client, query) do
+      interfaces = Enum.map(interfaces_result, &InterfaceDescriptor.from_db_result!/1)
+      {:ok, interfaces}
+    else
+      %{acc: _, msg: error_message} ->
+        _ = Logger.warning("Database error: #{error_message}.", tag: "db_error")
+        {:error, :database_error}
+
+      {:error, reason} ->
+        _ =
+          Logger.warning("Database error: failed with reason: #{inspect(reason)}.",
+            tag: "db_error"
+          )
+
+        {:error, :database_error}
+    end
+  end
+
+  defp fetch_mappings(client) do
+    all_endpoints_cols_statement = """
+    SELECT *
+    FROM endpoints
+    """
+
+    endpoints_query =
+      DatabaseQuery.new()
+      |> DatabaseQuery.statement(all_endpoints_cols_statement)
+      |> DatabaseQuery.consistency(:quorum)
+
+    with {:ok, endpoints_result} <- DatabaseQuery.call(client, endpoints_query) do
+      {:ok, Enum.map(endpoints_result, &Mapping.from_db_result!/1)}
+    else
+      :empty_dataset ->
+        {:error, :interface_not_found}
+
+      %{acc: _, msg: error_message} ->
+        _ = Logger.warning("Database error: #{error_message}.", tag: "db_error")
+        {:error, :database_error}
+
+      {:error, reason} ->
+        _ = Logger.warning("Failed, reason: #{inspect(reason)}.", tag: "db_error")
+        {:error, :database_error}
+    end
+  end
+
   def get_interfaces_list(client) do
     interfaces_list_statement = """
     SELECT DISTINCT name FROM interfaces

diff --git a/apps/astarte_realm_management/lib/astarte_realm_management/rpc/handler.ex b/apps/astarte_realm_management/lib/astarte_realm_management/rpc/handler.ex
@@ -55,7 +55,9 @@ defmodule Astarte.RealmManagement.RPC.Handler do
     GetTriggerPolicySource,
     GetTriggerPolicySourceReply,
     DeleteTriggerPolicy,
-    DeleteDevice
+    DeleteDevice,
+    GetDetailedInterfacesList,
+    GetDetailedInterfacesListReply
   }
 
   alias Astarte.Core.Triggers.Trigger
@@ -187,6 +189,14 @@ defmodule Astarte.RealmManagement.RPC.Handler do
     {:ok, Reply.encode(%Reply{error: false, reply: {:get_trigger_policy_source_reply, msg}})}
   end
 
+  def encode_reply(:get_detailed_interfaces_list, {:ok, reply}) do
+    msg = %GetDetailedInterfacesListReply{
+      interface_json: reply
+    }
+
+    {:ok, Reply.encode(%Reply{error: false, reply: {:get_detailed_interfaces_list_reply, msg}})}
+  end
+
   def encode_reply(:delete_trigger_policy, :ok) do
     {:ok, Reply.encode(%Reply{error: false, reply: {:generic_ok_reply, %GenericOkReply{}}})}
   end
@@ -295,6 +305,14 @@ defmodule Astarte.RealmManagement.RPC.Handler do
               _ = Logger.metadata(realm: realm_name)
               encode_reply(:get_interfaces_list, Engine.get_interfaces_list(realm_name))
 
+            {:get_detailed_interfaces_list, %GetDetailedInterfacesList{realm_name: realm_name}} ->
+              _ = Logger.metadata(realm: realm_name)
+
+              encode_reply(
+                :get_detailed_interfaces_list,
+                Engine.get_detailed_interfaces_list(realm_name)
+              )
+
             {:update_interface,
              %UpdateInterface{
                realm_name: realm_name,

diff --git a/apps/astarte_realm_management/mix.lock b/apps/astarte_realm_management/mix.lock
@@ -3,7 +3,7 @@
   "amqp_client": {:hex, :amqp_client, "3.12.10", "dcc0d5d0037fa2b486c6eb8b52695503765b96f919e38ca864a7b300b829742d", [:make, :rebar3], [{:credentials_obfuscation, "3.4.0", [hex: :credentials_obfuscation, repo: "hexpm", optional: false]}, {:rabbit_common, "3.12.10", [hex: :rabbit_common, repo: "hexpm", optional: false]}], "hexpm", "16a23959899a82d9c2534ed1dcf1fa281d3b660fb7f78426b880647f0a53731f"},
   "astarte_core": {:git, "https://github.com/astarte-platform/astarte_core.git", "dc964b7d9b3a3a4e20127b763705d9e53bd88890", []},
   "astarte_data_access": {:git, "https://github.com/astarte-platform/astarte_data_access.git", "6efd21dcab8c16affa1888cc5e9218ecf7df67f2", []},
-  "astarte_rpc": {:git, "https://github.com/astarte-platform/astarte_rpc.git", "c0d77760fb12256a23a2d00e1a119496a089fa9d", []},
+  "astarte_rpc": {:git, "https://github.com/astarte-platform/astarte_rpc.git", "225d179ed87e8a1899626d29d0d7b73f19325120", []},
   "castore": {:hex, :castore, "1.0.7", "b651241514e5f6956028147fe6637f7ac13802537e895a724f90bf3e36ddd1dd", [:mix], [], "hexpm", "da7785a4b0d2a021cd1292a60875a784b6caef71e76bf4917bdee1f390455cf5"},
   "certifi": {:hex, :certifi, "2.9.0", "6f2a475689dd47f19fb74334859d460a2dc4e3252a3324bd2111b8f0429e7e21", [:rebar3], [], "hexpm", "266da46bdb06d6c6d35fde799bcb28d36d985d424ad7c08b5bb48f5b5cdd4641"},
   "connection": {:hex, :connection, "1.1.0", "ff2a49c4b75b6fb3e674bfc5536451607270aac754ffd1bdfe175abe4a6d7a68", [:mix], [], "hexpm", "722c1eb0a418fbe91ba7bd59a47e28008a189d47e37e0e7bb85585a016b2869c"},

diff --git a/apps/astarte_realm_management/test/astarte_realm_management/queries_test.exs b/apps/astarte_realm_management/test/astarte_realm_management/queries_test.exs
@@ -258,6 +258,54 @@ defmodule Astarte.RealmManagement.QueriesTest do
     end)
   end
 
+  test "get_interfaces_details/1 returns interface with mappings" do
+    {:ok, _} = DatabaseTestHelper.connect_to_test_database()
+    client = connect_to_test_realm("autotestrealm")
+
+    interface_doc = %InterfaceDocument{
+      name: "org.astarte-platform.genericsensors.Values",
+      major_version: 1,
+      minor_version: 0,
+      interface_id: <<194, 56, 178, 68, 185, 15, 76, 109, 242, 118, 37, 118, 139, 246, 171, 172>>,
+      type: :datastream,
+      ownership: :device,
+      aggregation: :individual,
+      mappings: [
+        %Astarte.Core.Mapping{
+          allow_unset: false,
+          database_retention_policy: :no_ttl,
+          database_retention_ttl: nil,
+          description: "Sampled real value.",
+          doc: "Datastream of sampled real values.",
+          endpoint: "/%{sensor_id}/value",
+          endpoint_id: <<51, 117, 20, 18, 62, 119, 173, 31, 173, 87, 40, 12, 201, 250, 213, 129>>,
+          expiry: 0,
+          explicit_timestamp: true,
+          interface_id:
+            <<194, 56, 178, 68, 185, 15, 76, 109, 242, 118, 37, 118, 139, 246, 171, 172>>,
+          path: nil,
+          reliability: :unreliable,
+          retention: :discard,
+          type: nil,
+          value_type: :double
+        }
+      ]
+    }
+
+    {:ok, automaton} = Astarte.Core.Mapping.EndpointsAutomaton.build(interface_doc.mappings)
+
+    Queries.install_new_interface(client, interface_doc, automaton)
+
+    {:ok, interface_list} = Queries.get_detailed_interfaces_list(client)
+    {:ok, interface_list_decoded} = Jason.decode(List.first(interface_list))
+
+    interface_from_db = InterfaceDocument.changeset(%InterfaceDocument{}, interface_list_decoded)
+
+    {:ok, interface_doc_from_db} = Ecto.Changeset.apply_action(interface_from_db, :insert)
+
+    assert interface_doc_from_db == interface_doc
+  end
+
   test "object interface install" do
     {:ok, _} = DatabaseTestHelper.connect_to_test_database()
     client = connect_to_test_realm("autotestrealm")

diff --git a/apps/astarte_realm_management_api/lib/astarte_realm_management_api/interfaces/interfaces.ex b/apps/astarte_realm_management_api/lib/astarte_realm_management_api/interfaces/interfaces.ex
@@ -20,8 +20,12 @@ defmodule Astarte.RealmManagement.API.Interfaces do
   alias Astarte.Core.Interface
   alias Astarte.RealmManagement.API.RPC.RealmManagement
 
-  def list_interfaces(realm_name) do
-    RealmManagement.get_interfaces_list(realm_name)
+  def list_interfaces(realm_name, params \\ %{}) do
+    if params["detailed"] do
+      RealmManagement.get_detailed_interfaces_list(realm_name)
+    else
+      RealmManagement.get_interfaces_list(realm_name)
+    end
   end
 
   def list_interface_major_versions(realm_name, id) do

diff --git a/apps/astarte_realm_management_api/lib/astarte_realm_management_api/rpc/realm_management.ex b/apps/astarte_realm_management_api/lib/astarte_realm_management_api/rpc/realm_management.ex
@@ -53,7 +53,9 @@ defmodule Astarte.RealmManagement.API.RPC.RealmManagement do
     GetTriggerPolicySource,
     GetTriggerPolicySourceReply,
     DeleteTriggerPolicy,
-    DeleteDevice
+    DeleteDevice,
+    GetDetailedInterfacesList,
+    GetDetailedInterfacesListReply
   }
 
   alias Astarte.Core.Triggers.SimpleTriggersProtobuf.TaggedSimpleTrigger
@@ -86,6 +88,16 @@ defmodule Astarte.RealmManagement.API.RPC.RealmManagement do
     |> extract_reply()
   end
 
+  def get_detailed_interfaces_list(realm_name) do
+    %GetDetailedInterfacesList{
+      realm_name: realm_name
+    }
+    |> encode_call(:get_detailed_interfaces_list)
+    |> @rpc_client.rpc_call(@destination)
+    |> decode_reply()
+    |> extract_reply()
+  end
+
   def get_interface(realm_name, interface_name, interface_major_version) do
     %GetInterfaceSource{
       realm_name: realm_name,
@@ -356,6 +368,13 @@ defmodule Astarte.RealmManagement.API.RPC.RealmManagement do
     {:ok, list}
   end
 
+  defp extract_reply(
+         {:get_detailed_interfaces_list_reply,
+          %GetDetailedInterfacesListReply{interface_json: list}}
+       ) do
+    {:ok, list}
+  end
+
   defp extract_reply({:get_interface_source_reply, %GetInterfaceSourceReply{source: source}}) do
     {:ok, source}
   end

diff --git a/...m_management_api/lib/astarte_realm_management_api_web/controllers/interface_controller.ex b/...m_management_api/lib/astarte_realm_management_api_web/controllers/interface_controller.ex
@@ -24,9 +24,12 @@ defmodule Astarte.RealmManagement.APIWeb.InterfaceController do
 
   action_fallback Astarte.RealmManagement.APIWeb.FallbackController
 
-  def index(conn, %{"realm_name" => realm_name}) do
-    with {:ok, interfaces} <- Astarte.RealmManagement.API.Interfaces.list_interfaces(realm_name) do
-      render(conn, "index.json", interfaces: interfaces)
+  def index(conn, %{"realm_name" => realm_name} = params) do
+    detailed = Map.get(params, "detailed") == "true"
+
+    with {:ok, interfaces} <- Interfaces.list_interfaces(realm_name, %{"detailed" => detailed}) do
+      interface_list = if detailed, do: Enum.map(interfaces, &Jason.decode!/1), else: interfaces
+      render(conn, "index.json", interfaces: interface_list)
     end
   end
 

diff --git a/apps/astarte_realm_management_api/mix.lock b/apps/astarte_realm_management_api/mix.lock
@@ -2,7 +2,7 @@
   "amqp": {:hex, :amqp, "3.3.0", "056d9f4bac96c3ab5a904b321e70e78b91ba594766a1fc2f32afd9c016d9f43b", [:mix], [{:amqp_client, "~> 3.9", [hex: :amqp_client, repo: "hexpm", optional: false]}], "hexpm", "8d3ae139d2646c630d674a1b8d68c7f85134f9e8b2a1c3dd5621616994b10a8b"},
   "amqp_client": {:hex, :amqp_client, "3.12.10", "dcc0d5d0037fa2b486c6eb8b52695503765b96f919e38ca864a7b300b829742d", [:make, :rebar3], [{:credentials_obfuscation, "3.4.0", [hex: :credentials_obfuscation, repo: "hexpm", optional: false]}, {:rabbit_common, "3.12.10", [hex: :rabbit_common, repo: "hexpm", optional: false]}], "hexpm", "16a23959899a82d9c2534ed1dcf1fa281d3b660fb7f78426b880647f0a53731f"},
   "astarte_core": {:git, "https://github.com/astarte-platform/astarte_core.git", "dc964b7d9b3a3a4e20127b763705d9e53bd88890", []},
-  "astarte_rpc": {:git, "https://github.com/astarte-platform/astarte_rpc.git", "c0d77760fb12256a23a2d00e1a119496a089fa9d", []},
+  "astarte_rpc": {:git, "https://github.com/astarte-platform/astarte_rpc.git", "225d179ed87e8a1899626d29d0d7b73f19325120", []},
   "castore": {:hex, :castore, "1.0.7", "b651241514e5f6956028147fe6637f7ac13802537e895a724f90bf3e36ddd1dd", [:mix], [], "hexpm", "da7785a4b0d2a021cd1292a60875a784b6caef71e76bf4917bdee1f390455cf5"},
   "certifi": {:hex, :certifi, "2.9.0", "6f2a475689dd47f19fb74334859d460a2dc4e3252a3324bd2111b8f0429e7e21", [:rebar3], [], "hexpm", "266da46bdb06d6c6d35fde799bcb28d36d985d424ad7c08b5bb48f5b5cdd4641"},
   "cors_plug": {:hex, :cors_plug, "2.0.3", "316f806d10316e6d10f09473f19052d20ba0a0ce2a1d910ddf57d663dac402ae", [:mix], [{:plug, "~> 1.8", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "ee4ae1418e6ce117fc42c2ba3e6cbdca4e95ecd2fe59a05ec6884ca16d469aea"},

diff --git a/apps/astarte_realm_management_api/priv/static/astarte_realm_management_api.yaml b/apps/astarte_realm_management_api/priv/static/astarte_realm_management_api.yaml
@@ -124,10 +124,20 @@ paths:
       tags:
         - interface
       summary: Get interface list
-      description: Get a list of all installed interface names.
+      description: Get a list of all installed interfaces. By default a list of interface names 
+        is returned. The complete interface definitions list can be optionally retrieved rather than interfaces names list using
+        the `detailed` option.
       operationId: getInterfaceList
       security:
         - JWT: []
+      parameters:
+        - name: detailed
+          in: query
+          description: If true, interface definitions are returned instead of just names.
+          required: false
+          schema:
+            type: boolean
+            default: false
       responses:
         '200':
           $ref: '#/components/responses/GetInterfaceList'

diff --git a/...ement_api/test/astarte_realm_management_api_web/controllers/interface_controller_test.exs b/...ement_api/test/astarte_realm_management_api_web/controllers/interface_controller_test.exs
@@ -71,13 +71,27 @@ defmodule Astarte.RealmManagement.APIWeb.InterfaceControllerTest do
       assert json_response(conn, 200)["data"] == []
     end
 
+    test "lists empty interfaces with details", %{conn: conn} do
+      conn = get(conn, interface_path(conn, :index, @realm), detailed: true)
+      assert json_response(conn, 200)["data"] == []
+    end
+
     test "lists interface after installing it", %{conn: conn} do
       post_conn = post(conn, interface_path(conn, :create, @realm), data: @valid_attrs)
       assert response(post_conn, 201) == ""
 
       list_conn = get(conn, interface_path(conn, :index, @realm))
       assert json_response(list_conn, 200)["data"] == [@interface_name]
     end
+
+    test "lists interface definitions after installing it", %{conn: conn} do
+      post_conn = post(conn, interface_path(conn, :create, @realm), data: @valid_attrs)
+      assert response(post_conn, 201) == ""
+
+      list_conn = get(conn, interface_path(conn, :index, @realm), detailed: true)
+
+      assert json_response(list_conn, 200)["data"] == [@valid_attrs]
+    end
   end
 
   describe "show" do

diff --git a/apps/astarte_realm_management_api/test/support/astarte_realm_management_mock.ex b/apps/astarte_realm_management_api/test/support/astarte_realm_management_mock.ex
@@ -27,7 +27,9 @@ defmodule Astarte.RealmManagement.Mock do
     GetTriggerPolicySourceReply,
     DeleteDevice,
     GetDeviceRegistrationLimit,
-    GetDeviceRegistrationLimitReply
+    GetDeviceRegistrationLimitReply,
+    GetDetailedInterfacesList,
+    GetDetailedInterfacesListReply
   }
 
   alias Astarte.Core.Interface
@@ -74,6 +76,16 @@ defmodule Astarte.RealmManagement.Mock do
     |> ok_wrap
   end
 
+  defp execute_rpc(
+         {:get_detailed_interfaces_list, %GetDetailedInterfacesList{realm_name: realm_name}}
+       ) do
+    list = DB.get_detailed_interfaces_list(realm_name)
+
+    %GetDetailedInterfacesListReply{interface_json: list}
+    |> encode_reply(:get_detailed_interfaces_list_reply)
+    |> ok_wrap
+  end
+
   defp execute_rpc(
          {:get_interface_versions_list,
           %GetInterfaceVersionsList{realm_name: realm_name, interface_name: name}}