From 1853cb32033c5fc59c01a0cc5e3749e4b9e09a84 Mon Sep 17 00:00:00 2001 From: Ashutosh Benni Date: Tue, 21 Feb 2023 11:35:19 +0530 Subject: [PATCH 01/33] First commit --- README.md | 2 +- smsdk/Auth/auth.py | 7 +++++-- smsdk/client_v0.py | 2 +- smsdk/config/message_config.json | 1 + smsdk/ma_session.py | 1 + test.py | 19 +++++++++++++++++++ 6 files changed, 28 insertions(+), 4 deletions(-) create mode 100644 test.py diff --git a/README.md b/README.md index 83377ba..b2cceb6 100644 --- a/README.md +++ b/README.md @@ -62,7 +62,7 @@ For API Key authentication: key = api_key_from_platform secret = api_secret_from_platform -cli.login('apikey', secret_id=secret, key_id=key) +cli.login('apikey', secret_id=secret, key_id=key, schema_name="") ``` ### Proxy Servers and CA Certs diff --git a/smsdk/Auth/auth.py b/smsdk/Auth/auth.py index bd509cc..4a31d87 100644 --- a/smsdk/Auth/auth.py +++ b/smsdk/Auth/auth.py @@ -22,7 +22,7 @@ SM_AUTH_HEADER_SECRET_ID = RESOURCE_CONFIG["auth_header-api-secret"] SM_AUTH_HEADER_SECRET_ID_OLD = RESOURCE_CONFIG["auth_header-api-secret_old"] SM_AUTH_HEADER_KEY_ID = RESOURCE_CONFIG["auth_header-api-key"] - +SM_AUTH_HEADER_KEY_ID = RESOURCE_CONFIG["x_sm_db_schema"] class Authenticator(MaSession): """ @@ -89,7 +89,7 @@ def _auth_basic(self, email=None, password=None): return success - def _auth_apikey(self, secret_id, key_id): + def _auth_apikey(self, secret_id, key_id, db_schema=""): """ Authenticate by sending an API key. @@ -104,6 +104,8 @@ def _auth_apikey(self, secret_id, key_id): self.session.headers.update({SM_AUTH_HEADER_SECRET_ID: secret_id}) self.session.headers.update({SM_AUTH_HEADER_SECRET_ID_OLD: secret_id}) #add v0/v1 compat self.session.headers.update({SM_AUTH_HEADER_KEY_ID: key_id}) + if db_schema: + self.session.headers.update({SM_AUTH_HEADER_KEY_ID: db_schema}) if not self.check_auth(): raise RuntimeError( @@ -126,6 +128,7 @@ def login(self, method=None, **kwargs): if method == "basic": success = self._auth_basic(**kwargs) elif method == "apikey": + # import ipdb;ipdb.set_trace() success = self._auth_apikey(**kwargs) elif method is None: try: diff --git a/smsdk/client_v0.py b/smsdk/client_v0.py index b151f72..31b70e3 100644 --- a/smsdk/client_v0.py +++ b/smsdk/client_v0.py @@ -699,7 +699,7 @@ def get_machine_type_names(self, clean_strings_out=True): query_params = {'_only': ['source_type', 'source_type_clean'], '_order_by': 'source_type_clean'} machine_types = self.get_data('machine_type', 'get_machine_types', normalize=True, **query_params) - + # import ipdb;ipdb.set_trace() if clean_strings_out: return machine_types['source_type_clean'].to_list() else: diff --git a/smsdk/config/message_config.json b/smsdk/config/message_config.json index f5bb3e5..a5e29fd 100644 --- a/smsdk/config/message_config.json +++ b/smsdk/config/message_config.json @@ -2,6 +2,7 @@ "auth_header-api-secret": "X-SM-API-Secret", "auth_header-api-secret_old": "X-SM-API-Key", "auth_header-api-key": "X-SM-API-Key-Id", + "x_sm_db_schema":"x-sm-db-schema", "keywords": { "KEYWORD_MISSING_ACCOUNT": "No such account", "KEYWORD_INCORRECT_PASSWORD": "password does not match", diff --git a/smsdk/ma_session.py b/smsdk/ma_session.py index f194831..107c9cc 100644 --- a/smsdk/ma_session.py +++ b/smsdk/ma_session.py @@ -21,6 +21,7 @@ SM_AUTH_HEADER_SECRET_ID = RESOURCE_CONFIG["auth_header-api-secret"] SM_AUTH_HEADER_SECRET_ID_OLD = RESOURCE_CONFIG["auth_header-api-secret_old"] SM_AUTH_HEADER_KEY_ID = RESOURCE_CONFIG["auth_header-api-key"] +X_SM_DB_SCHEMA = RESOURCE_CONFIG['x_sm_db_schema'] import logging log = logging.getLogger(__name__) diff --git a/test.py b/test.py new file mode 100644 index 0000000..5cfb2cb --- /dev/null +++ b/test.py @@ -0,0 +1,19 @@ +from smsdk import client + + +# key = 'fe6cd4cd-7735-4634-be0e-ab3d497112b7' +# secret = 'sma_ZByjWtYzIA1wCalTRBDNOBljI9RRYYO7kN5RZOVSVOi_' + +# tenant = 'westrock-dallas' +# cli = client.Client(tenant) +# cli.login('apikey', secret_id=secret, key_id=key) +# types = cli.get_machine_type_names() +# print(types) + +tenant = "efmw-suzhou" +cli = client.Client(tenant) +cli.login('apikey', +key_id = '2c66709b-a691-4884-bef3-f4a89246faea', +secret_id = 'sma_oxmwHVNSvtpyeScktdeeZaRWgjplLQVazq7VfHjiKBk_') +machine_types = cli.get_machine_type_names() +print(f'Machine Types: {machine_types}') \ No newline at end of file From ef2b651b536e3a3146c8a65d89f204703db343c7 Mon Sep 17 00:00:00 2001 From: Ashutosh Benni Date: Tue, 21 Feb 2023 18:43:14 +0530 Subject: [PATCH 02/33] DATA-868 - Make SDK schema/workspace aware --- README.md | 3 ++- smsdk/Auth/auth.py | 5 ++--- smsdk/client_v0.py | 1 - test.py | 19 ------------------- 4 files changed, 4 insertions(+), 24 deletions(-) delete mode 100644 test.py diff --git a/README.md b/README.md index b2cceb6..90f406c 100644 --- a/README.md +++ b/README.md @@ -62,7 +62,8 @@ For API Key authentication: key = api_key_from_platform secret = api_secret_from_platform -cli.login('apikey', secret_id=secret, key_id=key, schema_name="") +# default value for db_schema is production pipeline schema +cli.login('apikey', secret_id=secret, key_id=key, db_schema=schema_name) ``` ### Proxy Servers and CA Certs diff --git a/smsdk/Auth/auth.py b/smsdk/Auth/auth.py index 4a31d87..057df6e 100644 --- a/smsdk/Auth/auth.py +++ b/smsdk/Auth/auth.py @@ -22,7 +22,7 @@ SM_AUTH_HEADER_SECRET_ID = RESOURCE_CONFIG["auth_header-api-secret"] SM_AUTH_HEADER_SECRET_ID_OLD = RESOURCE_CONFIG["auth_header-api-secret_old"] SM_AUTH_HEADER_KEY_ID = RESOURCE_CONFIG["auth_header-api-key"] -SM_AUTH_HEADER_KEY_ID = RESOURCE_CONFIG["x_sm_db_schema"] +X_SM_DB_SCHEMA = RESOURCE_CONFIG["x_sm_db_schema"] class Authenticator(MaSession): """ @@ -105,7 +105,7 @@ def _auth_apikey(self, secret_id, key_id, db_schema=""): self.session.headers.update({SM_AUTH_HEADER_SECRET_ID_OLD: secret_id}) #add v0/v1 compat self.session.headers.update({SM_AUTH_HEADER_KEY_ID: key_id}) if db_schema: - self.session.headers.update({SM_AUTH_HEADER_KEY_ID: db_schema}) + self.session.headers.update({X_SM_DB_SCHEMA: db_schema}) if not self.check_auth(): raise RuntimeError( @@ -128,7 +128,6 @@ def login(self, method=None, **kwargs): if method == "basic": success = self._auth_basic(**kwargs) elif method == "apikey": - # import ipdb;ipdb.set_trace() success = self._auth_apikey(**kwargs) elif method is None: try: diff --git a/smsdk/client_v0.py b/smsdk/client_v0.py index 31b70e3..b4d8129 100644 --- a/smsdk/client_v0.py +++ b/smsdk/client_v0.py @@ -699,7 +699,6 @@ def get_machine_type_names(self, clean_strings_out=True): query_params = {'_only': ['source_type', 'source_type_clean'], '_order_by': 'source_type_clean'} machine_types = self.get_data('machine_type', 'get_machine_types', normalize=True, **query_params) - # import ipdb;ipdb.set_trace() if clean_strings_out: return machine_types['source_type_clean'].to_list() else: diff --git a/test.py b/test.py deleted file mode 100644 index 5cfb2cb..0000000 --- a/test.py +++ /dev/null @@ -1,19 +0,0 @@ -from smsdk import client - - -# key = 'fe6cd4cd-7735-4634-be0e-ab3d497112b7' -# secret = 'sma_ZByjWtYzIA1wCalTRBDNOBljI9RRYYO7kN5RZOVSVOi_' - -# tenant = 'westrock-dallas' -# cli = client.Client(tenant) -# cli.login('apikey', secret_id=secret, key_id=key) -# types = cli.get_machine_type_names() -# print(types) - -tenant = "efmw-suzhou" -cli = client.Client(tenant) -cli.login('apikey', -key_id = '2c66709b-a691-4884-bef3-f4a89246faea', -secret_id = 'sma_oxmwHVNSvtpyeScktdeeZaRWgjplLQVazq7VfHjiKBk_') -machine_types = cli.get_machine_type_names() -print(f'Machine Types: {machine_types}') \ No newline at end of file From 0fc3eb34e46cb10ecdae5ac14d1a8d81979ef0c5 Mon Sep 17 00:00:00 2001 From: ashutoshbenni Date: Wed, 22 Feb 2023 13:14:00 +0530 Subject: [PATCH 03/33] DATA-868 - removeing schema_name from auth call --- README.md | 12 ++++++++++-- smsdk/Auth/auth.py | 4 +--- smsdk/client.py | 5 ++++- 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 90f406c..8a40eb5 100644 --- a/README.md +++ b/README.md @@ -62,8 +62,7 @@ For API Key authentication: key = api_key_from_platform secret = api_secret_from_platform -# default value for db_schema is production pipeline schema -cli.login('apikey', secret_id=secret, key_id=key, db_schema=schema_name) +cli.login('apikey', secret_id=secret, key_id=key) ``` ### Proxy Servers and CA Certs @@ -82,6 +81,15 @@ Similarly, if you need to use your own CA Certificates, set the CURL_CA_BUNDLE e CURL_CA_BUNDLE="/path/to/my/certificates" ``` +### Selecting database schema +By default, the production pipeline schema will be considered to retrive data from the tenant. +To select a perticular development pipeline schema you can use following function call: + +``` +db_schema = pipeline_id +cli.select_db_schema(schema_name=db_schema) +``` + ### Retrieving Data The typical workflow for getting data from Sight Machine is: diff --git a/smsdk/Auth/auth.py b/smsdk/Auth/auth.py index 057df6e..c3ff27e 100644 --- a/smsdk/Auth/auth.py +++ b/smsdk/Auth/auth.py @@ -89,7 +89,7 @@ def _auth_basic(self, email=None, password=None): return success - def _auth_apikey(self, secret_id, key_id, db_schema=""): + def _auth_apikey(self, secret_id, key_id): """ Authenticate by sending an API key. @@ -104,8 +104,6 @@ def _auth_apikey(self, secret_id, key_id, db_schema=""): self.session.headers.update({SM_AUTH_HEADER_SECRET_ID: secret_id}) self.session.headers.update({SM_AUTH_HEADER_SECRET_ID_OLD: secret_id}) #add v0/v1 compat self.session.headers.update({SM_AUTH_HEADER_KEY_ID: key_id}) - if db_schema: - self.session.headers.update({X_SM_DB_SCHEMA: db_schema}) if not self.check_auth(): raise RuntimeError( diff --git a/smsdk/client.py b/smsdk/client.py index 3d8113f..b03a677 100644 --- a/smsdk/client.py +++ b/smsdk/client.py @@ -12,7 +12,7 @@ from pandas.io.json import json_normalize from smsdk.utils import get_url -from smsdk.Auth.auth import Authenticator +from smsdk.Auth.auth import Authenticator, X_SM_DB_SCHEMA from smsdk.tool_register import smsdkentities from smsdk.client_v0 import ClientV0 @@ -115,6 +115,9 @@ def __init__(self, tenant, site_domain="sightmachine.io", protocol = "https"): # Setup Authenticator self.auth = Authenticator(self) self.session = self.auth.session + + def select_db_schema(self, schema_name): + self.session.headers.update({X_SM_DB_SCHEMA:schema_name}) def get_data_v1(self, ename, util_name, normalize=True, *args, **kwargs): """ From d3c764bdc30f3d4278048628d62d28c9a6bb6117 Mon Sep 17 00:00:00 2001 From: ashutoshbenni Date: Wed, 22 Feb 2023 13:42:49 +0530 Subject: [PATCH 04/33] DATA-868 - removeing schema_name from auth call --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 8a40eb5..c2d15b8 100644 --- a/README.md +++ b/README.md @@ -83,7 +83,7 @@ CURL_CA_BUNDLE="/path/to/my/certificates" ### Selecting database schema By default, the production pipeline schema will be considered to retrive data from the tenant. -To select a perticular development pipeline schema you can use following function call: +To select a particular development pipeline schema you can use following function call: ``` db_schema = pipeline_id From a0f443aa8b4226d3f603adeb3e7cacf9d405cb14 Mon Sep 17 00:00:00 2001 From: ashutoshbenni Date: Wed, 22 Feb 2023 19:46:20 +0530 Subject: [PATCH 05/33] DATA-868 - updated `Query Examples` notebook --- examples/Query Examples.ipynb | 37 +++++++++++++++++++++++++++++------ 1 file changed, 31 insertions(+), 6 deletions(-) diff --git a/examples/Query Examples.ipynb b/examples/Query Examples.ipynb index 4fe1cfa..df5b94c 100644 --- a/examples/Query Examples.ipynb +++ b/examples/Query Examples.ipynb @@ -50,6 +50,28 @@ "columns = cli.get_machine_schema(machines[0])['display'].to_list()" ] }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Selecting a Particular Development Pipeline schema\n", + "\n", + "You can select development pipeline schema using following code example:\n", + "\n", + "Note:- By default, the production pipeline schema will be considered to retrive data from the tenant" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "db_schema = 'pipeline_id' \n", + "cli.select_db_schema(schema_name=db_schema)" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -2069,11 +2091,9 @@ } ], "metadata": { - "interpreter": { - "hash": "767d51c1340bd893661ea55ea3124f6de3c7a262a8b4abca0554b478b1e2ff90" - }, "kernelspec": { - "display_name": "Python 3.7.5 64-bit ('usr')", + "display_name": "Python 3", + "language": "python", "name": "python3" }, "language_info": { @@ -2086,9 +2106,14 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.5" + "version": "3.10.9" }, - "orig_nbformat": 2 + "orig_nbformat": 2, + "vscode": { + "interpreter": { + "hash": "b0fa6594d8f4cbf19f97940f81e996739fb7646882a419484c72d19e05852a7e" + } + } }, "nbformat": 4, "nbformat_minor": 2 From c33e624914026161d47e9bf33b11a3c3d3034538 Mon Sep 17 00:00:00 2001 From: Michael Goldman Date: Tue, 28 Feb 2023 15:18:49 -0800 Subject: [PATCH 06/33] Get_kpis --- smsdk/client.py | 7 ++++ smsdk/config/api_endpoints.json | 6 ++- smsdk/smsdk_entities/kpi/__init__.py | 0 smsdk/smsdk_entities/kpi/kpi.py | 55 ++++++++++++++++++++++++++++ 4 files changed, 67 insertions(+), 1 deletion(-) create mode 100644 smsdk/smsdk_entities/kpi/__init__.py create mode 100644 smsdk/smsdk_entities/kpi/kpi.py diff --git a/smsdk/client.py b/smsdk/client.py index 3d8113f..03c0fc9 100644 --- a/smsdk/client.py +++ b/smsdk/client.py @@ -225,3 +225,10 @@ def get_machine_schema(self, machine_source, types=[], return_mtype=False, **kwa f"Unknow stat schema identified :: machine_type {machine_source} - " f"title_prefix :: {stat.get('display', {}).get('title_prefix', '')}") return fields + + def get_kpis(self, **kwargs): + kpis = smsdkentities.get('kpi') + base_url = get_url( + self.config["protocol"], self.tenant, self.config["site.domain"] + ) + return kpis(self.session, base_url).get_kpis(**kwargs) diff --git a/smsdk/config/api_endpoints.json b/smsdk/config/api_endpoints.json index c884290..81b744f 100644 --- a/smsdk/config/api_endpoints.json +++ b/smsdk/config/api_endpoints.json @@ -28,6 +28,10 @@ }, "DataViz": { "estimate_cycle": "/v1/selector/datavis/estimate/cycle", - "estimate_part": "/v1/selector/datavis/estimate/part" + "estimate_part": "/v1/selector/datavis/estimate/part", + "task": "/v1/datavis/task/async" + }, + "KPI": { + "availible_kpis": "/v1/selector/datavis/kpi/y_axis" } } diff --git a/smsdk/smsdk_entities/kpi/__init__.py b/smsdk/smsdk_entities/kpi/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/smsdk/smsdk_entities/kpi/kpi.py b/smsdk/smsdk_entities/kpi/kpi.py new file mode 100644 index 0000000..8577354 --- /dev/null +++ b/smsdk/smsdk_entities/kpi/kpi.py @@ -0,0 +1,55 @@ +import json +from datetime import datetime, timedelta +from typing import List + +import numpy as np + +try: + import importlib.resources as pkg_resources +except ImportError: + # Try backported to PY<37 `importlib_resources`. + import importlib_resources as pkg_resources + +from smsdk.tool_register import SmsdkEntities, smsdkentities +from smsdk.utils import module_utility +from smsdk import config +from smsdk.ma_session import MaSession + +import logging + +log = logging.getLogger(__name__) + +ENDPOINTS = json.loads(pkg_resources.read_text(config, "api_endpoints.json")) + + +@smsdkentities.register("kpi") +class KPI(SmsdkEntities, MaSession): + # Decorator to register a function as utility + # Only the registered utilites would be accessible + # to outside world via client.get_data() + mod_util = module_utility() + + def __init__(self, session, base_url) -> None: + self.session = session + self.base_url = base_url + + @mod_util + def get_utilities(self, *args, **kwargs) -> List: + """ + Get the list of registered utilites by name + """ + return [*self.mod_util.all] + + @mod_util + def get_kpis(self, *args, **kwargs): + """ + Utility function to get the downtimes + from the ma downtime API + Recommend to use 'enable_pagination':True for larger datasets + """ + url = "{}{}".format(self.base_url, ENDPOINTS["KPI"]["availible_kpis"]) + records = self._get_records_v1(url, **kwargs) + + if not isinstance(records, List): + raise ValueError("Error - {}".format(records)) + return records \ No newline at end of file From 8844f6369166ceba4638d3b7ad1134a9b19ef6a1 Mon Sep 17 00:00:00 2001 From: Michael Goldman Date: Wed, 1 Mar 2023 10:35:56 -0800 Subject: [PATCH 07/33] Working KPI data vis and aysnc task function --- smsdk/client.py | 7 ++++++ smsdk/ma_session.py | 42 +++++++++++++++++++++++++++++++++ smsdk/smsdk_entities/kpi/kpi.py | 14 +++++++++++ 3 files changed, 63 insertions(+) diff --git a/smsdk/client.py b/smsdk/client.py index 03c0fc9..ebf6ad2 100644 --- a/smsdk/client.py +++ b/smsdk/client.py @@ -232,3 +232,10 @@ def get_kpis(self, **kwargs): self.config["protocol"], self.tenant, self.config["site.domain"] ) return kpis(self.session, base_url).get_kpis(**kwargs) + + def get_kpi_data_viz(self, **kwargs): + kpis = smsdkentities.get('kpi') + base_url = get_url( + self.config["protocol"], self.tenant, self.config["site.domain"] + ) + return kpis(self.session, base_url).get_kpi_data_viz(**kwargs) diff --git a/smsdk/ma_session.py b/smsdk/ma_session.py index f194831..558845c 100644 --- a/smsdk/ma_session.py +++ b/smsdk/ma_session.py @@ -205,6 +205,48 @@ def _get_records_v1( print(traceback.print_exc()) return records + + + def _complete_async_task( + self, + endpoint, + method="post", + db_mode='sql', + **url_params + ): + if url_params.get('db_mode') == None: + url_params['db_mode'] = db_mode + try: + response = getattr(self.session, method.lower())( + endpoint, json=url_params + ) + data = response.json() + task_id = data['response']['task_id'] + while True: + try: + response = getattr(self.session, 'get')( + endpoint+'/'+task_id, json=url_params + ) + data = response.json() + state = data['response']['state'] + if state == 'SUCCESS': + return data['response']['meta']['results'] + + if state == 'FAILURE' or state == 'REVOKED': + import traceback + + print(traceback.print_exc()) + return [] + except: + import traceback + + print(traceback.print_exc()) + return [] + except: + import traceback + + print(traceback.print_exc()) + return [] def get_json_headers(self): """ diff --git a/smsdk/smsdk_entities/kpi/kpi.py b/smsdk/smsdk_entities/kpi/kpi.py index 8577354..ea50325 100644 --- a/smsdk/smsdk_entities/kpi/kpi.py +++ b/smsdk/smsdk_entities/kpi/kpi.py @@ -50,6 +50,20 @@ def get_kpis(self, *args, **kwargs): url = "{}{}".format(self.base_url, ENDPOINTS["KPI"]["availible_kpis"]) records = self._get_records_v1(url, **kwargs) + if not isinstance(records, List): + raise ValueError("Error - {}".format(records)) + return records + + @mod_util + def get_kpi_data_viz(self, *args, **kwargs): + """ + Utility function to get the downtimes + from the ma downtime API + Recommend to use 'enable_pagination':True for larger datasets + """ + url = "{}{}".format(self.base_url, ENDPOINTS["DataViz"]["task"]) + records = self._complete_async_task(url, **kwargs) + if not isinstance(records, List): raise ValueError("Error - {}".format(records)) return records \ No newline at end of file From 5f00dd216024e48594b802cfd748b247ae39e065 Mon Sep 17 00:00:00 2001 From: Michael Goldman Date: Thu, 2 Mar 2023 12:57:07 -0800 Subject: [PATCH 08/33] docs --- .../asset_selection.md | 17 ++ .../data_viz_query.md | 179 ++++++++++++++++++ docs/entities/kpis.md | 32 ++++ smsdk/ma_session.py | 9 +- smsdk/smsdk_entities/kpi/kpi.py | 1 + 5 files changed, 234 insertions(+), 4 deletions(-) create mode 100644 docs/commonly_used_data_types/asset_selection.md create mode 100644 docs/commonly_used_data_types/data_viz_query.md create mode 100644 docs/entities/kpis.md diff --git a/docs/commonly_used_data_types/asset_selection.md b/docs/commonly_used_data_types/asset_selection.md new file mode 100644 index 0000000..1f4b971 --- /dev/null +++ b/docs/commonly_used_data_types/asset_selection.md @@ -0,0 +1,17 @@ +# Asset Selection +Asset Selection is used in a few different places in order to tell the API which asset or assets you wish to perform an action on. Typically it will either select a machine_type or types or specific machines within a machine_type in order to select machine_types it should look like the following: +``` +asset_selection: { + machine_type: ["Lasercut", ...] + +} +``` + +In order to select machines within a type it should look like the follwoing: +``` +asset_selection: { + machine_type: ["Lasercut"], + machine_source: ["JB_AB_Lasercut_1", ...] + +} +``` \ No newline at end of file diff --git a/docs/commonly_used_data_types/data_viz_query.md b/docs/commonly_used_data_types/data_viz_query.md new file mode 100644 index 0000000..e81a93a --- /dev/null +++ b/docs/commonly_used_data_types/data_viz_query.md @@ -0,0 +1,179 @@ +# Data Vizualation Query +Data Viz Queries are used whenever the SDK calls our Data Vizualation APIs. The functions you call via SDK do some work on the query for you so this Query will look slightly different from the one our API uses directly. We will break down each field in some more detail futher along in this doc but as an example a full Data Viz Query looks like the following: +``` +{ + "asset_selection": { + "machine_source": [ + "JB_AB_Lasercut_1" + ], + "machine_type": [ + "Lasercut" + ] + }, + "d_vars": [ + { + "name": "quality", + "aggregate": [ + "avg" + ] + } + ], + "i_vars": [ + { + "name": "endtime", + "time_resolution": "day", + "query_tz": "America/Los_Angeles", + "output_tz": "America/Los_Angeles", + "bin_strategy": "user_defined2", + "bin_count": 50 + } + ], + "time_selection": { + "time_type": "relative", + "relative_start": 7, + "relative_unit": "year", + "ctime_tz": "America/Los_Angeles" + }, + "where": [], + "db_mode": "sql" +} +``` + +## Asset_selection +Used to select the asset(s) you want to recieve data from see the [asset_selection doc](/docs/commonly_used_data_types/asset_selection.md) for more information. + +## d_vars +The Dependent variables. These will change depending on the entity you are trying to access. But will always be a list in the following form: +``` + { + "name": "quality", + "aggregate": [ + "avg" + ] + } +``` +### name +This is the name of dependent variable you wish to view. This typically the name of a value we store on a machine or the name of a KPI. + +### aggregate +This is how you wish to aggregate the data of the named value in the time_resolution you have selected. The options for this are: +* avg +* sum +* min +* max + +## i_vars +The indepent variables. These should typically be time based values that are stored on the machine_type you are using. They will always be a list in the following form: +``` +{ + "name": "endtime", + "time_resolution": "day", + "query_tz": "America/Los_Angeles", + "output_tz": "America/Los_Angeles", + "bin_strategy": "user_defined2", + "bin_count": 50 + } +``` +### name +This is the name of idependent varaible you are using. + +### time_resolution +This is optional and is how detailed of a time breakdown you want in the variable the options for time_resolution are as follows: +* year +* month +* week +* day +* hour +* minute +* second + +### query_tz +This is optional and tells the system what time zone the query is in. + +### output_tz +This is optional and tells the system what time zone to return the data in. + +### bin_strategy +This is optional and tells the sytem how you wish to bin the data. You have the following options: +* user_defined2 +* none +* categorical + +### bin_count +This is optoinal and tels the system how many bins you wish to put the data into. + +## time_selection +This is the time frame you want to grab data from there are two different ways to make this time selection, Relative and Absolute + +### Relative Time Selection +Relative Time Selections goes back from now a certain amount of time based on what you tell it. The format for this time selection is the following: +``` +{ + "time_type": "relative", + "relative_start": 7, + "relative_unit": "year", + "ctime_tz": "America/Los_Angeles" +} +``` +#### Time Type +For a relative time selection this needs to be set to "relative". + +#### Relative Start +The amount of units from now you wish to go back to start your time selection. + +#### Relative Unit +The unit of time you wish to go back from now. Your options for this are as follows: +* year +* month +* week +* day +* hour +* minute +* second + +#### ctime_tz +The time zone for your time selection + +### Absolute Time Selection +Absolute Time Selections have a start and end time and will gather data from between the two. The format for this time selection is as follows: +``` +{ + "time_type": "absolute", + "start_time": "2023-02-23T08:00:00.000Z", + "end_time": "2023-03-01T21:35:35.499Z", + "time_zone": "America/Los_Angeles" +} +``` +#### Time Type +For absolute time selecitons this must be set to absolute. + +#### Start Time +The time you wish to start the time selection at. + +#### End Time +The time you wish to end the time selection at. + +#### Time Zone +The time zone for the time selection. + +## Where +This is optional. It will narrow down the data return based on criteria given. The list will be anded together. This is a list in the following format: +``` +{ + "name": "type__part_type", + "op": "eq", + "value": "EngineBlock" +} +``` + +### Name +The name of the field you are using in this criteria. + +### Op +The type of operation you are doing. + +### Value +The value to compare with the operation. + +## db_mode +This is optional. It will defualt to 'sql' and usually should be but we have a 'monogo' mode as well. You will likely ever need to tset this. \ No newline at end of file diff --git a/docs/entities/kpis.md b/docs/entities/kpis.md new file mode 100644 index 0000000..871bd12 --- /dev/null +++ b/docs/entities/kpis.md @@ -0,0 +1,32 @@ +# KPIs + +KPIs are user defined calculated fields in the Sight Machine software. + +## Functions +The SDK has two functions related to KPIs. The first of which allows a user to see which KPIs are availible for a particular asset. The second makes use of our Data Visulation api which allows a user to see these KPIs over a timeframe. + +### Get KPIs +This is the first KPI function allowing you to see which KPIs are availible for a particular asset. In order to call this function you must first have a logged in client see the [quick start guide](/README.md) for more information on logging in. Once you have a logged in client you can call the function as follows: + +``` +cli.get_kpis(**asset_selection) +``` + +For more info on [asset_selection](/docs/commonly_used_data_types/asset_selection.md) click on the previous link. After a moment the SDK should return a list that looks something like this: + +``` +[{'name': 'quality', 'display_name': 'Quality', 'unit': '', 'type': 'continuous', 'data_type': 'float', 'stream_types': [], 'raw_data_field': ''},...] +``` + +In order to make use of the data viz function you'll need the name of the KPI you wish to get. + +### Get KPI Data Viz +Once you have the name of the KPI you wish to access and a logged in client you can make a call to the data viz api with the following SDK function call: +``` +cli.get_kpi_data_viz(**data_viz_query) +``` + +The d_vars section of your data_viz_query should include an d_var with the name of the KPI you wish to query. For more information on [data_viz_queries](/docs/commonly_used_data_types/data_viz_query.md) click on the previous link. After some time the SDK should return a list that looks something like this: +``` +[{'i_vals': {'endtime': {'i_pos': 0, 'bin_no': 0, 'bin_min': '2022-10-20T00:00:00-07:00', 'bin_max': '2022-10-20T00:00:00-07:00', 'bin_avg': '2022-10-20T00:00:00-07:00'}}, 'd_vals': {'quality': {'avg': 95.18072289156626}}, '_count': 418, 'kpi_dependencies': {'quality': {'Output': 395.0, 'ScrapQuantity': 20.0}}},...] +``` \ No newline at end of file diff --git a/smsdk/ma_session.py b/smsdk/ma_session.py index 558845c..00f8ce1 100644 --- a/smsdk/ma_session.py +++ b/smsdk/ma_session.py @@ -220,6 +220,8 @@ def _complete_async_task( response = getattr(self.session, method.lower())( endpoint, json=url_params ) + if response.status_code not in [200, 201]: + raise ValueError("Error - {}".format(response.text)) data = response.json() task_id = data['response']['task_id'] while True: @@ -227,16 +229,15 @@ def _complete_async_task( response = getattr(self.session, 'get')( endpoint+'/'+task_id, json=url_params ) + if response.status_code not in [200, 201]: + raise ValueError("Error - {}".format(response.text)) data = response.json() state = data['response']['state'] if state == 'SUCCESS': return data['response']['meta']['results'] if state == 'FAILURE' or state == 'REVOKED': - import traceback - - print(traceback.print_exc()) - return [] + raise ValueError("Error - {}".format(response.text)) except: import traceback diff --git a/smsdk/smsdk_entities/kpi/kpi.py b/smsdk/smsdk_entities/kpi/kpi.py index ea50325..8920f36 100644 --- a/smsdk/smsdk_entities/kpi/kpi.py +++ b/smsdk/smsdk_entities/kpi/kpi.py @@ -62,6 +62,7 @@ def get_kpi_data_viz(self, *args, **kwargs): Recommend to use 'enable_pagination':True for larger datasets """ url = "{}{}".format(self.base_url, ENDPOINTS["DataViz"]["task"]) + kwargs['model']='kpi' records = self._complete_async_task(url, **kwargs) if not isinstance(records, List): From 2547a26282cb16a9cd0157418c13d7a1a4d6c7c0 Mon Sep 17 00:00:00 2001 From: Michael Goldman Date: Thu, 2 Mar 2023 15:37:42 -0800 Subject: [PATCH 09/33] Tests --- smsdk/ma_session.py | 2 +- tests/kpi/__init__.py | 0 tests/kpi/kpi_data.py | 3 +++ tests/kpi/test_kpi.py | 57 +++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 61 insertions(+), 1 deletion(-) create mode 100644 tests/kpi/__init__.py create mode 100644 tests/kpi/kpi_data.py create mode 100644 tests/kpi/test_kpi.py diff --git a/smsdk/ma_session.py b/smsdk/ma_session.py index 00f8ce1..735e8b0 100644 --- a/smsdk/ma_session.py +++ b/smsdk/ma_session.py @@ -226,7 +226,7 @@ def _complete_async_task( task_id = data['response']['task_id'] while True: try: - response = getattr(self.session, 'get')( + response = self.session.get( endpoint+'/'+task_id, json=url_params ) if response.status_code not in [200, 201]: diff --git a/tests/kpi/__init__.py b/tests/kpi/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/kpi/kpi_data.py b/tests/kpi/kpi_data.py new file mode 100644 index 0000000..7abc42e --- /dev/null +++ b/tests/kpi/kpi_data.py @@ -0,0 +1,3 @@ +AVALIBLE_KPI_JSON = [{'name': 'Scrap_Rate', 'display_name': 'Scrap Rate', 'unit': '', 'type': 'continuous', 'data_type': 'float', 'stream_types': [], 'raw_data_field': ''}, {'name': 'oee', 'display_name': 'OEE', 'unit': '', 'type': 'continuous', 'data_type': 'float', 'stream_types': [], 'raw_data_field': ''}, {'name': 'quality', 'display_name': 'Quality', 'unit': '', 'type': 'continuous', 'data_type': 'float', 'stream_types': [], 'raw_data_field': ''}, {'name': 'performance', 'display_name': 'Performance', 'unit': '', 'type': 'continuous', 'data_type': 'float', 'stream_types': [], 'raw_data_field': ''}, {'name': 'availability', 'display_name': 'Availability', 'unit': '', 'type': 'continuous', 'data_type': 'float', 'stream_types': [], 'raw_data_field': ''}] + +KPI_DATA_VIZ_JSON =[{'i_vals': {'endtime': {'i_pos': 0, 'bin_no': 0, 'bin_min': '2022-10-20T00:00:00-07:00', 'bin_max': '2022-10-20T00:00:00-07:00', 'bin_avg': '2022-10-20T00:00:00-07:00'}}, 'd_vals': {'quality': {'avg': 95.18072289156626}}, '_count': 418, 'kpi_dependencies': {'quality': {'Output': 395.0, 'ScrapQuantity': 20.0}}}, {'i_vals': {'endtime': {'i_pos': 0, 'bin_no': 1, 'bin_min': '2022-10-21T00:00:00-07:00', 'bin_max': '2022-10-21T00:00:00-07:00', 'bin_avg': '2022-10-21T00:00:00-07:00'}}, 'd_vals': {'quality': {'avg': 93.29123914759275}}, '_count': 1282, 'kpi_dependencies': {'quality': {'Output': 1182.0, 'ScrapQuantity': 85.0}}}, {'i_vals': {'endtime': {'i_pos': 0, 'bin_no': 2, 'bin_min': '2022-10-22T00:00:00-07:00', 'bin_max': '2022-10-22T00:00:00-07:00', 'bin_avg': '2022-10-22T00:00:00-07:00'}}, 'd_vals': {'quality': {'avg': 92.7477840451249}}, '_count': 1260, 'kpi_dependencies': {'quality': {'Output': 1151.0, 'ScrapQuantity': 90.0}}}] \ No newline at end of file diff --git a/tests/kpi/test_kpi.py b/tests/kpi/test_kpi.py new file mode 100644 index 0000000..2ed1dc2 --- /dev/null +++ b/tests/kpi/test_kpi.py @@ -0,0 +1,57 @@ +import pandas as pd +from requests.sessions import Session +from smsdk.client import Client +from tests.kpi.kpi_data import AVALIBLE_KPI_JSON, KPI_DATA_VIZ_JSON +from smsdk.smsdk_entities.kpi.kpi import KPI +from mock import mock_open, MagicMock, patch + +def test_get_kpi(monkeypatch): + # Setup + def mockapi(self, session, endpoint): + if endpoint == "/v1/selector/datavis/kpi/y_axis": + return pd.DataFrame(AVALIBLE_KPI_JSON) + return pd.DataFrame() + + monkeypatch.setattr(KPI, "get_kpis", mockapi) + + dt = KPI(Session(), "demo") + + # Run + df = dt.get_kpis(Session(), "/v1/selector/datavis/kpi/y_axis") + + # Verify + assert df.shape == (5, 7) + + names = ['Scrap_Rate', 'availability', 'oee', 'performance', 'quality'] + + assert names == df.name.sort_values().tolist() + + +@patch("smsdk.ma_session.Session") +def test_get_kpi_data_viz(mocked): + class ResponsePost: + ok = True + text = "Success" + status_code=200 + + @staticmethod + def json(): + return {"response": {"task_id": "test"}} + + class ResponseGet: + ok = True + text = "Success" + status_code=200 + + @staticmethod + def json(): + return {"response":{"state":"SUCCESS", "meta":{"results":KPI_DATA_VIZ_JSON}}} + + mocked.return_value = MagicMock( + post=MagicMock(return_value=ResponsePost()), get=MagicMock(return_value=ResponseGet()) + ) + + dt = Client("demo") + data = dt.get_kpi_data_viz() + assert len(data) == 3 + assert data[0]['d_vals']['quality']['avg'] == 95.18072289156626 \ No newline at end of file From b772b44aca051a4b380f90378eb35412b12b901a Mon Sep 17 00:00:00 2001 From: Michael Goldman Date: Mon, 6 Mar 2023 12:39:53 -0800 Subject: [PATCH 10/33] Sean comments --- docs/entities/kpis.md | 19 ++++++++++++++++--- smsdk/client.py | 7 +++++++ smsdk/config/api_endpoints.json | 3 ++- smsdk/smsdk_entities/kpi/kpi.py | 22 ++++++++++++++++------ tests/kpi/test_kpi.py | 29 ++++++++++++++++++++++++++--- 5 files changed, 67 insertions(+), 13 deletions(-) diff --git a/docs/entities/kpis.md b/docs/entities/kpis.md index 871bd12..24b2524 100644 --- a/docs/entities/kpis.md +++ b/docs/entities/kpis.md @@ -3,13 +3,26 @@ KPIs are user defined calculated fields in the Sight Machine software. ## Functions -The SDK has two functions related to KPIs. The first of which allows a user to see which KPIs are availible for a particular asset. The second makes use of our Data Visulation api which allows a user to see these KPIs over a timeframe. +The SDK has three functions related to KPIs. The first returns a list of all availible KPis. The second of which allows a user to see which KPIs are availible for a particular asset. The thrid makes use of our Data Visulation api which allows a user to see these KPIs over a timeframe. ### Get KPIs -This is the first KPI function allowing you to see which KPIs are availible for a particular asset. In order to call this function you must first have a logged in client see the [quick start guide](/README.md) for more information on logging in. Once you have a logged in client you can call the function as follows: +This is the first KPI function allowing you to see which all KPIs. In order to call this function you must first have a logged in client see the [quick start guide](/README.md) for more information on logging in. Once you have a logged in client you can call the function as follows: ``` -cli.get_kpis(**asset_selection) +cli.get_kpis() +``` +This will return a full list of KPIs which will look something like this example: +``` +[{'name': 'performance', 'display_name': 'Performance', 'formula': '( IdealCycle / Recorded_time ) * 100 if ( Recorded_time > 0 ) else None', 'data_type': '', 'dependencies': [{'aggregate': 'sum', 'name': 'Recorded_time'}, {'aggregate': 'sum', 'name': 'IdealCycle'}]}, ...] +``` + +In order to make use of the data viz function you'll need the name of the KPI you wish to get. + +### Get KPIs For Asset +This is the second KPI function allowing you to see which KPIs are availible for a particular asset. Once you have a logged in client you can call the function as follows: + +``` +cli.get_kpis_for_asset(**asset_selection) ``` For more info on [asset_selection](/docs/commonly_used_data_types/asset_selection.md) click on the previous link. After a moment the SDK should return a list that looks something like this: diff --git a/smsdk/client.py b/smsdk/client.py index ebf6ad2..ca7b5ac 100644 --- a/smsdk/client.py +++ b/smsdk/client.py @@ -233,6 +233,13 @@ def get_kpis(self, **kwargs): ) return kpis(self.session, base_url).get_kpis(**kwargs) + def get_kpis_for_asset(self, **kwargs): + kpis = smsdkentities.get('kpi') + base_url = get_url( + self.config["protocol"], self.tenant, self.config["site.domain"] + ) + return kpis(self.session, base_url).get_kpis_for_asset(**kwargs) + def get_kpi_data_viz(self, **kwargs): kpis = smsdkentities.get('kpi') base_url = get_url( diff --git a/smsdk/config/api_endpoints.json b/smsdk/config/api_endpoints.json index 81b744f..3114b76 100644 --- a/smsdk/config/api_endpoints.json +++ b/smsdk/config/api_endpoints.json @@ -32,6 +32,7 @@ "task": "/v1/datavis/task/async" }, "KPI": { - "availible_kpis": "/v1/selector/datavis/kpi/y_axis" + "availible_kpis": "/v1/selector/assets", + "availible_kpis_for_asset": "/v1/selector/datavis/kpi/y_axis" } } diff --git a/smsdk/smsdk_entities/kpi/kpi.py b/smsdk/smsdk_entities/kpi/kpi.py index 8920f36..2a8c520 100644 --- a/smsdk/smsdk_entities/kpi/kpi.py +++ b/smsdk/smsdk_entities/kpi/kpi.py @@ -43,11 +43,22 @@ def get_utilities(self, *args, **kwargs) -> List: @mod_util def get_kpis(self, *args, **kwargs): """ - Utility function to get the downtimes - from the ma downtime API - Recommend to use 'enable_pagination':True for larger datasets + Returns a list of all KPIs """ url = "{}{}".format(self.base_url, ENDPOINTS["KPI"]["availible_kpis"]) + records = self._get_records_v1(url, method="get", **kwargs)[0]["kpi"] + + if not isinstance(records, List): + raise ValueError("Error - {}".format(records)) + return records + + @mod_util + def get_kpis_for_asset(self, *args, **kwargs): + """ + Takes an asset selection + returns a list of KPIs avaible to that asset + """ + url = "{}{}".format(self.base_url, ENDPOINTS["KPI"]["availible_kpis_for_asset"]) records = self._get_records_v1(url, **kwargs) if not isinstance(records, List): @@ -57,9 +68,8 @@ def get_kpis(self, *args, **kwargs): @mod_util def get_kpi_data_viz(self, *args, **kwargs): """ - Utility function to get the downtimes - from the ma downtime API - Recommend to use 'enable_pagination':True for larger datasets + Takes a Data Viz query for the KPI model + Returns Data Viz info for that query """ url = "{}{}".format(self.base_url, ENDPOINTS["DataViz"]["task"]) kwargs['model']='kpi' diff --git a/tests/kpi/test_kpi.py b/tests/kpi/test_kpi.py index 2ed1dc2..3314fb2 100644 --- a/tests/kpi/test_kpi.py +++ b/tests/kpi/test_kpi.py @@ -5,19 +5,42 @@ from smsdk.smsdk_entities.kpi.kpi import KPI from mock import mock_open, MagicMock, patch -def test_get_kpi(monkeypatch): +@patch("smsdk.ma_session.Session") +def test_get_kpi(mocked): + class ResponseGet: + ok = True + text = "Success" + status_code=200 + + @staticmethod + def json(): + return {"results": [{"kpi":AVALIBLE_KPI_JSON}]} + mocked.return_value = MagicMock( + get=MagicMock(return_value=ResponseGet()) + ) + + dt = Client("demo") + + # Run + kpis = dt.get_kpis() + + # Verify + assert len(kpis) == 5 + + assert kpis[0]["name"] == 'Scrap_Rate' +def test_get_kpi_for_asset(monkeypatch): # Setup def mockapi(self, session, endpoint): if endpoint == "/v1/selector/datavis/kpi/y_axis": return pd.DataFrame(AVALIBLE_KPI_JSON) return pd.DataFrame() - monkeypatch.setattr(KPI, "get_kpis", mockapi) + monkeypatch.setattr(KPI, "get_kpis_for_asset", mockapi) dt = KPI(Session(), "demo") # Run - df = dt.get_kpis(Session(), "/v1/selector/datavis/kpi/y_axis") + df = dt.get_kpis_for_asset(Session(), "/v1/selector/datavis/kpi/y_axis") # Verify assert df.shape == (5, 7) From c9c539bea98fdfea68d6b6ba50307260b7a319ef Mon Sep 17 00:00:00 2001 From: Michael Goldman Date: Mon, 6 Mar 2023 18:21:49 -0800 Subject: [PATCH 11/33] Data viz taking in variables --- smsdk/client.py | 33 ++++++++++++++++++++++--- smsdk/config/api_endpoints.json | 4 ++- smsdk/ma_session.py | 2 ++ smsdk/smsdk_entities/kpi/kpi.py | 2 +- smsdk/smsdk_entities/machine/machine.py | 12 +++++++++ 5 files changed, 48 insertions(+), 5 deletions(-) diff --git a/smsdk/client.py b/smsdk/client.py index ca7b5ac..c96ec44 100644 --- a/smsdk/client.py +++ b/smsdk/client.py @@ -240,9 +240,36 @@ def get_kpis_for_asset(self, **kwargs): ) return kpis(self.session, base_url).get_kpis_for_asset(**kwargs) - def get_kpi_data_viz(self, **kwargs): - kpis = smsdkentities.get('kpi') + def get_kpi_data_viz(self, machine_source=None, kpis=None, i_vars=None, time_selection=None, **kwargs): + kpi_entity = smsdkentities.get('kpi') + if machine_source: + machine_type = self.get_type_from_machine(machine_source, **kwargs) + kwargs["asset_selection"]= { + "machine_source": [machine_source], + "machine_type": [machine_type] + } + + if kpis: + d_vars = [] + for kpi in kpis: + d_vars.append({"name": kpi, "aggregate": ["avg"]}) + kwargs['d_vars'] = d_vars + + if i_vars: + kwargs['i_vars'] = i_vars + + if time_selection: + kwargs["time_selection"] = time_selection + base_url = get_url( self.config["protocol"], self.tenant, self.config["site.domain"] ) - return kpis(self.session, base_url).get_kpi_data_viz(**kwargs) + return kpi_entity(self.session, base_url).get_kpi_data_viz(**kwargs) + + def get_type_from_machine(self, machine_source=None, **kwargs): + machine = smsdkentities.get('machine') + base_url = get_url( + self.config["protocol"], self.tenant, self.config["site.domain"] + ) + return machine(self.session, base_url).get_type_from_machine_name(machine_source, **kwargs) + diff --git a/smsdk/config/api_endpoints.json b/smsdk/config/api_endpoints.json index 3114b76..6217b77 100644 --- a/smsdk/config/api_endpoints.json +++ b/smsdk/config/api_endpoints.json @@ -32,7 +32,9 @@ "task": "/v1/datavis/task/async" }, "KPI": { - "availible_kpis": "/v1/selector/assets", "availible_kpis_for_asset": "/v1/selector/datavis/kpi/y_axis" + }, + "Assets": { + "url":"/v1/selector/assets" } } diff --git a/smsdk/ma_session.py b/smsdk/ma_session.py index 735e8b0..4c2131e 100644 --- a/smsdk/ma_session.py +++ b/smsdk/ma_session.py @@ -1,4 +1,5 @@ from json.decoder import JSONDecodeError +from time import sleep from typing import List import json import requests @@ -229,6 +230,7 @@ def _complete_async_task( response = self.session.get( endpoint+'/'+task_id, json=url_params ) + sleep(1) if response.status_code not in [200, 201]: raise ValueError("Error - {}".format(response.text)) data = response.json() diff --git a/smsdk/smsdk_entities/kpi/kpi.py b/smsdk/smsdk_entities/kpi/kpi.py index 2a8c520..b96aead 100644 --- a/smsdk/smsdk_entities/kpi/kpi.py +++ b/smsdk/smsdk_entities/kpi/kpi.py @@ -45,7 +45,7 @@ def get_kpis(self, *args, **kwargs): """ Returns a list of all KPIs """ - url = "{}{}".format(self.base_url, ENDPOINTS["KPI"]["availible_kpis"]) + url = "{}{}".format(self.base_url, ENDPOINTS["Assets"]["url"]) records = self._get_records_v1(url, method="get", **kwargs)[0]["kpi"] if not isinstance(records, List): diff --git a/smsdk/smsdk_entities/machine/machine.py b/smsdk/smsdk_entities/machine/machine.py index 3885111..bfccec1 100644 --- a/smsdk/smsdk_entities/machine/machine.py +++ b/smsdk/smsdk_entities/machine/machine.py @@ -46,3 +46,15 @@ def get_machines(self, *args, **kwargs): if not isinstance(records, List): raise ValueError("Error - {}".format(records)) return records + + def get_type_from_machine_name(self, machine_source, *args, **kwargs): + """ + Function that gives the name of a machine from it's type + """ + url = "{}{}".format(self.base_url, ENDPOINTS["Assets"]["url"]) + records = self._get_records_v1(url, method="get", **kwargs)[0]["machine"] + machine_type = '' + for record in records: + if record['name'] == machine_source: + machine_type = record['type'] + return machine_type From 6bb34a00c8a0713ab48d06a9d8d3c21622f7af70 Mon Sep 17 00:00:00 2001 From: Michael Goldman Date: Tue, 7 Mar 2023 13:14:53 -0800 Subject: [PATCH 12/33] Tests and docs --- docs/entities/kpis.md | 36 +++++++++++++++++++++++++++++++++-- docs/entities/machine.md | 15 +++++++++++++++ tests/kpi/test_kpi.py | 3 ++- tests/machine/test_machine.py | 27 ++++++++++++++++++++++++++ 4 files changed, 78 insertions(+), 3 deletions(-) create mode 100644 docs/entities/machine.md diff --git a/docs/entities/kpis.md b/docs/entities/kpis.md index 24b2524..2bb1fc1 100644 --- a/docs/entities/kpis.md +++ b/docs/entities/kpis.md @@ -36,10 +36,42 @@ In order to make use of the data viz function you'll need the name of the KPI yo ### Get KPI Data Viz Once you have the name of the KPI you wish to access and a logged in client you can make a call to the data viz api with the following SDK function call: ``` -cli.get_kpi_data_viz(**data_viz_query) +cli.get_kpi_data_viz(machine_source, kpis, i_vars, time_selection, **optional_data_viz_query) ``` -The d_vars section of your data_viz_query should include an d_var with the name of the KPI you wish to query. For more information on [data_viz_queries](/docs/commonly_used_data_types/data_viz_query.md) click on the previous link. After some time the SDK should return a list that looks something like this: +After some time the SDK should return a list that looks something like this: ``` [{'i_vals': {'endtime': {'i_pos': 0, 'bin_no': 0, 'bin_min': '2022-10-20T00:00:00-07:00', 'bin_max': '2022-10-20T00:00:00-07:00', 'bin_avg': '2022-10-20T00:00:00-07:00'}}, 'd_vals': {'quality': {'avg': 95.18072289156626}}, '_count': 418, 'kpi_dependencies': {'quality': {'Output': 395.0, 'ScrapQuantity': 20.0}}},...] +``` + +There's two ways to call this function you can use a data_viz_query,For more information on [data_viz_queries](/docs/commonly_used_data_types/data_viz_query.md) click on the previous link, or have the function fill out the query for you by passing in a few variable we will now go over one at a time. + +#### machine_source +This is a string and is the name of machine you wish to run a query on. + +#### kpis +This is a list of the names of all the kpis you wish to run this query on. + +#### i_vars +This is a list, this is the same as the i_vars object in the data_viz_query and is the axis you are querying the kpis against it will look like the following: +``` +[ + { + "name": "endtime", + "time_resolution": "day", + "query_tz": "America/Los_Angeles", + "output_tz": "America/Los_Angeles" + } +] +``` + +#### time_selection +This is an object, this is the same as the [time_selection](/docs/commonly_used_data_types/data_viz_query.md#time_selection) object in the data_viz_query and more info can be found at that link. The most common form is the relative time selection and looks like this: +``` +{ + "time_type": "relative", + "relative_start": 7, + "relative_unit": "day", + "ctime_tz": "America/Los_Angeles" +} ``` \ No newline at end of file diff --git a/docs/entities/machine.md b/docs/entities/machine.md new file mode 100644 index 0000000..1bf7381 --- /dev/null +++ b/docs/entities/machine.md @@ -0,0 +1,15 @@ +# Machines +Machines are just that, machines in factories. The machine object is how the Sight Machine software replesents and is the key to accessing data that we collect from them + +## Functions + +### get_type_from_machine +The get_type_from_machine function allows you to get the type of any machine from it's name and is called this way: +``` +cli.get_type_from_machine(machine_name) +``` + +And will return something like the following: +``` +'Lasercut' +``` \ No newline at end of file diff --git a/tests/kpi/test_kpi.py b/tests/kpi/test_kpi.py index 3314fb2..fc49946 100644 --- a/tests/kpi/test_kpi.py +++ b/tests/kpi/test_kpi.py @@ -28,6 +28,7 @@ def json(): assert len(kpis) == 5 assert kpis[0]["name"] == 'Scrap_Rate' + def test_get_kpi_for_asset(monkeypatch): # Setup def mockapi(self, session, endpoint): @@ -77,4 +78,4 @@ def json(): dt = Client("demo") data = dt.get_kpi_data_viz() assert len(data) == 3 - assert data[0]['d_vals']['quality']['avg'] == 95.18072289156626 \ No newline at end of file + assert data[0]['d_vals']['quality']['avg'] == 95.18072289156626 diff --git a/tests/machine/test_machine.py b/tests/machine/test_machine.py index 5efe803..f23d470 100644 --- a/tests/machine/test_machine.py +++ b/tests/machine/test_machine.py @@ -1,5 +1,8 @@ +from unittest.mock import MagicMock +from mock import patch import pandas as pd from requests.sessions import Session +from smsdk.client import Client from tests.machine.machine_data import JSON_MACHINE from smsdk.smsdk_entities.machine.machine import Machine @@ -32,3 +35,27 @@ def mockapi(self, session, endpoint): ] assert cols == df.columns.sort_values().tolist() + + +@patch("smsdk.ma_session.Session") +def test_get_type(mocked): + class ResponseGet: + ok = True + text = "Success" + status_code=200 + + @staticmethod + def json(): + return {'results':[{"machine":[{"name":"test", "type":'test_type'}]}]} + + mocked.return_value = MagicMock( + get=MagicMock(return_value=ResponseGet()) + ) + + dt = Client("demo") + + # Run + type = dt.get_type_from_machine('test') + + # Verify + assert type == 'test_type' From 0816f4c71d424c5dac61842f95197997cc9bc9d6 Mon Sep 17 00:00:00 2001 From: Michael Goldman Date: Tue, 7 Mar 2023 16:52:26 -0800 Subject: [PATCH 13/33] Sean comments --- docs/entities/kpis.md | 8 ++++---- smsdk/client.py | 12 +++++++----- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/docs/entities/kpis.md b/docs/entities/kpis.md index 2bb1fc1..97ddeb1 100644 --- a/docs/entities/kpis.md +++ b/docs/entities/kpis.md @@ -3,10 +3,10 @@ KPIs are user defined calculated fields in the Sight Machine software. ## Functions -The SDK has three functions related to KPIs. The first returns a list of all availible KPis. The second of which allows a user to see which KPIs are availible for a particular asset. The thrid makes use of our Data Visulation api which allows a user to see these KPIs over a timeframe. +The SDK has three functions related to KPIs. The first returns a list of all availible KPis. The second of which allows a user to see which KPIs are availible for a particular asset. The third makes use of our Data Visualization api which allows a user to see these KPIs over a timeframe. ### Get KPIs -This is the first KPI function allowing you to see which all KPIs. In order to call this function you must first have a logged in client see the [quick start guide](/README.md) for more information on logging in. Once you have a logged in client you can call the function as follows: +This is the first KPI function allowing you to see all KPIs availible to you. In order to call this function you must first have a logged in client see the [quick start guide](/README.md) for more information on logging in. Once you have a logged in client you can call the function as follows: ``` cli.get_kpis() @@ -46,8 +46,8 @@ After some time the SDK should return a list that looks something like this: There's two ways to call this function you can use a data_viz_query,For more information on [data_viz_queries](/docs/commonly_used_data_types/data_viz_query.md) click on the previous link, or have the function fill out the query for you by passing in a few variable we will now go over one at a time. -#### machine_source -This is a string and is the name of machine you wish to run a query on. +#### machine_sources +This is a list of strings and is the name of machine(s) you wish to run a query on. #### kpis This is a list of the names of all the kpis you wish to run this query on. diff --git a/smsdk/client.py b/smsdk/client.py index c96ec44..3fcb433 100644 --- a/smsdk/client.py +++ b/smsdk/client.py @@ -240,13 +240,15 @@ def get_kpis_for_asset(self, **kwargs): ) return kpis(self.session, base_url).get_kpis_for_asset(**kwargs) - def get_kpi_data_viz(self, machine_source=None, kpis=None, i_vars=None, time_selection=None, **kwargs): + def get_kpi_data_viz(self, machine_sources=None, kpis=None, i_vars=None, time_selection=None, **kwargs): kpi_entity = smsdkentities.get('kpi') - if machine_source: - machine_type = self.get_type_from_machine(machine_source, **kwargs) + if machine_sources: + machine_types = [] + for machine_source in machine_sources: + machine_types.append(self.get_type_from_machine(machine_source, **kwargs)) kwargs["asset_selection"]= { - "machine_source": [machine_source], - "machine_type": [machine_type] + "machine_source": machine_sources, + "machine_type": list(set(machine_types)) } if kpis: From 68d16c02c1da43397df4da486bb1783e1a8eefb2 Mon Sep 17 00:00:00 2001 From: Michael Goldman Date: Tue, 7 Mar 2023 18:41:20 -0800 Subject: [PATCH 14/33] Actually hit save --- docs/commonly_used_data_types/data_viz_query.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/commonly_used_data_types/data_viz_query.md b/docs/commonly_used_data_types/data_viz_query.md index e81a93a..3a8a32b 100644 --- a/docs/commonly_used_data_types/data_viz_query.md +++ b/docs/commonly_used_data_types/data_viz_query.md @@ -176,4 +176,4 @@ The type of operation you are doing. The value to compare with the operation. ## db_mode -This is optional. It will defualt to 'sql' and usually should be but we have a 'monogo' mode as well. You will likely ever need to tset this. \ No newline at end of file +This is optional. It will default to 'sql' and usually should be but we have a 'mongo' mode as well. You will likely ever need to set this. \ No newline at end of file From 261cedc12dfb3a7e20a4d8c736d415a2798fb39d Mon Sep 17 00:00:00 2001 From: Michael Goldman Date: Wed, 8 Mar 2023 11:09:40 -0800 Subject: [PATCH 15/33] Bug fixed, show_hidden added, more documenation --- docs/entities/machine.md | 38 ++++++++++++++++++++++++++++++++++++++ smsdk/client.py | 4 ++-- smsdk/client_v0.py | 4 ++-- 3 files changed, 42 insertions(+), 4 deletions(-) diff --git a/docs/entities/machine.md b/docs/entities/machine.md index 1bf7381..5067755 100644 --- a/docs/entities/machine.md +++ b/docs/entities/machine.md @@ -12,4 +12,42 @@ cli.get_type_from_machine(machine_name) And will return something like the following: ``` 'Lasercut' +``` + +### get_machine_schema +The get_machine_schema functions returns the fields of the machine schema for a given machine source and is called this way: +``` +cli.get_machine_schema(machine_source, types, show_hidden, return_mtype) +``` + +The only required field in this case is the machine_source, we will go over each variable in a second. The function will return a pandas dataframe that looks like the following: +``` + name display type +0 stats__Alarms__val Alarms float +1 stats__BLOCKED__val BLOCKED float +2 stats__DOWN__val DOWN float +3 stats__DefectCategory__val Defect Category string +``` + +#### machine_source +This is the name of the machine that you are trying to grab the schema of. This is the only required parameter for this function. + +#### types +This is an optional parameter and is a list of strings. If this is set the function will only return colomns that match the types given. For example if we were to pass ['string'] as our types parameter in the previous example we would instead have returned: +``` + name display type +0 stats__DefectCategory__val Defect Category string +``` + +#### show_hidden +This is an optional parameter and is a boolean. There are a few fields we have set to be hiddden from our ui in our application and by defualt these are also hidden from the return in this function if set to True we will also return these fields. + +#### return_mtype +This is an optional parameter and is a boolean. If set to True this will instead of returning just the pandas dataframe will return a Tupple with a string that is the machine type of the machine_soure for example: +``` +('Lasercut', + name display type +0 stats__Alarms__val Alarms float +1 stats__BLOCKED__val BLOCKED float +...) ``` \ No newline at end of file diff --git a/smsdk/client.py b/smsdk/client.py index e5d494e..56bf99e 100644 --- a/smsdk/client.py +++ b/smsdk/client.py @@ -213,11 +213,11 @@ def get_parts(self, normalize=True, clean_strings_in=True, clean_strings_out=Tru @ClientV0.get_machine_schema_decorator - def get_machine_schema(self, machine_source, types=[], return_mtype=False, **kwargs): + def get_machine_schema(self, machine_source, types=[], show_hidden=False, return_mtype=False, **kwargs): stats = kwargs.get('stats', []) fields = [] for stat in stats: - if not stat.get('display', {}).get('ui_hidden', False): + if not stat.get('display', {}).get('ui_hidden', False) or show_hidden: if len(types) == 0 or stat['analytics']['columns'][0]['type'] in types: try: fields.append({'name': stat['analytics']['columns'][0]['name'], diff --git a/smsdk/client_v0.py b/smsdk/client_v0.py index b4d8129..9f3ff98 100644 --- a/smsdk/client_v0.py +++ b/smsdk/client_v0.py @@ -424,7 +424,7 @@ def inner(self, normalize=True, clean_strings_in=True, clean_strings_out=True, d def get_machine_schema_decorator(func): @functools.wraps(func) - def inner(self, machine_source, types=[], return_mtype=False, **kwargs): + def inner(self, machine_source, types=[], show_hidden=False, return_mtype=False, **kwargs): try: machine_type = self.get_machines(source=machine_source)['source_type'][0] @@ -448,7 +448,7 @@ def inner(self, machine_source, types=[], return_mtype=False, **kwargs): print(f"Exception in getting machine type stats {ex}") kwargs['stats'] = stats - fields = func(self, machine_source, types=[], return_mtype=False, **kwargs) + fields = func(self, machine_source, types, show_hidden, return_mtype, **kwargs) if return_mtype: return machine_type, pd.DataFrame(fields) From 67ebd6818b758ffba8bf778d4c936807ceedf7e6 Mon Sep 17 00:00:00 2001 From: Michael Goldman Date: Wed, 8 Mar 2023 12:38:07 -0800 Subject: [PATCH 16/33] Unit test --- tests/machine/machine_data.py | 31 +++++++++++++++++++ tests/machine/test_machine.py | 58 ++++++++++++++++++++++++++++++++++- 2 files changed, 88 insertions(+), 1 deletion(-) diff --git a/tests/machine/machine_data.py b/tests/machine/machine_data.py index e48317c..1f91bdf 100644 --- a/tests/machine/machine_data.py +++ b/tests/machine/machine_data.py @@ -255,3 +255,34 @@ "source_type": "UpperComponentBacking", }, ] + +MACHINE_TYPE = { + 'stats': [ + [ + { + 'analytics' : {'columns' : [ + {'type': 'float', 'name': 'stat__test_float'} + ] + }, + 'display':{'title_prefix': 'test float'} + }, + { + 'analytics' : {'columns' : [ + {'type': 'string', 'name': 'stat__test_string'} + ] + }, + 'display':{'title_prefix': 'test string'} + }, + { + 'analytics' : {'columns' : [ + {'type': 'string', 'name': 'stat__test_hidden'} + ] + }, + 'display':{ + 'title_prefix': 'test hidden', + 'ui_hidden' : True + } + } + ] + ] +} diff --git a/tests/machine/test_machine.py b/tests/machine/test_machine.py index f23d470..2bc0970 100644 --- a/tests/machine/test_machine.py +++ b/tests/machine/test_machine.py @@ -3,7 +3,7 @@ import pandas as pd from requests.sessions import Session from smsdk.client import Client -from tests.machine.machine_data import JSON_MACHINE +from tests.machine.machine_data import JSON_MACHINE, MACHINE_TYPE from smsdk.smsdk_entities.machine.machine import Machine @@ -59,3 +59,59 @@ def json(): # Verify assert type == 'test_type' + + +@patch("smsdk.client_v0.ClientV0.get_machines") +@patch("smsdk.client_v0.ClientV0.get_machine_types") +def test_get_machine_schema(mocked_types, mocked_machines): + mocked_machines.return_value = {'source_type': ['test']} + mocked_types.return_value = MACHINE_TYPE + dt = Client("demo") + + # Run + schema = dt.get_machine_schema('test') + + # Verify + assert schema.name.sort_values().tolist() == ['stat__test_float', 'stat__test_string'] + + +@patch("smsdk.client_v0.ClientV0.get_machines") +@patch("smsdk.client_v0.ClientV0.get_machine_types") +def test_get_machine_schema_hidden(mocked_types, mocked_machines): + mocked_machines.return_value = {'source_type': ['test']} + mocked_types.return_value = MACHINE_TYPE + dt = Client("demo") + + # Run + schema = dt.get_machine_schema('test', show_hidden=True) + + # Verify + assert schema.name.sort_values().tolist() == ['stat__test_float', 'stat__test_hidden', 'stat__test_string'] + +@patch("smsdk.client_v0.ClientV0.get_machines") +@patch("smsdk.client_v0.ClientV0.get_machine_types") +def test_get_machine_schema_types(mocked_types, mocked_machines): + mocked_machines.return_value = {'source_type': ['test']} + mocked_types.return_value = MACHINE_TYPE + dt = Client("demo") + + # Run + schema = dt.get_machine_schema('test', types=["float"]) + + # Verify + assert schema.name.sort_values().tolist() == ['stat__test_float'] + +@patch("smsdk.client_v0.ClientV0.get_machines") +@patch("smsdk.client_v0.ClientV0.get_machine_types") +def test_get_machine_schema_types_return_mtype(mocked_types, mocked_machines): + mocked_machines.return_value = {'source_type': ['test_type']} + mocked_types.return_value = MACHINE_TYPE + dt = Client("demo") + + # Run + schema = dt.get_machine_schema('test', return_mtype=True) + + # Verify + assert schema[0] == 'test_type' + assert schema[1].name.sort_values().tolist() == ['stat__test_float', 'stat__test_string'] + From c7a4ff76f8a43925d2a277e8f7f1cb1b7e4e6016 Mon Sep 17 00:00:00 2001 From: Michael Goldman Date: Wed, 8 Mar 2023 13:22:18 -0800 Subject: [PATCH 17/33] New endpoint --- smsdk/client.py | 8 ++++++++ smsdk/config/api_endpoints.json | 3 ++- smsdk/smsdk_entities/machine_type/machinetype.py | 13 +++++++++++++ 3 files changed, 23 insertions(+), 1 deletion(-) diff --git a/smsdk/client.py b/smsdk/client.py index 56bf99e..b6444c8 100644 --- a/smsdk/client.py +++ b/smsdk/client.py @@ -277,4 +277,12 @@ def get_type_from_machine(self, machine_source=None, **kwargs): self.config["protocol"], self.tenant, self.config["site.domain"] ) return machine(self.session, base_url).get_type_from_machine_name(machine_source, **kwargs) + + def get_fields_of_machine(self, machine_source=None): + machineType= smsdkentities.get('machine_type') + machine_type = self.get_type_from_machine(machine_source) + base_url = get_url( + self.config["protocol"], self.tenant, self.config["site.domain"] + ) + return machineType(self.session, base_url).get_fields(machine_type) diff --git a/smsdk/config/api_endpoints.json b/smsdk/config/api_endpoints.json index 6217b77..f6fe0fb 100644 --- a/smsdk/config/api_endpoints.json +++ b/smsdk/config/api_endpoints.json @@ -11,7 +11,8 @@ "url" : "/api/factory" }, "MachineType": { - "url" : "/api/machinetype" + "url" : "/api/machinetype", + "fields": "/v1/selector/datatab/cycle/{}/field" }, "Machine": { "url" : "/api/machine" diff --git a/smsdk/smsdk_entities/machine_type/machinetype.py b/smsdk/smsdk_entities/machine_type/machinetype.py index c62f2a6..efc2d5d 100644 --- a/smsdk/smsdk_entities/machine_type/machinetype.py +++ b/smsdk/smsdk_entities/machine_type/machinetype.py @@ -43,3 +43,16 @@ def get_machine_types(self, *args, **kwargs): if not isinstance(records, List): raise ValueError("Error - {}".format(records)) return records + + @mod_util + def get_fields(self, machine_type, *args, **kwargs): + """ + Utility function to get the machine types + from the ma machine API + Recommend to use 'enable_pagination':True for larger datasets + """ + url = "{}{}".format(self.base_url, ENDPOINTS["MachineType"]["fields"].format(machine_type)) + records = self._get_records(url, **kwargs) + if not isinstance(records, List): + raise ValueError("Error - {}".format(records)) + return records From b47d69de70f33464cd3f17e1dec0660e9d9f9547 Mon Sep 17 00:00:00 2001 From: Michael Goldman Date: Wed, 8 Mar 2023 13:35:01 -0800 Subject: [PATCH 18/33] Feature Parity --- smsdk/client.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/smsdk/client.py b/smsdk/client.py index b6444c8..be8c5aa 100644 --- a/smsdk/client.py +++ b/smsdk/client.py @@ -278,11 +278,18 @@ def get_type_from_machine(self, machine_source=None, **kwargs): ) return machine(self.session, base_url).get_type_from_machine_name(machine_source, **kwargs) - def get_fields_of_machine(self, machine_source=None): + def get_fields_of_machine(self, machine_source, types=[], show_hidden=False, return_mtype=False, **kwargs): machineType= smsdkentities.get('machine_type') machine_type = self.get_type_from_machine(machine_source) base_url = get_url( self.config["protocol"], self.tenant, self.config["site.domain"] ) - return machineType(self.session, base_url).get_fields(machine_type) + fields = machineType(self.session, base_url).get_fields(machine_type, **kwargs) + fields = [field for field in fields if not field.get('ui_hidden') or show_hidden] + if len(types) > 0: + fields = [field for field in fields if field.get('type') in types] + if return_mtype: + return (machine_type, fields) + + return fields From 19790d4354b72d10840902eef1af45965e3f8cab Mon Sep 17 00:00:00 2001 From: Michael Goldman Date: Thu, 9 Mar 2023 11:36:12 -0800 Subject: [PATCH 19/33] Replacement and tests done --- docs/entities/machine.md | 16 ++----- smsdk/client.py | 32 ++++++------- tests/machine/machine_data.py | 34 +++++--------- tests/machine/test_machine.py | 62 ++++++++++++++----------- tests/machine_type/machine_type_data.py | 19 ++++++++ tests/machine_type/test_machine_type.py | 44 +++++++++++++++++- 6 files changed, 125 insertions(+), 82 deletions(-) diff --git a/docs/entities/machine.md b/docs/entities/machine.md index 5067755..498faf3 100644 --- a/docs/entities/machine.md +++ b/docs/entities/machine.md @@ -20,13 +20,9 @@ The get_machine_schema functions returns the fields of the machine schema for a cli.get_machine_schema(machine_source, types, show_hidden, return_mtype) ``` -The only required field in this case is the machine_source, we will go over each variable in a second. The function will return a pandas dataframe that looks like the following: +The only required field in this case is the machine_source, we will go over each variable in a second. The function will return a pandas list that looks like the following: ``` - name display type -0 stats__Alarms__val Alarms float -1 stats__BLOCKED__val BLOCKED float -2 stats__DOWN__val DOWN float -3 stats__DefectCategory__val Defect Category string +[{'display_name': 'Machine', 'unit': '', 'type': 'categorical', 'data_type': 'string', 'stream_types': [], 'raw_data_field': '', 'name': 'machine__source'}, {'display_name': 'Cycle Start Time', 'unit': '', 'type': 'datetime', 'data_type': 'datetime', 'stream_types': [], 'raw_data_field': '', 'name': 'starttime'},..] ``` #### machine_source @@ -35,8 +31,7 @@ This is the name of the machine that you are trying to grab the schema of. This #### types This is an optional parameter and is a list of strings. If this is set the function will only return colomns that match the types given. For example if we were to pass ['string'] as our types parameter in the previous example we would instead have returned: ``` - name display type -0 stats__DefectCategory__val Defect Category string +[{'display_name': 'Machine', 'unit': '', 'type': 'categorical', 'data_type': 'string', 'stream_types': [], 'raw_data_field': '', 'name': 'machine__source'}] ``` #### show_hidden @@ -46,8 +41,5 @@ This is an optional parameter and is a boolean. There are a few fields we have This is an optional parameter and is a boolean. If set to True this will instead of returning just the pandas dataframe will return a Tupple with a string that is the machine type of the machine_soure for example: ``` ('Lasercut', - name display type -0 stats__Alarms__val Alarms float -1 stats__BLOCKED__val BLOCKED float -...) +[{'display_name': 'Machine', 'unit': '', 'type': 'categorical', 'data_type': 'string', 'stream_types': [], 'raw_data_field': '', 'name': 'machine__source'}, {'display_name': 'Cycle Start Time', 'unit': '', 'type': 'datetime', 'data_type': 'datetime', 'stream_types': [], 'raw_data_field': '', 'name': 'starttime'},..] ``` \ No newline at end of file diff --git a/smsdk/client.py b/smsdk/client.py index be8c5aa..e9d58ae 100644 --- a/smsdk/client.py +++ b/smsdk/client.py @@ -210,24 +210,6 @@ def get_parts(self, normalize=True, clean_strings_in=True, clean_strings_out=Tru df = self.get_data_v1('part_v1', 'get_parts', normalize, *args, **kwargs) return df - - - @ClientV0.get_machine_schema_decorator - def get_machine_schema(self, machine_source, types=[], show_hidden=False, return_mtype=False, **kwargs): - stats = kwargs.get('stats', []) - fields = [] - for stat in stats: - if not stat.get('display', {}).get('ui_hidden', False) or show_hidden: - if len(types) == 0 or stat['analytics']['columns'][0]['type'] in types: - try: - fields.append({'name': stat['analytics']['columns'][0]['name'], - 'display': stat['display']['title_prefix'], - 'type': stat['analytics']['columns'][0]['type']}) - except: - log.warning( - f"Unknow stat schema identified :: machine_type {machine_source} - " - f"title_prefix :: {stat.get('display', {}).get('title_prefix', '')}") - return fields def get_kpis(self, **kwargs): kpis = smsdkentities.get('kpi') @@ -278,7 +260,7 @@ def get_type_from_machine(self, machine_source=None, **kwargs): ) return machine(self.session, base_url).get_type_from_machine_name(machine_source, **kwargs) - def get_fields_of_machine(self, machine_source, types=[], show_hidden=False, return_mtype=False, **kwargs): + def get_machine_schema(self, machine_source, types=[], show_hidden=False, return_mtype=False, **kwargs): machineType= smsdkentities.get('machine_type') machine_type = self.get_type_from_machine(machine_source) base_url = get_url( @@ -293,3 +275,15 @@ def get_fields_of_machine(self, machine_source, types=[], show_hidden=False, ret return (machine_type, fields) return fields + + def get_fields_of_machine_type(self, machine_type, types=[], show_hidden=False, **kwargs): + machineType= smsdkentities.get('machine_type') + base_url = get_url( + self.config["protocol"], self.tenant, self.config["site.domain"] + ) + fields = machineType(self.session, base_url).get_fields(machine_type, **kwargs) + fields = [field for field in fields if not field.get('ui_hidden') or show_hidden] + if len(types) > 0: + fields = [field for field in fields if field.get('type') in types] + + return fields diff --git a/tests/machine/machine_data.py b/tests/machine/machine_data.py index 1f91bdf..a83dc27 100644 --- a/tests/machine/machine_data.py +++ b/tests/machine/machine_data.py @@ -256,33 +256,21 @@ }, ] -MACHINE_TYPE = { - 'stats': [ - [ +MACHINE_TYPE =[ { - 'analytics' : {'columns' : [ - {'type': 'float', 'name': 'stat__test_float'} - ] - }, - 'display':{'title_prefix': 'test float'} + 'type': 'float', + 'name': 'stat__test_float', + 'title_prefix': 'test float' }, { - 'analytics' : {'columns' : [ - {'type': 'string', 'name': 'stat__test_string'} - ] - }, - 'display':{'title_prefix': 'test string'} + 'type': 'string', + 'name': 'stat__test_string', + 'title_prefix': 'test string' }, { - 'analytics' : {'columns' : [ - {'type': 'string', 'name': 'stat__test_hidden'} - ] - }, - 'display':{ - 'title_prefix': 'test hidden', - 'ui_hidden' : True - } + 'type': 'string', + 'name': 'stat__test_hidden', + 'title_prefix': 'test hidden', + 'ui_hidden' : True } ] - ] -} diff --git a/tests/machine/test_machine.py b/tests/machine/test_machine.py index 2bc0970..d960de0 100644 --- a/tests/machine/test_machine.py +++ b/tests/machine/test_machine.py @@ -61,57 +61,65 @@ def json(): assert type == 'test_type' -@patch("smsdk.client_v0.ClientV0.get_machines") -@patch("smsdk.client_v0.ClientV0.get_machine_types") +@patch("smsdk.smsdk_entities.machine_type.machinetype.MachineType.get_fields") +@patch("smsdk.client.Client.get_type_from_machine") def test_get_machine_schema(mocked_types, mocked_machines): - mocked_machines.return_value = {'source_type': ['test']} - mocked_types.return_value = MACHINE_TYPE + mocked_machines.return_value = MACHINE_TYPE + mocked_types.return_value = 'test' dt = Client("demo") # Run - schema = dt.get_machine_schema('test') - + fields = dt.get_machine_schema('test') + assert len(fields) == 2 + names = [field['name'] for field in fields] # Verify - assert schema.name.sort_values().tolist() == ['stat__test_float', 'stat__test_string'] + assert names == ['stat__test_float', 'stat__test_string'] -@patch("smsdk.client_v0.ClientV0.get_machines") -@patch("smsdk.client_v0.ClientV0.get_machine_types") +@patch("smsdk.smsdk_entities.machine_type.machinetype.MachineType.get_fields") +@patch("smsdk.client.Client.get_type_from_machine") def test_get_machine_schema_hidden(mocked_types, mocked_machines): - mocked_machines.return_value = {'source_type': ['test']} - mocked_types.return_value = MACHINE_TYPE + mocked_machines.return_value = MACHINE_TYPE + mocked_types.return_value = 'test' dt = Client("demo") # Run - schema = dt.get_machine_schema('test', show_hidden=True) + fields = dt.get_machine_schema('test', show_hidden=True) + assert len(fields) == 3 + names = [field['name'] for field in fields] + names.sort() # Verify - assert schema.name.sort_values().tolist() == ['stat__test_float', 'stat__test_hidden', 'stat__test_string'] + assert names == ['stat__test_float', 'stat__test_hidden', 'stat__test_string'] -@patch("smsdk.client_v0.ClientV0.get_machines") -@patch("smsdk.client_v0.ClientV0.get_machine_types") +@patch("smsdk.smsdk_entities.machine_type.machinetype.MachineType.get_fields") +@patch("smsdk.client.Client.get_type_from_machine") def test_get_machine_schema_types(mocked_types, mocked_machines): - mocked_machines.return_value = {'source_type': ['test']} - mocked_types.return_value = MACHINE_TYPE + mocked_machines.return_value = MACHINE_TYPE + mocked_types.return_value = 'test' dt = Client("demo") # Run - schema = dt.get_machine_schema('test', types=["float"]) + fields = dt.get_machine_schema('test', types=['float']) + assert len(fields) == 1 + names = [field['name'] for field in fields] # Verify - assert schema.name.sort_values().tolist() == ['stat__test_float'] + assert names == ['stat__test_float'] -@patch("smsdk.client_v0.ClientV0.get_machines") -@patch("smsdk.client_v0.ClientV0.get_machine_types") +@patch("smsdk.smsdk_entities.machine_type.machinetype.MachineType.get_fields") +@patch("smsdk.client.Client.get_type_from_machine") def test_get_machine_schema_types_return_mtype(mocked_types, mocked_machines): - mocked_machines.return_value = {'source_type': ['test_type']} - mocked_types.return_value = MACHINE_TYPE + mocked_machines.return_value = MACHINE_TYPE + mocked_types.return_value = 'test' dt = Client("demo") # Run - schema = dt.get_machine_schema('test', return_mtype=True) - + fields = dt.get_machine_schema('test', return_mtype=True) + assert fields[0] == 'test' + assert len(fields[1]) == 2 + names = [field['name'] for field in fields[1]] # Verify - assert schema[0] == 'test_type' - assert schema[1].name.sort_values().tolist() == ['stat__test_float', 'stat__test_string'] + assert names == ['stat__test_float', 'stat__test_string'] + diff --git a/tests/machine_type/machine_type_data.py b/tests/machine_type/machine_type_data.py index b9a7569..104c7d2 100644 --- a/tests/machine_type/machine_type_data.py +++ b/tests/machine_type/machine_type_data.py @@ -4411,3 +4411,22 @@ }, }, ] + +MACHINE_TYPE_FIELDS =[ + { + 'type': 'float', + 'name': 'stat__test_float', + 'title_prefix': 'test float' + }, + { + 'type': 'string', + 'name': 'stat__test_string', + 'title_prefix': 'test string' + }, + { + 'type': 'string', + 'name': 'stat__test_hidden', + 'title_prefix': 'test hidden', + 'ui_hidden' : True + } + ] diff --git a/tests/machine_type/test_machine_type.py b/tests/machine_type/test_machine_type.py index 0e1bb2d..bd47c13 100644 --- a/tests/machine_type/test_machine_type.py +++ b/tests/machine_type/test_machine_type.py @@ -1,6 +1,8 @@ +from mock import patch import pandas as pd from requests.sessions import Session -from tests.machine_type.machine_type_data import JSON_MACHINETYPE +from smsdk.client import Client +from tests.machine_type.machine_type_data import JSON_MACHINETYPE, MACHINE_TYPE_FIELDS from smsdk.smsdk_entities.machine_type.machinetype import MachineType @@ -47,3 +49,43 @@ def mockapi(self, session, endpoint): ] assert cols == df.columns.sort_values().tolist() + +@patch("smsdk.smsdk_entities.machine_type.machinetype.MachineType.get_fields") +def test_get_fields_of_machine_type(mocked_machines): + mocked_machines.return_value = MACHINE_TYPE_FIELDS + dt = Client("demo") + + # Run + fields = dt.get_fields_of_machine_type('test') + assert len(fields) == 2 + names = [field['name'] for field in fields] + # Verify + assert names == ['stat__test_float', 'stat__test_string'] + + +@patch("smsdk.smsdk_entities.machine_type.machinetype.MachineType.get_fields") +def test_get_fields_of_machine_type_hidden(mocked_machines): + mocked_machines.return_value = MACHINE_TYPE_FIELDS + dt = Client("demo") + + # Run + fields = dt.get_fields_of_machine_type('test', show_hidden=True) + assert len(fields) == 3 + names = [field['name'] for field in fields] + names.sort() + + # Verify + assert names == ['stat__test_float', 'stat__test_hidden', 'stat__test_string'] + +@patch("smsdk.smsdk_entities.machine_type.machinetype.MachineType.get_fields") +def test_get_fields_of_machine_type_types(mocked_machines): + mocked_machines.return_value = MACHINE_TYPE_FIELDS + dt = Client("demo") + + # Run + fields = dt.get_fields_of_machine_type('test', types=['float']) + assert len(fields) == 1 + names = [field['name'] for field in fields] + + # Verify + assert names == ['stat__test_float'] From 03ca7274ea2774455e9389e96e213f4f5b6356c8 Mon Sep 17 00:00:00 2001 From: Michael Goldman Date: Thu, 9 Mar 2023 11:42:37 -0800 Subject: [PATCH 20/33] New doc --- docs/entities/machine_type.md | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 docs/entities/machine_type.md diff --git a/docs/entities/machine_type.md b/docs/entities/machine_type.md new file mode 100644 index 0000000..52a21a9 --- /dev/null +++ b/docs/entities/machine_type.md @@ -0,0 +1,27 @@ +# Machine Types +Machine Types are the schema for the various machines in a factory. + +## Functions + +### get_fields_of_machine_type +The get_fields_of_machine_type function returns the fields of the machine schema for a given machine type and is called this way: +``` +cli.get_fields_of_machine_type(machine_type, types, show_hidden) +``` + +The only required field in this case is the machine_type, we will go over each variable in a second. The function will return a pandas list that looks like the following: +``` +[{'display_name': 'Machine', 'unit': '', 'type': 'categorical', 'data_type': 'string', 'stream_types': [], 'raw_data_field': '', 'name': 'machine__source'}, {'display_name': 'Cycle Start Time', 'unit': '', 'type': 'datetime', 'data_type': 'datetime', 'stream_types': [], 'raw_data_field': '', 'name': 'starttime'},..] +``` + +#### machine_type +This is the name of the machine type that you are trying to grab the fields of. This is the only required parameter for this function. + +#### types +This is an optional parameter and is a list of strings. If this is set the function will only return colomns that match the types given. For example if we were to pass ['string'] as our types parameter in the previous example we would instead have returned: +``` +[{'display_name': 'Machine', 'unit': '', 'type': 'categorical', 'data_type': 'string', 'stream_types': [], 'raw_data_field': '', 'name': 'machine__source'}] +``` + +#### show_hidden +This is an optional parameter and is a boolean. There are a few fields we have set to be hiddden from our ui in our application and by defualt these are also hidden from the return in this function if set to True we will also return these fields. From 05ba332a296faa56235c894f3227176a35069430 Mon Sep 17 00:00:00 2001 From: Michael Goldman Date: Fri, 10 Mar 2023 11:12:12 -0800 Subject: [PATCH 21/33] Get cookbooks --- smsdk/client.py | 7 +++ smsdk/config/api_endpoints.json | 3 ++ smsdk/ma_session.py | 3 +- smsdk/smsdk_entities/cookbooks/__init__.py | 0 smsdk/smsdk_entities/cookbooks/cookbook.py | 53 ++++++++++++++++++++++ 5 files changed, 65 insertions(+), 1 deletion(-) create mode 100644 smsdk/smsdk_entities/cookbooks/__init__.py create mode 100644 smsdk/smsdk_entities/cookbooks/cookbook.py diff --git a/smsdk/client.py b/smsdk/client.py index e5d494e..f36ddb2 100644 --- a/smsdk/client.py +++ b/smsdk/client.py @@ -277,4 +277,11 @@ def get_type_from_machine(self, machine_source=None, **kwargs): self.config["protocol"], self.tenant, self.config["site.domain"] ) return machine(self.session, base_url).get_type_from_machine_name(machine_source, **kwargs) + + def get_cookbooks(self): + cookbook = smsdkentities.get('cookbook') + base_url = get_url( + self.config["protocol"], self.tenant, self.config["site.domain"] + ) + return cookbook(self.session, base_url).get_cookbooks() diff --git a/smsdk/config/api_endpoints.json b/smsdk/config/api_endpoints.json index 6217b77..32b3574 100644 --- a/smsdk/config/api_endpoints.json +++ b/smsdk/config/api_endpoints.json @@ -36,5 +36,8 @@ }, "Assets": { "url":"/v1/selector/assets" + }, + "Cookbook": { + "get_cookbooks": "/v1/obj/recipe/cookbook" } } diff --git a/smsdk/ma_session.py b/smsdk/ma_session.py index 4dda66a..8ad56e3 100644 --- a/smsdk/ma_session.py +++ b/smsdk/ma_session.py @@ -148,6 +148,7 @@ def _get_records_v1( limit=np.Inf, offset=0, db_mode='sql', + results_under='results', **url_params ): """ @@ -187,7 +188,7 @@ def _get_records_v1( raise ValueError("Error - {}".format(response.text)) try: data = response.json() - data = data['results'] + data = data[results_under] if isinstance(data, dict): data = [data] except JSONDecodeError as e: diff --git a/smsdk/smsdk_entities/cookbooks/__init__.py b/smsdk/smsdk_entities/cookbooks/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/smsdk/smsdk_entities/cookbooks/cookbook.py b/smsdk/smsdk_entities/cookbooks/cookbook.py new file mode 100644 index 0000000..34870dd --- /dev/null +++ b/smsdk/smsdk_entities/cookbooks/cookbook.py @@ -0,0 +1,53 @@ +import json +from datetime import datetime, timedelta +from typing import List + +import numpy as np + +try: + import importlib.resources as pkg_resources +except ImportError: + # Try backported to PY<37 `importlib_resources`. + import importlib_resources as pkg_resources + +from smsdk.tool_register import SmsdkEntities, smsdkentities +from smsdk.utils import module_utility +from smsdk import config +from smsdk.ma_session import MaSession + +import logging + +log = logging.getLogger(__name__) + +ENDPOINTS = json.loads(pkg_resources.read_text(config, "api_endpoints.json")) + + +@smsdkentities.register("cookbook") +class Cookbook(SmsdkEntities, MaSession): + # Decorator to register a function as utility + # Only the registered utilites would be accessible + # to outside world via client.get_data() + mod_util = module_utility() + + def __init__(self, session, base_url) -> None: + self.session = session + self.base_url = base_url + + @mod_util + def get_utilities(self, *args, **kwargs) -> List: + """ + Get the list of registered utilites by name + """ + return [*self.mod_util.all] + + @mod_util + def get_cookbooks(self, *args, **kwargs): + """ + Returns a list of all KPIs + """ + url = "{}{}".format(self.base_url, ENDPOINTS["Cookbook"]["get_cookbooks"]) + records = self._get_records_v1(url, method="get", results_under="objects", **kwargs) + + if not isinstance(records, List): + raise ValueError("Error - {}".format(records)) + return records \ No newline at end of file From eb57a868daa63c8b836f13771609eac4a685a6be Mon Sep 17 00:00:00 2001 From: Michael Goldman Date: Fri, 10 Mar 2023 11:23:53 -0800 Subject: [PATCH 22/33] Pandas dataframe --- docs/entities/machine.md | 16 ++++++++++++---- smsdk/client.py | 4 ++-- tests/machine/test_machine.py | 21 ++++++++------------- 3 files changed, 22 insertions(+), 19 deletions(-) diff --git a/docs/entities/machine.md b/docs/entities/machine.md index 498faf3..d9c89b4 100644 --- a/docs/entities/machine.md +++ b/docs/entities/machine.md @@ -20,9 +20,13 @@ The get_machine_schema functions returns the fields of the machine schema for a cli.get_machine_schema(machine_source, types, show_hidden, return_mtype) ``` -The only required field in this case is the machine_source, we will go over each variable in a second. The function will return a pandas list that looks like the following: +The only required field in this case is the machine_source, we will go over each variable in a second. The function will return a pandas data frame that looks like the following: ``` -[{'display_name': 'Machine', 'unit': '', 'type': 'categorical', 'data_type': 'string', 'stream_types': [], 'raw_data_field': '', 'name': 'machine__source'}, {'display_name': 'Cycle Start Time', 'unit': '', 'type': 'datetime', 'data_type': 'datetime', 'stream_types': [], 'raw_data_field': '', 'name': 'starttime'},..] + name display type +0 stats__Alarms__val Alarms float +1 stats__BLOCKED__val BLOCKED float +2 stats__DOWN__val DOWN float +3 stats__DefectCategory__val Defect Category string ``` #### machine_source @@ -31,7 +35,8 @@ This is the name of the machine that you are trying to grab the schema of. This #### types This is an optional parameter and is a list of strings. If this is set the function will only return colomns that match the types given. For example if we were to pass ['string'] as our types parameter in the previous example we would instead have returned: ``` -[{'display_name': 'Machine', 'unit': '', 'type': 'categorical', 'data_type': 'string', 'stream_types': [], 'raw_data_field': '', 'name': 'machine__source'}] + name display type +0 stats__DefectCategory__val Defect Category string ``` #### show_hidden @@ -41,5 +46,8 @@ This is an optional parameter and is a boolean. There are a few fields we have This is an optional parameter and is a boolean. If set to True this will instead of returning just the pandas dataframe will return a Tupple with a string that is the machine type of the machine_soure for example: ``` ('Lasercut', -[{'display_name': 'Machine', 'unit': '', 'type': 'categorical', 'data_type': 'string', 'stream_types': [], 'raw_data_field': '', 'name': 'machine__source'}, {'display_name': 'Cycle Start Time', 'unit': '', 'type': 'datetime', 'data_type': 'datetime', 'stream_types': [], 'raw_data_field': '', 'name': 'starttime'},..] + name display type +0 stats__Alarms__val Alarms float +1 stats__BLOCKED__val BLOCKED float +...) ``` \ No newline at end of file diff --git a/smsdk/client.py b/smsdk/client.py index e9d58ae..faa08cc 100644 --- a/smsdk/client.py +++ b/smsdk/client.py @@ -272,9 +272,9 @@ def get_machine_schema(self, machine_source, types=[], show_hidden=False, return fields = [field for field in fields if field.get('type') in types] if return_mtype: - return (machine_type, fields) + return (machine_type, pd.DataFrame(fields)) - return fields + return pd.DataFrame(fields) def get_fields_of_machine_type(self, machine_type, types=[], show_hidden=False, **kwargs): machineType= smsdkentities.get('machine_type') diff --git a/tests/machine/test_machine.py b/tests/machine/test_machine.py index d960de0..39d5b27 100644 --- a/tests/machine/test_machine.py +++ b/tests/machine/test_machine.py @@ -70,10 +70,9 @@ def test_get_machine_schema(mocked_types, mocked_machines): # Run fields = dt.get_machine_schema('test') - assert len(fields) == 2 - names = [field['name'] for field in fields] + assert fields.shape == (2, 3) # Verify - assert names == ['stat__test_float', 'stat__test_string'] + assert fields.name.sort_values().tolist() == ['stat__test_float', 'stat__test_string'] @patch("smsdk.smsdk_entities.machine_type.machinetype.MachineType.get_fields") @@ -85,12 +84,10 @@ def test_get_machine_schema_hidden(mocked_types, mocked_machines): # Run fields = dt.get_machine_schema('test', show_hidden=True) - assert len(fields) == 3 - names = [field['name'] for field in fields] - names.sort() + assert fields.shape == (3, 4) # Verify - assert names == ['stat__test_float', 'stat__test_hidden', 'stat__test_string'] + assert fields.name.sort_values().tolist() == ['stat__test_float', 'stat__test_hidden', 'stat__test_string'] @patch("smsdk.smsdk_entities.machine_type.machinetype.MachineType.get_fields") @patch("smsdk.client.Client.get_type_from_machine") @@ -101,11 +98,10 @@ def test_get_machine_schema_types(mocked_types, mocked_machines): # Run fields = dt.get_machine_schema('test', types=['float']) - assert len(fields) == 1 - names = [field['name'] for field in fields] + assert fields.shape == (1, 3) # Verify - assert names == ['stat__test_float'] + assert fields.name.sort_values().tolist() == ['stat__test_float'] @patch("smsdk.smsdk_entities.machine_type.machinetype.MachineType.get_fields") @patch("smsdk.client.Client.get_type_from_machine") @@ -117,9 +113,8 @@ def test_get_machine_schema_types_return_mtype(mocked_types, mocked_machines): # Run fields = dt.get_machine_schema('test', return_mtype=True) assert fields[0] == 'test' - assert len(fields[1]) == 2 - names = [field['name'] for field in fields[1]] + assert fields[1].shape == (2, 3) # Verify - assert names == ['stat__test_float', 'stat__test_string'] + assert fields[1].name.sort_values().tolist() == ['stat__test_float', 'stat__test_string'] From c17592de13ce0494375199c3cec25373543c09d6 Mon Sep 17 00:00:00 2001 From: Michael Goldman Date: Mon, 13 Mar 2023 11:47:40 -0700 Subject: [PATCH 23/33] Source_clean can be passed --- docs/entities/machine.md | 4 ++-- smsdk/smsdk_entities/machine/machine.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/entities/machine.md b/docs/entities/machine.md index d9c89b4..5fb389f 100644 --- a/docs/entities/machine.md +++ b/docs/entities/machine.md @@ -4,7 +4,7 @@ Machines are just that, machines in factories. The machine object is how the Si ## Functions ### get_type_from_machine -The get_type_from_machine function allows you to get the type of any machine from it's name and is called this way: +The get_type_from_machine function allows you to get the type of any machine from it's name(or display name) and is called this way: ``` cli.get_type_from_machine(machine_name) ``` @@ -30,7 +30,7 @@ The only required field in this case is the machine_source, we will go over each ``` #### machine_source -This is the name of the machine that you are trying to grab the schema of. This is the only required parameter for this function. +This is the name of the machine that you are trying to grab the schema of. This will also work with it's display name or source_clean. This is the only required parameter for this function. #### types This is an optional parameter and is a list of strings. If this is set the function will only return colomns that match the types given. For example if we were to pass ['string'] as our types parameter in the previous example we would instead have returned: diff --git a/smsdk/smsdk_entities/machine/machine.py b/smsdk/smsdk_entities/machine/machine.py index bfccec1..d8a4f63 100644 --- a/smsdk/smsdk_entities/machine/machine.py +++ b/smsdk/smsdk_entities/machine/machine.py @@ -55,6 +55,6 @@ def get_type_from_machine_name(self, machine_source, *args, **kwargs): records = self._get_records_v1(url, method="get", **kwargs)[0]["machine"] machine_type = '' for record in records: - if record['name'] == machine_source: + if record['name'] == machine_source or record['display_name'] == machine_source: machine_type = record['type'] return machine_type From 45727dbe2fb8b5bfd8add9557081c94797d8a5eb Mon Sep 17 00:00:00 2001 From: Michael Goldman Date: Wed, 15 Mar 2023 11:52:51 -0700 Subject: [PATCH 24/33] All cookbooks apis --- smsdk/client.py | 28 ++++++++++++++++------ smsdk/config/api_endpoints.json | 4 +++- smsdk/ma_session.py | 24 ++++++++++++------- smsdk/smsdk_entities/cookbooks/cookbook.py | 28 +++++++++++++++++++++- 4 files changed, 66 insertions(+), 18 deletions(-) diff --git a/smsdk/client.py b/smsdk/client.py index f846640..cdd0003 100644 --- a/smsdk/client.py +++ b/smsdk/client.py @@ -259,13 +259,6 @@ def get_type_from_machine(self, machine_source=None, **kwargs): self.config["protocol"], self.tenant, self.config["site.domain"] ) return machine(self.session, base_url).get_type_from_machine_name(machine_source, **kwargs) - - def get_cookbooks(self): - cookbook = smsdkentities.get('cookbook') - base_url = get_url( - self.config["protocol"], self.tenant, self.config["site.domain"] - ) - return cookbook(self.session, base_url).get_cookbooks() def get_machine_schema(self, machine_source, types=[], show_hidden=False, return_mtype=False, **kwargs): machineType= smsdkentities.get('machine_type') @@ -294,3 +287,24 @@ def get_fields_of_machine_type(self, machine_type, types=[], show_hidden=False, fields = [field for field in fields if field.get('type') in types] return fields + + def get_cookbooks(self, **kwargs): + cookbook = smsdkentities.get('cookbook') + base_url = get_url( + self.config["protocol"], self.tenant, self.config["site.domain"] + ) + return cookbook(self.session, base_url).get_cookbooks(**kwargs) + + def get_top_results(self, recipe_group_id, limit=10, **kwargs): + cookbook = smsdkentities.get('cookbook') + base_url = get_url( + self.config["protocol"], self.tenant, self.config["site.domain"] + ) + return cookbook(self.session, base_url).get_top_results(recipe_group_id, limit, **kwargs) + + def get_current_value(self, minutes=10, variables=[], **kwargs): + cookbook = smsdkentities.get('cookbook') + base_url = get_url( + self.config["protocol"], self.tenant, self.config["site.domain"] + ) + return cookbook(self.session, base_url).get_current_value(minutes, variables, **kwargs) diff --git a/smsdk/config/api_endpoints.json b/smsdk/config/api_endpoints.json index a02a9d2..95a9a32 100644 --- a/smsdk/config/api_endpoints.json +++ b/smsdk/config/api_endpoints.json @@ -39,6 +39,8 @@ "url":"/v1/selector/assets" }, "Cookbook": { - "get_cookbooks": "/v1/obj/recipe/cookbook" + "get_cookbooks": "/v1/obj/recipe/cookbook", + "top_results": "/v1/recipe/top_results/{}?limit={}", + "current_value": "/v1/recipe/current_value" } } diff --git a/smsdk/ma_session.py b/smsdk/ma_session.py index 8ad56e3..49cc5bc 100644 --- a/smsdk/ma_session.py +++ b/smsdk/ma_session.py @@ -167,16 +167,19 @@ def _get_records_v1( records: List = [] while True: try: - remaining_limit = limit - len(records) - this_loop_limit = min(remaining_limit, max_page_size) + if limit: + remaining_limit = limit - len(records) + this_loop_limit = min(remaining_limit, max_page_size) - # If we exactly hit our desired number of records -- limit is 0 -- then can stop - if this_loop_limit == 0: - return records + # If we exactly hit our desired number of records -- limit is 0 -- then can stop + if this_loop_limit == 0: + return records + url_params["limit"] = this_loop_limit - url_params["offset"] = offset - url_params["limit"] = this_loop_limit - url_params["db_mode"] = db_mode + if offset: + url_params["offset"] = offset + if db_mode: + url_params["db_mode"] = db_mode # print(f'Pulling up to {this_loop_limit} records ({remaining_limit} remain)') @@ -188,7 +191,8 @@ def _get_records_v1( raise ValueError("Error - {}".format(response.text)) try: data = response.json() - data = data[results_under] + if results_under: + data = data[results_under] if isinstance(data, dict): data = [data] except JSONDecodeError as e: @@ -198,6 +202,8 @@ def _get_records_v1( return [] records.extend(data) + if limit is None: + return records if len(data) < this_loop_limit: # Cursor exhausted, so just return return records diff --git a/smsdk/smsdk_entities/cookbooks/cookbook.py b/smsdk/smsdk_entities/cookbooks/cookbook.py index 34870dd..da2c34a 100644 --- a/smsdk/smsdk_entities/cookbooks/cookbook.py +++ b/smsdk/smsdk_entities/cookbooks/cookbook.py @@ -43,11 +43,37 @@ def get_utilities(self, *args, **kwargs) -> List: @mod_util def get_cookbooks(self, *args, **kwargs): """ - Returns a list of all KPIs + Returns a list of all Cookbooks """ url = "{}{}".format(self.base_url, ENDPOINTS["Cookbook"]["get_cookbooks"]) records = self._get_records_v1(url, method="get", results_under="objects", **kwargs) + if not isinstance(records, List): + raise ValueError("Error - {}".format(records)) + return records + + @mod_util + def get_top_results(self, recipe_group_id, limit=10, *args, **kwargs): + """ + Returns the top results from a recipe group + """ + url = "{}{}".format(self.base_url, ENDPOINTS["Cookbook"]["top_results"].format(recipe_group_id, limit)) + records = self._get_records_v1(url, method="get", results_under=None, **kwargs) + + if not isinstance(records, List): + raise ValueError("Error - {}".format(records)) + return records[0] + + @mod_util + def get_current_value(self, variables, minutes=1440, **kwargs): + """ + Gets the current value of levers and constraints + """ + kwargs['minutes'] = minutes + kwargs['variables'] = variables + url = "{}{}".format(self.base_url, ENDPOINTS["Cookbook"]["current_value"]) + records = self._get_records_v1(url, offset=None, limit=None, db_mode=None, **kwargs) + if not isinstance(records, List): raise ValueError("Error - {}".format(records)) return records \ No newline at end of file From 113e99d4f2068ddd4ecf693b88f4a01616000f5d Mon Sep 17 00:00:00 2001 From: Michael Goldman Date: Wed, 15 Mar 2023 11:57:07 -0700 Subject: [PATCH 25/33] Column name fix --- smsdk/client.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/smsdk/client.py b/smsdk/client.py index faa08cc..cdf276c 100644 --- a/smsdk/client.py +++ b/smsdk/client.py @@ -271,10 +271,12 @@ def get_machine_schema(self, machine_source, types=[], show_hidden=False, return if len(types) > 0: fields = [field for field in fields if field.get('type') in types] + frame = pd.DataFrame(fields).rename(columns={"display_name": "display", "type": "sight_type", "data_type": "type"}) + if return_mtype: - return (machine_type, pd.DataFrame(fields)) + return (machine_type, frame) - return pd.DataFrame(fields) + return frame def get_fields_of_machine_type(self, machine_type, types=[], show_hidden=False, **kwargs): machineType= smsdkentities.get('machine_type') From fccb8b9518225d1ab8b88e399d74208159947b3b Mon Sep 17 00:00:00 2001 From: Michael Goldman Date: Mon, 20 Mar 2023 13:40:48 -0700 Subject: [PATCH 26/33] Lots of docs --- docs/commonly_used_data_types/cookbook.md | 236 ++++++++++++++++++++++ docs/commonly_used_data_types/run.md | 224 ++++++++++++++++++++ docs/entities/cookbook.md | 71 +++++++ smsdk/client.py | 15 +- 4 files changed, 544 insertions(+), 2 deletions(-) create mode 100644 docs/commonly_used_data_types/cookbook.md create mode 100644 docs/commonly_used_data_types/run.md create mode 100644 docs/entities/cookbook.md diff --git a/docs/commonly_used_data_types/cookbook.md b/docs/commonly_used_data_types/cookbook.md new file mode 100644 index 0000000..4685466 --- /dev/null +++ b/docs/commonly_used_data_types/cookbook.md @@ -0,0 +1,236 @@ +# Cookbook + This is how a cookbook is stored in the data base it looks like this: + ``` +{ + "hash": "hash", + "auto_generated_parent": "", + "name": "name", + "assetNames": ['machine names'], + "key_constraint":{ + 'field': { + 'fieldName': 'stats__Cylinders__val', + 'machineId': 'e2df2b4f115b763f45d04fa2', + 'machineName': 'JB_HM_Diecast_1', + 'machineDisplayName': 'Hamilton - Diecast 1', + 'fieldType': 'categorical', + 'machineType': 'Diecast', + 'fieldDisplayName': 'Cylinders', + 'fieldUnit': '' + }, + 'valueMap': {'4': 1, '6': 0} + } + "recipe_groups": [ + recipe_groups + ] + "metadata":{'created_by': {'id': '5a1c9f9798214579dacc917a', 'email': 'ahome@sightmachine.com', 'metadata': {'first_name': 'Andrew', 'last_name': 'Home', 'tenant': 'demo'}}}, + "updatetime": '2023-03-16 17:48:55.355000', + "assets": [], + "id": '63ab6b263fa4880c06334b03' + +} + ``` + + ## Hash + A hash of the cookbook object. + + ## auto_generated_parent + + ## name + The name of the cookbook. + + ## assetNames + A list of names of assets used in this cookbook these are often just machine_names. + + ## key_constraint + + ## recipe_groups + A list of recipe groups, They look like the following and will be described in more details + ``` + { + "id": "id" + "values": [1], + "runBoundaries": [], + "maxDuration": {'isEnabled': False, 'minimum': 0, 'unit': 'second'}, + "topRun": 10, + "constraints": [], + "levers": [{'fieldName': 'stats__AluminumTempAvg__val', 'machineId': 'e2df2b4f115b763f45d04fa2', 'machineName': 'JB_HM_Diecast_1', 'machineDisplayName': 'Hamilton - Diecast 1', 'fieldType': 'continuous', 'machineType': 'Diecast', 'fieldDisplayName': 'AluminumTemp - Average', 'fieldUnit': 'celsius'},...], + "outcomes": [{'field': {'fieldName': 'quality', 'machineId': 'e2df2b4f115b763f45d04fa2', 'machineName': 'JB_HM_Diecast_1', 'machineDisplayName': 'Hamilton - Diecast 1', 'fieldType': 'kpi', 'machineType': 'Diecast', 'fieldDisplayName': 'Quality'}, 'weight': 1, 'optimization_func': 'maximize'}, ...] + "filters": {'duration': {'isEnabled': False, 'minimum': 0, 'unit': 'second'}, 'recordFilters': []}, + "dateRange": {'value': {'relativeAmount': 7, 'relativeUnit': 'day'}, 'config': {'mode': 'relative', 'selectableRelativeUnits': ['minute', 'hour', 'day', 'week', 'month', 'year'], 'enableTimeTypeSelection': True, 'showQuarterShortcuts': True}} + "computeDeployedDateRange": None, + "statsCalculationSetting": "defualt", + "deployed":{recipe_group - deployed field} + } + ``` + + ### id + The id of the recipe group. + + ### values + List of values that currently in the goup? + + ### runBoundaries + List of constraints of boundaries? + + ### maxDuration + Max duration of a run. Looks like the following: + ``` + {'isEnabled': False, 'minimum': 0, 'unit': 'second'} + ``` + Here is a break down of each key: + + #### isEnabled + Boolean where or not this is enabled + + #### minimum + Minium run length? + + #### unit + Unit of run length + + ### topRun + The value of the topRun in this recipe group. + + ### constraints + List of fields and values that are constraints on the group? + + ### levers + List of fields that go into recipe runs. Looks like the following: + ``` + { + 'fieldName': 'stats__AluminumTempAvg__val', + 'machineId': 'e2df2b4f115b763f45d04fa2', + 'machineName': 'JB_HM_Diecast_1', + 'machineDisplayName': 'Hamilton - Diecast 1', + 'fieldType': 'continuous', + 'machineType': 'Diecast', + 'fieldDisplayName': 'AluminumTemp - Average', + 'fieldUnit': 'celsius' +} + ``` +And here is a break down of each field: + +#### fieldName +The name of field being looking. + +#### machineId +The id of the machine this field belongs to. + +#### machineName +The name of the machine this field belongs to. + +#### machineDisplayName +The display name of the machine this field belongs to. + +#### fieldType +Whieter the field is contionuous or discrete or kpi? + +#### machineType +The machineType of the machine this field bleongs to. + +#### fieldDisplayName +The display name of the field. + +#### fieldUnit +The unit of the field. + +### outcomes +A list of outcomes we are trying to optimize with this recipe_group: +``` +{ + 'field': + { + 'fieldName': 'quality', + 'machineId': 'e2df2b4f115b763f45d04fa2', + 'machineName': 'JB_HM_Diecast_1', + 'machineDisplayName': 'Hamilton - Diecast 1', + 'fieldType': 'kpi', + 'machineType': 'Diecast', + 'fieldDisplayName': 'Quality' + }, + 'weight': 1, + 'optimization_func': 'maximize' +} +``` +Here is more detail about each key: + +#### field +The field you are trying to optimize with this outcome. This is the same set up as [levers](#levers). + +#### wieght +The wieght you are putting on this outcome as compared to the others. + +#### optimization_func +How you wish to optimize this field usually 'maximize' or 'minize' + +### filters +Object containing a duration filter and list of record filters +``` +{'duration': {'isEnabled': False, 'minimum': 0, 'unit': 'second'}, 'recordFilters': []} +``` + +#### duration +Same as maxDuration.? + +#### recordFilters +List of recordFilters? + +### dateRange +The date range that runs can look at. Looks like this: +``` +{ + 'value': { + 'relativeAmount': 7, + 'relativeUnit': 'day' + }, + 'config': { + 'mode': 'relative', + 'selectableRelativeUnits': ['minute', 'hour', 'day', 'week', 'month', 'year'], + 'enableTimeTypeSelection': True, + 'showQuarterShortcuts': True + } +} +``` +Here is a breakdown of the keys: + +#### value +The value of the dateRange. For relative config we have two keys, relativeAmount and relativeUnit. + +##### relativeAmount +The amount of the relative range. + +##### relativeUnit +The unit of the amount of the relative range. + +#### config +The config for the daterange. This does not include the value as that is in it's value. Here is a breakdown of the keysk + +##### mode +The mode for the dateRange this can be relative or absolute? + +##### selectableRelativeUnits +The units you can select for a relative date range, relativeUnit will be one of these. + +##### enableTimeTypeSelection +where or not time type selection is availible on the UI? + +##### showQuarterShortcuts +Where or not the user can select quarters on the UI? + +### statsCalculationSetting +How the recipe group runs calculations? Can be set to defualt? + +### deployed +The deployed version of this recipe_group? It's a [recipe_group](#recipe_groups) minus this field + +## metadata +This is created by info, it includes the id of the user that created the cookbook along with their name and email + +## updatetime +The time the cookbook was last updated. + +## assets +A list of assets used in the cookbook? + +## id +The id of the cookbook \ No newline at end of file diff --git a/docs/commonly_used_data_types/run.md b/docs/commonly_used_data_types/run.md new file mode 100644 index 0000000..60378e1 --- /dev/null +++ b/docs/commonly_used_data_types/run.md @@ -0,0 +1,224 @@ +# Run +A recipe run or just run, is one set of inputs and outcomes for a cookbook recipe. Runs look like the following: +``` +{'_count': 12, + '_count_muted': 0, + '_duration_seconds': 649.0, + '_earliest': '2022-10-21T00:35:32+00:00', + '_latest': '2022-10-21T00:46:21+00:00', + '_score': 1.0, + 'constraint_group_id': '0', + 'constraints': [], + 'cookbook': '63ab6b263fa4880c06334b03', + 'filters': [], + 'i_vals': [{'asset': 'SHARED', 'name': 'group', 'value': '0'}, + {'asset': 'SHARED', 'name': 'sequence', 'value': 2}], + 'levers': [{'asset': 'JB_HM_Diecast_1', + 'd_pos': 2, + 'name': 'stats__AluminumTempAvg__val', + 'value': {'avg': 659.8448127439334, + 'count': 9.0, + 'max': 671.1048565509, + 'min': 653.718308813, + 'var_pop': 29.816738449355437}}, + {'asset': 'JB_HM_Diecast_1', + 'd_pos': 3, + 'name': 'stats__AluminumTempMax__val', + 'value': {'avg': 659.8448127439334, + 'count': 9.0, + 'max': 671.1048565509, + 'min': 653.718308813, + 'var_pop': 29.816738449355437}}, + {'asset': 'JB_HM_Diecast_1', + 'd_pos': 4, + 'name': 'stats__DieTemp__val', + 'value': {'avg': 290.8228674053778, + 'count': 9.0, + 'max': 295.2332287849, + 'min': 284.84446548529996, + 'var_pop': 11.993829414162574}}, + {'asset': 'JB_HM_Diecast_1', + 'd_pos': 5, + 'name': 'stats__InjectionCurveDifference__val', + 'value': {'avg': 1.576386980588889, + 'count': 9.0, + 'max': 2.8598220936, + 'min': 0.3786822327000001, + 'var_pop': 0.810658290414194}}, + {'asset': 'JB_HM_Diecast_1', + 'd_pos': 6, + 'name': 'stats__InjectionPressureMax__val', + 'value': {'avg': 39.537370569911104, + 'count': 9.0, + 'max': 44.0884060904, + 'min': 35.4952094968, + 'var_pop': 5.452849234986407}}, + {'asset': 'JB_HM_Diecast_1', + 'd_pos': 7, + 'name': 'stats__InjectionPressureMin__val', + 'value': {'avg': 36.76283137485556, + 'count': 9.0, + 'max': 38.8721003798, + 'min': 35.0169147198, + 'var_pop': 1.2194962602188073}}], + 'outcomes': [{'asset': 'JB_HM_Diecast_1', + 'd_pos': 0, + 'kpi': {'aggregates': {'Output': 'sum', 'ScrapQuantity': 'sum'}, + 'dependencies': {'Output': 9.0, 'ScrapQuantity': 0.0}, + 'formula': '((Output) / (Output + ScrapQuantity)) * 100 ' + 'if ((Output + ScrapQuantity) > 0) else ' + 'None'}, + 'name': 'quality', + 'value': {'avg': 100.0, + 'count': 100.0, + 'max': 100.0, + 'min': 100.0, + 'normal': 1.0000000000000002, + 'var_pop': 100.0}}]} +``` + +We will go over each key in more detail + +## _count +The number of total runs? + +## _count_muted +The number of runs muted? + +## _duration_seconds +The duration of the run in seconds. + +## _earliest +The start time of the run. + +## _latest +The end time of the run. + +## _score +The 'score' this run achieved (out of 1?) + +## constraint_group_id +The id of the group of constraints in the cookbook + +## constraints +A list of the constraints on this run + +## cookbook +The id of the cookbook this run relates to? + +## filters +A list of filters on this run + +## i_vals +?? + +## levers +A list of the levers attached to this run and the values these levers where at. A lever is in this format: +``` +{ +'asset': 'JB_HM_Diecast_1', +'d_pos': 2, +'name': 'stats__AluminumTempAvg__val', +'value': { + 'avg': 659.8448127439334, + 'count': 9.0, + 'max': 671.1048565509, + 'min': 653.718308813, + 'var_pop': 29.816738449355437 + } +} +``` + +### asset +The name of asset this lever is on, this is usually a machine_name + +### d_pos +?? + +### name +The actual name of the field being looked at. + +### value +The values that this lever was measured at during this run. It is futher broken down into the following keys. + +#### avg +Average value during the run. + +#### count +The amount of measurements during the run. + +#### max +The maxium value recorded during the run. + +#### min +The minimum value recorded during the run. + +#### var_pop +?? + +## outcomes +A list of outcomes recorded during the run. It looks like: +``` +{ + 'asset': 'JB_HM_Diecast_1', + 'd_pos': 0, + 'kpi': { + 'aggregates': {'Output': 'sum', 'ScrapQuantity': 'sum'}, + 'dependencies': {'Output': 9.0, 'ScrapQuantity': 0.0}, + 'formula': '((Output) / (Output + ScrapQuantity)) * 100 ' + 'if ((Output + ScrapQuantity) > 0) else ' + 'None' + }, + 'name': 'quality', + 'value': { + 'avg': 100.0, + 'count': 100.0, + 'max': 100.0, + 'min': 100.0, + 'normal': 1.0000000000000002, + 'var_pop': 100.0 + } +} +``` + +### asset +The name of the asset that outcome is attached to. Usually a machine name. + +### d_pos +?? + +### kpi +If the outcome is a kpi this field will descripe the kpi. It has the following keys: + +#### aggregates +How we aggregate each field being feed into our kpi formula, can be sum or avg. + +#### dependencies +The fields that feed into our kpi formula? + +#### formula +The actual formula the kpi uses + +### name +The name of the field that the outcome is being tracked from. + +### value +The values that this outcome was measured at during this run. It is futher broken down into the following keys. + +#### avg +Average value during the run. + +#### count +The amount of measurements during the run. + +#### max +The maxium value recorded during the run. + +#### min +The minimum value recorded during the run. + +#### normal +A measure of the normal distrabution of the values recorded during the + +#### var_pop +?? \ No newline at end of file diff --git a/docs/entities/cookbook.md b/docs/entities/cookbook.md new file mode 100644 index 0000000..30fce56 --- /dev/null +++ b/docs/entities/cookbook.md @@ -0,0 +1,71 @@ +# Cookbooks +Cookbooks are how the Sight Machine software provides recondemations and insights into various inputs and outcomes in a line. + +## Functions +The Sight Machine SDK has several functions related to cookbooks allowing you to do things like look at the configuration of cookbooks and see the top runs of various recipes. + +### Get Cookbooks +This functions gets the configuration of all cookbooks availible to your loged in user. It is called like the following: +``` +cli.get_cookbooks() +``` +This will return a list of [cookbooks](/docs/commonly_used_data_types/cookbook.md), see link for more details on what that response looks like. + +### Get Top Results +This function will get you the top runs of the recipe group you input and is called like the following: +``` +cli.get_top_results(recipe_group_id, limit) +``` + +This will return the following: +``` +[{ + "runs": [runs], + "constraint_group": [runs] +}] +``` + +For more info on the runs returned in each key see [runs](/docs/commonly_used_data_types/run.md). The two parameter inputs do the following: + +#### recipe_group_id +This is a string and is the only required parameter. This is the id of the recipe group you are trying to return runs for. + +#### limit +This is an int and is optional, if not entered it will default to 10. This is the max number of runs you wish to return. + +### Get Current Value +This function gets the current values of the fields passed into it. It is called like the following: +``` +get_current_value(variables, minutes) +``` + +It returns something like: +``` +[{'asset': 'JB_HM_Diecast_1', 'name': 'stats__InjectionPressureMin__val', 'values': {'latest': 35.1775016539}}] +``` + +The two parameter inputs do the following: + +#### variables +A list of variables to get the value of, they are in the following form: +``` +{'asset': 'JB_HM_Diecast_1', 'name': 'stats__InjectionPressureMin__val'} +``` + +##### asset +Asset is the name of the asset you are getting the field from. It is usually a machine_name + +##### name +Name is the name of the field you are grabing the value of. + +#### minutes +This is an optional parameter and is passed in as integer. This is the number of minutes you want to look bak for the current value. + +### Normalize Constraints +This is a function to return a clean string version of constraints. And is called like this: +``` +cli.normalize_constraints(constraints) +``` + +#### constraints +Are the constraints on a recipe group. \ No newline at end of file diff --git a/smsdk/client.py b/smsdk/client.py index cdd0003..a691d40 100644 --- a/smsdk/client.py +++ b/smsdk/client.py @@ -302,9 +302,20 @@ def get_top_results(self, recipe_group_id, limit=10, **kwargs): ) return cookbook(self.session, base_url).get_top_results(recipe_group_id, limit, **kwargs) - def get_current_value(self, minutes=10, variables=[], **kwargs): + def get_current_value(self, variables=[], minutes=1440, **kwargs): cookbook = smsdkentities.get('cookbook') base_url = get_url( self.config["protocol"], self.tenant, self.config["site.domain"] ) - return cookbook(self.session, base_url).get_current_value(minutes, variables, **kwargs) + return cookbook(self.session, base_url).get_current_value(variables, minutes, **kwargs) + + def normalize_constraint(self, constraint): + to = constraint.get("to") + from_constraint = constraint.get("from") + return "({},{})".format(to, from_constraint) + + def normalize_constraints(self, constraints): + constraints_normal = [] + for constraint in constraints: + constraints_normal.append(self.normalize_constraint(constraint)) + return constraints_normal From 391733999926ec4f7def433e7c5c10ab5632d706 Mon Sep 17 00:00:00 2001 From: Michael Goldman Date: Tue, 21 Mar 2023 10:48:29 -0700 Subject: [PATCH 27/33] Starting test --- tests/cookbook/__init__.py | 0 tests/cookbook/cookbook_data.py | 1 + tests/cookbook/test_cookbook.py | 30 ++++++++++++++++++++++++++++++ 3 files changed, 31 insertions(+) create mode 100644 tests/cookbook/__init__.py create mode 100644 tests/cookbook/cookbook_data.py create mode 100644 tests/cookbook/test_cookbook.py diff --git a/tests/cookbook/__init__.py b/tests/cookbook/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/cookbook/cookbook_data.py b/tests/cookbook/cookbook_data.py new file mode 100644 index 0000000..13b6494 --- /dev/null +++ b/tests/cookbook/cookbook_data.py @@ -0,0 +1 @@ +AVALIBLE_COOKBOOK_JSON=[{'hash': '64hujs5ffb40124475e15d9852ba7e69be121a6b435eae4879392acc970f80f8', 'tag': 'demo', 'auto_generated_parent': {}, 'name': 'Test cookbook', 'assetNames': ['asset_test'], 'key_constraint': {'field': {'fieldName': 'stats__Cylinders__val', 'machineId': 'e2mj2b4f215b2323f45d04fa2', 'machineName': 'asset_test', 'machineDisplayName': 'Asset - test', 'fieldType': 'categorical', 'machineType': 'Diecast', 'fieldDisplayName': 'Cylinders', 'fieldUnit': ''}, 'valueMap': {'4': 1, '6': 0}}, 'recipe_groups': [{'id': 'DJy4jyEEi', 'values': ['6'], 'runBoundaries': [], 'maxDuration': {'isEnabled': False, 'minimum': 0, 'unit': 'second'}, 'topRun': 10, 'constraints': [], 'levers': [{'fieldName': 'stats__AluminumTempAvg__val', 'machineId': 'e2df2b4f876b763f45d04se2', 'machineName': 'asset_test', 'machineDisplayName': 'Asset-Test', 'fieldType': 'continuous', 'machineType': 'Diecast', 'fieldDisplayName': 'AluminumTemp - Average', 'fieldUnit': 'celsius'}, {'fieldName': 'stats__AluminumTempMax__val', 'machineId': 'asfdalkfjlkfji', 'machineName': 'asset_test', 'machineDisplayName': 'Asset - test', 'fieldType': 'continuous', 'machineType': 'Diecast', 'fieldDisplayName': 'AluminumTemp - Max', 'fieldUnit': 'celsius'}, {'fieldName': 'stats__DieTemp__val', 'machineId': '435tsfrdsvxcvr', 'machineName': 'asset_test', 'machineDisplayName': 'asset - test', 'fieldType': 'continuous', 'machineType': 'Diecast', 'fieldDisplayName': 'Die Temperature', 'fieldUnit': 'celsius'}, {'fieldName': 'stats__InjectionCurveDifference__val', 'machineId': 'sfawrfefasf', 'machineName': 'JB_HM_Diecast_1', 'machineDisplayName': 'Hamilton - Diecast 1', 'fieldType': 'continuous', 'machineType': 'Diecast', 'fieldDisplayName': 'Injection Curve Difference', 'fieldUnit': 'pascal'}, {'fieldName': 'stats__InjectionPressureMax__val', 'machineId': 'e2df2b4f115b763f45d04fa2', 'machineName': 'JB_HM_Diecast_1', 'machineDisplayName': 'Hamilton - Diecast 1', 'fieldType': 'continuous', 'machineType': 'Diecast', 'fieldDisplayName': 'Injection Pressure Max', 'fieldUnit': 'pascal'}, {'fieldName': 'stats__InjectionPressureMin__val', 'machineId': 'e2df2b4f115b763f45d04fa2', 'machineName': 'JB_HM_Diecast_1', 'machineDisplayName': 'Hamilton - Diecast 1', 'fieldType': 'continuous', 'machineType': 'Diecast', 'fieldDisplayName': 'Injection Pressure Min', 'fieldUnit': 'pascal'}], 'outcomes': [{'field': {'fieldName': 'quality', 'machineId': 'e2df2b4f115b763f45d04fa2', 'machineName': 'JB_HM_Diecast_1', 'machineDisplayName': 'Hamilton - Diecast 1', 'fieldType': 'kpi', 'machineType': 'Diecast', 'fieldDisplayName': 'Quality'}, 'weight': 1, 'optimization_func': 'maximize'}], 'filters': {'duration': {'isEnabled': False, 'minimum': 0, 'unit': 'second'}, 'recordFilters': []}, 'dateRange': {'value': {'relativeAmount': 7, 'relativeUnit': 'day'}, 'config': {'mode': 'relative', 'selectableRelativeUnits': ['minute', 'hour', 'day', 'week', 'month', 'year'], 'enableTimeTypeSelection': True, 'showQuarterShortcuts': True}}, 'computeDeployedDateRange': None, 'statsCalculationSetting': 'default', 'deployed': {'id': 'SJy4jyFFi', 'values': ['6'], 'runBoundaries': [], 'maxDuration': {'isEnabled': False, 'minimum': 0, 'unit': 'second'}, 'topRun': 10, 'constraints': [], 'levers': [{'fieldName': 'stats__AluminumTempAvg__val', 'machineId': 'e2df2b4f115b763f45d04fa2', 'machineName': 'JB_HM_Diecast_1', 'machineDisplayName': 'Hamilton - Diecast 1', 'fieldType': 'continuous', 'machineType': 'Diecast', 'fieldDisplayName': 'AluminumTemp - Average', 'fieldUnit': 'celsius'}, {'fieldName': 'stats__AluminumTempMax__val', 'machineId': 'e2df2b4f115b763f45d04fa2', 'machineName': 'JB_HM_Diecast_1', 'machineDisplayName': 'Hamilton - Diecast 1', 'fieldType': 'continuous', 'machineType': 'Diecast', 'fieldDisplayName': 'AluminumTemp - Max', 'fieldUnit': 'celsius'}, {'fieldName': 'stats__DieTemp__val', 'machineId': 'e2df2b4f115b763f45d04fa2', 'machineName': 'JB_HM_Diecast_1', 'machineDisplayName': 'Hamilton - Diecast 1', 'fieldType': 'continuous', 'machineType': 'Diecast', 'fieldDisplayName': 'Die Temperature', 'fieldUnit': 'celsius'}, {'fieldName': 'stats__InjectionCurveDifference__val', 'machineId': 'e2df2b4f115b763f45d04fa2', 'machineName': 'JB_HM_Diecast_1', 'machineDisplayName': 'Hamilton - Diecast 1', 'fieldType': 'continuous', 'machineType': 'Diecast', 'fieldDisplayName': 'Injection Curve Difference', 'fieldUnit': 'pascal'}, {'fieldName': 'stats__InjectionPressureMax__val', 'machineId': 'e2df2b4f115b763f45d04fa2', 'machineName': 'JB_HM_Diecast_1', 'machineDisplayName': 'Hamilton - Diecast 1', 'fieldType': 'continuous', 'machineType': 'Diecast', 'fieldDisplayName': 'Injection Pressure Max', 'fieldUnit': 'pascal'}, {'fieldName': 'stats__InjectionPressureMin__val', 'machineId': 'e2df2b4f115b763f45d04fa2', 'machineName': 'JB_HM_Diecast_1', 'machineDisplayName': 'Hamilton - Diecast 1', 'fieldType': 'continuous', 'machineType': 'Diecast', 'fieldDisplayName': 'Injection Pressure Min', 'fieldUnit': 'pascal'}], 'outcomes': [{'field': {'fieldName': 'quality', 'machineId': 'e2df2b4f115b763f45d04fa2', 'machineName': 'JB_HM_Diecast_1', 'machineDisplayName': 'Hamilton - Diecast 1', 'fieldType': 'kpi', 'machineType': 'Diecast', 'fieldDisplayName': 'Quality'}, 'weight': 1, 'optimization_func': 'maximize'}], 'filters': {'duration': {'isEnabled': False, 'minimum': 0, 'unit': 'second'}, 'recordFilters': []}, 'dateRange': {'value': {'relativeAmount': 7, 'relativeUnit': 'day'}, 'config': {'mode': 'relative', 'selectableRelativeUnits': ['minute', 'hour', 'day', 'week', 'month', 'year'], 'enableTimeTypeSelection': True, 'showQuarterShortcuts': True}}, 'computeDeployedDateRange': None, 'statsCalculationSetting': 'default', 'deployed_date': '2023-01-20T17:02:06.797Z', 'deployed_status': {'success': True, 'run_count_saved': 10215, 'seconds_to_exec': 5874, 'timestamp': '2023-03-21 03:59:54.633000'}}}, {'constraints': [], 'levers': [{'fieldName': 'stats__AluminumTempAvg__val', 'machineId': 'e2df2b4f115b763f45d04fa2', 'machineName': 'JB_HM_Diecast_1', 'machineDisplayName': 'Hamilton - Diecast 1', 'fieldType': 'continuous', 'machineType': 'Diecast', 'fieldDisplayName': 'AluminumTemp - Average', 'fieldUnit': 'celsius'}, {'fieldName': 'stats__AluminumTempMax__val', 'machineId': 'e2df2b4f115b763f45d04fa2', 'machineName': 'JB_HM_Diecast_1', 'machineDisplayName': 'Hamilton - Diecast 1', 'fieldType': 'continuous', 'machineType': 'Diecast', 'fieldDisplayName': 'AluminumTemp - Max', 'fieldUnit': 'celsius'}, {'fieldName': 'stats__DieTemp__val', 'machineId': 'e2df2b4f115b763f45d04fa2', 'machineName': 'JB_HM_Diecast_1', 'machineDisplayName': 'Hamilton - Diecast 1', 'fieldType': 'continuous', 'machineType': 'Diecast', 'fieldDisplayName': 'Die Temperature', 'fieldUnit': 'celsius'}, {'fieldName': 'stats__InjectionCurveDifference__val', 'machineId': 'e2df2b4f115b763f45d04fa2', 'machineName': 'JB_HM_Diecast_1', 'machineDisplayName': 'Hamilton - Diecast 1', 'fieldType': 'continuous', 'machineType': 'Diecast', 'fieldDisplayName': 'Injection Curve Difference', 'fieldUnit': 'pascal'}, {'fieldName': 'stats__InjectionPressureMax__val', 'machineId': 'e2df2b4f115b763f45d04fa2', 'machineName': 'JB_HM_Diecast_1', 'machineDisplayName': 'Hamilton - Diecast 1', 'fieldType': 'continuous', 'machineType': 'Diecast', 'fieldDisplayName': 'Injection Pressure Max', 'fieldUnit': 'pascal'}, {'fieldName': 'stats__InjectionPressureMin__val', 'machineId': 'e2df2b4f115b763f45d04fa2', 'machineName': 'JB_HM_Diecast_1', 'machineDisplayName': 'Hamilton - Diecast 1', 'fieldType': 'continuous', 'machineType': 'Diecast', 'fieldDisplayName': 'Injection Pressure Min', 'fieldUnit': 'pascal'}], 'outcomes': [{'field': {'fieldName': 'quality', 'machineId': 'e2df2b4f115b763f45d04fa2', 'machineName': 'JB_HM_Diecast_1', 'machineDisplayName': 'Hamilton - Diecast 1', 'fieldType': 'kpi', 'machineType': 'Diecast', 'fieldDisplayName': 'Quality'}, 'weight': 1, 'optimization_func': 'maximize'}], 'dateRange': {'value': {'relativeAmount': 7, 'relativeUnit': 'day'}, 'config': {'mode': 'relative', 'selectableRelativeUnits': ['minute', 'hour', 'day', 'week', 'month', 'year'], 'enableTimeTypeSelection': True, 'showQuarterShortcuts': True}}, 'filters': {'duration': {'isEnabled': False, 'minimum': 0, 'unit': 'second'}, 'recordFilters': []}, 'runBoundaries': [], 'maxDuration': {'isEnabled': False, 'minimum': 0, 'unit': 'second'}, 'topRun': 10, 'computeDeployedDateRange': None, 'statsCalculationSetting': 'default', 'id': 'ByxJEiJFKj', 'values': ['4']}], 'metadata': {'created_by': {'id': '5a1c9f9798214579dacc917a', 'email': 'ahome@sightmachine.com', 'metadata': {'first_name': 'Andrew', 'last_name': 'Home', 'tenant': 'demo'}}}, 'updatetime': '2023-03-21 03:59:54.634000', 'assets': [], 'id': 'liafalkfjdlak'}] \ No newline at end of file diff --git a/tests/cookbook/test_cookbook.py b/tests/cookbook/test_cookbook.py new file mode 100644 index 0000000..75591f3 --- /dev/null +++ b/tests/cookbook/test_cookbook.py @@ -0,0 +1,30 @@ +import pandas as pd +from requests.sessions import Session +from smsdk.client import Client +from tests.cookbook.cookbook_data import AVALIBLE_KPI_JSON, KPI_DATA_VIZ_JSON +from smsdk.smsdk_entities.kpi.kpi import KPI +from mock import mock_open, MagicMock, patch + +@patch("smsdk.ma_session.Session") +def test_get_cookbooks(mocked): + class ResponseGet: + ok = True + text = "Success" + status_code=200 + + @staticmethod + def json(): + return {"results": [{"kpi":AVALIBLE_KPI_JSON}]} + mocked.return_value = MagicMock( + get=MagicMock(return_value=ResponseGet()) + ) + + dt = Client("demo") + + # Run + kpis = dt.get_cookbooks() + + # Verify + assert len(kpis) == 5 + + assert kpis[0]["name"] == 'Scrap_Rate' \ No newline at end of file From 69f690eebcd0813e5f3f6b03bb8a1fb2c629ae01 Mon Sep 17 00:00:00 2001 From: Michael Goldman Date: Tue, 21 Mar 2023 14:30:13 -0700 Subject: [PATCH 28/33] Better docs --- docs/commonly_used_data_types/cookbook.md | 123 +++++++++++++++------- docs/entities/cookbook.md | 6 +- 2 files changed, 86 insertions(+), 43 deletions(-) diff --git a/docs/commonly_used_data_types/cookbook.md b/docs/commonly_used_data_types/cookbook.md index 4685466..bf512fe 100644 --- a/docs/commonly_used_data_types/cookbook.md +++ b/docs/commonly_used_data_types/cookbook.md @@ -3,7 +3,6 @@ ``` { "hash": "hash", - "auto_generated_parent": "", "name": "name", "assetNames": ['machine names'], "key_constraint":{ @@ -33,8 +32,6 @@ ## Hash A hash of the cookbook object. - ## auto_generated_parent - ## name The name of the cookbook. @@ -42,6 +39,13 @@ A list of names of assets used in this cookbook these are often just machine_names. ## key_constraint + The field that specfices the product. This also how the system knows which recipe_group to use. + + ### field + This is detailed information on the field being used as the Key Constraint for more info see [levers](#levers). + + ### valueMap + This how the system maps the values of the key constraint on to recipe groups. Value for each key is the index of the recipe group to use. ## recipe_groups A list of recipe groups, They look like the following and will be described in more details @@ -73,26 +77,51 @@ List of constraints of boundaries? ### maxDuration - Max duration of a run. Looks like the following: - ``` - {'isEnabled': False, 'minimum': 0, 'unit': 'second'} - ``` - Here is a break down of each key: - - #### isEnabled - Boolean where or not this is enabled - - #### minimum - Minium run length? - - #### unit - Unit of run length + This field is ignored by the system currently ### topRun The value of the topRun in this recipe group. ### constraints - List of fields and values that are constraints on the group? + List of fields and values that are used to breakup runs They look like the following: + ``` +{ + "asset": "F1_010_BodyMaker_4", + "name": "stats__BM 001: Cans Out__val", + "type": "continuous", + "values": [ + { + "from": null, + "from_is_inclusive": false, + "to": 340, + "to_is_inclusive": false + }, + { + "from": 340, + "from_is_inclusive": true, + "to": 6000, + "to_is_inclusive": true + }, + { + "from": 6000, + "from_is_inclusive": false, + "to": null, + "to_is_inclusive": false + } + ] +} + ``` + #### asset + The name of the asset the field used in the constraint. + + #### name + The name of the field used for the constraint. + + #### type + The data type of the constraint, mostly commonly continuous or categorical + + #### values + These are the values to break up runs into. ### levers List of fields that go into recipe runs. Looks like the following: @@ -123,7 +152,7 @@ The name of the machine this field belongs to. The display name of the machine this field belongs to. #### fieldType -Whieter the field is contionuous or discrete or kpi? +What the type of the field is, commonly KPI, Continous or Categorical. #### machineType The machineType of the machine this field bleongs to. @@ -170,10 +199,39 @@ Object containing a duration filter and list of record filters ``` #### duration -Same as maxDuration.? +This is the minimum run duration. + +##### isEnabled +Whieter or not minimum run duration is enabled + +##### minimum +The amount of time of what ever unit the minimum run duration is. + +##### unit +The unit of time the minimum run duration uses. #### recordFilters -List of recordFilters? +List of record based filters. They look like the following: +``` +{ + asset: "F1_010_BodyMaker_4" + name: "stats__0_BM: CPM__val" + op: "gt" + value: 1 +} +``` + +##### asset +Name of the asset the field is on, usually a machine_name. + +##### name +Name of the field. + +##### op +The type of operation used to criteria match this filter. + +##### value +The value of used for the criteria matching operation. ### dateRange The date range that runs can look at. Looks like this: @@ -184,17 +242,14 @@ The date range that runs can look at. Looks like this: 'relativeUnit': 'day' }, 'config': { - 'mode': 'relative', - 'selectableRelativeUnits': ['minute', 'hour', 'day', 'week', 'month', 'year'], - 'enableTimeTypeSelection': True, - 'showQuarterShortcuts': True + ... } } ``` Here is a breakdown of the keys: #### value -The value of the dateRange. For relative config we have two keys, relativeAmount and relativeUnit. +The value of the dateRange. For relative config we have two keys, relativeAmount and relativeUnit. This very similiar to [Data Viz time Selection](/docs/commonly_used_data_types/data_viz_query.md#time_selection). ##### relativeAmount The amount of the relative range. @@ -203,25 +258,13 @@ The amount of the relative range. The unit of the amount of the relative range. #### config -The config for the daterange. This does not include the value as that is in it's value. Here is a breakdown of the keysk - -##### mode -The mode for the dateRange this can be relative or absolute? - -##### selectableRelativeUnits -The units you can select for a relative date range, relativeUnit will be one of these. - -##### enableTimeTypeSelection -where or not time type selection is availible on the UI? - -##### showQuarterShortcuts -Where or not the user can select quarters on the UI? +This is used by the frontend UI and can be safely ignored for our purposes. ### statsCalculationSetting How the recipe group runs calculations? Can be set to defualt? ### deployed -The deployed version of this recipe_group? It's a [recipe_group](#recipe_groups) minus this field +The deployed version of this recipe_group. It's a [recipe_group](#recipe_groups) minus this field. ## metadata This is created by info, it includes the id of the user that created the cookbook along with their name and email diff --git a/docs/entities/cookbook.md b/docs/entities/cookbook.md index 30fce56..c69004c 100644 --- a/docs/entities/cookbook.md +++ b/docs/entities/cookbook.md @@ -62,10 +62,10 @@ Name is the name of the field you are grabing the value of. This is an optional parameter and is passed in as integer. This is the number of minutes you want to look bak for the current value. ### Normalize Constraints -This is a function to return a clean string version of constraints. And is called like this: +This is a function to return a clean string version of constraints values. And is called like this: ``` -cli.normalize_constraints(constraints) +cli.normalize_constraints(constraint_values) ``` #### constraints -Are the constraints on a recipe group. \ No newline at end of file +The constraint values you wish to normalize, this only works for range values. \ No newline at end of file From e2d2e251146308d0e0611a08947d1cc621e67212 Mon Sep 17 00:00:00 2001 From: Michael Goldman Date: Wed, 22 Mar 2023 10:31:47 -0700 Subject: [PATCH 29/33] Tests --- docs/commonly_used_data_types/run.md | 11 +- smsdk/ma_session.py | 1 + tests/cookbook/cookbook_data.py | 155 ++++++++++++++++++++++++++- tests/cookbook/test_cookbook.py | 54 +++++++++- 4 files changed, 208 insertions(+), 13 deletions(-) diff --git a/docs/commonly_used_data_types/run.md b/docs/commonly_used_data_types/run.md index 60378e1..299af50 100644 --- a/docs/commonly_used_data_types/run.md +++ b/docs/commonly_used_data_types/run.md @@ -80,10 +80,10 @@ A recipe run or just run, is one set of inputs and outcomes for a cookbook recip We will go over each key in more detail ## _count -The number of total runs? +The number of total runs. ## _count_muted -The number of runs muted? +The number of runs muted. ## _duration_seconds The duration of the run in seconds. @@ -95,7 +95,7 @@ The start time of the run. The end time of the run. ## _score -The 'score' this run achieved (out of 1?) +The 'score' this run achieved. ## constraint_group_id The id of the group of constraints in the cookbook @@ -110,7 +110,7 @@ The id of the cookbook this run relates to? A list of filters on this run ## i_vals -?? +This is used by the backend to tell which run is which mostly not needed for our purposes ## levers A list of the levers attached to this run and the values these levers where at. A lever is in this format: @@ -219,6 +219,3 @@ The minimum value recorded during the run. #### normal A measure of the normal distrabution of the values recorded during the - -#### var_pop -?? \ No newline at end of file diff --git a/smsdk/ma_session.py b/smsdk/ma_session.py index 49cc5bc..881e513 100644 --- a/smsdk/ma_session.py +++ b/smsdk/ma_session.py @@ -186,6 +186,7 @@ def _get_records_v1( response = getattr(self.session, method.lower())( endpoint, json=url_params ) + if response.text: if response.status_code not in [200, 201]: raise ValueError("Error - {}".format(response.text)) diff --git a/tests/cookbook/cookbook_data.py b/tests/cookbook/cookbook_data.py index 13b6494..d8145eb 100644 --- a/tests/cookbook/cookbook_data.py +++ b/tests/cookbook/cookbook_data.py @@ -1 +1,154 @@ -AVALIBLE_COOKBOOK_JSON=[{'hash': '64hujs5ffb40124475e15d9852ba7e69be121a6b435eae4879392acc970f80f8', 'tag': 'demo', 'auto_generated_parent': {}, 'name': 'Test cookbook', 'assetNames': ['asset_test'], 'key_constraint': {'field': {'fieldName': 'stats__Cylinders__val', 'machineId': 'e2mj2b4f215b2323f45d04fa2', 'machineName': 'asset_test', 'machineDisplayName': 'Asset - test', 'fieldType': 'categorical', 'machineType': 'Diecast', 'fieldDisplayName': 'Cylinders', 'fieldUnit': ''}, 'valueMap': {'4': 1, '6': 0}}, 'recipe_groups': [{'id': 'DJy4jyEEi', 'values': ['6'], 'runBoundaries': [], 'maxDuration': {'isEnabled': False, 'minimum': 0, 'unit': 'second'}, 'topRun': 10, 'constraints': [], 'levers': [{'fieldName': 'stats__AluminumTempAvg__val', 'machineId': 'e2df2b4f876b763f45d04se2', 'machineName': 'asset_test', 'machineDisplayName': 'Asset-Test', 'fieldType': 'continuous', 'machineType': 'Diecast', 'fieldDisplayName': 'AluminumTemp - Average', 'fieldUnit': 'celsius'}, {'fieldName': 'stats__AluminumTempMax__val', 'machineId': 'asfdalkfjlkfji', 'machineName': 'asset_test', 'machineDisplayName': 'Asset - test', 'fieldType': 'continuous', 'machineType': 'Diecast', 'fieldDisplayName': 'AluminumTemp - Max', 'fieldUnit': 'celsius'}, {'fieldName': 'stats__DieTemp__val', 'machineId': '435tsfrdsvxcvr', 'machineName': 'asset_test', 'machineDisplayName': 'asset - test', 'fieldType': 'continuous', 'machineType': 'Diecast', 'fieldDisplayName': 'Die Temperature', 'fieldUnit': 'celsius'}, {'fieldName': 'stats__InjectionCurveDifference__val', 'machineId': 'sfawrfefasf', 'machineName': 'JB_HM_Diecast_1', 'machineDisplayName': 'Hamilton - Diecast 1', 'fieldType': 'continuous', 'machineType': 'Diecast', 'fieldDisplayName': 'Injection Curve Difference', 'fieldUnit': 'pascal'}, {'fieldName': 'stats__InjectionPressureMax__val', 'machineId': 'e2df2b4f115b763f45d04fa2', 'machineName': 'JB_HM_Diecast_1', 'machineDisplayName': 'Hamilton - Diecast 1', 'fieldType': 'continuous', 'machineType': 'Diecast', 'fieldDisplayName': 'Injection Pressure Max', 'fieldUnit': 'pascal'}, {'fieldName': 'stats__InjectionPressureMin__val', 'machineId': 'e2df2b4f115b763f45d04fa2', 'machineName': 'JB_HM_Diecast_1', 'machineDisplayName': 'Hamilton - Diecast 1', 'fieldType': 'continuous', 'machineType': 'Diecast', 'fieldDisplayName': 'Injection Pressure Min', 'fieldUnit': 'pascal'}], 'outcomes': [{'field': {'fieldName': 'quality', 'machineId': 'e2df2b4f115b763f45d04fa2', 'machineName': 'JB_HM_Diecast_1', 'machineDisplayName': 'Hamilton - Diecast 1', 'fieldType': 'kpi', 'machineType': 'Diecast', 'fieldDisplayName': 'Quality'}, 'weight': 1, 'optimization_func': 'maximize'}], 'filters': {'duration': {'isEnabled': False, 'minimum': 0, 'unit': 'second'}, 'recordFilters': []}, 'dateRange': {'value': {'relativeAmount': 7, 'relativeUnit': 'day'}, 'config': {'mode': 'relative', 'selectableRelativeUnits': ['minute', 'hour', 'day', 'week', 'month', 'year'], 'enableTimeTypeSelection': True, 'showQuarterShortcuts': True}}, 'computeDeployedDateRange': None, 'statsCalculationSetting': 'default', 'deployed': {'id': 'SJy4jyFFi', 'values': ['6'], 'runBoundaries': [], 'maxDuration': {'isEnabled': False, 'minimum': 0, 'unit': 'second'}, 'topRun': 10, 'constraints': [], 'levers': [{'fieldName': 'stats__AluminumTempAvg__val', 'machineId': 'e2df2b4f115b763f45d04fa2', 'machineName': 'JB_HM_Diecast_1', 'machineDisplayName': 'Hamilton - Diecast 1', 'fieldType': 'continuous', 'machineType': 'Diecast', 'fieldDisplayName': 'AluminumTemp - Average', 'fieldUnit': 'celsius'}, {'fieldName': 'stats__AluminumTempMax__val', 'machineId': 'e2df2b4f115b763f45d04fa2', 'machineName': 'JB_HM_Diecast_1', 'machineDisplayName': 'Hamilton - Diecast 1', 'fieldType': 'continuous', 'machineType': 'Diecast', 'fieldDisplayName': 'AluminumTemp - Max', 'fieldUnit': 'celsius'}, {'fieldName': 'stats__DieTemp__val', 'machineId': 'e2df2b4f115b763f45d04fa2', 'machineName': 'JB_HM_Diecast_1', 'machineDisplayName': 'Hamilton - Diecast 1', 'fieldType': 'continuous', 'machineType': 'Diecast', 'fieldDisplayName': 'Die Temperature', 'fieldUnit': 'celsius'}, {'fieldName': 'stats__InjectionCurveDifference__val', 'machineId': 'e2df2b4f115b763f45d04fa2', 'machineName': 'JB_HM_Diecast_1', 'machineDisplayName': 'Hamilton - Diecast 1', 'fieldType': 'continuous', 'machineType': 'Diecast', 'fieldDisplayName': 'Injection Curve Difference', 'fieldUnit': 'pascal'}, {'fieldName': 'stats__InjectionPressureMax__val', 'machineId': 'e2df2b4f115b763f45d04fa2', 'machineName': 'JB_HM_Diecast_1', 'machineDisplayName': 'Hamilton - Diecast 1', 'fieldType': 'continuous', 'machineType': 'Diecast', 'fieldDisplayName': 'Injection Pressure Max', 'fieldUnit': 'pascal'}, {'fieldName': 'stats__InjectionPressureMin__val', 'machineId': 'e2df2b4f115b763f45d04fa2', 'machineName': 'JB_HM_Diecast_1', 'machineDisplayName': 'Hamilton - Diecast 1', 'fieldType': 'continuous', 'machineType': 'Diecast', 'fieldDisplayName': 'Injection Pressure Min', 'fieldUnit': 'pascal'}], 'outcomes': [{'field': {'fieldName': 'quality', 'machineId': 'e2df2b4f115b763f45d04fa2', 'machineName': 'JB_HM_Diecast_1', 'machineDisplayName': 'Hamilton - Diecast 1', 'fieldType': 'kpi', 'machineType': 'Diecast', 'fieldDisplayName': 'Quality'}, 'weight': 1, 'optimization_func': 'maximize'}], 'filters': {'duration': {'isEnabled': False, 'minimum': 0, 'unit': 'second'}, 'recordFilters': []}, 'dateRange': {'value': {'relativeAmount': 7, 'relativeUnit': 'day'}, 'config': {'mode': 'relative', 'selectableRelativeUnits': ['minute', 'hour', 'day', 'week', 'month', 'year'], 'enableTimeTypeSelection': True, 'showQuarterShortcuts': True}}, 'computeDeployedDateRange': None, 'statsCalculationSetting': 'default', 'deployed_date': '2023-01-20T17:02:06.797Z', 'deployed_status': {'success': True, 'run_count_saved': 10215, 'seconds_to_exec': 5874, 'timestamp': '2023-03-21 03:59:54.633000'}}}, {'constraints': [], 'levers': [{'fieldName': 'stats__AluminumTempAvg__val', 'machineId': 'e2df2b4f115b763f45d04fa2', 'machineName': 'JB_HM_Diecast_1', 'machineDisplayName': 'Hamilton - Diecast 1', 'fieldType': 'continuous', 'machineType': 'Diecast', 'fieldDisplayName': 'AluminumTemp - Average', 'fieldUnit': 'celsius'}, {'fieldName': 'stats__AluminumTempMax__val', 'machineId': 'e2df2b4f115b763f45d04fa2', 'machineName': 'JB_HM_Diecast_1', 'machineDisplayName': 'Hamilton - Diecast 1', 'fieldType': 'continuous', 'machineType': 'Diecast', 'fieldDisplayName': 'AluminumTemp - Max', 'fieldUnit': 'celsius'}, {'fieldName': 'stats__DieTemp__val', 'machineId': 'e2df2b4f115b763f45d04fa2', 'machineName': 'JB_HM_Diecast_1', 'machineDisplayName': 'Hamilton - Diecast 1', 'fieldType': 'continuous', 'machineType': 'Diecast', 'fieldDisplayName': 'Die Temperature', 'fieldUnit': 'celsius'}, {'fieldName': 'stats__InjectionCurveDifference__val', 'machineId': 'e2df2b4f115b763f45d04fa2', 'machineName': 'JB_HM_Diecast_1', 'machineDisplayName': 'Hamilton - Diecast 1', 'fieldType': 'continuous', 'machineType': 'Diecast', 'fieldDisplayName': 'Injection Curve Difference', 'fieldUnit': 'pascal'}, {'fieldName': 'stats__InjectionPressureMax__val', 'machineId': 'e2df2b4f115b763f45d04fa2', 'machineName': 'JB_HM_Diecast_1', 'machineDisplayName': 'Hamilton - Diecast 1', 'fieldType': 'continuous', 'machineType': 'Diecast', 'fieldDisplayName': 'Injection Pressure Max', 'fieldUnit': 'pascal'}, {'fieldName': 'stats__InjectionPressureMin__val', 'machineId': 'e2df2b4f115b763f45d04fa2', 'machineName': 'JB_HM_Diecast_1', 'machineDisplayName': 'Hamilton - Diecast 1', 'fieldType': 'continuous', 'machineType': 'Diecast', 'fieldDisplayName': 'Injection Pressure Min', 'fieldUnit': 'pascal'}], 'outcomes': [{'field': {'fieldName': 'quality', 'machineId': 'e2df2b4f115b763f45d04fa2', 'machineName': 'JB_HM_Diecast_1', 'machineDisplayName': 'Hamilton - Diecast 1', 'fieldType': 'kpi', 'machineType': 'Diecast', 'fieldDisplayName': 'Quality'}, 'weight': 1, 'optimization_func': 'maximize'}], 'dateRange': {'value': {'relativeAmount': 7, 'relativeUnit': 'day'}, 'config': {'mode': 'relative', 'selectableRelativeUnits': ['minute', 'hour', 'day', 'week', 'month', 'year'], 'enableTimeTypeSelection': True, 'showQuarterShortcuts': True}}, 'filters': {'duration': {'isEnabled': False, 'minimum': 0, 'unit': 'second'}, 'recordFilters': []}, 'runBoundaries': [], 'maxDuration': {'isEnabled': False, 'minimum': 0, 'unit': 'second'}, 'topRun': 10, 'computeDeployedDateRange': None, 'statsCalculationSetting': 'default', 'id': 'ByxJEiJFKj', 'values': ['4']}], 'metadata': {'created_by': {'id': '5a1c9f9798214579dacc917a', 'email': 'ahome@sightmachine.com', 'metadata': {'first_name': 'Andrew', 'last_name': 'Home', 'tenant': 'demo'}}}, 'updatetime': '2023-03-21 03:59:54.634000', 'assets': [], 'id': 'liafalkfjdlak'}] \ No newline at end of file +AVALIBLE_COOKBOOK_JSON=[{'hash': '64hujs5ffb40124475e15d9852ba7e69be121a6b435eae4879392acc970f80f8', 'tag': 'demo', 'auto_generated_parent': {}, 'name': 'Test cookbook', 'assetNames': ['asset_test'], 'key_constraint': {'field': {'fieldName': 'stats__Cylinders__val', 'machineId': 'e2mj2b4f215b2323f45d04fa2', 'machineName': 'asset_test', 'machineDisplayName': 'Asset - test', 'fieldType': 'categorical', 'machineType': 'Diecast', 'fieldDisplayName': 'Cylinders', 'fieldUnit': ''}, 'valueMap': {'4': 1, '6': 0}}, 'recipe_groups': [{'id': 'DJy4jyEEi', 'values': ['6'], 'runBoundaries': [], 'maxDuration': {'isEnabled': False, 'minimum': 0, 'unit': 'second'}, 'topRun': 10, 'constraints': [], 'levers': [{'fieldName': 'stats__AluminumTempAvg__val', 'machineId': 'e2df2b4f876b763f45d04se2', 'machineName': 'asset_test', 'machineDisplayName': 'Asset-Test', 'fieldType': 'continuous', 'machineType': 'Diecast', 'fieldDisplayName': 'AluminumTemp - Average', 'fieldUnit': 'celsius'}, {'fieldName': 'stats__AluminumTempMax__val', 'machineId': 'asfdalkfjlkfji', 'machineName': 'asset_test', 'machineDisplayName': 'Asset - test', 'fieldType': 'continuous', 'machineType': 'Diecast', 'fieldDisplayName': 'AluminumTemp - Max', 'fieldUnit': 'celsius'}, {'fieldName': 'stats__DieTemp__val', 'machineId': '435tsfrdsvxcvr', 'machineName': 'asset_test', 'machineDisplayName': 'asset - test', 'fieldType': 'continuous', 'machineType': 'Diecast', 'fieldDisplayName': 'Die Temperature', 'fieldUnit': 'celsius'}, {'fieldName': 'stats__InjectionCurveDifference__val', 'machineId': 'sfawrfefasf', 'machineName': 'JB_HM_Diecast_1', 'machineDisplayName': 'Hamilton - Diecast 1', 'fieldType': 'continuous', 'machineType': 'Diecast', 'fieldDisplayName': 'Injection Curve Difference', 'fieldUnit': 'pascal'}, {'fieldName': 'stats__InjectionPressureMax__val', 'machineId': 'e2df2b4f115b763f45d04fa2', 'machineName': 'JB_HM_Diecast_1', 'machineDisplayName': 'Hamilton - Diecast 1', 'fieldType': 'continuous', 'machineType': 'Diecast', 'fieldDisplayName': 'Injection Pressure Max', 'fieldUnit': 'pascal'}, {'fieldName': 'stats__InjectionPressureMin__val', 'machineId': 'e2df2b4f115b763f45d04fa2', 'machineName': 'JB_HM_Diecast_1', 'machineDisplayName': 'Hamilton - Diecast 1', 'fieldType': 'continuous', 'machineType': 'Diecast', 'fieldDisplayName': 'Injection Pressure Min', 'fieldUnit': 'pascal'}], 'outcomes': [{'field': {'fieldName': 'quality', 'machineId': 'e2df2b4f115b763f45d04fa2', 'machineName': 'JB_HM_Diecast_1', 'machineDisplayName': 'Hamilton - Diecast 1', 'fieldType': 'kpi', 'machineType': 'Diecast', 'fieldDisplayName': 'Quality'}, 'weight': 1, 'optimization_func': 'maximize'}], 'filters': {'duration': {'isEnabled': False, 'minimum': 0, 'unit': 'second'}, 'recordFilters': []}, 'dateRange': {'value': {'relativeAmount': 7, 'relativeUnit': 'day'}, 'config': {'mode': 'relative', 'selectableRelativeUnits': ['minute', 'hour', 'day', 'week', 'month', 'year'], 'enableTimeTypeSelection': True, 'showQuarterShortcuts': True}}, 'computeDeployedDateRange': None, 'statsCalculationSetting': 'default', 'deployed': {'id': 'SJy4jyFFi', 'values': ['6'], 'runBoundaries': [], 'maxDuration': {'isEnabled': False, 'minimum': 0, 'unit': 'second'}, 'topRun': 10, 'constraints': [], 'levers': [{'fieldName': 'stats__AluminumTempAvg__val', 'machineId': 'e2df2b4f115b763f45d04fa2', 'machineName': 'JB_HM_Diecast_1', 'machineDisplayName': 'Hamilton - Diecast 1', 'fieldType': 'continuous', 'machineType': 'Diecast', 'fieldDisplayName': 'AluminumTemp - Average', 'fieldUnit': 'celsius'}, {'fieldName': 'stats__AluminumTempMax__val', 'machineId': 'e2df2b4f115b763f45d04fa2', 'machineName': 'JB_HM_Diecast_1', 'machineDisplayName': 'Hamilton - Diecast 1', 'fieldType': 'continuous', 'machineType': 'Diecast', 'fieldDisplayName': 'AluminumTemp - Max', 'fieldUnit': 'celsius'}, {'fieldName': 'stats__DieTemp__val', 'machineId': 'e2df2b4f115b763f45d04fa2', 'machineName': 'JB_HM_Diecast_1', 'machineDisplayName': 'Hamilton - Diecast 1', 'fieldType': 'continuous', 'machineType': 'Diecast', 'fieldDisplayName': 'Die Temperature', 'fieldUnit': 'celsius'}, {'fieldName': 'stats__InjectionCurveDifference__val', 'machineId': 'e2df2b4f115b763f45d04fa2', 'machineName': 'JB_HM_Diecast_1', 'machineDisplayName': 'Hamilton - Diecast 1', 'fieldType': 'continuous', 'machineType': 'Diecast', 'fieldDisplayName': 'Injection Curve Difference', 'fieldUnit': 'pascal'}, {'fieldName': 'stats__InjectionPressureMax__val', 'machineId': 'e2df2b4f115b763f45d04fa2', 'machineName': 'JB_HM_Diecast_1', 'machineDisplayName': 'Hamilton - Diecast 1', 'fieldType': 'continuous', 'machineType': 'Diecast', 'fieldDisplayName': 'Injection Pressure Max', 'fieldUnit': 'pascal'}, {'fieldName': 'stats__InjectionPressureMin__val', 'machineId': 'e2df2b4f115b763f45d04fa2', 'machineName': 'JB_HM_Diecast_1', 'machineDisplayName': 'Hamilton - Diecast 1', 'fieldType': 'continuous', 'machineType': 'Diecast', 'fieldDisplayName': 'Injection Pressure Min', 'fieldUnit': 'pascal'}], 'outcomes': [{'field': {'fieldName': 'quality', 'machineId': 'e2df2b4f115b763f45d04fa2', 'machineName': 'JB_HM_Diecast_1', 'machineDisplayName': 'Hamilton - Diecast 1', 'fieldType': 'kpi', 'machineType': 'Diecast', 'fieldDisplayName': 'Quality'}, 'weight': 1, 'optimization_func': 'maximize'}], 'filters': {'duration': {'isEnabled': False, 'minimum': 0, 'unit': 'second'}, 'recordFilters': []}, 'dateRange': {'value': {'relativeAmount': 7, 'relativeUnit': 'day'}, 'config': {'mode': 'relative', 'selectableRelativeUnits': ['minute', 'hour', 'day', 'week', 'month', 'year'], 'enableTimeTypeSelection': True, 'showQuarterShortcuts': True}}, 'computeDeployedDateRange': None, 'statsCalculationSetting': 'default', 'deployed_date': '2023-01-20T17:02:06.797Z', 'deployed_status': {'success': True, 'run_count_saved': 10215, 'seconds_to_exec': 5874, 'timestamp': '2023-03-21 03:59:54.633000'}}}, {'constraints': [], 'levers': [{'fieldName': 'stats__AluminumTempAvg__val', 'machineId': 'e2df2b4f115b763f45d04fa2', 'machineName': 'JB_HM_Diecast_1', 'machineDisplayName': 'Hamilton - Diecast 1', 'fieldType': 'continuous', 'machineType': 'Diecast', 'fieldDisplayName': 'AluminumTemp - Average', 'fieldUnit': 'celsius'}, {'fieldName': 'stats__AluminumTempMax__val', 'machineId': 'e2df2b4f115b763f45d04fa2', 'machineName': 'JB_HM_Diecast_1', 'machineDisplayName': 'Hamilton - Diecast 1', 'fieldType': 'continuous', 'machineType': 'Diecast', 'fieldDisplayName': 'AluminumTemp - Max', 'fieldUnit': 'celsius'}, {'fieldName': 'stats__DieTemp__val', 'machineId': 'e2df2b4f115b763f45d04fa2', 'machineName': 'JB_HM_Diecast_1', 'machineDisplayName': 'Hamilton - Diecast 1', 'fieldType': 'continuous', 'machineType': 'Diecast', 'fieldDisplayName': 'Die Temperature', 'fieldUnit': 'celsius'}, {'fieldName': 'stats__InjectionCurveDifference__val', 'machineId': 'e2df2b4f115b763f45d04fa2', 'machineName': 'JB_HM_Diecast_1', 'machineDisplayName': 'Hamilton - Diecast 1', 'fieldType': 'continuous', 'machineType': 'Diecast', 'fieldDisplayName': 'Injection Curve Difference', 'fieldUnit': 'pascal'}, {'fieldName': 'stats__InjectionPressureMax__val', 'machineId': 'e2df2b4f115b763f45d04fa2', 'machineName': 'JB_HM_Diecast_1', 'machineDisplayName': 'Hamilton - Diecast 1', 'fieldType': 'continuous', 'machineType': 'Diecast', 'fieldDisplayName': 'Injection Pressure Max', 'fieldUnit': 'pascal'}, {'fieldName': 'stats__InjectionPressureMin__val', 'machineId': 'e2df2b4f115b763f45d04fa2', 'machineName': 'JB_HM_Diecast_1', 'machineDisplayName': 'Hamilton - Diecast 1', 'fieldType': 'continuous', 'machineType': 'Diecast', 'fieldDisplayName': 'Injection Pressure Min', 'fieldUnit': 'pascal'}], 'outcomes': [{'field': {'fieldName': 'quality', 'machineId': 'e2df2b4f115b763f45d04fa2', 'machineName': 'JB_HM_Diecast_1', 'machineDisplayName': 'Hamilton - Diecast 1', 'fieldType': 'kpi', 'machineType': 'Diecast', 'fieldDisplayName': 'Quality'}, 'weight': 1, 'optimization_func': 'maximize'}], 'dateRange': {'value': {'relativeAmount': 7, 'relativeUnit': 'day'}, 'config': {'mode': 'relative', 'selectableRelativeUnits': ['minute', 'hour', 'day', 'week', 'month', 'year'], 'enableTimeTypeSelection': True, 'showQuarterShortcuts': True}}, 'filters': {'duration': {'isEnabled': False, 'minimum': 0, 'unit': 'second'}, 'recordFilters': []}, 'runBoundaries': [], 'maxDuration': {'isEnabled': False, 'minimum': 0, 'unit': 'second'}, 'topRun': 10, 'computeDeployedDateRange': None, 'statsCalculationSetting': 'default', 'id': 'ByxJEiJFKj', 'values': ['4']}], 'metadata': {'created_by': {'id': '5a1c9f9798214579dacc917a', 'email': 'ahome@sightmachine.com', 'metadata': {'first_name': 'Andrew', 'last_name': 'Home', 'tenant': 'demo'}}}, 'updatetime': '2023-03-21 03:59:54.634000', 'assets': [], 'id': 'liafalkfjdlak'}] +RUNS = {'runs':[ + {'_count': 12, + '_count_muted': 0, + '_duration_seconds': 649.0, + '_earliest': '2022-10-21T00:35:32+00:00', + '_latest': '2022-10-21T00:46:21+00:00', + '_score': 1.0, + 'constraint_group_id': '0', + 'constraints': [], + 'cookbook': '63ab6b263fa4880c06334b03', + 'filters': [], + 'i_vals': [{'asset': 'SHARED', 'name': 'group', 'value': '0'}, + {'asset': 'SHARED', 'name': 'sequence', 'value': 2}], + 'levers': [{'asset': 'JB_HM_Diecast_1', + 'd_pos': 2, + 'name': 'stats__AluminumTempAvg__val', + 'value': {'avg': 659.8448127439334, + 'count': 9.0, + 'max': 671.1048565509, + 'min': 653.718308813, + 'var_pop': 29.816738449355437}}, + {'asset': 'JB_HM_Diecast_1', + 'd_pos': 3, + 'name': 'stats__AluminumTempMax__val', + 'value': {'avg': 659.8448127439334, + 'count': 9.0, + 'max': 671.1048565509, + 'min': 653.718308813, + 'var_pop': 29.816738449355437}}, + {'asset': 'JB_HM_Diecast_1', + 'd_pos': 4, + 'name': 'stats__DieTemp__val', + 'value': {'avg': 290.8228674053778, + 'count': 9.0, + 'max': 295.2332287849, + 'min': 284.84446548529996, + 'var_pop': 11.993829414162574}}, + {'asset': 'JB_HM_Diecast_1', + 'd_pos': 5, + 'name': 'stats__InjectionCurveDifference__val', + 'value': {'avg': 1.576386980588889, + 'count': 9.0, + 'max': 2.8598220936, + 'min': 0.3786822327000001, + 'var_pop': 0.810658290414194}}, + {'asset': 'JB_HM_Diecast_1', + 'd_pos': 6, + 'name': 'stats__InjectionPressureMax__val', + 'value': {'avg': 39.537370569911104, + 'count': 9.0, + 'max': 44.0884060904, + 'min': 35.4952094968, + 'var_pop': 5.452849234986407}}, + {'asset': 'JB_HM_Diecast_1', + 'd_pos': 7, + 'name': 'stats__InjectionPressureMin__val', + 'value': {'avg': 36.76283137485556, + 'count': 9.0, + 'max': 38.8721003798, + 'min': 35.0169147198, + 'var_pop': 1.2194962602188073}}], + 'outcomes': [{'asset': 'JB_HM_Diecast_1', + 'd_pos': 0, + 'kpi': {'aggregates': {'Output': 'sum', 'ScrapQuantity': 'sum'}, + 'dependencies': {'Output': 9.0, 'ScrapQuantity': 0.0}, + 'formula': '((Output) / (Output + ScrapQuantity)) * 100 ' + 'if ((Output + ScrapQuantity) > 0) else ' + 'None'}, + 'name': 'quality', + 'value': {'avg': 100.0, + 'count': 100.0, + 'max': 100.0, + 'min': 100.0, + 'normal': 1.0000000000000002, + 'var_pop': 100.0}}]} +], + 'constraint_group':[{'_count': 12, + '_count_muted': 0, + '_duration_seconds': 649.0, + '_earliest': '2022-10-21T00:35:32+00:00', + '_latest': '2022-10-21T00:46:21+00:00', + '_score': 1.0, + 'constraint_group_id': '0', + 'constraints': [], + 'cookbook': '63ab6b263fa4880c06334b03', + 'filters': [], + 'i_vals': [{'asset': 'SHARED', 'name': 'group', 'value': '0'}, + {'asset': 'SHARED', 'name': 'sequence', 'value': 2}], + 'levers': [{'asset': 'JB_HM_Diecast_1', + 'd_pos': 2, + 'name': 'stats__AluminumTempAvg__val', + 'value': {'avg': 659.8448127439334, + 'count': 9.0, + 'max': 671.1048565509, + 'min': 653.718308813, + 'var_pop': 29.816738449355437}}, + {'asset': 'JB_HM_Diecast_1', + 'd_pos': 3, + 'name': 'stats__AluminumTempMax__val', + 'value': {'avg': 659.8448127439334, + 'count': 9.0, + 'max': 671.1048565509, + 'min': 653.718308813, + 'var_pop': 29.816738449355437}}, + {'asset': 'JB_HM_Diecast_1', + 'd_pos': 4, + 'name': 'stats__DieTemp__val', + 'value': {'avg': 290.8228674053778, + 'count': 9.0, + 'max': 295.2332287849, + 'min': 284.84446548529996, + 'var_pop': 11.993829414162574}}, + {'asset': 'JB_HM_Diecast_1', + 'd_pos': 5, + 'name': 'stats__InjectionCurveDifference__val', + 'value': {'avg': 1.576386980588889, + 'count': 9.0, + 'max': 2.8598220936, + 'min': 0.3786822327000001, + 'var_pop': 0.810658290414194}}, + {'asset': 'JB_HM_Diecast_1', + 'd_pos': 6, + 'name': 'stats__InjectionPressureMax__val', + 'value': {'avg': 39.537370569911104, + 'count': 9.0, + 'max': 44.0884060904, + 'min': 35.4952094968, + 'var_pop': 5.452849234986407}}, + {'asset': 'JB_HM_Diecast_1', + 'd_pos': 7, + 'name': 'stats__InjectionPressureMin__val', + 'value': {'avg': 36.76283137485556, + 'count': 9.0, + 'max': 38.8721003798, + 'min': 35.0169147198, + 'var_pop': 1.2194962602188073}}], + 'outcomes': [{'asset': 'JB_HM_Diecast_1', + 'd_pos': 0, + 'kpi': {'aggregates': {'Output': 'sum', 'ScrapQuantity': 'sum'}, + 'dependencies': {'Output': 9.0, 'ScrapQuantity': 0.0}, + 'formula': '((Output) / (Output + ScrapQuantity)) * 100 ' + 'if ((Output + ScrapQuantity) > 0) else ' + 'None'}, + 'name': 'quality', + 'value': {'avg': 100.0, + 'count': 100.0, + 'max': 100.0, + 'min': 100.0, + 'normal': 1.0000000000000002, + 'var_pop': 100.0}}]}] +} + +CURRENT_VALUE = [{'asset': 'test', 'name': 'test_field', 'values': {'latest': 42.42}}] \ No newline at end of file diff --git a/tests/cookbook/test_cookbook.py b/tests/cookbook/test_cookbook.py index 75591f3..dd8d0ad 100644 --- a/tests/cookbook/test_cookbook.py +++ b/tests/cookbook/test_cookbook.py @@ -1,7 +1,7 @@ import pandas as pd from requests.sessions import Session from smsdk.client import Client -from tests.cookbook.cookbook_data import AVALIBLE_KPI_JSON, KPI_DATA_VIZ_JSON +from tests.cookbook.cookbook_data import AVALIBLE_COOKBOOK_JSON, RUNS, CURRENT_VALUE from smsdk.smsdk_entities.kpi.kpi import KPI from mock import mock_open, MagicMock, patch @@ -14,7 +14,7 @@ class ResponseGet: @staticmethod def json(): - return {"results": [{"kpi":AVALIBLE_KPI_JSON}]} + return {"objects": AVALIBLE_COOKBOOK_JSON} mocked.return_value = MagicMock( get=MagicMock(return_value=ResponseGet()) ) @@ -22,9 +22,53 @@ def json(): dt = Client("demo") # Run - kpis = dt.get_cookbooks() + cookbooks = dt.get_cookbooks() # Verify - assert len(kpis) == 5 + assert len(cookbooks) == 1 - assert kpis[0]["name"] == 'Scrap_Rate' \ No newline at end of file + assert cookbooks[0]["name"] == 'Test cookbook' + +@patch("smsdk.ma_session.Session") +def test_get_top_results(mocked): + class ResponseGet: + ok = True + text = "Success" + status_code=200 + + @staticmethod + def json(): + return RUNS + mocked.return_value = MagicMock( + get=MagicMock(return_value=ResponseGet()) + ) + + dt = Client("demo") + + # Run + runs = dt.get_top_results('recipe_group_id', 1) + + # Verify + assert len(runs['runs']) == 1 + +@patch("smsdk.ma_session.Session") +def test_get_current_value(mocked): + class ResponseGet: + ok = True + text = "Success" + status_code=200 + + @staticmethod + def json(): + return {"results": CURRENT_VALUE} + mocked.return_value = MagicMock( + post=MagicMock(return_value=ResponseGet()) + ) + + dt = Client("demo") + + # Run + value = dt.get_current_value([{"asset": "test", "name": "test_field"}]) + + # Verify + assert value[0]['values']['latest'] == 42.42 \ No newline at end of file From c7282be808ded10d5093b57a207d42cb1c9e5688 Mon Sep 17 00:00:00 2001 From: Michael Goldman Date: Wed, 22 Mar 2023 12:26:00 -0700 Subject: [PATCH 30/33] Fixes --- docs/commonly_used_data_types/cookbook.md | 16 ++++++------ docs/commonly_used_data_types/run.md | 32 +++++++++++------------ docs/entities/cookbook.md | 4 +-- smsdk/client.py | 30 +++++++++++++++++++-- tests/cookbook/test_cookbook.py | 4 +-- 5 files changed, 56 insertions(+), 30 deletions(-) diff --git a/docs/commonly_used_data_types/cookbook.md b/docs/commonly_used_data_types/cookbook.md index bf512fe..c3f1237 100644 --- a/docs/commonly_used_data_types/cookbook.md +++ b/docs/commonly_used_data_types/cookbook.md @@ -62,7 +62,7 @@ "filters": {'duration': {'isEnabled': False, 'minimum': 0, 'unit': 'second'}, 'recordFilters': []}, "dateRange": {'value': {'relativeAmount': 7, 'relativeUnit': 'day'}, 'config': {'mode': 'relative', 'selectableRelativeUnits': ['minute', 'hour', 'day', 'week', 'month', 'year'], 'enableTimeTypeSelection': True, 'showQuarterShortcuts': True}} "computeDeployedDateRange": None, - "statsCalculationSetting": "defualt", + "statsCalculationSetting": "default", "deployed":{recipe_group - deployed field} } ``` @@ -74,13 +74,13 @@ List of values that currently in the goup? ### runBoundaries - List of constraints of boundaries? + List of boundaries on runs. ### maxDuration This field is ignored by the system currently ### topRun - The value of the topRun in this recipe group. + The number of top runs to consider when computing lever statistics. ### constraints List of fields and values that are used to breakup runs They look like the following: @@ -111,14 +111,14 @@ ] } ``` - #### asset + #### assets The name of the asset the field used in the constraint. #### name The name of the field used for the constraint. #### type - The data type of the constraint, mostly commonly continuous or categorical + The data type of the constraint, most commonly continuous or categorical #### values These are the values to break up runs into. @@ -190,7 +190,7 @@ The field you are trying to optimize with this outcome. This is the same set up The wieght you are putting on this outcome as compared to the others. #### optimization_func -How you wish to optimize this field usually 'maximize' or 'minize' +How you wish to optimize this field usually 'maximize' or 'minimize' ### filters Object containing a duration filter and list of record filters @@ -261,7 +261,7 @@ The unit of the amount of the relative range. This is used by the frontend UI and can be safely ignored for our purposes. ### statsCalculationSetting -How the recipe group runs calculations? Can be set to defualt? +A legacy setting currently unused. ### deployed The deployed version of this recipe_group. It's a [recipe_group](#recipe_groups) minus this field. @@ -273,7 +273,7 @@ This is created by info, it includes the id of the user that created the cookboo The time the cookbook was last updated. ## assets -A list of assets used in the cookbook? +A list of assets used in the cookbook. ## id The id of the cookbook \ No newline at end of file diff --git a/docs/commonly_used_data_types/run.md b/docs/commonly_used_data_types/run.md index 299af50..d610ab6 100644 --- a/docs/commonly_used_data_types/run.md +++ b/docs/commonly_used_data_types/run.md @@ -80,10 +80,10 @@ A recipe run or just run, is one set of inputs and outcomes for a cookbook recip We will go over each key in more detail ## _count -The number of total runs. +The number of total records in the run. ## _count_muted -The number of runs muted. +The number of records filtered out in the run. ## _duration_seconds The duration of the run in seconds. @@ -98,22 +98,22 @@ The end time of the run. The 'score' this run achieved. ## constraint_group_id -The id of the group of constraints in the cookbook +The id of the group of constraints in the cookbook. ## constraints -A list of the constraints on this run +A list of the constraints on this run. ## cookbook -The id of the cookbook this run relates to? +The id of the cookbook this run relates to. ## filters -A list of filters on this run +A list of filters on this run. ## i_vals -This is used by the backend to tell which run is which mostly not needed for our purposes +Lists the constraint and/or run boundary values that were used to delimit this run. ## levers -A list of the levers attached to this run and the values these levers where at. A lever is in this format: +A list of the levers attached to this run and their values. A lever is in this format: ``` { 'asset': 'JB_HM_Diecast_1', @@ -130,10 +130,10 @@ A list of the levers attached to this run and the values these levers where at. ``` ### asset -The name of asset this lever is on, this is usually a machine_name +The name of asset this lever is on, this is a machine_name ### d_pos -?? +The index of the corresponding dependent variable in the linevis query. Internal use only. ### name The actual name of the field being looked at. @@ -145,7 +145,7 @@ The values that this lever was measured at during this run. It is futher broken Average value during the run. #### count -The amount of measurements during the run. +The amount of records during the run. #### max The maxium value recorded during the run. @@ -154,7 +154,7 @@ The maxium value recorded during the run. The minimum value recorded during the run. #### var_pop -?? +Population variance. ## outcomes A list of outcomes recorded during the run. It looks like: @@ -185,16 +185,16 @@ A list of outcomes recorded during the run. It looks like: The name of the asset that outcome is attached to. Usually a machine name. ### d_pos -?? +The index of the corresponding dependent variable in the linevis query. Internal use only. ### kpi If the outcome is a kpi this field will descripe the kpi. It has the following keys: #### aggregates -How we aggregate each field being feed into our kpi formula, can be sum or avg. +How we aggregate each field being feed into our kpi formula, can be sum, avg, min or max. #### dependencies -The fields that feed into our kpi formula? +The variables in the KPI formula. #### formula The actual formula the kpi uses @@ -218,4 +218,4 @@ The maxium value recorded during the run. The minimum value recorded during the run. #### normal -A measure of the normal distrabution of the values recorded during the +A measure of the normal distribution of the values recorded during the run diff --git a/docs/entities/cookbook.md b/docs/entities/cookbook.md index c69004c..c504ce1 100644 --- a/docs/entities/cookbook.md +++ b/docs/entities/cookbook.md @@ -14,7 +14,7 @@ This will return a list of [cookbooks](/docs/commonly_used_data_types/cookbook.m ### Get Top Results This function will get you the top runs of the recipe group you input and is called like the following: ``` -cli.get_top_results(recipe_group_id, limit) +cli.get_cookbook_top_results(recipe_group_id, limit) ``` This will return the following: @@ -36,7 +36,7 @@ This is an int and is optional, if not entered it will default to 10. This is t ### Get Current Value This function gets the current values of the fields passed into it. It is called like the following: ``` -get_current_value(variables, minutes) +cli.get_cookbook_current_value(variables, minutes) ``` It returns something like: diff --git a/smsdk/client.py b/smsdk/client.py index a691d40..25aeb48 100644 --- a/smsdk/client.py +++ b/smsdk/client.py @@ -289,20 +289,36 @@ def get_fields_of_machine_type(self, machine_type, types=[], show_hidden=False, return fields def get_cookbooks(self, **kwargs): + """ + Gets all of the cookbooks accessable to the logged in user. + :return: list of cookbooks + """ cookbook = smsdkentities.get('cookbook') base_url = get_url( self.config["protocol"], self.tenant, self.config["site.domain"] ) return cookbook(self.session, base_url).get_cookbooks(**kwargs) - def get_top_results(self, recipe_group_id, limit=10, **kwargs): + def get_cookbook_top_results(self, recipe_group_id, limit=10, **kwargs): + """ + Gets the top runs for a recipe group. + :param recipe_group_id: The id of the recipe group to get runs for. + :param limit: The max number of runs wished to return. Defaults to 10. + :return: List of runs + """ cookbook = smsdkentities.get('cookbook') base_url = get_url( self.config["protocol"], self.tenant, self.config["site.domain"] ) return cookbook(self.session, base_url).get_top_results(recipe_group_id, limit, **kwargs) - def get_current_value(self, variables=[], minutes=1440, **kwargs): + def get_cookbook_current_value(self, variables=[], minutes=1440, **kwargs): + """ + Gets the current value of a field. + :param variables: A list of fields to return values for in the format {'asset': machine_name, 'name': field_name} + :param minutes: The number of minutes to consider when grabing the current value, defaults to 1440 or 1 day + :return: A list of values associated with the proper fields. + """ cookbook = smsdkentities.get('cookbook') base_url = get_url( self.config["protocol"], self.tenant, self.config["site.domain"] @@ -310,11 +326,21 @@ def get_current_value(self, variables=[], minutes=1440, **kwargs): return cookbook(self.session, base_url).get_current_value(variables, minutes, **kwargs) def normalize_constraint(self, constraint): + """ + Takes a constraint and returns a string version of it's to and from fields. + :param constraint: A range constraint field most have a to and from key. + :return: A string + """ to = constraint.get("to") from_constraint = constraint.get("from") return "({},{})".format(to, from_constraint) def normalize_constraints(self, constraints): + """ + Takes a list of constraint and returns string versions of their to and from fields. + :param constraint: A list range constraint field each most have a to and from key. + :return: A list of strings + """ constraints_normal = [] for constraint in constraints: constraints_normal.append(self.normalize_constraint(constraint)) diff --git a/tests/cookbook/test_cookbook.py b/tests/cookbook/test_cookbook.py index dd8d0ad..323cee5 100644 --- a/tests/cookbook/test_cookbook.py +++ b/tests/cookbook/test_cookbook.py @@ -46,7 +46,7 @@ def json(): dt = Client("demo") # Run - runs = dt.get_top_results('recipe_group_id', 1) + runs = dt.get_cookbook_top_results('recipe_group_id', 1) # Verify assert len(runs['runs']) == 1 @@ -68,7 +68,7 @@ def json(): dt = Client("demo") # Run - value = dt.get_current_value([{"asset": "test", "name": "test_field"}]) + value = dt.get_cookbook_current_value([{"asset": "test", "name": "test_field"}]) # Verify assert value[0]['values']['latest'] == 42.42 \ No newline at end of file From 16c2bfda567fa641deb73c5316d9c66b00ec8ecd Mon Sep 17 00:00:00 2001 From: Michael Goldman Date: Wed, 22 Mar 2023 12:30:14 -0700 Subject: [PATCH 31/33] Gramer check --- docs/commonly_used_data_types/cookbook.md | 18 ++++++++---------- docs/commonly_used_data_types/run.md | 8 +++----- 2 files changed, 11 insertions(+), 15 deletions(-) diff --git a/docs/commonly_used_data_types/cookbook.md b/docs/commonly_used_data_types/cookbook.md index c3f1237..2612270 100644 --- a/docs/commonly_used_data_types/cookbook.md +++ b/docs/commonly_used_data_types/cookbook.md @@ -48,7 +48,7 @@ This how the system maps the values of the key constraint on to recipe groups. Value for each key is the index of the recipe group to use. ## recipe_groups - A list of recipe groups, They look like the following and will be described in more details + A list of recipe groups. They look like the following: ``` { "id": "id" @@ -71,7 +71,7 @@ The id of the recipe group. ### values - List of values that currently in the goup? + List of values that currently in the goup. ### runBoundaries List of boundaries on runs. @@ -83,7 +83,7 @@ The number of top runs to consider when computing lever statistics. ### constraints - List of fields and values that are used to breakup runs They look like the following: + List of fields and values that are used to breakup runs they look like the following: ``` { "asset": "F1_010_BodyMaker_4", @@ -118,7 +118,7 @@ The name of the field used for the constraint. #### type - The data type of the constraint, most commonly continuous or categorical + The data type of the constraint, most commonly continuous or categorical. #### values These are the values to break up runs into. @@ -137,7 +137,6 @@ 'fieldUnit': 'celsius' } ``` -And here is a break down of each field: #### fieldName The name of field being looking. @@ -190,10 +189,10 @@ The field you are trying to optimize with this outcome. This is the same set up The wieght you are putting on this outcome as compared to the others. #### optimization_func -How you wish to optimize this field usually 'maximize' or 'minimize' +How you wish to optimize this field usually 'maximize' or 'minimize'. ### filters -Object containing a duration filter and list of record filters +Object containing a duration filter and list of record filters. It looks like the following: ``` {'duration': {'isEnabled': False, 'minimum': 0, 'unit': 'second'}, 'recordFilters': []} ``` @@ -202,7 +201,7 @@ Object containing a duration filter and list of record filters This is the minimum run duration. ##### isEnabled -Whieter or not minimum run duration is enabled +Whieter or not minimum run duration is enabled. ##### minimum The amount of time of what ever unit the minimum run duration is. @@ -246,7 +245,6 @@ The date range that runs can look at. Looks like this: } } ``` -Here is a breakdown of the keys: #### value The value of the dateRange. For relative config we have two keys, relativeAmount and relativeUnit. This very similiar to [Data Viz time Selection](/docs/commonly_used_data_types/data_viz_query.md#time_selection). @@ -267,7 +265,7 @@ A legacy setting currently unused. The deployed version of this recipe_group. It's a [recipe_group](#recipe_groups) minus this field. ## metadata -This is created by info, it includes the id of the user that created the cookbook along with their name and email +This is created by info, it includes the id of the user that created the cookbook along with their name and email. ## updatetime The time the cookbook was last updated. diff --git a/docs/commonly_used_data_types/run.md b/docs/commonly_used_data_types/run.md index d610ab6..a7e2ab2 100644 --- a/docs/commonly_used_data_types/run.md +++ b/docs/commonly_used_data_types/run.md @@ -77,8 +77,6 @@ A recipe run or just run, is one set of inputs and outcomes for a cookbook recip 'var_pop': 100.0}}]} ``` -We will go over each key in more detail - ## _count The number of total records in the run. @@ -130,7 +128,7 @@ A list of the levers attached to this run and their values. A lever is in this ``` ### asset -The name of asset this lever is on, this is a machine_name +The name of asset this lever is on, this is a machine_name. ### d_pos The index of the corresponding dependent variable in the linevis query. Internal use only. @@ -197,7 +195,7 @@ How we aggregate each field being feed into our kpi formula, can be sum, avg, mi The variables in the KPI formula. #### formula -The actual formula the kpi uses +The actual formula the kpi uses. ### name The name of the field that the outcome is being tracked from. @@ -218,4 +216,4 @@ The maxium value recorded during the run. The minimum value recorded during the run. #### normal -A measure of the normal distribution of the values recorded during the run +A measure of the normal distribution of the values recorded during the run. From 4314d10b8d647f104cc675e08df198963efd30eb Mon Sep 17 00:00:00 2001 From: Michael Goldman Date: Wed, 22 Mar 2023 19:54:18 -0700 Subject: [PATCH 32/33] specifies --- docs/commonly_used_data_types/cookbook.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/commonly_used_data_types/cookbook.md b/docs/commonly_used_data_types/cookbook.md index 2612270..4b48319 100644 --- a/docs/commonly_used_data_types/cookbook.md +++ b/docs/commonly_used_data_types/cookbook.md @@ -39,7 +39,7 @@ A list of names of assets used in this cookbook these are often just machine_names. ## key_constraint - The field that specfices the product. This also how the system knows which recipe_group to use. + The field that specifies the product. This also how the system knows which recipe_group to use. ### field This is detailed information on the field being used as the Key Constraint for more info see [levers](#levers). From e971249a836917ab9d3730f9b60e5649c25211b4 Mon Sep 17 00:00:00 2001 From: Michael Goldman Date: Thu, 23 Mar 2023 11:02:15 -0700 Subject: [PATCH 33/33] Fixes get cycles --- smsdk/client_v0.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/smsdk/client_v0.py b/smsdk/client_v0.py index 9f3ff98..97e91b3 100644 --- a/smsdk/client_v0.py +++ b/smsdk/client_v0.py @@ -290,9 +290,7 @@ def inner(self, normalize=True, clean_strings_in=True, clean_strings_out=True, * if not '_only' in kwargs: print('_only not specified. Selecting first 50 fields.') only_names = schema['name'].tolist()[:50] - toplevel = ['machine__source', 'starttime', 'endtime', 'total', 'record_time', 'shift', 'output'] - - kwargs['_only'] = only_names + toplevel + kwargs['_only'] = only_names else: if ('Machine' not in kwargs['_only']) and ('machine__source' not in kwargs['_only']): print("Adding Machine to _only")