diff --git a/CHANGELOG.md b/CHANGELOG.md index 9b63ff82..1b538757 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +# [2.5.0-dev.1](https://github.com/kuzzleio/kuzzle-device-manager/compare/v2.4.3...v2.5.0-dev.1) (2024-11-08) + + +### Features + +* **measure:** allow measures to be pushed on Assets via API (no devices) ([#344](https://github.com/kuzzleio/kuzzle-device-manager/issues/344)) ([c1073c1](https://github.com/kuzzleio/kuzzle-device-manager/commit/c1073c1f0ccb4cfc7cee64d86c51a4999617fd41)) + ## [2.4.3](https://github.com/kuzzleio/kuzzle-device-manager/compare/v2.4.2...v2.4.3) (2024-10-25) diff --git a/doc/2/controllers/assets/ingest-measure/index.md b/doc/2/controllers/assets/ingest-measure/index.md new file mode 100644 index 00000000..efe8b50c --- /dev/null +++ b/doc/2/controllers/assets/ingest-measure/index.md @@ -0,0 +1,88 @@ +--- +code: true +type: page +title: ingestMeasure +description: Kuzzle IoT Platform - Device Manager - Assets Controller +--- + +# ingestMeasure + +Ingest a single measure into an asset. + +--- + +## Query Syntax + +### HTTP + +```http +URL: http://kuzzle:7512/_/device-manager/:engineId/assets/:assetId/measures/:slotName +Method: POST +``` + +### Other protocols + +```js +{ + "controller": "device-manager/assets", + "action": "measureIngest", + "assetId": "", + "engineId": "", + "slotName": "" + "body": { + "dataSource": { + "id": "", + // optional: + "metadata": { + // ... + } + }, + "measuredAt": "" + "values": { + "": "", + // ... + } + }, + + // optional: + "engineGroup": "" +} +``` + +--- + +## Arguments + +- `engineId`: target engine id +- `assetId`: target asset id +- `slotName`: target measure slot name +- `engineGroup` (optional): target engine group + +## Body properties +- `dataSource`: the measure source +- `measuredAt`: the timestamp of when the measure was collected +- `values`: the measure values + +# Datasource properties + +- `id`: the measure source unique identifier +- `metadata`: (optional) additional metadata for the source + +--- + +## Response + +```js +{ + "status": 200, + "error": null, + "controller": "device-manager/assets", + "action": "measureIngest", + "requestId": "", + "result": null, +} +``` + +## Errors + +Ingesting a measure with incorrect values will throw a [ MeasureValidationError ](../../../errors/measure-validation/index.md) with the HTTP code **400**. \ No newline at end of file diff --git a/doc/2/controllers/assets/ingest-measures/index.md b/doc/2/controllers/assets/ingest-measures/index.md new file mode 100644 index 00000000..5649d92c --- /dev/null +++ b/doc/2/controllers/assets/ingest-measures/index.md @@ -0,0 +1,98 @@ +--- +code: true +type: page +title: ingestMeasures +description: Kuzzle IoT Platform - Device Manager - Assets Controller +--- + +# ingestMeasures + +Ingest measures from a data source into an asset. + +--- + +## Query Syntax + +### HTTP + +```http +URL: http://kuzzle:7512/_/device-manager/:engineId/assets/:assetId/_mMeasureIngest +Method: POST +``` + +### Other protocols + +```js +{ + "controller": "device-manager/assets", + "action": "_mMeasureIngest", + "assetId": "", + "engineId": "", + "body": { + "dataSource": { + "id": "", + // optional: + "metadata": { + // ... + } + }, + "measurements": [ + { + "slotName": "", + "measuredAt": "", + "values": { + "": "", + // ... + } + } + // ... + ] + }, + + // optional: + "engineGroup": "" +} +``` + +--- + +## Arguments + +- `engineId`: target engine id +- `assetId`: target asset id +- `engineGroup`: (optional): target engine group + +## Body properties + +- `dataSource`: the measures source +- `measurements`: the list of measurements to ingest + +# Datasource properties + +- `id`: the measure source unique identifier +- `metadata`: (optional) additional metadata for the source + +# Measurement properties + +- `slotName`: target measure slot name +- `measuredAt`: the timestamp of when the measure was collected +- `values`: the measure values + +--- + +## Response + +```js +{ + "status": 200, + "error": null, + "controller": "device-manager/assets", + "action": "mMeasureIngest", + "requestId": "", + "result": null, +} +``` + +## Errors + +Ingesting measures with incorrect values will throw a [ MeasureValidationError ](../../../errors/measure-validation/index.md) with the HTTP code **400**. \ No newline at end of file diff --git a/doc/2/controllers/models/get-asset/index.md b/doc/2/controllers/models/get-asset/index.md index ce0178c6..6e51f90a 100644 --- a/doc/2/controllers/models/get-asset/index.md +++ b/doc/2/controllers/models/get-asset/index.md @@ -16,7 +16,7 @@ Gets an asset model. ### HTTP ```http -URL: http://kuzzle:7512/_/device-manager/models/asset/:_id +URL: http://kuzzle:7512/_/device-manager/models/asset/:model Method: GET ``` diff --git a/doc/2/controllers/models/get-device/index.md b/doc/2/controllers/models/get-device/index.md index 6d5a1ff8..80cf0323 100644 --- a/doc/2/controllers/models/get-device/index.md +++ b/doc/2/controllers/models/get-device/index.md @@ -16,7 +16,7 @@ Gets a device model. ### HTTP ```http -URL: http://kuzzle:7512/_/device-manager/models/device/:id +URL: http://kuzzle:7512/_/device-manager/models/device/:model Method: GET ``` diff --git a/doc/2/controllers/models/get-measure/index.md b/doc/2/controllers/models/get-measure/index.md index bce3f647..33548149 100644 --- a/doc/2/controllers/models/get-measure/index.md +++ b/doc/2/controllers/models/get-measure/index.md @@ -16,7 +16,7 @@ Gets a measure model. ### HTTP ```http -URL: http://kuzzle:7512/_/device-manager/models/measure/:_id +URL: http://kuzzle:7512/_/device-manager/models/measure/:type Method: GET ``` diff --git a/doc/2/controllers/models/write-asset/index.md b/doc/2/controllers/models/write-asset/index.md index 830e142c..44a6460f 100644 --- a/doc/2/controllers/models/write-asset/index.md +++ b/doc/2/controllers/models/write-asset/index.md @@ -160,4 +160,8 @@ Method: POST ## Errors -Writing an asset with metadata mappings can cause conflicts, in this case a [ MappingsConflictsError ](../../../errors/mappings-conflicts/index.md) will be thrown with the HTTP code **409**. +| error | code | cause | +| ------------------------------------------------------------------------------ | ------- | --------------------------------------------------- | +| [ MappingsConflictsError ](../../../errors/mappings-conflicts/index.md) | **409** | Writing an asset with conflicting metadata mappings | +| [ MeasuresNamesDuplicatesError ](../../../errors/measures-duplicates/index.md) | **400** | Defining a measure name more than once | + diff --git a/doc/2/controllers/models/write-measure/index.md b/doc/2/controllers/models/write-measure/index.md index 781a417e..1a5212c6 100644 --- a/doc/2/controllers/models/write-measure/index.md +++ b/doc/2/controllers/models/write-measure/index.md @@ -34,6 +34,10 @@ Method: POST // Optional "valuesDetails":{ // Values details and translation + }, + // Optional + "validationSchema": { + // Valid JSON Schema } } } @@ -46,6 +50,7 @@ Method: POST - `model`: Measure model name - `valuesMappings`: Mappings of the measure values in Elasticsearch format - `valuesDetails`: (optional) Measurement translations and units +- `validationSchema`: (optional) Measurement validation JSON schema --- diff --git a/doc/2/errors/mappings-conflicts/index.md b/doc/2/errors/mappings-conflicts/index.md index ae301694..71f3aedf 100644 --- a/doc/2/errors/mappings-conflicts/index.md +++ b/doc/2/errors/mappings-conflicts/index.md @@ -2,7 +2,7 @@ code: false type: page title: Mappings Conflicts -description: Mappings Conflicts +description: Mappings Conflicts | Kuzzle Documentation --- # Mappings Conflicts diff --git a/doc/2/errors/measure-validation/index.md b/doc/2/errors/measure-validation/index.md new file mode 100644 index 00000000..aee65fac --- /dev/null +++ b/doc/2/errors/measure-validation/index.md @@ -0,0 +1,61 @@ +--- +code: false +type: page +title: Measure Validation +description: Measure Validation | Kuzzle Documentation +--- + +# Measure Validation + +A `MeasureValidationError` is thrown when the provided measures values could not be validated by the JSON schema. It can occur on creation of a measure. + +**HTTP status**: 400 + +**Additional Properties:** + +| property | type | description | +| -------- | ---------------- | ---------------------------------------------------- | +| `errors` | array of objects | List of invalid data from measures, by measures names. | + +Here is an example of a `errors` field: +```js +[ + { + "measureName": "magiculeExt", + "validationErrors": [ + { + "instancePath": "/magicule", + "schemaPath": "#/properties/magicule/type", + "keyword": "type", + "params": { + "type": "integer" + }, + "message": "must be integer" + } + ] + }, + { + "measureName": "magiculeInt", + "validationErrors": [ + { + "instancePath": "/magicule", + "schemaPath": "#/properties/magicule/type", + "keyword": "type", + "params": { + "type": "integer" + }, + "message": "must be integer" + } + ] + } +] +``` + +Errors fields: +| field | type | description | +| -------------------- | ---------------- | ------------------------------------------------ | +| `measureName` | string | The measure name where validation errors occured | +| `validationErrors` | array of objects | The list of validation errors (AJV formated) | + + +The validation errors array contain standard AJV errors, please refer to their documentation about [errors](https://ajv.js.org/api.html#error-objects) for more informations. `instancePath`, in our case, refer to the model name. \ No newline at end of file diff --git a/doc/2/errors/measures-duplicates/index.md b/doc/2/errors/measures-duplicates/index.md new file mode 100644 index 00000000..6d8e6dd8 --- /dev/null +++ b/doc/2/errors/measures-duplicates/index.md @@ -0,0 +1,18 @@ +--- +code: false +type: page +title: Measures Names Duplicates +description: Measures Names Duplicates | Kuzzle Documentation +--- + +# Measures Duplicates + +A `MeasuresNamesDuplicatesError` is thrown when one or multiple measures names is defined more than once inside the same model. + +**HTTP status**: 400 + +**Additional Properties:** + +| property | type | description | +| ------------ | --------------- | --------------------------------- | +| `duplicates` | array of string | List of duplicated measures names | \ No newline at end of file diff --git a/lib/modules/asset/AssetsController.ts b/lib/modules/asset/AssetsController.ts index 808a94ad..f6207498 100644 --- a/lib/modules/asset/AssetsController.ts +++ b/lib/modules/asset/AssetsController.ts @@ -1,11 +1,17 @@ import { + BadRequestError, ControllerDefinition, HttpStream, + JSONObject, KuzzleError, KuzzleRequest, } from "kuzzle"; -import { MeasureExporter } from "../measure/"; +import { + AskMeasureSourceIngest, + DecodedMeasurement, + MeasureExporter, +} from "../measure/"; import { DeviceManagerPlugin, InternalCollection } from "../plugin"; import { DigitalTwinExporter, EmbeddedMeasure } from "../shared"; @@ -26,6 +32,17 @@ import { ApiAssetMGetLastMeasuresResult, ApiAssetGetLastMeasuredAtResult, } from "./types/AssetApi"; +import { isSourceApi } from "../measure/types/MeasureSources"; +import { getValidator } from "../shared/utils/AJValidator"; +import { ask } from "kuzzle-plugin-commons"; +import { toApiTarget } from "../measure/MeasureTargetBuilder"; +import { DATA_SOURCE_METADATA_TYPE } from "../measure/MeasureSourcesBuilder"; +import { + MeasureValidationError, + MeasureValidationChunks, +} from "../measure/MeasureValidationError"; +import { AskModelAssetGet } from "../model"; +import { AssetContent } from "./exports"; export class AssetsController { public definition: ControllerDefinition; @@ -114,6 +131,24 @@ export class AssetsController { }, ], }, + mMeasureIngest: { + handler: this.mMeasureIngest.bind(this), + http: [ + { + path: "device-manager/:engineId/assets/:assetId/_mMeasureIngest", + verb: "post", + }, + ], + }, + measureIngest: { + handler: this.measureIngest.bind(this), + http: [ + { + path: "device-manager/:engineId/assets/:assetId/measures/:slotName", + verb: "post", + }, + ], + }, exportMeasures: { handler: this.exportMeasures.bind(this), http: [ @@ -383,6 +418,179 @@ export class AssetsController { return response; } + /** + * + * @param indexId The asset index + * @param assetId The target asset ID + * @param measureName The measureName to get the type from + * @param engineGroup The target engine group + * @returns The measure type if used in the asset, null otherwise + * @throws If the asset does not exists + */ + private async getTypeFromMeasureSlot( + indexId: string, + assetId: string, + measureName: string, + engineGroup: string, + ) { + let asset: AssetContent; + try { + const assetDocument = + await this.plugin.context.accessors.sdk.document.get( + indexId, + InternalCollection.ASSETS, + assetId, + ); + + asset = assetDocument._source; + } catch (error) { + throw new BadRequestError( + `Asset "${assetId}" does not exists on index "${indexId}"`, + ); + } + + const assetModel = await ask( + "ask:device-manager:model:asset:get", + { + engineGroup, + model: asset.model, + }, + ); + + return ( + assetModel.asset.measures.find((elt) => elt.name === measureName)?.type ?? + null + ); + } + + async mMeasureIngest(request: KuzzleRequest) { + const assetId = request.getString("assetId"); + const indexId = request.getString("engineId"); + const engineGroup = request.getString("engineGroup", "commons"); + const rawMeasurements = request.getBodyArray("measurements"); + const source = request.getBodyObject("dataSource"); + source.type = DATA_SOURCE_METADATA_TYPE.API; + + if (!isSourceApi(source)) { + throw new BadRequestError( + "The provided data source does not match the API source format", + ); + } + + const measurements = rawMeasurements.map((elt) => { + return { + measureName: elt.slotName, + measuredAt: elt.measuredAt, + type: elt.type, + values: elt.values, + }; + }) as DecodedMeasurement[]; + + const target = toApiTarget(indexId, assetId, engineGroup); + + const errors: MeasureValidationChunks[] = []; + for (const measure of measurements) { + const type = await this.getTypeFromMeasureSlot( + indexId, + assetId, + measure.measureName, + engineGroup, + ); + + const validator = getValidator(type); + + if (validator) { + const valid = validator(measure.values); + + if (!valid) { + errors.push({ + measureName: measure.measureName, + validationErrors: validator.errors ?? [], + }); + } + } + } + + if (errors.length > 0) { + throw new MeasureValidationError( + "The provided measures do not comply with their respective schemas", + errors, + ); + } + + await ask("device-manager:measures:sourceIngest", { + measurements, + payloadUuids: [], + source, + target, + }); + } + + async measureIngest(request: KuzzleRequest) { + const assetId = request.getString("assetId"); + const indexId = request.getString("engineId"); + const measureName = request.getString("slotName"); + const engineGroup = request.getString("engineGroup", "commons"); + const source = request.getBodyObject("dataSource"); + source.type = DATA_SOURCE_METADATA_TYPE.API; + + if (!isSourceApi(source)) { + throw new BadRequestError( + "The provided data source does not match the API source format", + ); + } + + const measuredAt = request.getBodyNumber("measuredAt"); + const values = request.getBodyObject("values"); + + const type = await this.getTypeFromMeasureSlot( + indexId, + assetId, + measureName, + engineGroup, + ); + + if (!type) { + throw new BadRequestError( + `Slot name ${measureName} does not exist on Asset ${assetId}`, + ); + } + + const measurement = { + measureName, + measuredAt, + type, + values, + } as DecodedMeasurement; + + const target = toApiTarget(indexId, assetId, engineGroup); + + const validator = getValidator(type); + + if (validator) { + const valid = validator(values); + + if (!valid) { + throw new MeasureValidationError( + "The provided measure does not respect its schema", + [ + { + measureName: measureName, + validationErrors: validator.errors ?? [], + }, + ], + ); + } + } + + await ask("device-manager:measures:sourceIngest", { + measurements: [measurement], + payloadUuids: [], + source, + target, + }); + } + async exportMeasures(request: KuzzleRequest) { const engineId = request.getString("engineId"); diff --git a/lib/modules/asset/types/AssetApi.ts b/lib/modules/asset/types/AssetApi.ts index 3dd8521e..2bac7454 100644 --- a/lib/modules/asset/types/AssetApi.ts +++ b/lib/modules/asset/types/AssetApi.ts @@ -1,6 +1,6 @@ import { JSONObject, KDocument, KHit, SearchResult } from "kuzzle-sdk"; -import { MeasureContent } from "../../../modules/measure"; +import { MeasureContent, Measurement } from "../../../modules/measure"; import { ApiDigitalTwinGetLastMeasuredAtRequest, ApiDigitalTwinGetLastMeasuredAtResult, @@ -16,6 +16,7 @@ import { import { AssetContent } from "./AssetContent"; type AssetsControllerName = "device-manager/assets"; +import { ApiMeasureSource } from "../../measure/types/MeasureSources"; interface AssetsControllerRequest { controller: AssetsControllerName; @@ -142,6 +143,43 @@ export type ApiAssetGetMeasuresResult = { total: number; }; +type TypelessApiMeasureSource = Omit; + +export interface ApiAssetMeasureIngestRequest extends AssetsControllerRequest { + action: "measureIngest"; + + assetId: string; + + engineId: string; + engineGroup?: string; + slotName: string; + + body: { + dataSource: TypelessApiMeasureSource; + measuredAt: number; + values: JSONObject; + }; +} +export type ApiAssetMeasureIngestResult = void; + +type APIDecodedMeasurement = Omit & { slotName: string }; + +export interface ApiAssetmMeasureIngestRequest extends AssetsControllerRequest { + action: "mMeasureIngest"; + + assetId: string; + + engineId: string; + engineGroup?: string; + + body: { + dataSource: TypelessApiMeasureSource; + measurements: APIDecodedMeasurement[]; + }; +} + +export type ApiAssetmMeasureIngestResult = void; + export type ApiAssetGetLastMeasuresRequest = ApiDigitalTwinGetLastMeasuresRequest; export type ApiAssetGetLastMeasuresResult = ApiDigitalTwinGetLastMeasuresResult; diff --git a/lib/modules/device/DevicesController.ts b/lib/modules/device/DevicesController.ts index dd2acdf3..91465d78 100644 --- a/lib/modules/device/DevicesController.ts +++ b/lib/modules/device/DevicesController.ts @@ -446,6 +446,10 @@ export class DevicesController { return results.reduce( (accumulator, result) => { + if (result.origin.type !== "device") { + return accumulator; + } + const measure: EmbeddedMeasure = { measuredAt: result.measuredAt, name: result.origin.measureName, @@ -482,6 +486,10 @@ export class DevicesController { for (const [deviceId, measures] of Object.entries(results)) { response[deviceId] = measures.reduce( (accumulator, result) => { + if (result.origin.type !== "device") { + return accumulator; + } + const measure: EmbeddedMeasure = { measuredAt: result.measuredAt, name: result.origin.measureName, diff --git a/lib/modules/measure/MeasureService.ts b/lib/modules/measure/MeasureService.ts index 1f0ddc11..eaf1c041 100644 --- a/lib/modules/measure/MeasureService.ts +++ b/lib/modules/measure/MeasureService.ts @@ -17,21 +17,56 @@ import { BaseService, Metadata, keepStack, lock, objectDiff } from "../shared"; import { DecodedMeasurement, MeasureContent } from "./types/MeasureContent"; import { AskMeasureIngest, + AskMeasureSourceIngest, EventMeasurePersistBefore, + EventMeasurePersistSourceBefore, EventMeasureProcessAfter, EventMeasureProcessBefore, + EventMeasureProcessSourceAfter, + EventMeasureProcessSourceBefore, TenantEventMeasurePersistBefore, + TenantEventMeasurePersistSourceBefore, TenantEventMeasureProcessAfter, TenantEventMeasureProcessBefore, + TenantEventMeasureProcessSourceAfter, + TenantEventMeasureProcessSourceBefore, } from "./types/MeasureEvents"; +import { ApiMeasureSource, isSourceApi } from "./types/MeasureSources"; +import { apiSourceToOriginApi, toDeviceSource } from "./MeasureSourcesBuilder"; +import { AskModelAssetGet } from "../model"; +import { ApiMeasureTarget, isTargetApi } from "./types/MeasureTarget"; +import { toDeviceTarget } from "./MeasureTargetBuilder"; export class MeasureService extends BaseService { constructor(plugin: DeviceManagerPlugin) { super(plugin); + onAsk( + "device-manager:measures:sourceIngest", + + async (payload) => { + if (!payload) { + return; + } + + if (isSourceApi(payload.source) && isTargetApi(payload.target)) { + await this.ingestApi( + payload.source, + payload.target, + payload.measurements, + payload.payloadUuids, + ); + } + }, + ); + onAsk( "device-manager:measures:ingest", async (payload) => { + if (!payload) { + return; + } + await this.ingest( payload.device, payload.measurements, @@ -42,6 +77,155 @@ export class MeasureService extends BaseService { ); } + /** + * Register new measures from an API, updates : + * - asset + * - engine measures + * + * This method represents the ingestion pipeline: + * - trigger events `before` (measure enrichment) + * - save documents (measures and asset) + * - trigger events `after` + */ + public async ingestApi( + source: ApiMeasureSource, + target: ApiMeasureTarget, + measurements: DecodedMeasurement[], + payloadUuids: string[], + ) { + const { id: dataSourceId } = source; + const { indexId, assetId } = target; + + if (!measurements) { + this.app.log.warn( + `No measurements provided for "${dataSourceId}" API measures ingest`, + ); + return; + } + + const assetDocument = await this.findAsset(indexId, assetId); + + if (!assetDocument) { + throw new BadRequestError( + `Asset "${assetId}" does not exists on index "${indexId}"`, + ); + } + + const asset = assetDocument._source; + + const measures = await this.buildApiMeasures( + source, + assetDocument, + measurements, + payloadUuids, + target.engineGroup, + ); + + asset.measures ||= {}; + + /** + * Event before starting to process new measures. + * + * Useful to enrich measures before they are saved. + */ + await this.app.trigger( + "device-manager:measures:process:sourceBefore", + { asset, measures, source, target }, + ); + + if (indexId) { + await this.app.trigger( + `engine:${indexId}:device-manager:measures:process:sourceBefore`, + { asset, measures, source, target }, + ); + } + + const assetStates = this.updateAssetMeasures(assetDocument, measures); + + await this.app.trigger( + "device-manager:measures:persist:sourceBefore", + { asset, measures, source, target }, + ); + + if (indexId) { + await this.app.trigger( + `engine:${indexId}:device-manager:measures:persist:sourceBefore`, + { asset, measures, source, target }, + ); + } + + const promises: Promise[] = []; + + if (indexId) { + promises.push( + this.sdk.document + .mCreate( + indexId, + InternalCollection.MEASURES, + measures.map((measure) => ({ body: measure })), + ) + .then(({ errors }) => { + if (errors.length !== 0) { + throw new BadRequestError( + `Cannot save measures: ${errors[0].reason}`, + ); + } + }), + ); + + // @todo potential race condition if 2 differents device are linked + // to the same asset and get processed at the same time + // asset measures update could be protected by mutex + promises.push( + this.sdk.document + .update( + indexId, + InternalCollection.ASSETS, + assetId, + asset, + ) + .catch((error) => { + throw keepStack( + error, + new BadRequestError( + `Cannot update asset "${assetId}": ${error.message}`, + ), + ); + }), + ); + + promises.push( + historizeAssetStates( + assetStates, + indexId, + JSON.parse(JSON.stringify(asset.metadata)), + asset.metadata, + ), + ); + } + + await Promise.all(promises); + + /** + * Event at the end of the measure process pipeline. + * + * Useful to trigger business rules like alerts + * + * @todo test this + */ + await this.app.trigger( + "device-manager:measures:process:sourceAfter", + { asset, measures, source, target }, + ); + + if (indexId) { + await this.app.trigger( + `engine:${indexId}:device-manager:measures:process:sourceAfter`, + { asset, measures, source, target }, + ); + } + } + /** * Register new measures from a device, updates : * - admin device @@ -56,6 +240,8 @@ export class MeasureService extends BaseService { * - trigger events `before` (measure enrichment) * - save documents (measures, device and asset) * - trigger events `after` + * + * @deprecated */ public async ingest( device: KDocument, @@ -71,11 +257,13 @@ export class MeasureService extends BaseService { return; } - const engineId = device._source.engineId; - const asset = await this.tryGetLinkedAsset( - engineId, - device._source.assetId, - ); + const { engineId, reference, model, assetId, lastMeasuredAt } = + device._source; + + const asset = assetId ? await this.findAsset(engineId, assetId) : null; + + const assetContent = asset?._source; + const originalAssetMetadata: Metadata = asset === null ? {} @@ -94,6 +282,19 @@ export class MeasureService extends BaseService { asset._source.measures ||= {}; } + const source = toDeviceSource( + device._id, + reference, + model, + device._source.metadata, + lastMeasuredAt, + ); + + const target = toDeviceTarget( + device._source.engineId, + device._source.assetId ?? undefined, + ); + /** * Event before starting to process new measures. * @@ -111,6 +312,25 @@ export class MeasureService extends BaseService { ); } + /** + * Here are the new process before triggers using sources + * + * Event before starting to process new measures. + * + * Useful to enrich measures before they are saved. + */ + await this.app.trigger( + "device-manager:measures:process:sourceBefore", + { asset: assetContent, measures, source, target }, + ); + + if (engineId) { + await this.app.trigger( + `engine:${engineId}:device-manager:measures:process:sourceBefore`, + { asset: assetContent, measures, source, target }, + ); + } + await this.updateDeviceMeasures(device, measures); let assetStates = new Map>(); @@ -134,7 +354,22 @@ export class MeasureService extends BaseService { ); } - const promises = []; + /** + * Here are the new persist before triggers using sources + */ + await this.app.trigger( + "device-manager:measures:persist:sourceBefore", + { asset: assetContent, measures, source, target }, + ); + + if (engineId) { + await this.app.trigger( + `engine:${engineId}:device-manager:measures:persist:sourceBefore`, + { asset: assetContent, measures, source, target }, + ); + } + + const promises: Promise[] = []; promises.push( this.sdk.document @@ -246,6 +481,27 @@ export class MeasureService extends BaseService { { asset, device, measures }, ); } + + /** + * Here are the new process after triggers using sources + * + * Event at the end of the measure process pipeline. + * + * Useful to trigger business rules like alerts + * + * @todo test this + */ + await this.app.trigger( + "device-manager:measures:process:sourceAfter", + { asset: assetContent, measures, source, target }, + ); + + if (engineId) { + await this.app.trigger( + `engine:${engineId}:device-manager:measures:process:sourceAfter`, + { asset: assetContent, measures, source, target }, + ); + } }); } @@ -260,7 +516,7 @@ export class MeasureService extends BaseService { let lastMeasuredAt = 0; for (const measurement of measurements) { - if (measurement.origin.type === "computed") { + if (measurement.origin.type !== "device") { continue; } @@ -319,7 +575,7 @@ export class MeasureService extends BaseService { continue; } - const measureName = measurement.asset.measureName; + const measureName = measurement.asset?.measureName ?? null; // The measurement was not present in the asset device links so it should // not be saved in the asset measures if (measureName === null) { @@ -358,6 +614,54 @@ export class MeasureService extends BaseService { return assetStates; } + /** + * Build the measures document received from the API + * + * @param source The API data source + * @param asset The target asset to build the measure for + * @param measure The decoded raw measures + * @param payloadUuids The uuid's of the payloads used to create the measure + * + * @returns A MeasurementContent builded from parameters + */ + private async buildApiMeasures( + source: ApiMeasureSource, + asset: KDocument, + measures: DecodedMeasurement[], + payloadUuids: string[], + engineGroup?: string, + ): Promise { + const apiMeasures: MeasureContent[] = []; + + for (const measure of measures) { + let assetContext = null; + const isInModel = await this.isMeasureNameInModel( + measure.measureName, + asset._source.model, + engineGroup, + ); + + if (isInModel) { + assetContext = AssetSerializer.measureContext( + asset, + measure.measureName, + ); + } + + const measureSource = apiSourceToOriginApi(source, payloadUuids); + + apiMeasures.push({ + asset: assetContext, + measuredAt: measure.measuredAt, + origin: measureSource, + type: measure.type, + values: measure.values, + }); + } + + return apiMeasures; + } + /** * Build the measures documents to save */ @@ -371,16 +675,21 @@ export class MeasureService extends BaseService { for (const measurement of measurements) { // @todo check if measure type exists - const assetMeasureName = this.tryFindAssetMeasureName( - device, - asset, - measurement.measureName, - ); + let assetContext = null; + if (asset) { + const assetMeasureName = this.findAssetMeasureNameFromDevice( + device._id, + measurement.measureName, + asset._source, + ); - const assetContext = - asset === null || assetMeasureName === null - ? null - : AssetSerializer.measureContext(asset, assetMeasureName); + if (assetMeasureName) { + assetContext = AssetSerializer.measureContext( + asset, + assetMeasureName, + ); + } + } const measureContent: MeasureContent = { asset: assetContext, @@ -406,14 +715,17 @@ export class MeasureService extends BaseService { ); } - private async tryGetLinkedAsset( + /** + * Find an asset by its ID and its engine ID. + * + * @param engineId The target index ID + * @param assetId the target asset ID + * @returns The asset or null if not found + */ + private async findAsset( engineId: string, assetId: string, - ): Promise> { - if (!assetId) { - return null; - } - + ): Promise | null> { try { const asset = await this.sdk.document.get( engineId, @@ -430,36 +742,68 @@ export class MeasureService extends BaseService { } /** - * Retrieve the measure name for the asset + * Check if the asset measure name is associated to the asset model + * + * @param measureName The measure name to check + * @param model The asset model the measureName should belong + * + * @returns True if the asset measure name belongs to the asset model, false otherwise + * @throws If the model does not exists */ - private tryFindAssetMeasureName( - device: KDocument, - asset: KDocument, - deviceMeasureName: string, - ): string | null { - if (!asset) { - return null; + private async isMeasureNameInModel( + measureName: string, + model: string, + engineGroup = "commons", + ): Promise { + const assetModel = await ask( + "ask:device-manager:model:asset:get", + { + engineGroup, + model: model, + }, + ); + + if (!assetModel) { + throw new BadRequestError(`Model "${model}" does not exists`); } - const deviceLink = asset._source.linkedDevices.find( - (link) => link._id === device._id, + const assetMeasureName = assetModel.asset.measures.find( + (m) => m.name === measureName, ); - if (!deviceLink) { + return assetMeasureName?.name ? true : false; + } + + /** + * Find the asset measure name from the device and its measure type + * + * @param deviceId The source device ID + * @param measureType The device measure type + * @param asset The asset the device is linked to + * + * @returns The asset measure name or null if it does not belong to the link + * @throws If the device is not linked to the asset + */ + private findAssetMeasureNameFromDevice( + deviceId: string, + measureType: string, + asset: AssetContent, + ): string | null { + const linkedDevice = asset.linkedDevices.find( + (link) => link._id === deviceId, + ); + + if (!linkedDevice) { throw new BadRequestError( - `Device "${device._id}" is not linked to asset "${asset._id}"`, + `Device "${deviceId}" is not linked to "${asset.model}" asset: "${asset.reference}"`, ); } - const measureName = deviceLink.measureNames.find( - (m) => m.device === deviceMeasureName, + const assetMeasureName = linkedDevice.measureNames.find( + (m) => m.device === measureType, ); - // The measure is decoded by the device but is not linked to the asset - if (!measureName) { - return null; - } - return measureName.asset; + return assetMeasureName?.asset ?? null; } } diff --git a/lib/modules/measure/MeasureSourcesBuilder.ts b/lib/modules/measure/MeasureSourcesBuilder.ts new file mode 100644 index 00000000..75b69eea --- /dev/null +++ b/lib/modules/measure/MeasureSourcesBuilder.ts @@ -0,0 +1,54 @@ +import { Metadata } from "../shared"; +import { MeasureOriginApi, MeasureOriginDevice } from "./types/MeasureContent"; +import { ApiMeasureSource, DeviceMeasureSource } from "./types/MeasureSources"; + +export const enum DATA_SOURCE_METADATA_TYPE { + API = "api", + DEVICE = "device", +} + +export function apiSourceToOriginApi( + source: ApiMeasureSource, + payloadUuids: string[], +): MeasureOriginApi { + return { + _id: source.id, + apiMetadata: source.metadata, + payloadUuids: payloadUuids, + type: DATA_SOURCE_METADATA_TYPE.API, + }; +} + +export function toDeviceSource( + dataSourceId: string, + reference: string, + model: string, + metadata?: Metadata, + lastMeasuredAt?: number, +): DeviceMeasureSource { + return { + id: dataSourceId, + lastMeasuredAt, + metadata, + model, + reference, + type: DATA_SOURCE_METADATA_TYPE.DEVICE, + }; +} + +export function deviceSourceToOriginDevice( + source: DeviceMeasureSource, + measureName: string, + payloadUuids: string[], +): MeasureOriginDevice { + const { id: dataSourceId, model, reference } = source; + return { + _id: dataSourceId, + deviceModel: model, + measureName, + metadata: source.metadata, + payloadUuids, + reference: reference, + type: DATA_SOURCE_METADATA_TYPE.DEVICE, + }; +} diff --git a/lib/modules/measure/MeasureTargetBuilder.ts b/lib/modules/measure/MeasureTargetBuilder.ts new file mode 100644 index 00000000..05f76832 --- /dev/null +++ b/lib/modules/measure/MeasureTargetBuilder.ts @@ -0,0 +1,25 @@ +import { ApiMeasureTarget, DeviceMeasureTarget } from "./types/MeasureTarget"; + +export function toApiTarget( + indexId: string, + assetId: string, + engineGroup?: string, +): ApiMeasureTarget { + return { + assetId, + engineGroup, + indexId, + type: "api", + }; +} + +export function toDeviceTarget( + indexId: string, + assetId?: string, +): DeviceMeasureTarget { + return { + assetId, + indexId, + type: "device", + }; +} diff --git a/lib/modules/measure/MeasureValidationError.ts b/lib/modules/measure/MeasureValidationError.ts new file mode 100644 index 00000000..1e2d45b2 --- /dev/null +++ b/lib/modules/measure/MeasureValidationError.ts @@ -0,0 +1,32 @@ +import { ErrorObject } from "ajv"; +import { KuzzleError } from "kuzzle"; + +export interface MeasureValidationChunks { + measureName: string; + validationErrors: ErrorObject[]; +} + +export class MeasureValidationError extends KuzzleError { + private errors: MeasureValidationChunks[]; + constructor( + message: string, + errors: MeasureValidationChunks[], + id?: string, + code?: number, + ) { + super(message, 400, id, code); + this.errors = errors; + } + + get name() { + return "MeasureValidationError"; + } + + public toJSON() { + const json = super.toJSON(); + + json.errors = this.errors; + + return json; + } +} diff --git a/lib/modules/measure/collections/measuresMappings.ts b/lib/modules/measure/collections/measuresMappings.ts index 1c9626b7..c25aa303 100644 --- a/lib/modules/measure/collections/measuresMappings.ts +++ b/lib/modules/measure/collections/measuresMappings.ts @@ -57,6 +57,11 @@ export const measuresMappings = { }, }, + apiMetadata: { + dynamic: "false", + properties: {}, + }, + payloadUuids: { type: "keyword" }, deviceModel: { type: "keyword" }, diff --git a/lib/modules/measure/types/MeasureContent.ts b/lib/modules/measure/types/MeasureContent.ts index b04e2e89..b256719a 100644 --- a/lib/modules/measure/types/MeasureContent.ts +++ b/lib/modules/measure/types/MeasureContent.ts @@ -3,26 +3,25 @@ import { JSONObject, KDocumentContent } from "kuzzle-sdk"; import { Metadata } from "../../../modules/shared"; import { AssetMeasureContext } from "../../../modules/asset"; -export type MeasureOriginDevice = { +interface AbstractMeasureOrigin { /** * Origin of the measure */ - type: "device"; + type: string; /** - * Name of the measure in the device + * Payload uuids that were used to create this measure. */ - measureName: string; + payloadUuids: Array; /** - * Origin device metadata + * Custom metadata provided by the user */ - deviceMetadata?: Metadata; + metadata?: Metadata; +} - /** - * Payload uuids that were used to create this measure. - */ - payloadUuids: Array; +export interface MeasureOriginDevice extends AbstractMeasureOrigin { + type: "device"; /** * Model of the device @@ -31,18 +30,28 @@ export type MeasureOriginDevice = { */ deviceModel: string; + /** + * Name of the measure + */ + measureName: string; + /** * Reference of the device */ reference: string; + /** + * Origin device metadata + */ + deviceMetadata?: Metadata; + /** * Device ID */ _id: string; -}; +} -export type MeasureOriginComputed = { +export interface MeasureOriginComputed extends AbstractMeasureOrigin { /** * Computed measures are not automatically added into the asset and device * documents at the end of the ingestion pipeline. @@ -50,22 +59,31 @@ export type MeasureOriginComputed = { type: "computed"; /** - * String that identify the rule used to compute the measure + * Name of the measure */ - _id: string; + measureName: string; /** - * Name of the measure + * String that identify the rule used to compute the measure */ - measureName: string; + _id: string; +} + +export interface MeasureOriginApi extends AbstractMeasureOrigin { + type: "api"; + + apiMetadata?: Metadata; /** - * Payload uuids that were used to create this measure for traceability. + * API ID */ - payloadUuids: Array; -}; + _id: string; +} -export type MeasureOrigin = MeasureOriginDevice | MeasureOriginComputed; +export type MeasureOrigin = + | MeasureOriginDevice + | MeasureOriginComputed + | MeasureOriginApi; /** * Represents the content of a measure document. diff --git a/lib/modules/measure/types/MeasureDefinition.ts b/lib/modules/measure/types/MeasureDefinition.ts index 3015227a..468c9dbd 100644 --- a/lib/modules/measure/types/MeasureDefinition.ts +++ b/lib/modules/measure/types/MeasureDefinition.ts @@ -1,3 +1,4 @@ +import { SchemaObject } from "ajv"; import { JSONObject } from "kuzzle-sdk"; /* * @@ -30,6 +31,20 @@ export interface MeasureValuesDetails { * Represents a measure definition registered by the Device Manager * * @example + * { + * valuesMappings: { temperature: { type: 'float' } }, + * validationSchema: { + type: "object", + properties: { + temperature: { + type: "number", + multipleOf: 0.01 + } + }, + required: ["temperature"], + additionalProperties: false + } + * }, *{ * valuesMappings: { temperature: { type: "float" } }, * valuesDetails: { @@ -52,6 +67,11 @@ export interface MeasureDefinition { * Mappings for the measurement values in order to index the fields */ valuesMappings: JSONObject; + /** + * Schema to validate the values against + */ + validationSchema?: SchemaObject; + valuesDetails?: { [valueName: string]: MeasureLocales; }; diff --git a/lib/modules/measure/types/MeasureEvents.ts b/lib/modules/measure/types/MeasureEvents.ts index 32100cb3..411a51db 100644 --- a/lib/modules/measure/types/MeasureEvents.ts +++ b/lib/modules/measure/types/MeasureEvents.ts @@ -5,9 +5,13 @@ import { AssetContent } from "../../../modules/asset"; import { Metadata } from "../../../modules/shared"; import { DecodedMeasurement, MeasureContent } from "./MeasureContent"; +import { MeasureSource } from "./MeasureSources"; +import { MeasureTarget } from "./MeasureTarget"; /** * @internal + * + * @deprecated Replaced by new Ask implementing data sources */ export type AskMeasureIngest = { name: "device-manager:measures:ingest"; @@ -22,10 +26,28 @@ export type AskMeasureIngest = { result: void; }; +/** + * @internal + */ +export type AskMeasureSourceIngest = { + name: "device-manager:measures:sourceIngest"; + + payload: { + source: MeasureSource; + target: MeasureTarget; + measurements: DecodedMeasurement[]; + payloadUuids: string[]; + }; + + result: void; +}; + /** * Event before starting to process new measures. * * Useful to enrich measures before they are saved. + * + * @deprecated Replaced by new triggers implementing data sources */ export type EventMeasureProcessBefore = { name: "device-manager:measures:process:before"; @@ -39,10 +61,30 @@ export type EventMeasureProcessBefore = { ]; }; +/** + * Event before starting to process new measures from data source. + * + * Useful to enrich measures before they are saved. + */ +export type EventMeasureProcessSourceBefore = { + name: "device-manager:measures:process:sourceBefore"; + + args: [ + { + source: MeasureSource; + target: MeasureTarget; + asset?: AssetContent; + measures: MeasureContent[]; + }, + ]; +}; + /** * Tenant event before starting to process new measures. * * Useful to enrich measures before they are saved. + * + * @deprecated Replaced by new triggers implementing data sources */ export type TenantEventMeasureProcessBefore = { name: `engine:${string}:device-manager:measures:process:before`; @@ -56,9 +98,29 @@ export type TenantEventMeasureProcessBefore = { ]; }; +/** + * Tenant event before starting to process new measures from data source. + * + * Useful to enrich measures before they are saved. + */ +export type TenantEventMeasureProcessSourceBefore = { + name: `engine:${string}:device-manager:measures:process:sourceBefore`; + + args: [ + { + source: MeasureSource; + target: MeasureTarget; + asset?: AssetContent; + measures: MeasureContent[]; + }, + ]; +}; + /** * Event triggered after updating device and asset with new measures but * before persistence in database. + * + * @deprecated Replaced by new triggers implementing data sources */ export type EventMeasurePersistBefore = { name: "device-manager:measures:persist:before"; @@ -72,9 +134,28 @@ export type EventMeasurePersistBefore = { ]; }; +/** + * Event triggered after updating the data source and asset with new measures but + * before persistence in database. + */ +export type EventMeasurePersistSourceBefore = { + name: "device-manager:measures:persist:sourceBefore"; + + args: [ + { + source: MeasureSource; + target: MeasureTarget; + asset?: AssetContent; + measures: MeasureContent[]; + }, + ]; +}; + /** * Tenant event triggered after updating device and asset with new measures but * before persistence in database. + * + * @deprecated Replaced by new triggers implementing data sources */ export type TenantEventMeasurePersistBefore = { name: `engine:${string}:device-manager:measures:persist:before`; @@ -88,6 +169,28 @@ export type TenantEventMeasurePersistBefore = { ]; }; +/** + * Tenant event triggered after updating the data source and asset with new measures but + * before persistence in database. + */ +export type TenantEventMeasurePersistSourceBefore = { + name: `engine:${string}:device-manager:measures:persist:sourceBefore`; + + args: [ + { + source: MeasureSource; + target: MeasureTarget; + asset?: AssetContent; + measures: MeasureContent[]; + }, + ]; +}; + +/** + * Event after processing new measures. + * + * @deprecated Replaced by new triggers implementing data sources + */ export type EventMeasureProcessAfter = { name: "device-manager:measures:process:after"; @@ -100,6 +203,27 @@ export type EventMeasureProcessAfter = { ]; }; +/** + * Event after processing new measures from data source. + */ +export type EventMeasureProcessSourceAfter = { + name: "device-manager:measures:process:sourceAfter"; + + args: [ + { + source: MeasureSource; + target: MeasureTarget; + asset?: AssetContent; + measures: MeasureContent[]; + }, + ]; +}; + +/** + * Tenant event after processing new measures. + * + * @deprecated Replaced by new triggers implementing data sources + */ export type TenantEventMeasureProcessAfter = { name: `engine:${string}:device-manager:measures:process:after`; @@ -111,3 +235,20 @@ export type TenantEventMeasureProcessAfter = { }, ]; }; + +/** + * Tenant event after processing new measures from data source. + * + */ +export type TenantEventMeasureProcessSourceAfter = { + name: `engine:${string}:device-manager:measures:process:sourceAfter`; + + args: [ + { + source: MeasureSource; + target: MeasureTarget; + asset?: AssetContent; + measures: MeasureContent[]; + }, + ]; +}; diff --git a/lib/modules/measure/types/MeasureSources.ts b/lib/modules/measure/types/MeasureSources.ts new file mode 100644 index 00000000..0cab3d78 --- /dev/null +++ b/lib/modules/measure/types/MeasureSources.ts @@ -0,0 +1,53 @@ +import { Metadata } from "../../shared"; + +interface AbstractMeasureSource { + type: string; + id: string; + metadata?: Metadata; +} + +export interface DeviceMeasureSource extends AbstractMeasureSource { + type: "device"; + reference: string; + model: string; + lastMeasuredAt?: number; +} + +export interface ApiMeasureSource extends AbstractMeasureSource { + type: "api"; +} + +export function isSource(source: any): source is AbstractMeasureSource { + if (!source) { + return false; + } + + if (source.metadata !== undefined && typeof source.metadata !== "object") { + return false; + } + + return typeof source.type === "string" && typeof source.id === "string"; +} + +export function isSourceDevice(source: any): source is DeviceMeasureSource { + if (!isSource(source) && source.type !== "device") { + return false; + } + + if ( + source.lastMeasuredAt !== undefined && + typeof source.lastMeasuredAt !== "number" + ) { + return false; + } + + return ( + typeof source.reference === "string" && typeof source.model === "string" + ); +} + +export function isSourceApi(source: any): source is ApiMeasureSource { + return isSource(source) && source.type === "api"; +} + +export type MeasureSource = DeviceMeasureSource | ApiMeasureSource; diff --git a/lib/modules/measure/types/MeasureTarget.ts b/lib/modules/measure/types/MeasureTarget.ts new file mode 100644 index 00000000..8fa4cd5c --- /dev/null +++ b/lib/modules/measure/types/MeasureTarget.ts @@ -0,0 +1,45 @@ +interface AbstractMeasureTarget { + type: string; + assetId?: string; + indexId: string; +} + +export interface DeviceMeasureTarget extends AbstractMeasureTarget { + type: "device"; +} + +export interface ApiMeasureTarget extends AbstractMeasureTarget { + type: "api"; + assetId: string; + engineGroup?: string; +} + +export function isTarget(target: any): target is AbstractMeasureTarget { + if (!target) { + return false; + } + + if (target.assetId && typeof target.assetId !== "string") { + return false; + } + + return typeof target.type === "string" && typeof target.indexId === "string"; +} + +export function isTargetDevice(target: any): target is DeviceMeasureTarget { + return isTarget(target) && target.type === "device"; +} + +export function isTargetApi(target: any): target is ApiMeasureTarget { + if (!isTarget(target) && target.type !== "api") { + return false; + } + + if (target.engineGroup && typeof target.engineGroup !== "string") { + return false; + } + + return typeof target.assetId === "string"; +} + +export type MeasureTarget = DeviceMeasureTarget | ApiMeasureTarget; diff --git a/lib/modules/model/MappingsConflictsError.ts b/lib/modules/model/MappingsConflictsError.ts index 65ff4f1a..7f04cbf8 100644 --- a/lib/modules/model/MappingsConflictsError.ts +++ b/lib/modules/model/MappingsConflictsError.ts @@ -1,10 +1,11 @@ -import { JSONObject, KuzzleError } from "kuzzle"; +import { KuzzleError } from "kuzzle"; +import { ConflictChunk } from "./ModelsConflicts"; export class MappingsConflictsError extends KuzzleError { - private conflicts: JSONObject; + private conflicts: ConflictChunk[]; constructor( message: string, - conflicts: JSONObject, + conflicts: ConflictChunk[], id?: string, code?: number, ) { diff --git a/lib/modules/model/MeasuresDuplicates.ts b/lib/modules/model/MeasuresDuplicates.ts new file mode 100644 index 00000000..dace78ef --- /dev/null +++ b/lib/modules/model/MeasuresDuplicates.ts @@ -0,0 +1,17 @@ +import _ from "lodash"; +import { NamedMeasures } from "../decoder"; + +function getNamedMeasuresDuplicates(measures: NamedMeasures): string[] { + const duplicates: string[] = []; + + const groups = _.groupBy(measures, (elt) => elt.name); + for (const group in groups) { + if (groups[group].length > 1) { + duplicates.push(group); + } + } + + return duplicates; +} + +export { getNamedMeasuresDuplicates }; diff --git a/lib/modules/model/MeasuresNamesDuplicatesError.ts b/lib/modules/model/MeasuresNamesDuplicatesError.ts new file mode 100644 index 00000000..4bf7ec67 --- /dev/null +++ b/lib/modules/model/MeasuresNamesDuplicatesError.ts @@ -0,0 +1,26 @@ +import { KuzzleError } from "kuzzle"; + +export class MeasuresNamesDuplicatesError extends KuzzleError { + private duplicates: string[]; + constructor( + message: string, + duplicates: string[], + id?: string, + code?: number, + ) { + super(message, 400, id, code); + this.duplicates = duplicates; + } + + get name() { + return "MeasuresNamesDuplicatesError"; + } + + public toJSON() { + const json = super.toJSON(); + + json.duplicates = this.duplicates; + + return json; + } +} diff --git a/lib/modules/model/ModelService.ts b/lib/modules/model/ModelService.ts index d5dc9800..1ac3edf5 100644 --- a/lib/modules/model/ModelService.ts +++ b/lib/modules/model/ModelService.ts @@ -37,7 +37,13 @@ import { AskModelMeasureGet, } from "./types/ModelEvents"; import { MappingsConflictsError } from "./MappingsConflictsError"; +import { SchemaObject } from "ajv"; +import { addSchemaToCache, getAJVErrors } from "../shared/utils/AJValidator"; +import { SchemaValidationError } from "../shared/errors/SchemaValidationError"; import { MeasureValuesDetails } from "../measure"; +import { NamedMeasures } from "../decoder"; +import { getNamedMeasuresDuplicates } from "./MeasuresDuplicates"; +import { MeasuresNamesDuplicatesError } from "./MeasuresNamesDuplicatesError"; export class ModelService extends BaseService { constructor(plugin: DeviceManagerPlugin) { @@ -185,12 +191,20 @@ export class ModelService extends BaseService { defaultMetadata: JSONObject, metadataDetails: MetadataDetails, metadataGroups: MetadataGroups, - measures: AssetModelContent["asset"]["measures"], + measures: NamedMeasures, tooltipModels: TooltipModels, ): Promise> { if (Inflector.pascalCase(model) !== model) { throw new BadRequestError(`Asset model "${model}" must be PascalCase.`); } + const duplicates = getNamedMeasuresDuplicates(measures); + + if (duplicates.length > 0) { + throw new MeasuresNamesDuplicatesError( + "Asset model measures contain one or multiple duplicate measure name", + duplicates, + ); + } const modelContent: AssetModelContent = { asset: { @@ -294,12 +308,21 @@ export class ModelService extends BaseService { defaultMetadata: JSONObject, metadataDetails: MetadataDetails, metadataGroups: MetadataGroups, - measures: DeviceModelContent["device"]["measures"], + measures: NamedMeasures, ): Promise> { if (Inflector.pascalCase(model) !== model) { throw new BadRequestError(`Device model "${model}" must be PascalCase.`); } + const duplicates = getNamedMeasuresDuplicates(measures); + + if (duplicates.length > 0) { + throw new MeasuresNamesDuplicatesError( + "Device model measures contain one or multiple duplicate measure name", + duplicates, + ); + } + const modelContent: DeviceModelContent = { device: { defaultMetadata, @@ -344,6 +367,7 @@ export class ModelService extends BaseService { async writeMeasure( type: string, valuesMappings: JSONObject, + validationSchema?: SchemaObject, valuesDetails?: MeasureValuesDetails, ): Promise> { const modelContent: MeasureModelContent = { @@ -355,6 +379,18 @@ export class ModelService extends BaseService { type: "measure", }; + if (validationSchema) { + try { + addSchemaToCache(type, validationSchema); + modelContent.measure.validationSchema = validationSchema; + } catch (error) { + throw new SchemaValidationError( + "Provided schema is not valid", + getAJVErrors(), + ); + } + } + const conflicts = await ask( "ask:device-manager:engine:doesUpdateConflict", { measuresModels: [modelContent] }, diff --git a/lib/modules/model/ModelsController.ts b/lib/modules/model/ModelsController.ts index f462e098..299f5cdd 100644 --- a/lib/modules/model/ModelsController.ts +++ b/lib/modules/model/ModelsController.ts @@ -46,15 +46,15 @@ export class ModelsController { }, getAsset: { handler: this.getAsset.bind(this), - http: [{ path: "device-manager/models/asset/:_id", verb: "get" }], + http: [{ path: "device-manager/models/asset/:model", verb: "get" }], }, getDevice: { handler: this.getDevice.bind(this), - http: [{ path: "device-manager/models/device/:_id", verb: "get" }], + http: [{ path: "device-manager/models/device/:model", verb: "get" }], }, getMeasure: { handler: this.getMeasure.bind(this), - http: [{ path: "device-manager/models/measure/:_id", verb: "get" }], + http: [{ path: "device-manager/models/measure/:type", verb: "get" }], }, listAssets: { handler: this.listAssets.bind(this), @@ -110,7 +110,7 @@ export class ModelsController { async getAsset(request: KuzzleRequest): Promise { const model = request.getString("model"); - const engineGroup = request.getString("engineGroup"); + const engineGroup = request.getString("engineGroup", "commons"); const assetModel = await this.modelService.getAsset(engineGroup, model); @@ -184,10 +184,13 @@ export class ModelsController { ): Promise { const type = request.getBodyString("type"); const valuesMappings = request.getBodyObject("valuesMappings"); + const validationSchema = request.getBodyObject("validationSchema", {}); const valuesDetails = request.getBodyObject("valuesDetails", {}); + const measureModel = await this.modelService.writeMeasure( type, valuesMappings, + validationSchema, valuesDetails, ); diff --git a/lib/modules/model/ModelsRegister.ts b/lib/modules/model/ModelsRegister.ts index d82152b2..280aa96c 100644 --- a/lib/modules/model/ModelsRegister.ts +++ b/lib/modules/model/ModelsRegister.ts @@ -20,6 +20,10 @@ import { } from "./types/ModelContent"; import { ModelSerializer } from "./ModelSerializer"; import { JSONObject } from "kuzzle-sdk"; +import { addSchemaToCache, getAJVErrors } from "../shared/utils/AJValidator"; +import { SchemaValidationError } from "../shared/errors/SchemaValidationError"; +import { getNamedMeasuresDuplicates } from "./MeasuresDuplicates"; +import { MeasuresNamesDuplicatesError } from "./MeasuresNamesDuplicatesError"; export class ModelsRegister { private config: DeviceManagerConfiguration; @@ -79,6 +83,15 @@ export class ModelsRegister { ); } + const duplicates = getNamedMeasuresDuplicates(measures); + + if (duplicates.length > 0) { + throw new MeasuresNamesDuplicatesError( + "Asset model measures contain one or multiple duplicate measure name", + duplicates, + ); + } + // Construct and push the new asset model to the assetModels array this.assetModels.push({ asset: { @@ -120,6 +133,15 @@ export class ModelsRegister { ); } + const duplicates = getNamedMeasuresDuplicates(measures); + + if (duplicates.length > 0) { + throw new MeasuresNamesDuplicatesError( + "Device model measures contain one or multiple duplicate measure name", + duplicates, + ); + } + // Construct and push the new device model to the deviceModels array this.deviceModels.push({ device: { @@ -135,11 +157,25 @@ export class ModelsRegister { } registerMeasure(type: string, measureDefinition: MeasureDefinition) { + const { validationSchema, valuesMappings, valuesDetails } = + measureDefinition; + if (validationSchema) { + try { + addSchemaToCache(type, validationSchema); + } catch (error) { + throw new SchemaValidationError( + "Provided schema is not valid", + getAJVErrors(), + ); + } + } + this.measureModels.push({ measure: { type, - valuesDetails: measureDefinition.valuesDetails, - valuesMappings: measureDefinition.valuesMappings, + validationSchema, + valuesDetails, + valuesMappings, }, type: "measure", }); diff --git a/lib/modules/model/collections/modelsMappings.ts b/lib/modules/model/collections/modelsMappings.ts index 24460127..d1da6802 100644 --- a/lib/modules/model/collections/modelsMappings.ts +++ b/lib/modules/model/collections/modelsMappings.ts @@ -21,6 +21,10 @@ export const modelsMappings: CollectionMappings = { dynamic: "false", properties: {}, }, + validationSchema: { + dynamic: "false", + properties: {}, + }, valuesDetails: { dynamic: "false", properties: {}, diff --git a/lib/modules/model/types/ModelApi.ts b/lib/modules/model/types/ModelApi.ts index d7ce9a8f..bfb7af53 100644 --- a/lib/modules/model/types/ModelApi.ts +++ b/lib/modules/model/types/ModelApi.ts @@ -9,6 +9,7 @@ import { MetadataMappings, TooltipModels, } from "./ModelContent"; +import { SchemaObject } from "ajv"; import { MeasureValuesDetails } from "../../measure/types/MeasureDefinition"; interface ModelsControllerRequest { @@ -70,6 +71,7 @@ export interface ApiModelWriteMeasureRequest extends ModelsControllerRequest { body: { type: string; valuesMappings: JSONObject; + validationSchema?: SchemaObject; valuesDetails?: MeasureValuesDetails; }; } diff --git a/lib/modules/plugin/DeviceManagerEngine.ts b/lib/modules/plugin/DeviceManagerEngine.ts index 7e920828..177383e6 100644 --- a/lib/modules/plugin/DeviceManagerEngine.ts +++ b/lib/modules/plugin/DeviceManagerEngine.ts @@ -32,6 +32,7 @@ import { getMeasureConflicts, getTwinConflicts, } from "../model/ModelsConflicts"; +import { addSchemaToCache } from "../shared/utils/AJValidator"; export type TwinType = "asset" | "device"; @@ -108,6 +109,7 @@ export class DeviceManagerEngine extends AbstractEngine { "ask:device-manager:engine:updateAll", async () => { await this.updateEngines(); + await this.updateMeasuresSchema(); await this.createDevicesCollection(this.config.adminIndex); }, ); @@ -259,6 +261,26 @@ export class DeviceManagerEngine extends AbstractEngine { return result.hits.map((elt) => elt._source); } + async updateMeasuresSchema() { + const models = await this.getModels( + this.adminIndex, + "measure", + ); + + for (const measure of models) { + const { validationSchema } = measure.measure; + if (validationSchema) { + try { + addSchemaToCache(measure.measure.type, validationSchema); + } catch (error) { + this.app.log.error( + `The validation schema for the "${measure.type}" measure model does not comply with the JSON Schema standard, so its update has been skipped`, + ); + } + } + } + } + async onCreate(index: string, group = "commons") { const promises = []; diff --git a/lib/modules/plugin/DeviceManagerPlugin.ts b/lib/modules/plugin/DeviceManagerPlugin.ts index 2186bf5e..e97995b6 100644 --- a/lib/modules/plugin/DeviceManagerPlugin.ts +++ b/lib/modules/plugin/DeviceManagerPlugin.ts @@ -255,7 +255,7 @@ export class DeviceManagerPlugin extends Plugin { * Register a new measure * * @param name Name of the measure - * @param valuesMappings Mappings for the measure values + * @param measureDefinition Values of the measure * * @example * ``` @@ -513,6 +513,7 @@ export class DeviceManagerPlugin extends Plugin { if (this.config.engine.autoUpdate) { try { await this.deviceManagerEngine.updateEngines(); + await this.deviceManagerEngine.updateMeasuresSchema(); } catch (error) { this.context.log.error( `An error occured while updating the engines during startup: ${error}`, diff --git a/lib/modules/shared/errors/SchemaValidationError.ts b/lib/modules/shared/errors/SchemaValidationError.ts new file mode 100644 index 00000000..f29cf970 --- /dev/null +++ b/lib/modules/shared/errors/SchemaValidationError.ts @@ -0,0 +1,27 @@ +import { ErrorObject } from "ajv"; +import { KuzzleError } from "kuzzle"; + +export class SchemaValidationError extends KuzzleError { + private schemaErrors: ErrorObject[]; + constructor( + message: string, + schemaErrors: ErrorObject[], + id?: string, + code?: number, + ) { + super(message, 400, id, code); + this.schemaErrors = schemaErrors; + } + + get name() { + return "SchemaValidationError"; + } + + public toJSON() { + const json = super.toJSON(); + + json.schemaErrors = this.schemaErrors; + + return json; + } +} diff --git a/lib/modules/shared/utils/AJValidator.ts b/lib/modules/shared/utils/AJValidator.ts new file mode 100644 index 00000000..414a76eb --- /dev/null +++ b/lib/modules/shared/utils/AJValidator.ts @@ -0,0 +1,59 @@ +import addFormats from "ajv-formats"; +import Ajv, { SchemaObject } from "ajv"; + +/** + * TODO: add stricter TypeScript rules to allow the use of JSONSchemaType + */ +const ajv = addFormats(new Ajv({}), [ + "date-time", + "time", + "date", + "email", + "hostname", + "ipv4", + "ipv6", + "uri", + "uri-reference", + "uuid", + "uri-template", + "json-pointer", + "relative-json-pointer", + "regex", +]); + +/** + * Compile and add the schema to the AJV cache + * + * @param scemaID The schema unique identifier + * @param schema The schema to add + * + * @returns The newly created validator for the schema + * @throws If the provided schema is not valid + */ +export function addSchemaToCache(schemaID: string, schema: SchemaObject) { + ajv.validateSchema(schema, true); + + ajv.removeSchema(schemaID); + return ajv.addSchema(schema, schemaID); +} + +/** + * Search the corresponding validator in the AJV cache. + * + * @param schemaID The desired schema identifier + * + * @returns The desired schema validator + * @throws If the schema cannot be found + */ +export function getValidator(schemaID: string) { + return ajv.getSchema(schemaID); +} + +/** + * Get and return the previous AJV errors + * + * @returns an array of ErrorObject + */ +export function getAJVErrors() { + return ajv.errors ?? []; +} diff --git a/package-lock.json b/package-lock.json index 92044755..dd0d6ac1 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,14 +1,16 @@ { "name": "kuzzle-device-manager", - "version": "2.4.3", + "version": "2.5.0-dev.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "kuzzle-device-manager", - "version": "2.4.3", + "version": "2.5.0-dev.1", "license": "Apache-2.0", "dependencies": { + "ajv": "^8.13.0", + "ajv-formats": "^3.0.1", "csv-stringify": "^6.4.5", "kuzzle-plugin-commons": "^1.2.0", "lodash": "^4.17.21", @@ -692,26 +694,6 @@ "node": ">=v18" } }, - "node_modules/@commitlint/config-validator/node_modules/ajv": { - "version": "8.17.1", - "dev": true, - "license": "MIT", - "dependencies": { - "fast-deep-equal": "^3.1.3", - "fast-uri": "^3.0.1", - "json-schema-traverse": "^1.0.0", - "require-from-string": "^2.0.2" - }, - "funding": { - "type": "github", - "url": "https://github.com/sponsors/epoberezkin" - } - }, - "node_modules/@commitlint/config-validator/node_modules/json-schema-traverse": { - "version": "1.0.0", - "dev": true, - "license": "MIT" - }, "node_modules/@commitlint/ensure": { "version": "18.6.1", "dev": true, @@ -1160,6 +1142,28 @@ "url": "https://opencollective.com/eslint" } }, + "node_modules/@eslint/eslintrc/node_modules/ajv": { + "version": "6.12.6", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", + "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", + "dev": true, + "dependencies": { + "fast-deep-equal": "^3.1.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/@eslint/eslintrc/node_modules/json-schema-traverse": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", + "dev": true + }, "node_modules/@eslint/js": { "version": "8.57.1", "dev": true, @@ -3180,20 +3184,36 @@ } }, "node_modules/ajv": { - "version": "6.12.6", - "dev": true, - "license": "MIT", + "version": "8.17.1", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.17.1.tgz", + "integrity": "sha512-B/gBuNg5SiMTrPkC+A2+cW0RszwxYmn6VYxB/inlBStS5nx6xHIt/ehKRhIMhqusl7a8LjQoZnjCs5vhwxOQ1g==", "dependencies": { - "fast-deep-equal": "^3.1.1", - "fast-json-stable-stringify": "^2.0.0", - "json-schema-traverse": "^0.4.1", - "uri-js": "^4.2.2" + "fast-deep-equal": "^3.1.3", + "fast-uri": "^3.0.1", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2" }, "funding": { "type": "github", "url": "https://github.com/sponsors/epoberezkin" } }, + "node_modules/ajv-formats": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/ajv-formats/-/ajv-formats-3.0.1.tgz", + "integrity": "sha512-8iUql50EUR+uUcdRQ3HDqa6EVyo3docL8g5WJ3FNcWmu62IbkGUue/pEyLBW8VGKKucTPgqeks4fIU1DA4yowQ==", + "dependencies": { + "ajv": "^8.0.0" + }, + "peerDependencies": { + "ajv": "^8.0.0" + }, + "peerDependenciesMeta": { + "ajv": { + "optional": true + } + } + }, "node_modules/ansi-escapes": { "version": "4.3.2", "license": "MIT", @@ -5513,6 +5533,22 @@ "url": "https://opencollective.com/eslint" } }, + "node_modules/eslint/node_modules/ajv": { + "version": "6.12.6", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", + "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", + "dev": true, + "dependencies": { + "fast-deep-equal": "^3.1.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, "node_modules/eslint/node_modules/escape-string-regexp": { "version": "4.0.0", "dev": true, @@ -5558,6 +5594,12 @@ "node": ">=10.13.0" } }, + "node_modules/eslint/node_modules/json-schema-traverse": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", + "dev": true + }, "node_modules/espree": { "version": "9.6.1", "dev": true, @@ -5852,7 +5894,6 @@ }, "node_modules/fast-deep-equal": { "version": "3.1.3", - "dev": true, "license": "MIT" }, "node_modules/fast-diff": { @@ -5908,7 +5949,6 @@ }, "node_modules/fast-uri": { "version": "3.0.1", - "dev": true, "license": "MIT" }, "node_modules/fastfall": { @@ -8020,9 +8060,9 @@ "license": "MIT" }, "node_modules/json-schema-traverse": { - "version": "0.4.1", - "dev": true, - "license": "MIT" + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==" }, "node_modules/json-stable-stringify": { "version": "1.1.0", @@ -14617,7 +14657,6 @@ }, "node_modules/require-from-string": { "version": "2.0.2", - "dev": true, "license": "MIT", "engines": { "node": ">=0.10.0" @@ -16402,8 +16441,9 @@ }, "node_modules/uri-js": { "version": "4.4.1", + "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.1.tgz", + "integrity": "sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==", "dev": true, - "license": "BSD-2-Clause", "dependencies": { "punycode": "^2.1.0" } diff --git a/package.json b/package.json index 982d517a..86cc1bef 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "kuzzle-device-manager", - "version": "2.4.3", + "version": "2.5.0-dev.1", "description": "Manage your IoT devices and assets. Choose a provisioning strategy, receive and decode payload, handle your IoT business logic.", "author": "The Kuzzle Team (support@kuzzle.io)", "repository": { @@ -24,6 +24,8 @@ }, "license": "Apache-2.0", "dependencies": { + "ajv": "^8.13.0", + "ajv-formats": "^3.0.1", "csv-stringify": "^6.4.5", "kuzzle-plugin-commons": "^1.2.0", "lodash": "^4.17.21", diff --git a/tests/application/app.ts b/tests/application/app.ts index 276aca7b..38bfe005 100644 --- a/tests/application/app.ts +++ b/tests/application/app.ts @@ -16,6 +16,8 @@ import { brightnessMeasureDefinition } from "./measures/BrightnessMeasure"; import { co2MeasureDefinition } from "./measures/CO2Measure"; import { illuminanceMeasureDefinition } from "./measures/IlluminanceMeasure"; import { powerConsumptionMeasureDefinition } from "./measures/PowerConsumptionMeasure"; +import { magicHouseAssetDefinition } from "./assets/MagicHouse"; +import { magiculeMeasureDefinition } from "./measures/Magicule"; const app = new Backend("kuzzle"); @@ -76,7 +78,14 @@ deviceManager.models.registerAsset( streetLampAssetDefinition ); +deviceManager.models.registerAsset( + "commons", + "MagicHouse", + magicHouseAssetDefinition +) + deviceManager.models.registerMeasure("acceleration", accelerationMeasureDefinition); +deviceManager.models.registerMeasure("magicule", magiculeMeasureDefinition) deviceManager.models.registerMeasure("brightness", brightnessMeasureDefinition); deviceManager.models.registerMeasure("co2", co2MeasureDefinition); deviceManager.models.registerMeasure("illuminance", illuminanceMeasureDefinition); diff --git a/tests/application/assets/MagicHouse.ts b/tests/application/assets/MagicHouse.ts new file mode 100644 index 00000000..71571f54 --- /dev/null +++ b/tests/application/assets/MagicHouse.ts @@ -0,0 +1,11 @@ +import { AssetModelDefinition } from "../../../index"; + +export const magicHouseAssetDefinition: AssetModelDefinition = { + measures: [ + { name: "magiculeExt", type: "magicule" }, + { name: "magiculeInt", type: "magicule" }, + ], + metadataMappings: { + power: { type: "integer" }, + }, +}; diff --git a/tests/application/measures/Magicule.ts b/tests/application/measures/Magicule.ts new file mode 100644 index 00000000..aa7c3627 --- /dev/null +++ b/tests/application/measures/Magicule.ts @@ -0,0 +1,19 @@ +import { MeasureDefinition } from "lib/modules/measure"; + +export type MagiculeMeasurement = { + magicule: number; +}; + +export const magiculeMeasureDefinition: MeasureDefinition = { + valuesMappings: { + magicule: { type: "integer" }, + }, + validationSchema: { + type: "object", + properties: { + magicule: { type: "integer" }, + }, + required: ["magicule"], + additionalProperties: false, + }, +}; diff --git a/tests/fixtures/fixtures.ts b/tests/fixtures/fixtures.ts index 251de19a..66e5bc65 100644 --- a/tests/fixtures/fixtures.ts +++ b/tests/fixtures/fixtures.ts @@ -175,6 +175,15 @@ const assetAyseGrouped2 = { }; const assetAyseGroupedId2 = `${assetAyseGrouped2.model}-${assetAyseGrouped2.reference}`; +const assetAyseDebug1 = { + model: "MagicHouse", + reference: "debug1", + linkedDevices: [], + metadata: [], +}; + +const assetAyseDebug1Id = `${assetAyseDebug1.model}-${assetAyseDebug1.reference}`; + export default { "device-manager": { devices: [ @@ -240,6 +249,9 @@ export default { { index: { _id: assetAyseWarehouseLinkedId } }, assetAyseWarehouseLinked, + + { index: { _id: assetAyseDebug1Id } }, + assetAyseDebug1, ], ...assetGroupFixtures, }, diff --git a/tests/hooks/collections.ts b/tests/hooks/collections.ts index d362516c..be599d63 100644 --- a/tests/hooks/collections.ts +++ b/tests/hooks/collections.ts @@ -21,13 +21,13 @@ async function deleteModels(sdk: Kuzzle) { "model-measure-presence", "model-asset-Plane", "model-asset-AdvancedPlane", - "model-asset-Vehicle", - "model-asset-CompanyAssetInvalid", + "model-asset-Car", "model-device-Zigbee", "model-device-Bluetooth", "model-device-Enginko", "model-asset-TestHouse", "model-asset-AdvancedWarehouse", + "model-measure-falseMagic", ], }, }, diff --git a/tests/scenario/modules/assets/action-export.test.ts b/tests/scenario/modules/assets/action-export.test.ts index 26af2d66..62d32cf6 100644 --- a/tests/scenario/modules/assets/action-export.test.ts +++ b/tests/scenario/modules/assets/action-export.test.ts @@ -86,7 +86,7 @@ describe("AssetsController:exportMeasures", () => { writeFileSync("./assets.csv", csv.join("")); expect(csv[0]).toBe( - "Model,Reference,brightness.lumens,co2,humidity,illuminance,position,position.accuracy,position.altitude,powerConsumption.watt,temperature,temperatureExt,temperatureInt,temperatureWeather,lastMeasuredAt,lastMeasuredAtISO\n", + "Model,Reference,brightness.lumens,co2,humidity,illuminance,magiculeExt,magiculeInt,position,position.accuracy,position.altitude,powerConsumption.watt,temperature,temperatureExt,temperatureInt,temperatureWeather,lastMeasuredAt,lastMeasuredAtISO\n", ); expect(csv).toHaveLength(assetCount + 1); diff --git a/tests/scenario/modules/assets/action-measure-ingest.test.ts b/tests/scenario/modules/assets/action-measure-ingest.test.ts new file mode 100644 index 00000000..046f0033 --- /dev/null +++ b/tests/scenario/modules/assets/action-measure-ingest.test.ts @@ -0,0 +1,302 @@ +import { InternalCollection } from "../../../../lib/modules/plugin"; +import { + ApiAssetMeasureIngestRequest, + ApiAssetMeasureIngestResult, +} from "../../../../index"; +import { setupHooks } from "../../../helpers"; +import axios from "axios"; + +jest.setTimeout(10000); + +describe("AssetsController:measureIngest", () => { + describe("AssetsController:measureIngest:sdk", () => { + const sdk = setupHooks(); + + it("should ingest measure", async () => { + const assetId = "MagicHouse-debug1"; + const indexId = "engine-ayse"; + + const query = sdk.query< + ApiAssetMeasureIngestRequest, + ApiAssetMeasureIngestResult + >({ + controller: "device-manager/assets", + action: "measureIngest", + assetId, + engineId: indexId, + slotName: "magiculeExt", + body: { + dataSource: { + id: "testApi1", + }, + measuredAt: 170000000, + values: { + magicule: 18, + }, + }, + }); + + await expect(query).resolves.not.toThrow(); + + await sdk.collection.refresh(indexId, InternalCollection.MEASURES); + + const total = await sdk.document.count( + indexId, + InternalCollection.MEASURES, + ); + + expect(total).toBe(1); + + const document = await sdk.document.search( + indexId, + InternalCollection.MEASURES, + { + query: { + equals: { + "origin._id": "testApi1", + }, + }, + }, + { lang: "koncorde" }, + ); + + expect(document.fetched).toBe(1); + + expect(document.hits[0]._source).toMatchObject({ + asset: { + _id: "MagicHouse-debug1", + measureName: "magiculeExt", + }, + values: { + magicule: 18, + }, + }); + }); + + it("should not ingest measure with incorrect values", async () => { + const assetId = "MagicHouse-debug1"; + const indexId = "engine-ayse"; + + const query = sdk.query< + ApiAssetMeasureIngestRequest, + ApiAssetMeasureIngestResult + >({ + controller: "device-manager/assets", + action: "measureIngest", + assetId, + engineId: "engine-ayse", + slotName: "magiculeExt", + body: { + dataSource: { + id: "testApi2", + }, + measuredAt: 170000000, + values: { + magicule: "99", + }, + }, + }); + + await expect(query).rejects.toThrow( + "The provided measure does not respect its schema", + ); + + const total = await sdk.document.count( + indexId, + InternalCollection.MEASURES, + { + query: { + equals: { + "origin._id": "testApi2", + }, + }, + }, + { lang: "koncorde" }, + ); + + expect(total).toBe(0); + }); + + it("should not ingest measure with unknow asset id", async () => { + const assetId = "MagicHouse-debug"; + const indexId = "engine-ayse"; + + const query = sdk.query< + ApiAssetMeasureIngestRequest, + ApiAssetMeasureIngestResult + >({ + controller: "device-manager/assets", + action: "measureIngest", + assetId, + engineId: "engine-ayse", + slotName: "magiculeExt", + body: { + dataSource: { + id: "testApi3", + }, + measuredAt: 170000000, + values: { + magicule: 99, + }, + }, + }); + + await expect(query).rejects.toThrow( + 'Asset "MagicHouse-debug" does not exists on index "engine-ayse"', + ); + + const total = await sdk.document.count( + indexId, + InternalCollection.MEASURES, + { + query: { + equals: { + "origin._id": "testApi3", + }, + }, + }, + { lang: "koncorde" }, + ); + + expect(total).toBe(0); + }); + }); + + describe("AssetsController:measureIngest:rest", () => { + const sdk = setupHooks(); + + it("should ingest measure", async () => { + const assetId = "MagicHouse-debug1"; + const indexId = "engine-ayse"; + + const query = await axios.post( + `http://localhost:7512/_/device-manager/${indexId}/assets/${assetId}/measures/magiculeExt`, + { + dataSource: { + id: "testApi1", + }, + measuredAt: 170000000, + values: { + magicule: 18, + }, + }, + ); + + expect(query.status).toBe(200); + + await sdk.collection.refresh(indexId, InternalCollection.MEASURES); + + const total = await sdk.document.count( + indexId, + InternalCollection.MEASURES, + ); + + expect(total).toBe(1); + + const document = await sdk.document.search( + indexId, + InternalCollection.MEASURES, + { + query: { + equals: { + "origin._id": "testApi1", + }, + }, + }, + { lang: "koncorde" }, + ); + + expect(document.fetched).toBe(1); + + expect(document.hits[0]._source).toMatchObject({ + asset: { + _id: "MagicHouse-debug1", + measureName: "magiculeExt", + }, + values: { + magicule: 18, + }, + }); + }); + + it("should not ingest measure with incorrect values", async () => { + const assetId = "MagicHouse-debug1"; + const indexId = "engine-ayse"; + + const query = await axios + .post( + `http://localhost:7512/_/device-manager/${indexId}/assets/${assetId}/measures/magiculeExt`, + { + dataSource: { + id: "testApi2", + }, + measuredAt: 170000000, + values: { + magicule: "99", + }, + }, + ) + .catch((e) => e.response); + + expect(query.status).toBe(400); + expect(query.data.error.message).toBe( + "The provided measure does not respect its schema", + ); + + const total = await sdk.document.count( + indexId, + InternalCollection.MEASURES, + { + query: { + equals: { + "origin._id": "testApi2", + }, + }, + }, + { lang: "koncorde" }, + ); + + expect(total).toBe(0); + }); + + it("should not ingest measure with unknow asset id", async () => { + const assetId = "MagicHouse-debug"; + const indexId = "engine-ayse"; + + const query = await axios + .post( + `http://localhost:7512/_/device-manager/${indexId}/assets/${assetId}/measures/magiculeExt`, + { + dataSource: { + id: "testApi3", + }, + measuredAt: 170000000, + values: { + magicule: 99, + }, + }, + ) + .catch((e) => e.response); + + expect(query.status).toBe(400); + expect(query.data.error.message).toMatch( + 'Asset "MagicHouse-debug" does not exists on index "engine-ayse"', + ); + + const total = await sdk.document.count( + indexId, + InternalCollection.MEASURES, + { + query: { + equals: { + "origin._id": "testApi3", + }, + }, + }, + { lang: "koncorde" }, + ); + + expect(total).toBe(0); + }); + }); +}); diff --git a/tests/scenario/modules/assets/action-mmeasure-ingest.test.ts b/tests/scenario/modules/assets/action-mmeasure-ingest.test.ts new file mode 100644 index 00000000..d03b2a44 --- /dev/null +++ b/tests/scenario/modules/assets/action-mmeasure-ingest.test.ts @@ -0,0 +1,366 @@ +import { InternalCollection } from "../../../../lib/modules/plugin"; +import { + ApiAssetmMeasureIngestRequest, + ApiAssetmMeasureIngestResult, +} from "../../../../index"; +import { setupHooks } from "../../../helpers"; +import axios from "axios"; + +jest.setTimeout(10000); + +describe("AssetsController:mMeasureIngest", () => { + describe("AssetsController:mMeasureIngest:sdk", () => { + const sdk = setupHooks(); + + it("should ingest measures from measurements and API source", async () => { + const assetId = "MagicHouse-debug1"; + const indexId = "engine-ayse"; + + const query = sdk.query< + ApiAssetmMeasureIngestRequest, + ApiAssetmMeasureIngestResult + >({ + controller: "device-manager/assets", + action: "mMeasureIngest", + assetId, + engineId: indexId, + body: { + dataSource: { + id: "testApi1", + }, + measurements: [ + { + slotName: "magiculeExt", + measuredAt: 170000000, + values: { + magicule: 18, + }, + }, + { + slotName: "magiculeInt", + measuredAt: 170000001, // ? Set this to ensure sorting order + values: { + magicule: 25, + }, + }, + ], + }, + }); + + await expect(query).resolves.not.toThrow(); + + await sdk.collection.refresh(indexId, InternalCollection.MEASURES); + + const total = await sdk.document.count( + indexId, + InternalCollection.MEASURES, + ); + + expect(total).toBe(2); + + const document = await sdk.document.search( + indexId, + InternalCollection.MEASURES, + { + query: { + equals: { + "origin._id": "testApi1", + }, + }, + }, + { lang: "koncorde" }, + ); + + expect(document.fetched).toBe(2); + + expect(document.hits[0]._source).toMatchObject({ + asset: { + _id: "MagicHouse-debug1", + measureName: "magiculeExt", + }, + values: { + magicule: 18, + }, + }); + + expect(document.hits[1]._source).toMatchObject({ + asset: { + _id: "MagicHouse-debug1", + measureName: "magiculeInt", + }, + values: { + magicule: 25, + }, + }); + }); + + it("should not ingest measures with incorrect values", async () => { + const assetId = "MagicHouse-debug1"; + const indexId = "engine-ayse"; + + const query = sdk.query< + ApiAssetmMeasureIngestRequest, + ApiAssetmMeasureIngestResult + >({ + controller: "device-manager/assets", + action: "mMeasureIngest", + assetId, + engineId: "engine-ayse", + body: { + dataSource: { + id: "testApi2", + }, + measurements: [ + { + slotName: "magiculeExt", + measuredAt: 170000000, + values: { + magicule: "99", + }, + }, + ], + }, + }); + + await expect(query).rejects.toThrow( + "The provided measures do not comply with their respective schemas", + ); + + const total = await sdk.document.count( + indexId, + InternalCollection.MEASURES, + { + query: { + equals: { + "origin._id": "testApi2", + }, + }, + }, + { lang: "koncorde" }, + ); + + expect(total).toBe(0); + }); + + it("should not ingest measures with unknow asset id", async () => { + const assetId = "MagicHouse-debug"; + const indexId = "engine-ayse"; + + const query = sdk.query< + ApiAssetmMeasureIngestRequest, + ApiAssetmMeasureIngestResult + >({ + controller: "device-manager/assets", + action: "mMeasureIngest", + assetId, + engineId: "engine-ayse", + body: { + dataSource: { + id: "testApi3", + }, + measurements: [ + { + slotName: "magiculeExt", + measuredAt: 170000000, + values: { + magicule: 59, + }, + }, + ], + }, + }); + + await expect(query).rejects.toThrow( + 'Asset "MagicHouse-debug" does not exists on index "engine-ayse"', + ); + + const total = await sdk.document.count( + indexId, + InternalCollection.MEASURES, + { + query: { + equals: { + "origin._id": "testApi3", + }, + }, + }, + { lang: "koncorde" }, + ); + + expect(total).toBe(0); + }); + }); + + describe("AssetsController:mMeasureIngest:rest", () => { + const sdk = setupHooks(); + + it("should ingest measures from measurements and API source", async () => { + const assetId = "MagicHouse-debug1"; + const indexId = "engine-ayse"; + + const query = await axios.post( + `http://localhost:7512/_/device-manager/${indexId}/assets/${assetId}/_mMeasureIngest`, + { + dataSource: { + type: "api", + id: "testApi1", + }, + measurements: [ + { + slotName: "magiculeExt", + measuredAt: 170000000, + values: { + magicule: 18, + }, + }, + { + slotName: "magiculeInt", + measuredAt: 170000001, // ? Set this to ensure sorting order + values: { + magicule: 25, + }, + }, + ], + }, + ); + + expect(query.status).toBe(200); + + await sdk.collection.refresh(indexId, InternalCollection.MEASURES); + + const total = await sdk.document.count( + indexId, + InternalCollection.MEASURES, + ); + + expect(total).toBe(2); + + const document = await sdk.document.search( + indexId, + InternalCollection.MEASURES, + { + query: { + equals: { + "origin._id": "testApi1", + }, + }, + }, + { lang: "koncorde" }, + ); + + expect(document.fetched).toBe(2); + + expect(document.hits[0]._source).toMatchObject({ + asset: { + _id: "MagicHouse-debug1", + measureName: "magiculeExt", + }, + values: { + magicule: 18, + }, + }); + + expect(document.hits[1]._source).toMatchObject({ + asset: { + _id: "MagicHouse-debug1", + measureName: "magiculeInt", + }, + values: { + magicule: 25, + }, + }); + }); + + it("should not ingest measures with incorrect values", async () => { + const assetId = "MagicHouse-debug1"; + const indexId = "engine-ayse"; + + const query = await axios + .post( + `http://localhost:7512/_/device-manager/${indexId}/assets/${assetId}/_mMeasureIngest`, + { + dataSource: { + type: "api", + id: "testApi2", + }, + measurements: [ + { + slotName: "magiculeExt", + measuredAt: 170000000, + values: { + magicule: "99", + }, + }, + ], + }, + ) + .catch((e) => e.response); + + expect(query.status).toBe(400); + expect(query.data.error.message).toMatch( + "The provided measures do not comply with their respective schemas", + ); + + const total = await sdk.document.count( + indexId, + InternalCollection.MEASURES, + { + query: { + equals: { + "origin._id": "testApi2", + }, + }, + }, + { lang: "koncorde" }, + ); + + expect(total).toBe(0); + }); + + it("should not ingest measures with unknow asset id", async () => { + const assetId = "MagicHouse-debug"; + const indexId = "engine-ayse"; + + const query = await axios + .post( + `http://localhost:7512/_/device-manager/${indexId}/assets/${assetId}/_mMeasureIngest`, + { + dataSource: { + type: "api", + id: "testApi3", + }, + measurements: [ + { + slotName: "magiculeExt", + measuredAt: 170000000, + values: { + magicule: 59, + }, + }, + ], + }, + ) + .catch((e) => e.response); + + expect(query.status).toBe(400); + expect(query.data.error.message).toMatch( + 'Asset "MagicHouse-debug" does not exists on index "engine-ayse"', + ); + + const total = await sdk.document.count( + indexId, + InternalCollection.MEASURES, + { + query: { + equals: { + "origin._id": "testApi3", + }, + }, + }, + { lang: "koncorde" }, + ); + + expect(total).toBe(0); + }); + }); +}); diff --git a/tests/scenario/modules/models/asset-model-metadata-propagation.test.ts b/tests/scenario/modules/models/asset-model-metadata-propagation.test.ts index ea43ceb5..430177a7 100644 --- a/tests/scenario/modules/models/asset-model-metadata-propagation.test.ts +++ b/tests/scenario/modules/models/asset-model-metadata-propagation.test.ts @@ -11,7 +11,7 @@ import { } from "../../../../lib/modules/model"; import { setupHooks } from "../../../helpers"; -jest.setTimeout(10000); +jest.setTimeout(20000); describe("Asset model metadata propagation", () => { const sdk = setupHooks(); diff --git a/tests/scenario/modules/models/asset-model.test.ts b/tests/scenario/modules/models/asset-model.test.ts index 38d47885..082341fe 100644 --- a/tests/scenario/modules/models/asset-model.test.ts +++ b/tests/scenario/modules/models/asset-model.test.ts @@ -1,7 +1,7 @@ import { EditorHintEnum } from "../../../../lib/modules/model"; import { setupHooks } from "../../../helpers"; -jest.setTimeout(10000); +jest.setTimeout(20000); describe("ModelsController:assets", () => { const sdk = setupHooks(); @@ -91,14 +91,13 @@ describe("ModelsController:assets", () => { engineGroup: "commons", }); - expect(listAssets.result).toMatchObject({ - total: 3, - models: [ - { _id: "model-asset-Container" }, - { _id: "model-asset-Plane" }, - { _id: "model-asset-Warehouse" }, - ], - }); + expect(listAssets.result.total).toBe(4); + expect(listAssets.result.models).toMatchObject([ + { _id: "model-asset-Container" }, + { _id: "model-asset-MagicHouse" }, + { _id: "model-asset-Plane" }, + { _id: "model-asset-Warehouse" }, + ]); const getAsset = await sdk.query({ controller: "device-manager/models", @@ -129,14 +128,13 @@ describe("ModelsController:assets", () => { engineGroup: "air_quality", }); - expect(listAssets.result).toMatchObject({ - total: 3, - models: [ - { _id: "model-asset-Container" }, - { _id: "model-asset-Room" }, - { _id: "model-asset-Warehouse" }, - ], - }); + expect(listAssets.result.total).toBe(4); + expect(listAssets.result.models).toMatchObject([ + { _id: "model-asset-Container" }, + { _id: "model-asset-MagicHouse" }, + { _id: "model-asset-Room" }, + { _id: "model-asset-Warehouse" }, + ]); }); it("Write and Search an Asset model", async () => { diff --git a/tests/scenario/modules/models/device-model.test.ts b/tests/scenario/modules/models/device-model.test.ts index bb2883b3..2aa7a25f 100644 --- a/tests/scenario/modules/models/device-model.test.ts +++ b/tests/scenario/modules/models/device-model.test.ts @@ -1,7 +1,7 @@ import { EditorHintEnum } from "../../../../lib/modules/model"; import { setupHooks } from "../../../helpers"; -jest.setTimeout(10000); +jest.setTimeout(20000); describe("ModelsController:devices", () => { const sdk = setupHooks(); diff --git a/tests/scenario/modules/models/measures-model.test.ts b/tests/scenario/modules/models/measures-model.test.ts index 89949c33..15a4e633 100644 --- a/tests/scenario/modules/models/measures-model.test.ts +++ b/tests/scenario/modules/models/measures-model.test.ts @@ -165,7 +165,7 @@ describe("ModelsController:measures", () => { action: "listMeasures", }); - expect(listMeasures.result.total).toBe(11); + expect(listMeasures.result.total).toBe(12); expect(listMeasures.result.models).toMatchObject([ { _id: "model-measure-acceleration" }, { _id: "model-measure-battery" }, @@ -173,6 +173,7 @@ describe("ModelsController:measures", () => { { _id: "model-measure-co2" }, { _id: "model-measure-humidity" }, { _id: "model-measure-illuminance" }, + { _id: "model-measure-magicule" }, { _id: "model-measure-movement" }, { _id: "model-measure-position" }, { _id: "model-measure-powerConsumption" }, diff --git a/types/package.json b/types/package.json index 409b6775..60ffd2fc 100644 --- a/types/package.json +++ b/types/package.json @@ -1,6 +1,6 @@ { "name": "kuzzle-device-manager-types", - "version": "2.4.3", + "version": "2.5.0-dev.1", "description": "Shared types for Kuzzle Device Manager", "author": "The Kuzzle Team (support@kuzzle.io)", "main": "index.js",