From e460ed9303f253b1a74bc222d31d574d3b28c444 Mon Sep 17 00:00:00 2001 From: TangoBeeAkto Date: Wed, 18 Sep 2024 22:47:08 +0530 Subject: [PATCH 1/4] feat: created metrics table for traffic processor and traffic collector --- .../traffic_metrics/TrafficMetricsAction.java | 13 +- apps/dashboard/src/main/resources/struts.xml | 11 + .../apps/dashboard/components/GraphMetric.jsx | 24 +- .../src/apps/dashboard/pages/settings/api.js | 20 ++ .../pages/settings/metrics/Metrics.jsx | 250 +++++++++++++++++- .../apps/dashboard/pages/settings/module.js | 14 + .../dto/traffic_metrics/RuntimeMetrics.java | 12 +- 7 files changed, 319 insertions(+), 25 deletions(-) diff --git a/apps/dashboard/src/main/java/com/akto/action/traffic_metrics/TrafficMetricsAction.java b/apps/dashboard/src/main/java/com/akto/action/traffic_metrics/TrafficMetricsAction.java index 95ef4bdfb4..9e15572e45 100644 --- a/apps/dashboard/src/main/java/com/akto/action/traffic_metrics/TrafficMetricsAction.java +++ b/apps/dashboard/src/main/java/com/akto/action/traffic_metrics/TrafficMetricsAction.java @@ -126,7 +126,7 @@ public String execute() { public String fetchRuntimeInstances() { instanceIds = new ArrayList<>(); Bson filters = RuntimeMetricsDao.buildFilters(startTimestamp, endTimestamp); - runtimeMetrics = RuntimeMetricsDao.instance.findAll(filters, 0, 0, Sorts.descending("timestamp"), Projections.include("instanceId")); + runtimeMetrics = RuntimeMetricsDao.instance.findAll(filters, 0, 0, Sorts.descending("timestamp")); for (RuntimeMetrics metric: runtimeMetrics) { instanceIds.add(metric.getInstanceId()); } @@ -149,13 +149,22 @@ public String fetchRuntimeMetrics() { while (cursor.hasNext()) { BasicDBObject basicDBObject = cursor.next(); BasicDBObject latestDoc = (BasicDBObject) basicDBObject.get("latestDoc"); - runtimeMetrics.add(new RuntimeMetrics(latestDoc.getString("name"), 0, instanceId, latestDoc.getDouble("val"))); + runtimeMetrics.add(new RuntimeMetrics(latestDoc.getString("name"), 0, instanceId, latestDoc.getString("version"), latestDoc.getDouble("val"))); } } return SUCCESS.toUpperCase(); } + public String fetchAllRuntimeMetrics() { + Bson filters = RuntimeMetricsDao.buildFilters(startTimestamp, endTimestamp, instanceId); + runtimeMetrics = new ArrayList<>(); + + runtimeMetrics.addAll(RuntimeMetricsDao.instance.findAll(filters)); + + return SUCCESS.toUpperCase(); + } + public String fetchTrafficMetricsDesciptions(){ names = Arrays.asList(TrafficMetrics.Name.values()); return SUCCESS.toUpperCase(); diff --git a/apps/dashboard/src/main/resources/struts.xml b/apps/dashboard/src/main/resources/struts.xml index 0afa269650..58bf1ff4a0 100644 --- a/apps/dashboard/src/main/resources/struts.xml +++ b/apps/dashboard/src/main/resources/struts.xml @@ -4996,6 +4996,17 @@ ^actionErrors.* + + + + + + + 422 + false + ^actionErrors.* + + diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/components/GraphMetric.jsx b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/components/GraphMetric.jsx index e3deb0ce6f..5541ebe527 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/components/GraphMetric.jsx +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/components/GraphMetric.jsx @@ -31,32 +31,16 @@ function GraphMetric(props) { const series = [ ...dataForChart, - inputMetrics.length > 0 && inputMetrics.map((x, i) => { - return { - data: x.data, - color: '#FF4DCA', - name: x.name, - marker: { - enabled: false, - symbol: 'circle', - }, - fillColor: { - linearGradient: { x1: 0, y1: 0, x2: 0, y2: 1 }, - stops: [ - [0, '#000000'], - [1, '#000000'], - ], - }, - yAxis: i + 1, - }; - }), ]; const chartOptions = { chart: { type, + zooming: { + type: 'x' + }, height: `${height}px`, - backgroundColor, + backgroundColor: backgroundColor || '#ffffff', }, credits:{ enabled: false, diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/settings/api.js b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/settings/api.js index 3b7ca1338f..3e39dc3e37 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/settings/api.js +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/settings/api.js @@ -137,6 +137,26 @@ const settingRequests = { data: {} }) }, + fetchRuntimeInstances(startTimestamp, endTimestamp) { + return request({ + url: '/api/fetchRuntimeInstances', + method: 'post', + data: { + startTimestamp, + endTimestamp + } + }) + }, + fetchRuntimeMetrics(startTimestamp, endTimestamp, instanceId) { + return request({ + url: '/api/fetchAllRuntimeMetrics', + method: 'post', + data: { + startTimestamp, endTimestamp, + instanceId + } + }) + }, fetchTrafficMetrics(groupBy, startTimestamp, endTimestamp, names, host) { return request({ url: '/api/fetchTrafficMetrics', diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/settings/metrics/Metrics.jsx b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/settings/metrics/Metrics.jsx index 22f173bea7..f950a1f328 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/settings/metrics/Metrics.jsx +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/settings/metrics/Metrics.jsx @@ -1,4 +1,4 @@ -import { EmptyState, LegacyCard, Page } from '@shopify/polaris' +import { Divider, EmptyState, LegacyCard, Page } from '@shopify/polaris' import React, { useEffect, useReducer, useState } from 'react' import DateRangeFilter from '../../../components/layouts/DateRangeFilter' import Dropdown from '../../../components/layouts/Dropdown' @@ -9,6 +9,8 @@ import settingFunctions from '../module' import GraphMetric from '../../../components/GraphMetric' import values from '@/util/values' import PersistStore from '../../../../main/PersistStore' +import GithubSimpleTable from '../../../components/tables/GithubSimpleTable' +import FlyLayout from '../../../components/layouts/FlyLayout' function Metrics() { @@ -19,6 +21,12 @@ function Metrics() { const [hostsActive, setHostsActive] = useState(false) const [currentHost, setCurrentHost] = useState(null) + const [runtimeMetricsData, setRuntimeMetricsData] = useState([]) + const [showRuntimeGraph, setShowRuntimeGraph] = useState(false) + const [graphs, setGraphs] = useState([]) + const [runtimeFilterVal, setRuntimeFilterVal] = useState('1day') + const [loading, setLoading] = useState(false) + const [currDateRange, dispatchCurrDateRange] = useReducer(produce((draft, action) => func.dateRangeReducer(draft, action)), values.ranges[2]); const getTimeEpoch = (key) => { return Math.floor(Date.parse(currDateRange.period[key]) / 1000) @@ -72,6 +80,10 @@ function Metrics() { setHosts(func.getListOfHosts(apiCollections)) },[]) + useEffect(() => { + getRuntimeMetrics() + }, [runtimeFilterVal]) + useEffect(()=>{ getGraphData(startTime,endTime) },[currDateRange,groupBy]) @@ -138,8 +150,234 @@ function Metrics() { )) ) + const runtimeMetricsNameMap = { + "rt_kafka_record_count": "Kafka Record Count", + "rt_kafka_record_size": "Kafka Record Size", + "rt_kafka_latency": "Kafka Latency", + "rt_kafka_records_lag_max": "Kafka Records Lag Max", + "rt_kafka_records_consumed_rate": "Kafka Records Consumed Rate", + "rt_kafka_fetch_avg_latency": "Kafka Fetch Average Latency", + "rt_kafka_bytes_consumed_rate": "Kafka Bytes Consumed Rate" + } + + const runtimeFilterOptionsValueMap = { + "15minutes": Math.floor((Date.now() - (15 * 60 * 1000)) / 1000), + "30minutes": Math.floor((Date.now() - (30 * 60 * 1000)) / 1000), + "1hour": Math.floor(Date.now() / 1000) - 3600, + "6hours": Math.floor(Date.now() / 1000) - 21600, + "1day": Math.floor((Date.now() - (24 * 60 * 60 * 1000)) / 1000), + "3days": Math.floor((Date.now() - (3 * 24 * 60 * 60 * 1000)) / 1000), + "last7days": Math.floor((Date.now() - (7 * 24 * 60 * 60 * 1000)) / 1000) + } + + const getRuntimeMetrics = async () => { + const currentEpoch = Math.floor(Date.now() / 1000) + let runtimeRes = await settingFunctions.fetchRuntimeInstances(runtimeFilterOptionsValueMap[runtimeFilterVal], currentEpoch) + + const uniqueInstanceIds = new Set(runtimeRes.instanceIds) + const runtimeMetrics = runtimeRes.runtimeMetrics + + const namesArray = Object.keys(runtimeMetricsNameMap) + + const groupedData = Array.from(uniqueInstanceIds).map(instanceId => { + const instanceData = runtimeMetrics.filter(item => {return item.instanceId === instanceId && namesArray.includes(item.name.toLowerCase())}) + + if (!instanceData.length) return null + + const startTime = Math.min(...instanceData.map(item => item.timestamp)) + const heartbeat = Math.max(...instanceData.map(item => item.timestamp)) + + const result = { + id: instanceId, + startTime: func.prettifyEpoch(startTime), + heartbeat: func.prettifyEpoch(heartbeat), + version: instanceData[0].version, + } + + const latestValuesByName = {}; + + instanceData.forEach(item => { + const name = item.name.toLowerCase(); + + if (!latestValuesByName[name] || latestValuesByName[name].timestamp < item.timestamp) { + latestValuesByName[name] = { + val: item.val, + timestamp: item.timestamp + }; + } + }); + + Object.keys(latestValuesByName).forEach(name => { + result[name] = latestValuesByName[name].val; + }); + + return result + }).filter(item => item !== null) + + setRuntimeMetricsData(groupedData) + } + const runtimeFilterOptions = [ + { label: '15 Minutes ago', value: '15minutes' }, + { label: '30 Minutes ago', value: '30minutes' }, + { label: '1 hour ago', value: '1hour' }, + { label: '6 hours ago', value: '6hours' }, + { label: '1 Day ago', value: '1day' }, + { label: '3 Days ago', value: '3days' }, + { label: 'Last 7 days', value: 'last7days' } + ] + + const fillMissingTimestamps = (data) => { + const sortedData = data.slice().sort((a, b) => a[0] - b[0]) + const smallestTime = sortedData[0][0] + const largestTime = sortedData[sortedData.length - 1][0] + + const result = [] + const timestampMap = new Map() + + for (let timestamp = smallestTime; timestamp <= largestTime; timestamp += 60000) { + timestampMap.set(timestamp, 0) + } + + sortedData.forEach(([timestamp, value]) => { + timestampMap.set(timestamp, value) + }) + + timestampMap.forEach((value, timestamp) => { + result.push([timestamp, value]) + }) + + result.sort((a, b) => a[0] - b[0]) + + return result + } + + const handleOnRuntimeRowClick = async (data) => { + setLoading(true) + const currentEpoch = Math.floor(Date.now() / 1000) + const instanceId = data.id + const runtimeMetricsRes = await settingFunctions.fetchRuntimeMetrics(runtimeFilterOptionsValueMap[runtimeFilterVal], currentEpoch, instanceId) + + const valuesByName = getRuntimeValuesByName(runtimeMetricsRes) + + setShowRuntimeGraph(true) + const componentsArray = [] + + Object.entries(valuesByName).forEach(([name, values]) => { + const readableName = runtimeMetricsNameMap[name.toLowerCase()] + const valuesWithMissingTimestamp = fillMissingTimestamps(values) + + const component = runtimeGraphContainer(valuesWithMissingTimestamp, readableName) + componentsArray.push() + componentsArray.push(component) + }) + + setGraphs(componentsArray) + + setTimeout(() => { + setLoading(false) + }, 100); + } + + const getRuntimeValuesByName = (data) => { + const valueByName = {}; + + data.forEach(item => { + if (!valueByName[item.name]) { + valueByName[item.name] = []; + } + valueByName[item.name].push([(item.timestamp*1000), item.val]) + }); + + for (const name in valueByName) { + if (Object.hasOwnProperty.call(valueByName, name)) { + valueByName[name].sort((a, b) => a.val - b.val); + } + } + + return valueByName; + } + + const headers = [ + { title: "Instance ID", text: "Instance ID", value: "id", showFilter: false }, + { title: "Heartbeat", text: "Heartbeat", value: "heartbeat", showFilter: false }, + { title: "Start Time", text: "Start Time", value: "startTime", showFilter: false }, + { title: "Runtime Version", text: "Runtime Version", value: "version", showFilter: false }, + { title: "Kafka Record Count", text: "Kafka Record Count", value: "rt_kafka_record_count", showFilter: false }, + { title: "Kafka Record Size", text: "Kafka Record Size", value: "rt_kafka_record_size", showFilter: false }, + { title: "Kafka Latency", text: "Kafka Latency", value: "rt_kafka_latency", showFilter: false }, + { title: "Kafka Records Lag Max", text: "Kafka Records Lag Max", value: "rt_kafka_records_lag_max", showFilter: false }, + { title: "Kafka Records Consumed Rate", text: "Kafka Records Consumed Rate", value: "rt_kafka_records_consumed_rate", showFilter: false }, + { title: "Kafka Fetch Average Latency", text: "Kafka Records Consumed Rate", value: "rt_kafka_fetch_avg_latency", showFilter: false }, + { title: "Kafka Bytes Consumed Rate", text: "Kafka Bytes Consumed Rate", value: "rt_kafka_bytes_consumed_rate", showFilter: false }, + ] + + const promotedBulkActions = (selectedResources) => { + const actions = [ + { + content: `Copy Instance ID${func.addPlurality(selectedResources.length)}`, + onAction: () => navigator.clipboard.writeText(selectedResources.toString()) + } + ] + + return actions; + } + + const runtimeTableContainer = ( + + ) + + const processChartData = (data) => { + return [ + { + data: data, + color: "#AEE9D1", + name: "Runtime Values" + }, + ] + } + + const runtimeGraphContainer = (data, title) => ( + + + + ) + return ( - + + + + + setRuntimeFilterVal(val)} /> + + + + { runtimeTableContainer } + @@ -151,6 +389,14 @@ function Metrics() { {graphContainer} + + ) } diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/settings/module.js b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/settings/module.js index 6927c40ef8..79685be02b 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/settings/module.js +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/settings/module.js @@ -228,6 +228,20 @@ const settingFunctions = { }) return arr }, + fetchRuntimeInstances: async function(startTimestamp, endTimestamp) { + let res = [] + await settingRequests.fetchRuntimeInstances(startTimestamp, endTimestamp).then((resp) => { + res = resp + }) + return res + }, + fetchRuntimeMetrics: async function(startTimestamp, endTimestamp, instanceId) { + let res = [] + await settingRequests.fetchRuntimeMetrics(startTimestamp, endTimestamp, instanceId).then((resp) => { + res = resp.runtimeMetrics + }) + return res + }, fetchGraphData: async function(groupBy, startTimestamp, endTimestamp, names, host){ let trafficData = {} await settingRequests.fetchTrafficMetrics(groupBy, startTimestamp, endTimestamp, names, host).then((resp)=>{ diff --git a/libs/dao/src/main/java/com/akto/dto/traffic_metrics/RuntimeMetrics.java b/libs/dao/src/main/java/com/akto/dto/traffic_metrics/RuntimeMetrics.java index 53899ac03d..844ed939a4 100644 --- a/libs/dao/src/main/java/com/akto/dto/traffic_metrics/RuntimeMetrics.java +++ b/libs/dao/src/main/java/com/akto/dto/traffic_metrics/RuntimeMetrics.java @@ -6,14 +6,16 @@ public class RuntimeMetrics { private int timestamp; private String instanceId; private Double val; + private String version; public RuntimeMetrics() { } - public RuntimeMetrics(String name, int timestamp, String instanceId, Double val) { + public RuntimeMetrics(String name, int timestamp, String instanceId, String version, Double val) { this.name = name; this.timestamp = timestamp; this.instanceId = instanceId; + this.version = version; this.val = val; } @@ -41,6 +43,14 @@ public void setInstanceId(String instanceId) { this.instanceId = instanceId; } + public String getVersion() { + return version; + } + + public void setVersion(String version) { + this.version = version; + } + public Double getVal() { return val; } From 67c85442b521625897e0ba3a664a1e1b81c02bb6 Mon Sep 17 00:00:00 2001 From: TangoBeeAkto Date: Thu, 19 Sep 2024 12:18:31 +0530 Subject: [PATCH 2/4] feat: created traffic collector table and endpoints to fetch metrics data --- .../traffic_metrics/TrafficMetricsAction.java | 44 ++++++ apps/dashboard/src/main/resources/struts.xml | 45 ++++++ .../src/apps/dashboard/pages/settings/api.js | 14 ++ .../pages/settings/metrics/Metrics.jsx | 133 ++++++++++++++---- .../apps/dashboard/pages/settings/module.js | 14 ++ .../TrafficCollectorInfoDao.java | 62 ++++++++ .../TrafficCollectorMetricsDao.java | 72 ++++++++++ .../TrafficCollectorInfo.java | 64 +++++++++ .../TrafficCollectorMetrics.java | 90 ++++++++++++ 9 files changed, 511 insertions(+), 27 deletions(-) create mode 100644 libs/dao/src/main/java/com/akto/dao/traffic_collector/TrafficCollectorInfoDao.java create mode 100644 libs/dao/src/main/java/com/akto/dao/traffic_collector/TrafficCollectorMetricsDao.java create mode 100644 libs/dao/src/main/java/com/akto/dto/traffic_collector/TrafficCollectorInfo.java create mode 100644 libs/dao/src/main/java/com/akto/dto/traffic_collector/TrafficCollectorMetrics.java diff --git a/apps/dashboard/src/main/java/com/akto/action/traffic_metrics/TrafficMetricsAction.java b/apps/dashboard/src/main/java/com/akto/action/traffic_metrics/TrafficMetricsAction.java index 9e15572e45..fae5ea737c 100644 --- a/apps/dashboard/src/main/java/com/akto/action/traffic_metrics/TrafficMetricsAction.java +++ b/apps/dashboard/src/main/java/com/akto/action/traffic_metrics/TrafficMetricsAction.java @@ -1,8 +1,12 @@ package com.akto.action.traffic_metrics; import com.akto.action.UserAction; +import com.akto.dao.traffic_collector.TrafficCollectorInfoDao; +import com.akto.dao.traffic_collector.TrafficCollectorMetricsDao; import com.akto.dao.traffic_metrics.RuntimeMetricsDao; import com.akto.dao.traffic_metrics.TrafficMetricsDao; +import com.akto.dto.traffic_collector.TrafficCollectorInfo; +import com.akto.dto.traffic_collector.TrafficCollectorMetrics; import com.akto.dto.traffic_metrics.RuntimeMetrics; import com.akto.dto.traffic_metrics.TrafficMetrics; import com.mongodb.BasicDBObject; @@ -170,6 +174,30 @@ public String fetchTrafficMetricsDesciptions(){ return SUCCESS.toUpperCase(); } + TrafficCollectorMetrics trafficCollectorMetrics; + public String fetchTrafficCollectorMetrics() { + Bson filters = Filters.and( + Filters.gte("bucketStartEpoch", startTimestamp), + Filters.lte("bucketStartEpoch", endTimestamp), + Filters.in("_id", instanceId) + ); + trafficCollectorMetrics = TrafficCollectorMetricsDao.instance.findOne(filters); + + return SUCCESS.toUpperCase(); + } + + List trafficCollectorInfos; + public String fetchTrafficCollectorInfos() { + trafficCollectorInfos = new ArrayList<>(); + Bson filters = Filters.and( + Filters.gte("startTime", startTimestamp), + Filters.lte("startTime", endTimestamp) + ); + trafficCollectorInfos = TrafficCollectorInfoDao.instance.findAll(filters); + + return SUCCESS.toUpperCase(); + } + public void setStartTimestamp(int startTimestamp) { this.startTimestamp = startTimestamp; } @@ -227,4 +255,20 @@ public void setInstanceId(String instanceId) { this.instanceId = instanceId; } + public List getTrafficCollectorInfos() { + return trafficCollectorInfos; + } + + public void setTrafficCollectorInfos(List trafficCollectorInfos) { + this.trafficCollectorInfos = trafficCollectorInfos; + } + + public TrafficCollectorMetrics getTrafficCollectorMetrics() { + return trafficCollectorMetrics; + } + + public void setTrafficCollectorMetrics(TrafficCollectorMetrics trafficCollectorMetrics) { + this.trafficCollectorMetrics = trafficCollectorMetrics; + } + } diff --git a/apps/dashboard/src/main/resources/struts.xml b/apps/dashboard/src/main/resources/struts.xml index 58bf1ff4a0..7321113c9e 100644 --- a/apps/dashboard/src/main/resources/struts.xml +++ b/apps/dashboard/src/main/resources/struts.xml @@ -5031,6 +5031,51 @@ + + + + + METRICS + READ_WRITE + + + + 403 + false + ^actionErrors.* + + + trafficCollectorInfos + + + 422 + false + ^actionErrors.* + + + + + + + METRICS + READ_WRITE + + + + 403 + false + ^actionErrors.* + + + trafficCollectorMetrics + + + 422 + false + ^actionErrors.* + + + diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/settings/api.js b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/settings/api.js index 3e39dc3e37..bd3c36f120 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/settings/api.js +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/settings/api.js @@ -164,6 +164,20 @@ const settingRequests = { data: {groupBy, startTimestamp, endTimestamp, names, host} }) }, + fetchTrafficCollectorInfos(startTimestamp, endTimestamp) { + return request({ + url: '/api/fetchTrafficCollectorInfos', + method: 'post', + data: {startTimestamp, endTimestamp} + }) + }, + fetchTrafficCollectorMetrics(instanceId, startTimestamp, endTimestamp) { + return request({ + url: '/api/fetchTrafficCollectorMetrics', + method: 'post', + data: {instanceId, startTimestamp, endTimestamp} + }) + }, addCustomWebhook(webhookName, url, queryParams, method, headerString, body, frequencyInSeconds, selectedWebhookOptions, newEndpointCollections, newSensitiveEndpointCollections) { return request({ diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/settings/metrics/Metrics.jsx b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/settings/metrics/Metrics.jsx index f950a1f328..1dd5ddc91a 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/settings/metrics/Metrics.jsx +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/settings/metrics/Metrics.jsx @@ -22,9 +22,13 @@ function Metrics() { const [currentHost, setCurrentHost] = useState(null) const [runtimeMetricsData, setRuntimeMetricsData] = useState([]) - const [showRuntimeGraph, setShowRuntimeGraph] = useState(false) - const [graphs, setGraphs] = useState([]) + const [collectorMetricsData, setCollectorMetricsData] = useState([]) + const [showFlyout, setShowFlyout] = useState(false) + const [currentFlyoutType, setCurrentFlyoutType] = useState(null) + const [runtimeGraphs, setRuntimeGraphs] = useState([]) + const [collectorGraphs, setCollectorGraphs] = useState([]) const [runtimeFilterVal, setRuntimeFilterVal] = useState('1day') + const [trafficCollectorFilterVal, setTrafficCollectorFilterVal] = useState('1day') const [loading, setLoading] = useState(false) const [currDateRange, dispatchCurrDateRange] = useReducer(produce((draft, action) => func.dateRangeReducer(draft, action)), values.ranges[2]); @@ -83,6 +87,9 @@ function Metrics() { useEffect(() => { getRuntimeMetrics() }, [runtimeFilterVal]) + useEffect(() => { + getTrafficCollectorMetrics() + }, [trafficCollectorFilterVal]) useEffect(()=>{ getGraphData(startTime,endTime) @@ -160,7 +167,7 @@ function Metrics() { "rt_kafka_bytes_consumed_rate": "Kafka Bytes Consumed Rate" } - const runtimeFilterOptionsValueMap = { + const metricsTimeFilterOptionsMap = { "15minutes": Math.floor((Date.now() - (15 * 60 * 1000)) / 1000), "30minutes": Math.floor((Date.now() - (30 * 60 * 1000)) / 1000), "1hour": Math.floor(Date.now() / 1000) - 3600, @@ -172,7 +179,7 @@ function Metrics() { const getRuntimeMetrics = async () => { const currentEpoch = Math.floor(Date.now() / 1000) - let runtimeRes = await settingFunctions.fetchRuntimeInstances(runtimeFilterOptionsValueMap[runtimeFilterVal], currentEpoch) + let runtimeRes = await settingFunctions.fetchRuntimeInstances(metricsTimeFilterOptionsMap[runtimeFilterVal], currentEpoch) const uniqueInstanceIds = new Set(runtimeRes.instanceIds) const runtimeMetrics = runtimeRes.runtimeMetrics @@ -216,7 +223,25 @@ function Metrics() { setRuntimeMetricsData(groupedData) } - const runtimeFilterOptions = [ + + const getTrafficCollectorMetrics = async () => { + const currentEpoch = Math.floor(Date.now() / 1000) + const res = await settingFunctions.fetchTrafficCollectorInfos(metricsTimeFilterOptionsMap[trafficCollectorFilterVal], currentEpoch) + + const trafficCollectorMetricRes = res.map(metrix => { + return { + "id": metrix.id, + "runtimeId": metrix.runtimeId, + "version": metrix.version, + "lastHeartbeat": func.prettifyEpoch(metrix.lastHeartbeat), + "startTime": func.prettifyEpoch(metrix.startTime) + } + }) + + setCollectorMetricsData(trafficCollectorMetricRes) + } + + const metricsTimeFilterOptions = [ { label: '15 Minutes ago', value: '15minutes' }, { label: '30 Minutes ago', value: '30minutes' }, { label: '1 hour ago', value: '1hour' }, @@ -255,29 +280,54 @@ function Metrics() { setLoading(true) const currentEpoch = Math.floor(Date.now() / 1000) const instanceId = data.id - const runtimeMetricsRes = await settingFunctions.fetchRuntimeMetrics(runtimeFilterOptionsValueMap[runtimeFilterVal], currentEpoch, instanceId) + const runtimeMetricsRes = await settingFunctions.fetchRuntimeMetrics(metricsTimeFilterOptionsMap[runtimeFilterVal], currentEpoch, instanceId) const valuesByName = getRuntimeValuesByName(runtimeMetricsRes) - setShowRuntimeGraph(true) const componentsArray = [] Object.entries(valuesByName).forEach(([name, values]) => { const readableName = runtimeMetricsNameMap[name.toLowerCase()] const valuesWithMissingTimestamp = fillMissingTimestamps(values) - const component = runtimeGraphContainer(valuesWithMissingTimestamp, readableName) + const component = metricsGraphContainer(valuesWithMissingTimestamp, readableName) componentsArray.push() componentsArray.push(component) }) - setGraphs(componentsArray) + setRuntimeGraphs(componentsArray) + setCurrentFlyoutType("runtime") + setShowFlyout(true) setTimeout(() => { setLoading(false) }, 100); } + const handleOnTrafficCollectorRowClick = async (data) => { + setLoading(true) + + const id = data.id + const currentEpoch = Math.floor(Date.now() / 1000) + const res = await settingFunctions.fetchTrafficCollectorMetrics(id, metricsTimeFilterOptionsMap[trafficCollectorFilterVal], currentEpoch) + + const requestsCountMap = Object.entries(res.requestsCountMapPerMinute || []).map(([timestamp, value]) => { + return [parseInt(timestamp) * 60, value] + }) + + const dataWithDefaultVal = fillMissingTimestamps(requestsCountMap) + + const graph = metricsGraphContainer(dataWithDefaultVal, "Traffic Collector") + + setCollectorGraphs([graph]) + setCurrentFlyoutType("collector"); + setShowFlyout(true); + + setTimeout(() => { + setLoading(false) + }, 100) + } + const getRuntimeValuesByName = (data) => { const valueByName = {}; @@ -297,7 +347,7 @@ function Metrics() { return valueByName; } - const headers = [ + const runtimeHeaders = [ { title: "Instance ID", text: "Instance ID", value: "id", showFilter: false }, { title: "Heartbeat", text: "Heartbeat", value: "heartbeat", showFilter: false }, { title: "Start Time", text: "Start Time", value: "startTime", showFilter: false }, @@ -311,6 +361,14 @@ function Metrics() { { title: "Kafka Bytes Consumed Rate", text: "Kafka Bytes Consumed Rate", value: "rt_kafka_bytes_consumed_rate", showFilter: false }, ] + const collectorHeaders = [ + { title: "ID", text: "ID", value: "id", showFilter: false }, + { title: "Runtime ID", text: "Runtime ID", value: "runtimeId", showFilter: false }, + { title: "Heartbeat", text: "Heartbeat", value: "lastHeartbeat", showFilter: false }, + { title: "Start Time", text: "Start Time", value: "startTime", showFilter: false }, + { title: "Runtime Version", text: "Runtime Version", value: "version", showFilter: false }, + ] + const promotedBulkActions = (selectedResources) => { const actions = [ { @@ -322,11 +380,11 @@ function Metrics() { return actions; } - const runtimeTableContainer = ( + const metricsTableContainer = (type, data, onRowClick, headers) => ( ) - const processChartData = (data) => { + const processChartData = (name, data) => { return [ { data: data, color: "#AEE9D1", - name: "Runtime Values" + name: `${name} Values` }, ] } - const runtimeGraphContainer = (data, title) => ( + const metricsGraphContainer = (data, title) => ( - ) + const flyoutTitle = currentFlyoutType === "runtime" + ? "Runtime Metrics Details" + : "Traffic Collector Metrics Details"; + + const flyoutComponents = currentFlyoutType === "runtime" + ? runtimeGraphs + : collectorGraphs; + return ( - setRuntimeFilterVal(val)} /> + setRuntimeFilterVal(val)} /> + + + + { metricsTableContainer("processor", runtimeMetricsData, handleOnRuntimeRowClick, runtimeHeaders) } + + + + + + setTrafficCollectorFilterVal(val)} /> - { runtimeTableContainer } + { metricsTableContainer("collector", collectorMetricsData, handleOnTrafficCollectorRowClick, collectorHeaders) } + @@ -390,13 +467,15 @@ function Metrics() { {graphContainer} - + {showFlyout && ( + + )} ) } diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/settings/module.js b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/settings/module.js index 79685be02b..998a716105 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/settings/module.js +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/settings/module.js @@ -249,6 +249,20 @@ const settingFunctions = { }) return trafficData }, + fetchTrafficCollectorInfos: async function(startTimestamp, endTimestamp) { + let res = [] + await settingRequests.fetchTrafficCollectorInfos(startTimestamp, endTimestamp).then((resp) => { + res = resp + }) + return res + }, + fetchTrafficCollectorMetrics: async function(id, startTimestamp, endTimestamp) { + let res = [] + await settingRequests.fetchTrafficCollectorMetrics(id, startTimestamp, endTimestamp).then((resp) => { + res = resp + }) + return res + }, testJiraIntegration: async function(userEmail, apiToken, baseUrl, projId){ let issueType = "" await settingRequests.testJiraIntegration(userEmail, apiToken, baseUrl, projId).then((resp)=>{ diff --git a/libs/dao/src/main/java/com/akto/dao/traffic_collector/TrafficCollectorInfoDao.java b/libs/dao/src/main/java/com/akto/dao/traffic_collector/TrafficCollectorInfoDao.java new file mode 100644 index 0000000000..730b673dfc --- /dev/null +++ b/libs/dao/src/main/java/com/akto/dao/traffic_collector/TrafficCollectorInfoDao.java @@ -0,0 +1,62 @@ +package com.akto.dao.traffic_collector; + +import com.akto.dao.AccountsContextDao; +import com.akto.dao.context.Context; +import com.akto.dto.traffic_collector.TrafficCollectorInfo; +import com.akto.dto.traffic_collector.TrafficCollectorMetrics; +import com.akto.util.DbMode; +import com.mongodb.client.MongoDatabase; +import com.mongodb.client.model.CreateCollectionOptions; +import com.mongodb.client.model.Filters; +import com.mongodb.client.model.Updates; + +public class TrafficCollectorInfoDao extends AccountsContextDao { + + public static final TrafficCollectorInfoDao instance = new TrafficCollectorInfoDao(); + + @Override + public String getCollName() { + return "traffic_collector_info"; + } + + @Override + public Class getClassT() { + return TrafficCollectorInfo.class; + } + + public static final int maxDocuments = 10_000; + public static final int sizeInBytes = 10_000_000; + + public void createIndicesIfAbsent() { + boolean exists = false; + String dbName = Context.accountId.get()+""; + MongoDatabase db = clients[0].getDatabase(dbName); + for (String col: db.listCollectionNames()){ + if (getCollName().equalsIgnoreCase(col)){ + exists = true; + break; + } + }; + + if (!exists) { + if (DbMode.allowCappedCollections()) { + db.createCollection(getCollName(), new CreateCollectionOptions().capped(true).maxDocuments(maxDocuments).sizeInBytes(sizeInBytes)); + } else { + db.createCollection(getCollName()); + } + } + } + + + public void updateHeartbeat(String id, String runtimeId, String version) { + instance.updateOne( + Filters.eq("_id", id), + Updates.combine( + Updates.set(TrafficCollectorInfo.LAST_HEARTBEAT, Context.now()), + Updates.setOnInsert(TrafficCollectorInfo.START_TIME, Context.now()), + Updates.setOnInsert(TrafficCollectorInfo.RUNTIME_ID, runtimeId), + Updates.set(TrafficCollectorMetrics.VERSION, version) + ) + ); + } +} \ No newline at end of file diff --git a/libs/dao/src/main/java/com/akto/dao/traffic_collector/TrafficCollectorMetricsDao.java b/libs/dao/src/main/java/com/akto/dao/traffic_collector/TrafficCollectorMetricsDao.java new file mode 100644 index 0000000000..d04ced42c1 --- /dev/null +++ b/libs/dao/src/main/java/com/akto/dao/traffic_collector/TrafficCollectorMetricsDao.java @@ -0,0 +1,72 @@ +package com.akto.dao.traffic_collector; + +import com.akto.dao.AccountsContextDao; +import com.akto.dao.context.Context; +import com.akto.dto.traffic_collector.TrafficCollectorInfo; +import com.akto.dto.traffic_collector.TrafficCollectorMetrics; +import com.akto.util.DbMode; +import com.mongodb.client.MongoDatabase; +import com.mongodb.client.model.CreateCollectionOptions; +import com.mongodb.client.model.Filters; +import com.mongodb.client.model.Updates; +import org.apache.commons.collections.ArrayStack; +import org.bson.conversions.Bson; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +public class TrafficCollectorMetricsDao extends AccountsContextDao { + + public static final TrafficCollectorMetricsDao instance = new TrafficCollectorMetricsDao(); + + @Override + public String getCollName() { + return "traffic_collector_metrics"; + } + + @Override + public Class getClassT() { + return TrafficCollectorMetrics.class; + } + + public static final int maxDocuments = 10_000; + public static final int sizeInBytes = 10_000_000; + + public void createIndicesIfAbsent() { + boolean exists = false; + String dbName = Context.accountId.get()+""; + MongoDatabase db = clients[0].getDatabase(dbName); + for (String col: db.listCollectionNames()){ + if (getCollName().equalsIgnoreCase(col)){ + exists = true; + break; + } + }; + + if (!exists) { + if (DbMode.allowCappedCollections()) { + db.createCollection(getCollName(), new CreateCollectionOptions().capped(true).maxDocuments(maxDocuments).sizeInBytes(sizeInBytes)); + } else { + db.createCollection(getCollName()); + } + } + } + + public void updateCount(TrafficCollectorMetrics trafficCollectorMetrics) { + List updates = new ArrayList<>(); + Map requestsCountMapPerMinute = trafficCollectorMetrics.getRequestsCountMapPerMinute(); + if (requestsCountMapPerMinute == null || requestsCountMapPerMinute.isEmpty()) return; + for (String key: requestsCountMapPerMinute.keySet()) { + updates.add(Updates.inc(TrafficCollectorMetrics.REQUESTS_COUNT_MAP_PER_MINUTE + "." + key, requestsCountMapPerMinute.getOrDefault(key, 0))); + } + instance.updateOne( + Filters.and( + Filters.eq("_id", trafficCollectorMetrics.getId()), + Filters.eq(TrafficCollectorMetrics.BUCKET_START_EPOCH, trafficCollectorMetrics.getBucketStartEpoch()), + Filters.eq(TrafficCollectorMetrics.BUCKET_END_EPOCH, trafficCollectorMetrics.getBucketEndEpoch()) + ), + Updates.combine(updates) + ); + } +} \ No newline at end of file diff --git a/libs/dao/src/main/java/com/akto/dto/traffic_collector/TrafficCollectorInfo.java b/libs/dao/src/main/java/com/akto/dto/traffic_collector/TrafficCollectorInfo.java new file mode 100644 index 0000000000..64b4f48d8d --- /dev/null +++ b/libs/dao/src/main/java/com/akto/dto/traffic_collector/TrafficCollectorInfo.java @@ -0,0 +1,64 @@ +package com.akto.dto.traffic_collector; + +public class TrafficCollectorInfo { + + private String id; + public static final String RUNTIME_ID = "runtimeId"; + private String runtimeId; + public static final String START_TIME = "startTime"; + private int startTime; + public static final String LAST_HEARTBEAT = "lastHeartbeat"; + private int lastHeartbeat; + public static final String VERSION = "version"; + private String version; + + public TrafficCollectorInfo() {} + + public TrafficCollectorInfo(String id, String runtimeId, int startTime, int lastHeartbeat, String version) { + this.id = id; + this.runtimeId = runtimeId; + this.startTime = startTime; + this.lastHeartbeat = lastHeartbeat; + this.version = version; + } + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public int getStartTime() { + return startTime; + } + + public void setStartTime(int startTime) { + this.startTime = startTime; + } + + public int getLastHeartbeat() { + return lastHeartbeat; + } + + public void setLastHeartbeat(int lastHeartbeat) { + this.lastHeartbeat = lastHeartbeat; + } + + public String getRuntimeId() { + return runtimeId; + } + + public void setRuntimeId(String runtimeId) { + this.runtimeId = runtimeId; + } + + public String getVersion() { + return version; + } + + public void setVersion(String version) { + this.version = version; + } +} \ No newline at end of file diff --git a/libs/dao/src/main/java/com/akto/dto/traffic_collector/TrafficCollectorMetrics.java b/libs/dao/src/main/java/com/akto/dto/traffic_collector/TrafficCollectorMetrics.java new file mode 100644 index 0000000000..62d515bac8 --- /dev/null +++ b/libs/dao/src/main/java/com/akto/dto/traffic_collector/TrafficCollectorMetrics.java @@ -0,0 +1,90 @@ +package com.akto.dto.traffic_collector; + +import java.util.Map; + +public class TrafficCollectorMetrics { + private String id; + public static final String RUNTIME_ID = "runtimeId"; + private String runtimeId; + public static final String REQUESTS_COUNT_MAP_PER_MINUTE = "requestsCountMapPerMinute"; + private Map requestsCountMapPerMinute; + public static final String BUCKET_START_EPOCH = "bucketStartEpoch"; + private int bucketStartEpoch; + public static final String BUCKET_END_EPOCH = "bucketEndEpoch"; + private int bucketEndEpoch; + public static final String VERSION = "version"; + private String version; + + + public TrafficCollectorMetrics(String id, String runtimeId, Map requestsCountMapPerMinute, int bucketStartEpoch, int bucketEndEpoch, String version) { + this.id = id; + this.runtimeId = runtimeId; + this.requestsCountMapPerMinute = requestsCountMapPerMinute; + this.bucketStartEpoch = bucketStartEpoch; + this.bucketEndEpoch = bucketEndEpoch; + this.version = version; + } + + public TrafficCollectorMetrics() { + } + + @Override + public String toString() { + return "TrafficCollectorMetrics{" + + "id='" + id + '\'' + + ", runtimeId='" + runtimeId + '\'' + + ", requestsCountMapPerMinute=" + requestsCountMapPerMinute + + ", bucketStartEpoch=" + bucketStartEpoch + + ", bucketEndEpoch=" + bucketEndEpoch + + ", runtimeVersion=" + version + + '}'; + } + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public Map getRequestsCountMapPerMinute() { + return requestsCountMapPerMinute; + } + + public void setRequestsCountMapPerMinute(Map requestsCountMapPerMinute) { + this.requestsCountMapPerMinute = requestsCountMapPerMinute; + } + + public int getBucketStartEpoch() { + return bucketStartEpoch; + } + + public void setBucketStartEpoch(int bucketStartEpoch) { + this.bucketStartEpoch = bucketStartEpoch; + } + + public int getBucketEndEpoch() { + return bucketEndEpoch; + } + + public void setBucketEndEpoch(int bucketEndEpoch) { + this.bucketEndEpoch = bucketEndEpoch; + } + + public String getRuntimeId() { + return runtimeId; + } + + public void setRuntimeId(String runtimeId) { + this.runtimeId = runtimeId; + } + + public String getVersion() { + return version; + } + + public void setVersion(String version) { + this.version = version; + } +} \ No newline at end of file From 779b669e505dfba13069c744dccd96114bafaccb Mon Sep 17 00:00:00 2001 From: TangoBeeAkto Date: Thu, 19 Sep 2024 13:35:02 +0530 Subject: [PATCH 3/4] fix: fixed traffic collector graph x-axis --- .../src/apps/dashboard/pages/settings/metrics/Metrics.jsx | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/settings/metrics/Metrics.jsx b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/settings/metrics/Metrics.jsx index 1dd5ddc91a..7f650893cc 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/settings/metrics/Metrics.jsx +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/settings/metrics/Metrics.jsx @@ -311,8 +311,9 @@ function Metrics() { const currentEpoch = Math.floor(Date.now() / 1000) const res = await settingFunctions.fetchTrafficCollectorMetrics(id, metricsTimeFilterOptionsMap[trafficCollectorFilterVal], currentEpoch) - const requestsCountMap = Object.entries(res.requestsCountMapPerMinute || []).map(([timestamp, value]) => { - return [parseInt(timestamp) * 60, value] + const requestsCountMap = Object.entries(res.requestsCountMapPerMinute).map(([timestamp, value]) => { + const epochInMilliSeconds = parseInt(timestamp)*60*1000 + return [epochInMilliSeconds, value] }) const dataWithDefaultVal = fillMissingTimestamps(requestsCountMap) From b1d68a61c0d16038c3919b451c9021d836a21f1b Mon Sep 17 00:00:00 2001 From: TangoBeeAkto Date: Fri, 20 Sep 2024 15:28:23 +0530 Subject: [PATCH 4/4] fix: passing day epoch instead of second epoch in traffic collector filter --- .../web/src/apps/dashboard/pages/settings/metrics/Metrics.jsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/settings/metrics/Metrics.jsx b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/settings/metrics/Metrics.jsx index 7f650893cc..2b5cd72097 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/settings/metrics/Metrics.jsx +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/settings/metrics/Metrics.jsx @@ -309,7 +309,7 @@ function Metrics() { const id = data.id const currentEpoch = Math.floor(Date.now() / 1000) - const res = await settingFunctions.fetchTrafficCollectorMetrics(id, metricsTimeFilterOptionsMap[trafficCollectorFilterVal], currentEpoch) + const res = await settingFunctions.fetchTrafficCollectorMetrics(id, metricsTimeFilterOptionsMap[trafficCollectorFilterVal]/86400, currentEpoch/86400) const requestsCountMap = Object.entries(res.requestsCountMapPerMinute).map(([timestamp, value]) => { const epochInMilliSeconds = parseInt(timestamp)*60*1000