From 8ba3353a01cf9a3b9dfed422ef9babfeed44f7d6 Mon Sep 17 00:00:00 2001 From: Xinrui Wu Date: Thu, 4 Jul 2024 17:24:16 -0700 Subject: [PATCH 01/33] init --- ui/src/views/Dashboard/index.vue | 163 ++++++++++++++++++++++++++++++- 1 file changed, 160 insertions(+), 3 deletions(-) diff --git a/ui/src/views/Dashboard/index.vue b/ui/src/views/Dashboard/index.vue index f6c1b98b8..09b50eceb 100644 --- a/ui/src/views/Dashboard/index.vue +++ b/ui/src/views/Dashboard/index.vue @@ -18,6 +18,138 @@ --> - \ No newline at end of file + \ No newline at end of file From 238456a96d23c124d1b80c2b2b19c2205cad2e62 Mon Sep 17 00:00:00 2001 From: Xinrui Wu Date: Fri, 5 Jul 2024 22:43:52 -0700 Subject: [PATCH 02/33] data processing --- ui/src/views/Dashboard/index.vue | 272 ++++++++++++++++++------------- 1 file changed, 163 insertions(+), 109 deletions(-) diff --git a/ui/src/views/Dashboard/index.vue b/ui/src/views/Dashboard/index.vue index 09b50eceb..ad2373cb2 100644 --- a/ui/src/views/Dashboard/index.vue +++ b/ui/src/views/Dashboard/index.vue @@ -25,131 +25,172 @@ const utcTime = ref({ now: '', thirtySecondsAgo: '' }); - -const upTimeDataContent = ref(null); -const memoryDataContent = ref(null); -const diskDataContent = ref(null); - -function getCurrentUTCTime() { - const now = new Date(); - utcTime.value.now = now.toISOString(); - - const FiftheenSecondsAgo = new Date(now.getTime() - 15000); - utcTime.value.thirtySecondsAgo = FiftheenSecondsAgo.toISOString(); -} - const commonParams = { groups: ["_monitoring"], offset: 0, - limit: 0, orderBy: { indexRuleName: "", sort: "SORT_UNSPECIFIED" }, - tagProjection: { - tagFamilies: [ - { - name: "default", - tags: ["node_type", "node_id", "grpc_address", "http_address"] - } - ] - }, fieldProjection: { names: [ "value" ] } }; - -function fetchData() { - getCurrentUTCTime(); - fetchUptime(); - fetchMemory(); - fetchDisk(); +const tagProjectionUptime = { + tagFamilies: [ + { + name: "default", + tags: ["node_type", "node_id", "grpc_address", "http_address"] + } + ] } - -function fetchUptime() { - const params = JSON.parse(JSON.stringify(commonParams)); - params.name = "up_time" - params.timeRange = { - begin: utcTime.value.thirtySecondsAgo, - end: utcTime.value.now, - }, - getTableList(params, "measure") - .then((res) => { - if (res.status === 200) { - upTimeDataContent.value = getLatestDataPointsForEachNode(res.data.dataPoints) +const tagProjection = { + tagFamilies: [ + { + name: "default", + tags: ["node_id", "kind"] + } + ] +} +const tagProjectionDisk = { + tagFamilies: [ + { + name: "default", + tags: ["node_id", "kind", "path"] + } + ] +} +const nodes = ref([]); + +async function fetchTableData() { + getCurrentUTCTime() + const [upTimeDataPoints, cpuDataPoints, memoryDataPoints, diskDataPoints] = await Promise.all([ + fetchDataPoints("up_time", tagProjectionUptime), + fetchDataPoints("cpu_state", tagProjection), + fetchDataPoints("memory_state", tagProjection), + fetchDataPoints("disk", tagProjectionDisk), + ]); + // create table rows using uptime datapoints + const rows = deduplicateByNodeId(upTimeDataPoints).map(item => { + const tags = item.tagFamilies[0].tags; + const nodeType = tags.find(tag => tag.key === 'node_type').value.str.value; + const nodeId = tags.find(tag => tag.key === 'node_id').value.str.value; + const grpcAddress = tags.find(tag => tag.key === 'grpc_address').value.str.value; + const httpAddress = tags.find(tag => tag.key === 'http_address').value.str.value; + const value = item.fields.find(field => field.name === 'value').value.float.value; + return { + node_id: nodeId, + node_type: nodeType, + grpc_address: grpcAddress, + http_address: httpAddress, + uptime: value + }; + }); + rows.sort((a, b) => { + return a.node_id.localeCompare(b.node_id); + }); + // group by other metrics + const cpuData = groupBy(cpuDataPoints, "kind"); + const memoryData = groupBy(memoryDataPoints, "kind") + const diskDataByPath = groupBy(diskDataPoints, "path") + const diskData = Object.keys(diskDataByPath).reduce((acc, path) => { + acc[path] = groupBy(diskDataByPath[path], 'kind'); + return acc; + }, {}); + // attach metrics to table + rows.forEach(node => { + node.cpu = getFieldValueByNodeId(cpuData.system, node.node_id); + node.memory = { + used: getFieldValueByNodeId(memoryData.used, node.node_id), + total: getFieldValueByNodeId(memoryData.total, node.node_id), + used_percent: getFieldValueByNodeId(memoryData.used_percent, node.node_id), + }; + node.disk = {} + for (const path in diskData) { + node.disk[path] = { + used: getFieldValueByNodeId(diskData[path].used, node.node_id), + total: getFieldValueByNodeId(diskData[path].total, node.node_id), + used_percent: getFieldValueByNodeId(diskData[path].used_percent, node.node_id) } - }) + } + }); + nodes.value = rows + console.log(rows) } -function fetchMemory() { - const params = JSON.parse(JSON.stringify(commonParams)); - params.name = "memory_state" - params.timeRange = { - begin: utcTime.value.thirtySecondsAgo, - end: utcTime.value.now, - }, - getTableList(params, "measure") - .then((res) => { - if (res.status === 200) { - memoryDataContent.value = getLatestDataPointsForEachNode(res.data.dataPoints) - } - }) +function getCurrentUTCTime() { + const now = new Date(); + utcTime.value.now = now.toISOString(); + + const FiftheenSecondsAgo = new Date(now.getTime() - 15000); + utcTime.value.thirtySecondsAgo = FiftheenSecondsAgo.toISOString(); } -function fetchDisk() { +async function fetchDataPoints(type, tagProjection) { const params = JSON.parse(JSON.stringify(commonParams)); - params.name = "disk" - params.timeRange = { + params.name = type; + params.timeRange = { begin: utcTime.value.thirtySecondsAgo, end: utcTime.value.now, - }, - getTableList(params, "measure") - .then((res) => { - if (res.status === 200) { - diskDataContent.value = getLatestDataPointsForEachNode(res.data.dataPoints) - } - }) + }; + params.tagProjection = tagProjection + const res = await getTableList(params, "measure"); + if (res.status === 200) { + return res.data.dataPoints; + } + return null; // Handle the case when status is not 200 } - -// get the latest data for each node id -function getLatestDataPointsForEachNode(data) { +function deduplicateByNodeId(data) { const nodeDataMap = {}; data.forEach(item => { const nodeIdTag = item.tagFamilies[0].tags.find(tag => tag.key === "node_id"); const nodeId = nodeIdTag.value.str.value; - const timestamp = new Date(item.timestamp).getTime(); - if (!nodeDataMap[nodeId] || timestamp > nodeDataMap[nodeId].timestamp) { - nodeDataMap[nodeId] = { ...item, timestamp }; + // Only add the item if it hasn't been added before + if (!nodeDataMap[nodeId]) { + nodeDataMap[nodeId] = item; } }); - const uniqueNodeData = Object.values(nodeDataMap).map(item => { - delete item.timestamp; // Remove the timestamp property added for comparison - return item; - }); - return uniqueNodeData + // The values in nodeDataMap are the first entries for each node ID + return Object.values(nodeDataMap); } -onMounted(() => { - fetchData(); - // Optional: Update the time every 15 seconds - setInterval(fetchData, 15000); -}); +function groupBy(data, key) { + return data.reduce((acc, obj) => { + const keyValue = obj.tagFamilies[0].tags.find(tag => tag.key === key).value.str.value; + if (!acc[keyValue]) { + acc[keyValue] = []; + } + acc[keyValue].push(obj); + return acc; + }, {}); +} + +function getFieldValueByNodeId(data, nodeId) { + // Find the object with the matching node_id + const item = data.find(item => { + const nodeIdTag = item.tagFamilies[0].tags.find(tag => tag.key === 'node_id'); + return nodeIdTag.value.str.value === nodeId; + }); + // Return the first field value if the object is found + if (item && item.fields.length > 0) { + return item.fields[0].value.float.value; + } -function getTagValue(tags, key) { - const tag = tags.find(tag => tag.key === key); - return tag ? tag.value.str.value : null; + // Return null if the object is not found or there are no fields + return null; } -function getFieldValue(fields, name) { - const field = fields.find(field => field.name === name); - return field ? field.value.float.value : null; -} +onMounted(() => { + fetchTableData(); + // Optional: Update the time every 15 seconds + setInterval(fetchTableData, 15000); +}); From b707034de21619ff36a6d93b6b00bf1899cab61d Mon Sep 17 00:00:00 2001 From: Xinrui Wu Date: Fri, 5 Jul 2024 23:08:35 -0700 Subject: [PATCH 03/33] get lastest --- ui/src/views/Dashboard/index.vue | 110 +++++++++++++++++-------------- 1 file changed, 62 insertions(+), 48 deletions(-) diff --git a/ui/src/views/Dashboard/index.vue b/ui/src/views/Dashboard/index.vue index ad2373cb2..6e71d6c48 100644 --- a/ui/src/views/Dashboard/index.vue +++ b/ui/src/views/Dashboard/index.vue @@ -23,7 +23,7 @@ import { getTableList } from '@/api/index' const utcTime = ref({ now: '', - thirtySecondsAgo: '' + fiftheenSecondsAgo: '' }); const commonParams = { groups: ["_monitoring"], @@ -64,7 +64,7 @@ const tagProjectionDisk = { } const nodes = ref([]); -async function fetchTableData() { +async function fetchNodes() { getCurrentUTCTime() const [upTimeDataPoints, cpuDataPoints, memoryDataPoints, diskDataPoints] = await Promise.all([ fetchDataPoints("up_time", tagProjectionUptime), @@ -73,7 +73,7 @@ async function fetchTableData() { fetchDataPoints("disk", tagProjectionDisk), ]); // create table rows using uptime datapoints - const rows = deduplicateByNodeId(upTimeDataPoints).map(item => { + const rows = getLatestForEachNode(upTimeDataPoints).map(item => { const tags = item.tagFamilies[0].tags; const nodeType = tags.find(tag => tag.key === 'node_type').value.str.value; const nodeId = tags.find(tag => tag.key === 'node_id').value.str.value; @@ -94,45 +94,44 @@ async function fetchTableData() { // group by other metrics const cpuData = groupBy(cpuDataPoints, "kind"); const memoryData = groupBy(memoryDataPoints, "kind") - const diskDataByPath = groupBy(diskDataPoints, "path") - const diskData = Object.keys(diskDataByPath).reduce((acc, path) => { - acc[path] = groupBy(diskDataByPath[path], 'kind'); + const paths = groupBy(diskDataPoints, "path") + const diskData = Object.keys(paths).reduce((acc, path) => { + acc[path] = groupBy(paths[path], 'kind'); return acc; }, {}); // attach metrics to table - rows.forEach(node => { - node.cpu = getFieldValueByNodeId(cpuData.system, node.node_id); - node.memory = { - used: getFieldValueByNodeId(memoryData.used, node.node_id), - total: getFieldValueByNodeId(memoryData.total, node.node_id), - used_percent: getFieldValueByNodeId(memoryData.used_percent, node.node_id), + rows.forEach(row => { + row.cpu = getLatestField(cpuData.system, row.node_id); + row.memory = { + used: getLatestField(memoryData.used, row.node_id), + total: getLatestField(memoryData.total, row.node_id), + used_percent: getLatestField(memoryData.used_percent, row.node_id), }; - node.disk = {} + row.disk = {} for (const path in diskData) { - node.disk[path] = { - used: getFieldValueByNodeId(diskData[path].used, node.node_id), - total: getFieldValueByNodeId(diskData[path].total, node.node_id), - used_percent: getFieldValueByNodeId(diskData[path].used_percent, node.node_id) + row.disk[path] = { + used: getLatestField(diskData[path].used, row.node_id), + total: getLatestField(diskData[path].total, row.node_id), + used_percent: getLatestField(diskData[path].used_percent, row.node_id) } } }); nodes.value = rows - console.log(rows) } function getCurrentUTCTime() { const now = new Date(); utcTime.value.now = now.toISOString(); - const FiftheenSecondsAgo = new Date(now.getTime() - 15000); - utcTime.value.thirtySecondsAgo = FiftheenSecondsAgo.toISOString(); + const fiftheenSecondsAgo = new Date(now.getTime() - 15000); + utcTime.value.fiftheenSecondsAgo = fiftheenSecondsAgo.toISOString(); } async function fetchDataPoints(type, tagProjection) { const params = JSON.parse(JSON.stringify(commonParams)); params.name = type; params.timeRange = { - begin: utcTime.value.thirtySecondsAgo, + begin: utcTime.value.fiftheenSecondsAgo, end: utcTime.value.now, }; params.tagProjection = tagProjection @@ -143,22 +142,6 @@ async function fetchDataPoints(type, tagProjection) { return null; // Handle the case when status is not 200 } -function deduplicateByNodeId(data) { - const nodeDataMap = {}; - data.forEach(item => { - const nodeIdTag = item.tagFamilies[0].tags.find(tag => tag.key === "node_id"); - const nodeId = nodeIdTag.value.str.value; - - // Only add the item if it hasn't been added before - if (!nodeDataMap[nodeId]) { - nodeDataMap[nodeId] = item; - } - }); - - // The values in nodeDataMap are the first entries for each node ID - return Object.values(nodeDataMap); -} - function groupBy(data, key) { return data.reduce((acc, obj) => { const keyValue = obj.tagFamilies[0].tags.find(tag => tag.key === key).value.str.value; @@ -170,26 +153,57 @@ function groupBy(data, key) { }, {}); } -function getFieldValueByNodeId(data, nodeId) { - // Find the object with the matching node_id - const item = data.find(item => { +// depuplicate by getting the latest data for each node id +function getLatestForEachNode(data) { + const nodeDataMap = {}; + data.forEach(item => { + const nodeIdTag = item.tagFamilies[0].tags.find(tag => tag.key === "node_id"); + const nodeId = nodeIdTag.value.str.value; + const timestamp = new Date(item.timestamp).getTime(); + + if (!nodeDataMap[nodeId] || timestamp > nodeDataMap[nodeId].timestamp) { + nodeDataMap[nodeId] = { ...item, timestamp }; + } + }); + + const uniqueNodeData = Object.values(nodeDataMap).map(item => { + delete item.timestamp; // Remove the timestamp property added for comparison + return item; + }); + return uniqueNodeData +} + +// get latest field value by nodeId +function getLatestField(data, nodeId) { + let latestItem = null; + let latestTimestamp = 0; + + // Iterate through each item in the data array + data.forEach(item => { const nodeIdTag = item.tagFamilies[0].tags.find(tag => tag.key === 'node_id'); - return nodeIdTag.value.str.value === nodeId; + const currentNodeId = nodeIdTag.value.str.value; + const timestamp = new Date(item.timestamp).getTime(); // Convert timestamp to milliseconds + + // Check if the current item matches the nodeId and is the latest + if (currentNodeId === nodeId && timestamp > latestTimestamp) { + latestTimestamp = timestamp; + latestItem = item; + } }); - // Return the first field value if the object is found - if (item && item.fields.length > 0) { - return item.fields[0].value.float.value; + // Return the first field value if a matching latest item is found + if (latestItem && latestItem.fields.length > 0) { + return latestItem.fields[0].value.float.value; } - // Return null if the object is not found or there are no fields + // Return null if no matching item is found or there are no fields return null; } onMounted(() => { - fetchTableData(); + fetchNodes(); // Optional: Update the time every 15 seconds - setInterval(fetchTableData, 15000); + setInterval(fetchNodes, 15000); }); @@ -201,7 +215,7 @@ onMounted(() => {

Current UTC Time

Now: {{ utcTime.now }}

-

30 Seconds Ago: {{ utcTime.thirtySecondsAgo }}

+

15 Seconds Ago: {{ utcTime.fiftheenSecondsAgo }}

From f0ae3f953d31b326bca2eac62414be99a3682802 Mon Sep 17 00:00:00 2001 From: Xinrui Wu Date: Sun, 14 Jul 2024 16:33:36 -0700 Subject: [PATCH 04/33] ui init change --- ui/src/views/Dashboard/index.vue | 120 +++++++++++++++++++++---------- 1 file changed, 84 insertions(+), 36 deletions(-) diff --git a/ui/src/views/Dashboard/index.vue b/ui/src/views/Dashboard/index.vue index 6e71d6c48..9bc45aa2d 100644 --- a/ui/src/views/Dashboard/index.vue +++ b/ui/src/views/Dashboard/index.vue @@ -64,6 +64,23 @@ const tagProjectionDisk = { } const nodes = ref([]); +function formatUptime(seconds) { + const hrs = Math.floor(seconds / 3600); + const mins = Math.floor((seconds % 3600) / 60); + const secs = Math.floor(seconds % 60); + return `${hrs > 0 ? `${hrs}h ` : ''}${mins}m ${secs}s`; +} +function extractAddress(fullAddress) { + const parts = fullAddress.split(':'); + return parts[parts.length - 1]; +} + +const colors = [ + { color: '#5cb87a', percentage: 51 }, + { color: '#edc374', percentage: 81 }, + { color: '#f56c6c', percentage: 100 }, +]; + async function fetchNodes() { getCurrentUTCTime() const [upTimeDataPoints, cpuDataPoints, memoryDataPoints, diskDataPoints] = await Promise.all([ @@ -77,8 +94,8 @@ async function fetchNodes() { const tags = item.tagFamilies[0].tags; const nodeType = tags.find(tag => tag.key === 'node_type').value.str.value; const nodeId = tags.find(tag => tag.key === 'node_id').value.str.value; - const grpcAddress = tags.find(tag => tag.key === 'grpc_address').value.str.value; - const httpAddress = tags.find(tag => tag.key === 'http_address').value.str.value; + const grpcAddress = extractAddress(tags.find(tag => tag.key === 'grpc_address').value.str.value); + const httpAddress = extractAddress(tags.find(tag => tag.key === 'http_address').value.str.value); const value = item.fields.find(field => field.name === 'value').value.float.value; return { node_id: nodeId, @@ -101,7 +118,7 @@ async function fetchNodes() { }, {}); // attach metrics to table rows.forEach(row => { - row.cpu = getLatestField(cpuData.system, row.node_id); + row.cpu = getLatestField(cpuData.user, row.node_id); row.memory = { used: getLatestField(memoryData.used, row.node_id), total: getLatestField(memoryData.total, row.node_id), @@ -116,6 +133,11 @@ async function fetchNodes() { } } }); + + // Post-process row data + rows.forEach(row => { + row.uptime = formatUptime(row.uptime); + }); nodes.value = rows } @@ -217,42 +239,68 @@ onMounted(() => {

Now: {{ utcTime.now }}

15 Seconds Ago: {{ utcTime.fiftheenSecondsAgo }}

-
- - - - - - - - - - - - - - - - - - - - - - - - - -
Node IDTypegRPC AddressHTTP AddressUptimeCPUMemory UsedMemory TotalMemory Used %Disk Details
{{ node.node_id }}{{ node.node_type }}{{ node.grpc_address }}{{ node.http_address || 'N/A' }}{{ node.uptime.toFixed(2) }} s{{ node.cpu }}{{ node.memory.used }}{{ node.memory.total }}{{ (node.memory.used_percent * 100).toFixed(2) }}% -
+ + + + + + + + + + + + + + + + + + +
- \ No newline at end of file + \ No newline at end of file From ee57cf42dd397079f39d2fa769e26733d180452d Mon Sep 17 00:00:00 2001 From: Xinrui Wu Date: Sun, 14 Jul 2024 18:04:30 -0700 Subject: [PATCH 05/33] memory style --- test/docker/base-compose.yml | 4 +- ui/src/views/Dashboard/index.vue | 79 +++++++++++++++++++++++--------- 2 files changed, 60 insertions(+), 23 deletions(-) diff --git a/test/docker/base-compose.yml b/test/docker/base-compose.yml index cc2d7f927..08ec87218 100644 --- a/test/docker/base-compose.yml +++ b/test/docker/base-compose.yml @@ -32,7 +32,7 @@ services: - 17912 - 2121 - 6060 - command: liaison --etcd-endpoints=http://etcd:2379 + command: liaison --etcd-endpoints=http://etcd:2379 --observability-modes=native healthcheck: test: ["CMD", "./bydbctl", "health", "--addr=http://liaison:17913"] interval: 30s @@ -45,7 +45,7 @@ services: - 17912 - 2121 - 6060 - command: data --etcd-endpoints=http://etcd:2379 + command: data --etcd-endpoints=http://etcd:2379 --observability-modes=native healthcheck: test: ["CMD", "./bydbctl", "health", "--grpc-addr=data:17912"] interval: 30s diff --git a/ui/src/views/Dashboard/index.vue b/ui/src/views/Dashboard/index.vue index 9bc45aa2d..ab5c8731f 100644 --- a/ui/src/views/Dashboard/index.vue +++ b/ui/src/views/Dashboard/index.vue @@ -70,15 +70,23 @@ function formatUptime(seconds) { const secs = Math.floor(seconds % 60); return `${hrs > 0 ? `${hrs}h ` : ''}${mins}m ${secs}s`; } + function extractAddress(fullAddress) { const parts = fullAddress.split(':'); return parts[parts.length - 1]; } +function formatBytes(bytes) { + if (bytes === 0 || bytes === undefined) return 'N/A'; + const sizes = ['Bytes', 'KB', 'MB', 'GB', 'TB']; + const i = Math.floor(Math.log(bytes) / Math.log(1024)); + return parseFloat((bytes / Math.pow(1024, i)).toFixed(2)) + ' ' + sizes[i]; +} + const colors = [ - { color: '#5cb87a', percentage: 51 }, - { color: '#edc374', percentage: 81 }, - { color: '#f56c6c', percentage: 100 }, + { color: '#5cb87a', percentage: 51 }, + { color: '#edc374', percentage: 81 }, + { color: '#f56c6c', percentage: 100 }, ]; async function fetchNodes() { @@ -247,24 +255,34 @@ onMounted(() => { - - + + +