From 6a9af61aad3e824bcf3ce0933177d7af536f3ac0 Mon Sep 17 00:00:00 2001 From: zacyanliu Date: Sat, 21 Oct 2023 09:26:55 +0800 Subject: [PATCH 01/15] =?UTF-8?q?feat:=20=E9=BB=98=E8=AE=A4=E6=9F=A5?= =?UTF-8?q?=E8=AF=A2=E6=97=B6=E9=97=B4=E8=B0=83=E6=95=B4=20#1314?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/tencent/bkrepo/opdata/pojo/ProjectMetricsOption.kt | 2 +- .../bkrepo/opdata/repository/ProjectMetricsRepository.kt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/backend/opdata/api-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/pojo/ProjectMetricsOption.kt b/src/backend/opdata/api-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/pojo/ProjectMetricsOption.kt index 91f8fa91c7..4bfc7c18f0 100644 --- a/src/backend/opdata/api-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/pojo/ProjectMetricsOption.kt +++ b/src/backend/opdata/api-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/pojo/ProjectMetricsOption.kt @@ -36,5 +36,5 @@ data class ProjectMetricsOption( var projectId: String? = null, val pageNumber: Int = DEFAULT_PAGE_NUMBER, val pageSize: Int = DEFAULT_PAGE_SIZE, - val createdDate: LocalDateTime = LocalDate.now().atStartOfDay() + val createdDate: LocalDateTime = LocalDate.now().minusDays(1).atStartOfDay() ) diff --git a/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/repository/ProjectMetricsRepository.kt b/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/repository/ProjectMetricsRepository.kt index 8a64dafa20..14c95d8719 100644 --- a/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/repository/ProjectMetricsRepository.kt +++ b/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/repository/ProjectMetricsRepository.kt @@ -48,6 +48,6 @@ interface ProjectMetricsRepository : MongoRepository{ createdDate: LocalDateTime, pageable: Pageable ): Page fun findAllByCreatedDate( - createdDate: LocalDateTime = LocalDate.now().atStartOfDay() + createdDate: LocalDateTime = LocalDate.now().minusDays(1).atStartOfDay() ): List } From 8a572c4b1d1ebdb957ccbe9a090edb1e77119ae1 Mon Sep 17 00:00:00 2001 From: zacyanliu Date: Sat, 21 Oct 2023 11:40:11 +0800 Subject: [PATCH 02/15] =?UTF-8?q?feat:=20=E6=9F=A5=E8=AF=A2=E5=87=BA?= =?UTF-8?q?=E6=8C=87=E5=AE=9A=E6=9D=A1=E4=BB=B6=E7=9A=84=E6=95=B0=E6=8D=AE?= =?UTF-8?q?=20#1314?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../bkrepo/opdata/pojo/ProjectMetrics.kt | 37 +++++++++++++++++++ .../opdata/pojo/ProjectMetricsRequest.kt | 33 +++++++++++++++++ .../opdata/controller/ProjectController.kt | 12 ++++++ .../repository/ProjectMetricsRepository.kt | 8 ++++ .../opdata/service/ProjectMetricsService.kt | 22 +++++++++++ 5 files changed, 112 insertions(+) create mode 100644 src/backend/opdata/api-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/pojo/ProjectMetrics.kt create mode 100644 src/backend/opdata/api-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/pojo/ProjectMetricsRequest.kt diff --git a/src/backend/opdata/api-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/pojo/ProjectMetrics.kt b/src/backend/opdata/api-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/pojo/ProjectMetrics.kt new file mode 100644 index 0000000000..a67344686d --- /dev/null +++ b/src/backend/opdata/api-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/pojo/ProjectMetrics.kt @@ -0,0 +1,37 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.opdata.pojo + +import java.time.LocalDateTime + +data class ProjectMetrics( + var projectId: String, + var nodeNum: Long, + var capSize: Long, + val createdDate: LocalDateTime? = LocalDateTime.now() +) diff --git a/src/backend/opdata/api-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/pojo/ProjectMetricsRequest.kt b/src/backend/opdata/api-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/pojo/ProjectMetricsRequest.kt new file mode 100644 index 0000000000..4a6db5079e --- /dev/null +++ b/src/backend/opdata/api-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/pojo/ProjectMetricsRequest.kt @@ -0,0 +1,33 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.opdata.pojo + +data class ProjectMetricsRequest( + val limitSize: Long = 1024 * 1024 * 1024L, + val oldDataFlag: Boolean = false +) diff --git a/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/controller/ProjectController.kt b/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/controller/ProjectController.kt index 7ebf8873b3..de105485f6 100644 --- a/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/controller/ProjectController.kt +++ b/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/controller/ProjectController.kt @@ -33,7 +33,9 @@ import com.tencent.bkrepo.common.security.permission.Principal import com.tencent.bkrepo.common.security.permission.PrincipalType import com.tencent.bkrepo.common.service.util.ResponseBuilder import com.tencent.bkrepo.opdata.model.TProjectMetrics +import com.tencent.bkrepo.opdata.pojo.ProjectMetrics import com.tencent.bkrepo.opdata.pojo.ProjectMetricsOption +import com.tencent.bkrepo.opdata.pojo.ProjectMetricsRequest import com.tencent.bkrepo.opdata.service.ProjectMetricsService import org.springframework.web.bind.annotation.GetMapping import org.springframework.web.bind.annotation.RequestMapping @@ -55,4 +57,14 @@ class ProjectController( ): Response> { return ResponseBuilder.success(projectMetricsService.page(option)) } + + /** + * 获取项目的统计数据 + */ + @GetMapping("/list/project/capSize") + fun getProjectCapSizeMetrics( + metricsRequest: ProjectMetricsRequest + ): Response> { + return ResponseBuilder.success(projectMetricsService.list(metricsRequest)) + } } diff --git a/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/repository/ProjectMetricsRepository.kt b/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/repository/ProjectMetricsRepository.kt index 14c95d8719..ab5d63aecc 100644 --- a/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/repository/ProjectMetricsRepository.kt +++ b/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/repository/ProjectMetricsRepository.kt @@ -50,4 +50,12 @@ interface ProjectMetricsRepository : MongoRepository{ fun findAllByCreatedDate( createdDate: LocalDateTime = LocalDate.now().minusDays(1).atStartOfDay() ): List + + fun findAllByCreatedDateNot( + createdDate: LocalDateTime = LocalDate.now().minusDays(1).atStartOfDay() + ): List + + fun findAllByCreatedDateAfter( + createdDate: LocalDateTime = LocalDate.now().minusDays(1).atStartOfDay() + ): List } diff --git a/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/service/ProjectMetricsService.kt b/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/service/ProjectMetricsService.kt index 88462d97bc..5fe6a09fca 100644 --- a/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/service/ProjectMetricsService.kt +++ b/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/service/ProjectMetricsService.kt @@ -31,7 +31,9 @@ import com.tencent.bkrepo.common.api.pojo.Page import com.tencent.bkrepo.common.mongo.dao.util.Pages import com.tencent.bkrepo.opdata.constant.TO_GIGABYTE import com.tencent.bkrepo.opdata.model.TProjectMetrics +import com.tencent.bkrepo.opdata.pojo.ProjectMetrics import com.tencent.bkrepo.opdata.pojo.ProjectMetricsOption +import com.tencent.bkrepo.opdata.pojo.ProjectMetricsRequest import com.tencent.bkrepo.opdata.repository.ProjectMetricsRepository import org.springframework.stereotype.Service @@ -59,4 +61,24 @@ class ProjectMetricsService ( return Pages.ofResponse(pageRequest, queryResult.totalElements, queryResult.content) } } + + fun list(metricsRequest: ProjectMetricsRequest): List { + val queryResult = if (metricsRequest.oldDataFlag) { + projectMetricsRepository.findAllByCreatedDateNot() + } else { + projectMetricsRepository.findAllByCreatedDateAfter() + } + val result = mutableListOf() + queryResult.map { + if (it.capSize >= metricsRequest.limitSize) { + result.add(ProjectMetrics( + projectId = it.projectId, + capSize = it.capSize / TO_GIGABYTE, + nodeNum = it.nodeNum, + createdDate = it.createdDate + )) + } + } + return result + } } \ No newline at end of file From 5d12367faecb05ae04583f2bf4f3330608ec449f Mon Sep 17 00:00:00 2001 From: zacyanliu Date: Mon, 23 Oct 2023 15:50:22 +0800 Subject: [PATCH 03/15] =?UTF-8?q?feat:=20=E5=8F=82=E6=95=B0=E8=B0=83?= =?UTF-8?q?=E6=95=B4=20#1314?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/tencent/bkrepo/opdata/pojo/ProjectMetricsRequest.kt | 3 ++- .../bkrepo/opdata/repository/ProjectMetricsRepository.kt | 2 +- .../tencent/bkrepo/opdata/service/ProjectMetricsService.kt | 4 +++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/backend/opdata/api-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/pojo/ProjectMetricsRequest.kt b/src/backend/opdata/api-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/pojo/ProjectMetricsRequest.kt index 4a6db5079e..e14f44c3cb 100644 --- a/src/backend/opdata/api-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/pojo/ProjectMetricsRequest.kt +++ b/src/backend/opdata/api-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/pojo/ProjectMetricsRequest.kt @@ -29,5 +29,6 @@ package com.tencent.bkrepo.opdata.pojo data class ProjectMetricsRequest( val limitSize: Long = 1024 * 1024 * 1024L, - val oldDataFlag: Boolean = false + val oldDataFlag: Boolean = false, + val minusDay: Long = 1 ) diff --git a/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/repository/ProjectMetricsRepository.kt b/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/repository/ProjectMetricsRepository.kt index ab5d63aecc..3fd8118623 100644 --- a/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/repository/ProjectMetricsRepository.kt +++ b/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/repository/ProjectMetricsRepository.kt @@ -56,6 +56,6 @@ interface ProjectMetricsRepository : MongoRepository{ ): List fun findAllByCreatedDateAfter( - createdDate: LocalDateTime = LocalDate.now().minusDays(1).atStartOfDay() + createdDate: LocalDateTime ): List } diff --git a/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/service/ProjectMetricsService.kt b/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/service/ProjectMetricsService.kt index 5fe6a09fca..60befaed23 100644 --- a/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/service/ProjectMetricsService.kt +++ b/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/service/ProjectMetricsService.kt @@ -36,6 +36,7 @@ import com.tencent.bkrepo.opdata.pojo.ProjectMetricsOption import com.tencent.bkrepo.opdata.pojo.ProjectMetricsRequest import com.tencent.bkrepo.opdata.repository.ProjectMetricsRepository import org.springframework.stereotype.Service +import java.time.LocalDate @Service class ProjectMetricsService ( @@ -66,7 +67,8 @@ class ProjectMetricsService ( val queryResult = if (metricsRequest.oldDataFlag) { projectMetricsRepository.findAllByCreatedDateNot() } else { - projectMetricsRepository.findAllByCreatedDateAfter() + val createdDate = LocalDate.now().minusDays(metricsRequest.minusDay).atStartOfDay() + projectMetricsRepository.findAllByCreatedDateAfter(createdDate) } val result = mutableListOf() queryResult.map { From db5e0c07351c8ad19eebd03c577b1e6ad40f5b7b Mon Sep 17 00:00:00 2001 From: zacyanliu Date: Mon, 23 Oct 2023 16:49:49 +0800 Subject: [PATCH 04/15] =?UTF-8?q?feat:=20=E9=80=BB=E8=BE=91=E8=B0=83?= =?UTF-8?q?=E6=95=B4=20#1314?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../bkrepo/opdata/pojo/ProjectMetricsRequest.kt | 1 - .../opdata/repository/ProjectMetricsRepository.kt | 8 -------- .../bkrepo/opdata/service/ProjectMetricsService.kt | 10 +++------- 3 files changed, 3 insertions(+), 16 deletions(-) diff --git a/src/backend/opdata/api-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/pojo/ProjectMetricsRequest.kt b/src/backend/opdata/api-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/pojo/ProjectMetricsRequest.kt index e14f44c3cb..7be94c7a0a 100644 --- a/src/backend/opdata/api-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/pojo/ProjectMetricsRequest.kt +++ b/src/backend/opdata/api-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/pojo/ProjectMetricsRequest.kt @@ -29,6 +29,5 @@ package com.tencent.bkrepo.opdata.pojo data class ProjectMetricsRequest( val limitSize: Long = 1024 * 1024 * 1024L, - val oldDataFlag: Boolean = false, val minusDay: Long = 1 ) diff --git a/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/repository/ProjectMetricsRepository.kt b/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/repository/ProjectMetricsRepository.kt index 3fd8118623..14c95d8719 100644 --- a/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/repository/ProjectMetricsRepository.kt +++ b/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/repository/ProjectMetricsRepository.kt @@ -50,12 +50,4 @@ interface ProjectMetricsRepository : MongoRepository{ fun findAllByCreatedDate( createdDate: LocalDateTime = LocalDate.now().minusDays(1).atStartOfDay() ): List - - fun findAllByCreatedDateNot( - createdDate: LocalDateTime = LocalDate.now().minusDays(1).atStartOfDay() - ): List - - fun findAllByCreatedDateAfter( - createdDate: LocalDateTime - ): List } diff --git a/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/service/ProjectMetricsService.kt b/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/service/ProjectMetricsService.kt index 60befaed23..794e6229c7 100644 --- a/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/service/ProjectMetricsService.kt +++ b/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/service/ProjectMetricsService.kt @@ -64,14 +64,10 @@ class ProjectMetricsService ( } fun list(metricsRequest: ProjectMetricsRequest): List { - val queryResult = if (metricsRequest.oldDataFlag) { - projectMetricsRepository.findAllByCreatedDateNot() - } else { - val createdDate = LocalDate.now().minusDays(metricsRequest.minusDay).atStartOfDay() - projectMetricsRepository.findAllByCreatedDateAfter(createdDate) - } + val createdDate = LocalDate.now().minusDays(metricsRequest.minusDay).atStartOfDay() + val queryResult = projectMetricsRepository.findAllByCreatedDate(createdDate) val result = mutableListOf() - queryResult.map { + queryResult.forEach { if (it.capSize >= metricsRequest.limitSize) { result.add(ProjectMetrics( projectId = it.projectId, From 6863be6a904b583a548b128ebe764ad0b6c4388f Mon Sep 17 00:00:00 2001 From: zacyanliu Date: Mon, 23 Oct 2023 19:35:38 +0800 Subject: [PATCH 05/15] =?UTF-8?q?feat:=20=E6=94=AF=E6=8C=81=E9=A1=B9?= =?UTF-8?q?=E7=9B=AE=E4=BF=A1=E6=81=AF=E5=AF=BC=E5=87=BA=20#1314?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../opdata/api-opdata/build.gradle.kts | 1 + .../bkrepo/opdata/pojo/ProjectMetrics.kt | 10 +++ .../opdata/pojo/ProjectMetricsRequest.kt | 3 +- .../opdata/controller/ProjectController.kt | 9 ++ .../bkrepo/opdata/model/StatDateModel.kt | 77 +++++++++++++++++ .../opdata/service/ProjectMetricsService.kt | 46 +++++++--- .../bkrepo/opdata/util/EasyExcelUtils.kt | 83 +++++++++++++++++++ 7 files changed, 217 insertions(+), 12 deletions(-) create mode 100644 src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/model/StatDateModel.kt create mode 100644 src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/util/EasyExcelUtils.kt diff --git a/src/backend/opdata/api-opdata/build.gradle.kts b/src/backend/opdata/api-opdata/build.gradle.kts index beff456627..654e9d98b6 100644 --- a/src/backend/opdata/api-opdata/build.gradle.kts +++ b/src/backend/opdata/api-opdata/build.gradle.kts @@ -32,6 +32,7 @@ dependencies { implementation(project(":common:common-api")) implementation(project(":common:common-operate:operate-annotation")) + api("com.alibaba:easyexcel:3.1.1") compileOnly("org.springframework.cloud:spring-cloud-openfeign-core") testImplementation(project(":common:common-operate:operate-service")) } diff --git a/src/backend/opdata/api-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/pojo/ProjectMetrics.kt b/src/backend/opdata/api-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/pojo/ProjectMetrics.kt index a67344686d..b7e7f9aada 100644 --- a/src/backend/opdata/api-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/pojo/ProjectMetrics.kt +++ b/src/backend/opdata/api-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/pojo/ProjectMetrics.kt @@ -27,11 +27,21 @@ package com.tencent.bkrepo.opdata.pojo +import com.alibaba.excel.annotation.ExcelProperty +import com.alibaba.excel.annotation.write.style.ColumnWidth import java.time.LocalDateTime data class ProjectMetrics( + @ColumnWidth(30) + @ExcelProperty(value = ["项目ID"], order = 0) var projectId: String, + @ColumnWidth(20) + @ExcelProperty(value = ["节点个数"], order = 1) var nodeNum: Long, + @ColumnWidth(20) + @ExcelProperty(value = ["项目大小(GB)"], order = 2) var capSize: Long, + @ColumnWidth(20) + @ExcelProperty(value = ["统计时间"], order = 3) val createdDate: LocalDateTime? = LocalDateTime.now() ) diff --git a/src/backend/opdata/api-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/pojo/ProjectMetricsRequest.kt b/src/backend/opdata/api-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/pojo/ProjectMetricsRequest.kt index 7be94c7a0a..26a474f178 100644 --- a/src/backend/opdata/api-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/pojo/ProjectMetricsRequest.kt +++ b/src/backend/opdata/api-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/pojo/ProjectMetricsRequest.kt @@ -29,5 +29,6 @@ package com.tencent.bkrepo.opdata.pojo data class ProjectMetricsRequest( val limitSize: Long = 1024 * 1024 * 1024L, - val minusDay: Long = 1 + val minusDay: Long = 1, + val default: Boolean = true ) diff --git a/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/controller/ProjectController.kt b/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/controller/ProjectController.kt index de105485f6..f36834c416 100644 --- a/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/controller/ProjectController.kt +++ b/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/controller/ProjectController.kt @@ -67,4 +67,13 @@ class ProjectController( ): Response> { return ResponseBuilder.success(projectMetricsService.list(metricsRequest)) } + /** + * 获取项目的统计数据 + */ + @GetMapping("/list/project/capSize/download") + fun downloadProjectCapSizeMetrics( + metricsRequest: ProjectMetricsRequest + ) { + projectMetricsService.download(metricsRequest) + } } diff --git a/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/model/StatDateModel.kt b/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/model/StatDateModel.kt new file mode 100644 index 0000000000..a0d3740467 --- /dev/null +++ b/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/model/StatDateModel.kt @@ -0,0 +1,77 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.opdata.model + +import com.tencent.bkrepo.common.job.JobAutoConfiguration.Companion.SHED_LOCK_COLLECTION_NAME +import com.tencent.bkrepo.common.mongo.constant.ID +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.data.mongodb.core.MongoTemplate +import org.springframework.data.mongodb.core.query.Criteria +import org.springframework.data.mongodb.core.query.Query +import org.springframework.data.mongodb.core.query.isEqualTo +import org.springframework.stereotype.Service +import java.time.LocalDate +import java.time.LocalDateTime + +@Service +class StatDateModel @Autowired constructor( + private val mongoTemplate: MongoTemplate +) { + + companion object { + private val JOB_NAME = "NodeStatCompositeMongoDbBatchJob" + } + + fun getShedLockInfo(id: String = JOB_NAME): LocalDateTime { + val query = Query(Criteria.where(ID).isEqualTo(id)) + val result = mongoTemplate.find(query, ShedlockInfo::class.java, SHED_LOCK_COLLECTION_NAME) + return if (result.isEmpty()) { + LocalDate.now().minusDays(1).atStartOfDay() + } else { + getLockedAtDate(result.first()) + } + } + + + private fun getLockedAtDate(lockInfo: ShedlockInfo): LocalDateTime { + with(lockInfo) { + return if (lockUntil!!.isBefore(LocalDateTime.now())) { + lockedAt!!.toLocalDate().atStartOfDay() + } else { + lockedAt!!.toLocalDate().minusDays(1).atStartOfDay() + } + } + } + + data class ShedlockInfo( + val id: String, + var lockUntil: LocalDateTime?, + var lockedAt: LocalDateTime?, + var lockedBy: String, + ) +} diff --git a/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/service/ProjectMetricsService.kt b/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/service/ProjectMetricsService.kt index 794e6229c7..52fd0dc340 100644 --- a/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/service/ProjectMetricsService.kt +++ b/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/service/ProjectMetricsService.kt @@ -30,17 +30,20 @@ package com.tencent.bkrepo.opdata.service import com.tencent.bkrepo.common.api.pojo.Page import com.tencent.bkrepo.common.mongo.dao.util.Pages import com.tencent.bkrepo.opdata.constant.TO_GIGABYTE +import com.tencent.bkrepo.opdata.model.StatDateModel import com.tencent.bkrepo.opdata.model.TProjectMetrics import com.tencent.bkrepo.opdata.pojo.ProjectMetrics import com.tencent.bkrepo.opdata.pojo.ProjectMetricsOption import com.tencent.bkrepo.opdata.pojo.ProjectMetricsRequest import com.tencent.bkrepo.opdata.repository.ProjectMetricsRepository +import com.tencent.bkrepo.opdata.util.EasyExcelUtils import org.springframework.stereotype.Service import java.time.LocalDate @Service class ProjectMetricsService ( - private val projectMetricsRepository: ProjectMetricsRepository + private val projectMetricsRepository: ProjectMetricsRepository, + private val statDateModel: StatDateModel ){ fun page(option: ProjectMetricsOption): Page { @@ -64,19 +67,40 @@ class ProjectMetricsService ( } fun list(metricsRequest: ProjectMetricsRequest): List { - val createdDate = LocalDate.now().minusDays(metricsRequest.minusDay).atStartOfDay() + return getProjectMetrics(metricsRequest) + } + + fun download(metricsRequest: ProjectMetricsRequest) { + val records = getProjectMetrics(metricsRequest) + // 导出 + val includeColumns = mutableSetOf( + ProjectMetrics::projectId.name, + ProjectMetrics::nodeNum.name, + ProjectMetrics::capSize.name, + ProjectMetrics::createdDate.name + ) + val fileName = "大于${metricsRequest.limitSize/TO_GIGABYTE}GB的项目信息" + EasyExcelUtils.download(records, fileName, ProjectMetrics::class.java, includeColumns) + } + + private fun getProjectMetrics(metricsRequest: ProjectMetricsRequest): List { + val createdDate = if (metricsRequest.default) { + statDateModel.getShedLockInfo() + } else { + LocalDate.now().minusDays(metricsRequest.minusDay).atStartOfDay() + } val queryResult = projectMetricsRepository.findAllByCreatedDate(createdDate) val result = mutableListOf() queryResult.forEach { - if (it.capSize >= metricsRequest.limitSize) { - result.add(ProjectMetrics( - projectId = it.projectId, - capSize = it.capSize / TO_GIGABYTE, - nodeNum = it.nodeNum, - createdDate = it.createdDate - )) - } + if (it.capSize >= metricsRequest.limitSize) { + result.add(ProjectMetrics( + projectId = it.projectId, + capSize = it.capSize / TO_GIGABYTE, + nodeNum = it.nodeNum, + createdDate = it.createdDate + )) } - return result + } + return result.sortedByDescending { it.capSize } } } \ No newline at end of file diff --git a/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/util/EasyExcelUtils.kt b/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/util/EasyExcelUtils.kt new file mode 100644 index 0000000000..0fbe82db54 --- /dev/null +++ b/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/util/EasyExcelUtils.kt @@ -0,0 +1,83 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.opdata.util + +import com.alibaba.excel.EasyExcel +import com.alibaba.excel.write.metadata.style.WriteCellStyle +import com.alibaba.excel.write.style.HorizontalCellStyleStrategy +import com.tencent.bkrepo.common.service.util.HttpContextHolder +import org.apache.poi.ss.usermodel.HorizontalAlignment +import org.apache.poi.ss.usermodel.VerticalAlignment +import org.slf4j.LoggerFactory +import java.io.IOException +import java.net.URLEncoder +import java.time.LocalDateTime +import java.time.format.DateTimeFormatter +import javax.servlet.http.HttpServletResponse + +object EasyExcelUtils { + + private val logger = LoggerFactory.getLogger(EasyExcelUtils::class.java) + + fun download(data: Collection<*>, name: String, dataClass: Class<*>, includeColumns: Set? = null) { + val response = getDownloadResponse(name) + try { + EasyExcel.write(response.outputStream, dataClass).build().use { excelWriter -> + val date = LocalDateTime.now().format(DateTimeFormatter.ofPattern("yyyy-MM-dd-HH-mm-ss")) + val cellStyleStrategy = HorizontalCellStyleStrategy(WriteCellStyle(), WriteCellStyle().apply { + this.horizontalAlignment = HorizontalAlignment.LEFT + this.verticalAlignment = VerticalAlignment.CENTER + this.wrapped = true + }) + val writerSheetBuilder = EasyExcel + .writerSheet(date) + .registerWriteHandler(cellStyleStrategy) + .head(dataClass) + includeColumns?.let { + writerSheetBuilder.includeColumnFieldNames(includeColumns) + } + excelWriter.write(data, writerSheetBuilder.build()) + } + } catch (e: IOException) { + logger.warn("download exception: $e") + } + } + + private fun getDownloadResponse(fileName: String): HttpServletResponse { + val response = HttpContextHolder.getResponse() + response.contentType = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" + response.characterEncoding = "utf-8" + val date = LocalDateTime.now().format(DateTimeFormatter.ofPattern("yyyy-MM-dd-HH-mm-ss")) + val exportFileName = URLEncoder.encode("$fileName-$date", "UTF-8") + response.setHeader( + "Content-disposition", + "attachment;filename*=utf-8''$exportFileName.xlsx" + ) + return response + } +} From 54112d4d3621bb9821ce5840648d7eeb1686d5e4 Mon Sep 17 00:00:00 2001 From: zacyanliu Date: Mon, 23 Oct 2023 19:45:48 +0800 Subject: [PATCH 06/15] =?UTF-8?q?feat:=20grafana=E6=8E=A5=E5=8F=A3?= =?UTF-8?q?=E6=9F=A5=E8=AF=A2=E6=97=B6=E9=97=B4=E8=B0=83=E6=95=B4=20#1314?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../bkrepo/opdata/handler/impl/CapSizeHandler.kt | 6 ++++-- .../handler/impl/EffectiveProjectNumHandler.kt | 6 ++++-- .../bkrepo/opdata/handler/impl/NodeNumHandler.kt | 6 ++++-- .../opdata/handler/impl/ProjectListHandler.kt | 6 ++++-- .../opdata/handler/impl/ProjectNodeNumHandler.kt | 6 ++++-- .../opdata/handler/impl/ProjectNodeSizeHandler.kt | 6 ++++-- .../opdata/handler/impl/RepoNodeNumHandler.kt | 6 ++++-- .../opdata/handler/impl/RepoNodeSizeHandler.kt | 6 ++++-- .../tencent/bkrepo/opdata/model/StatDateModel.kt | 14 +++++++++----- .../bkrepo/opdata/model/StorageCredentialsModel.kt | 5 +++-- 10 files changed, 44 insertions(+), 23 deletions(-) diff --git a/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/handler/impl/CapSizeHandler.kt b/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/handler/impl/CapSizeHandler.kt index 76e066dc1e..bde2fc95f1 100644 --- a/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/handler/impl/CapSizeHandler.kt +++ b/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/handler/impl/CapSizeHandler.kt @@ -35,6 +35,7 @@ import com.tencent.bkrepo.opdata.constant.OPDATA_CAP_SIZE import com.tencent.bkrepo.opdata.constant.OPDATA_GRAFANA_NUMBER import com.tencent.bkrepo.opdata.constant.TO_GIGABYTE import com.tencent.bkrepo.opdata.handler.QueryHandler +import com.tencent.bkrepo.opdata.model.StatDateModel import com.tencent.bkrepo.opdata.pojo.Columns import com.tencent.bkrepo.opdata.pojo.QueryResult import com.tencent.bkrepo.opdata.pojo.Target @@ -47,14 +48,15 @@ import org.springframework.stereotype.Component */ @Component class CapSizeHandler( - private val projectMetricsRepository: ProjectMetricsRepository + private val projectMetricsRepository: ProjectMetricsRepository, + private val statDateModel: StatDateModel ) : QueryHandler { override val metric: Metrics get() = Metrics.CAPSIZE override fun handle(target: Target, result: MutableList) { var size = 0L - val projects = projectMetricsRepository.findAllByCreatedDate() + val projects = projectMetricsRepository.findAllByCreatedDate(statDateModel.getShedLockInfo()) projects.forEach { size += it.capSize } diff --git a/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/handler/impl/EffectiveProjectNumHandler.kt b/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/handler/impl/EffectiveProjectNumHandler.kt index 0eaebd19b7..e2f526ec1a 100644 --- a/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/handler/impl/EffectiveProjectNumHandler.kt +++ b/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/handler/impl/EffectiveProjectNumHandler.kt @@ -34,6 +34,7 @@ package com.tencent.bkrepo.opdata.handler.impl import com.tencent.bkrepo.opdata.constant.OPDATA_GRAFANA_NUMBER import com.tencent.bkrepo.opdata.constant.OPDATA_PROJECT_NUM import com.tencent.bkrepo.opdata.handler.QueryHandler +import com.tencent.bkrepo.opdata.model.StatDateModel import com.tencent.bkrepo.opdata.pojo.Columns import com.tencent.bkrepo.opdata.pojo.QueryResult import com.tencent.bkrepo.opdata.pojo.Target @@ -46,13 +47,14 @@ import org.springframework.stereotype.Component */ @Component class EffectiveProjectNumHandler( - private val projectMetricsRepository: ProjectMetricsRepository + private val projectMetricsRepository: ProjectMetricsRepository, + private val statDateModel: StatDateModel ) : QueryHandler { override val metric: Metrics get() = Metrics.EFFECTIVEPROJECTNUM override fun handle(target: Target, result: MutableList) { - val projects = projectMetricsRepository.findAllByCreatedDate() + val projects = projectMetricsRepository.findAllByCreatedDate(statDateModel.getShedLockInfo()) val count = projects.filter { it.capSize > 0 }.size.toLong() val columns = Columns(OPDATA_PROJECT_NUM, OPDATA_GRAFANA_NUMBER) val row = listOf(count) diff --git a/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/handler/impl/NodeNumHandler.kt b/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/handler/impl/NodeNumHandler.kt index c6876eae36..e04b5dce7b 100644 --- a/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/handler/impl/NodeNumHandler.kt +++ b/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/handler/impl/NodeNumHandler.kt @@ -34,6 +34,7 @@ package com.tencent.bkrepo.opdata.handler.impl import com.tencent.bkrepo.opdata.constant.OPDATA_GRAFANA_NUMBER import com.tencent.bkrepo.opdata.constant.OPDATA_NODE_NUM import com.tencent.bkrepo.opdata.handler.QueryHandler +import com.tencent.bkrepo.opdata.model.StatDateModel import com.tencent.bkrepo.opdata.pojo.Columns import com.tencent.bkrepo.opdata.pojo.QueryResult import com.tencent.bkrepo.opdata.pojo.Target @@ -46,14 +47,15 @@ import org.springframework.stereotype.Component */ @Component class NodeNumHandler( - private val projectMetricsRepository: ProjectMetricsRepository + private val projectMetricsRepository: ProjectMetricsRepository, + private val statDateModel: StatDateModel ) : QueryHandler { override val metric: Metrics get() = Metrics.NODENUM override fun handle(target: Target, result: MutableList) { var num = 0L - val projects = projectMetricsRepository.findAllByCreatedDate() + val projects = projectMetricsRepository.findAllByCreatedDate(statDateModel.getShedLockInfo()) projects.forEach { num += it.nodeNum } diff --git a/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/handler/impl/ProjectListHandler.kt b/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/handler/impl/ProjectListHandler.kt index 1303937ccc..bf1525b472 100644 --- a/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/handler/impl/ProjectListHandler.kt +++ b/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/handler/impl/ProjectListHandler.kt @@ -41,6 +41,7 @@ import com.tencent.bkrepo.opdata.constant.OPDATA_PIPELINE_NUM import com.tencent.bkrepo.opdata.constant.OPDATA_PIPELINE_SIZE import com.tencent.bkrepo.opdata.constant.TO_GIGABYTE import com.tencent.bkrepo.opdata.handler.QueryHandler +import com.tencent.bkrepo.opdata.model.StatDateModel import com.tencent.bkrepo.opdata.model.TProjectMetrics import com.tencent.bkrepo.opdata.pojo.Columns import com.tencent.bkrepo.opdata.pojo.QueryResult @@ -54,7 +55,8 @@ import org.springframework.stereotype.Component */ @Component class ProjectListHandler( - private val projectMetricsRepository: ProjectMetricsRepository + private val projectMetricsRepository: ProjectMetricsRepository, + private val statDateModel: StatDateModel ) : QueryHandler { override val metric: Metrics get() = Metrics.PROJECTLIST @@ -62,7 +64,7 @@ class ProjectListHandler( override fun handle(target: Target, result: MutableList) { val rows = mutableListOf>() val columns = mutableListOf() - val info = projectMetricsRepository.findAllByCreatedDate() + val info = projectMetricsRepository.findAllByCreatedDate(statDateModel.getShedLockInfo()) columns.add(Columns(TProjectMetrics::projectId.name, OPDATA_GRAFANA_STRING)) columns.add(Columns(TProjectMetrics::nodeNum.name, OPDATA_GRAFANA_NUMBER)) columns.add(Columns(TProjectMetrics::capSize.name, OPDATA_GRAFANA_NUMBER)) diff --git a/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/handler/impl/ProjectNodeNumHandler.kt b/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/handler/impl/ProjectNodeNumHandler.kt index 85ddb97739..83e528059f 100644 --- a/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/handler/impl/ProjectNodeNumHandler.kt +++ b/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/handler/impl/ProjectNodeNumHandler.kt @@ -33,6 +33,7 @@ package com.tencent.bkrepo.opdata.handler.impl import com.tencent.bkrepo.opdata.constant.PROJECT_NAME import com.tencent.bkrepo.opdata.handler.QueryHandler +import com.tencent.bkrepo.opdata.model.StatDateModel import com.tencent.bkrepo.opdata.pojo.Target import com.tencent.bkrepo.opdata.pojo.enums.Metrics import com.tencent.bkrepo.opdata.repository.ProjectMetricsRepository @@ -43,13 +44,14 @@ import org.springframework.stereotype.Component */ @Component class ProjectNodeNumHandler( - private val projectMetricsRepository: ProjectMetricsRepository + private val projectMetricsRepository: ProjectMetricsRepository, + private val statDateModel: StatDateModel ) : QueryHandler { override val metric: Metrics get() = Metrics.PROJECTNODENUM override fun handle(target: Target, result: MutableList): List { - val projects = projectMetricsRepository.findAllByCreatedDate() + val projects = projectMetricsRepository.findAllByCreatedDate(statDateModel.getShedLockInfo()) val tmpMap = HashMap() projects.forEach { val projectId = it.projectId diff --git a/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/handler/impl/ProjectNodeSizeHandler.kt b/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/handler/impl/ProjectNodeSizeHandler.kt index b32cf51f13..5df5de0557 100644 --- a/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/handler/impl/ProjectNodeSizeHandler.kt +++ b/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/handler/impl/ProjectNodeSizeHandler.kt @@ -33,6 +33,7 @@ package com.tencent.bkrepo.opdata.handler.impl import com.tencent.bkrepo.opdata.constant.TO_GIGABYTE import com.tencent.bkrepo.opdata.handler.QueryHandler +import com.tencent.bkrepo.opdata.model.StatDateModel import com.tencent.bkrepo.opdata.pojo.Target import com.tencent.bkrepo.opdata.pojo.enums.Metrics import com.tencent.bkrepo.opdata.repository.ProjectMetricsRepository @@ -43,13 +44,14 @@ import org.springframework.stereotype.Component */ @Component class ProjectNodeSizeHandler( - private val projectMetricsRepository: ProjectMetricsRepository + private val projectMetricsRepository: ProjectMetricsRepository, + private val statDateModel: StatDateModel ) : QueryHandler { override val metric: Metrics get() = Metrics.PROJECTNODESIZE override fun handle(target: Target, result: MutableList): List { - val projects = projectMetricsRepository.findAllByCreatedDate() + val projects = projectMetricsRepository.findAllByCreatedDate(statDateModel.getShedLockInfo()) val tmpMap = HashMap() projects.forEach { val projectId = it.projectId diff --git a/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/handler/impl/RepoNodeNumHandler.kt b/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/handler/impl/RepoNodeNumHandler.kt index 7895b3730f..329e73f078 100644 --- a/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/handler/impl/RepoNodeNumHandler.kt +++ b/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/handler/impl/RepoNodeNumHandler.kt @@ -33,6 +33,7 @@ package com.tencent.bkrepo.opdata.handler.impl import com.tencent.bkrepo.opdata.constant.PROJECT_NAME import com.tencent.bkrepo.opdata.handler.QueryHandler +import com.tencent.bkrepo.opdata.model.StatDateModel import com.tencent.bkrepo.opdata.pojo.Target import com.tencent.bkrepo.opdata.pojo.enums.Metrics import com.tencent.bkrepo.opdata.repository.ProjectMetricsRepository @@ -43,13 +44,14 @@ import org.springframework.stereotype.Component */ @Component class RepoNodeNumHandler( - private val projectMetricsRepository: ProjectMetricsRepository + private val projectMetricsRepository: ProjectMetricsRepository, + private val statDateModel: StatDateModel ) : QueryHandler { override val metric: Metrics get() = Metrics.REPONODENUM override fun handle(target: Target, result: MutableList): List { - val projects = projectMetricsRepository.findAllByCreatedDate() + val projects = projectMetricsRepository.findAllByCreatedDate(statDateModel.getShedLockInfo()) val tmpMap = HashMap() projects.forEach { it -> val projectId = it.projectId diff --git a/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/handler/impl/RepoNodeSizeHandler.kt b/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/handler/impl/RepoNodeSizeHandler.kt index 9cb4fd4004..cd9fa1724a 100644 --- a/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/handler/impl/RepoNodeSizeHandler.kt +++ b/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/handler/impl/RepoNodeSizeHandler.kt @@ -33,6 +33,7 @@ package com.tencent.bkrepo.opdata.handler.impl import com.tencent.bkrepo.opdata.constant.TO_GIGABYTE import com.tencent.bkrepo.opdata.handler.QueryHandler +import com.tencent.bkrepo.opdata.model.StatDateModel import com.tencent.bkrepo.opdata.pojo.Target import com.tencent.bkrepo.opdata.pojo.enums.Metrics import com.tencent.bkrepo.opdata.repository.ProjectMetricsRepository @@ -43,13 +44,14 @@ import org.springframework.stereotype.Component */ @Component class RepoNodeSizeHandler( - private val projectMetricsRepository: ProjectMetricsRepository + private val projectMetricsRepository: ProjectMetricsRepository, + private val statDateModel: StatDateModel ) : QueryHandler { override val metric: Metrics get() = Metrics.REPONODESIZE override fun handle(target: Target, result: MutableList): List { - val projects = projectMetricsRepository.findAllByCreatedDate() + val projects = projectMetricsRepository.findAllByCreatedDate(statDateModel.getShedLockInfo()) val tmpMap = HashMap() projects.forEach { it -> val projectId = it.projectId diff --git a/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/model/StatDateModel.kt b/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/model/StatDateModel.kt index a0d3740467..4d035f82f1 100644 --- a/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/model/StatDateModel.kt +++ b/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/model/StatDateModel.kt @@ -48,12 +48,16 @@ class StatDateModel @Autowired constructor( } fun getShedLockInfo(id: String = JOB_NAME): LocalDateTime { - val query = Query(Criteria.where(ID).isEqualTo(id)) - val result = mongoTemplate.find(query, ShedlockInfo::class.java, SHED_LOCK_COLLECTION_NAME) - return if (result.isEmpty()) { + return try { + val query = Query(Criteria.where(ID).isEqualTo(id)) + val result = mongoTemplate.find(query, ShedlockInfo::class.java, SHED_LOCK_COLLECTION_NAME) + if (result.isEmpty()) { + LocalDate.now().minusDays(1).atStartOfDay() + } else { + getLockedAtDate(result.first()) + } + } catch (e: Exception) { LocalDate.now().minusDays(1).atStartOfDay() - } else { - getLockedAtDate(result.first()) } } diff --git a/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/model/StorageCredentialsModel.kt b/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/model/StorageCredentialsModel.kt index 1d45207b5b..3d64b8dbfd 100644 --- a/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/model/StorageCredentialsModel.kt +++ b/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/model/StorageCredentialsModel.kt @@ -39,12 +39,13 @@ import org.springframework.stereotype.Service @Service class StorageCredentialsModel @Autowired constructor( - private val projectMetricsRepository: ProjectMetricsRepository + private val projectMetricsRepository: ProjectMetricsRepository, + private val statDateModel: StatDateModel ) { fun getStorageCredentialsStat(metrics: StatMetrics): Map { val result = mutableMapOf() - val projectMetricsList = projectMetricsRepository.findAllByCreatedDate() + val projectMetricsList = projectMetricsRepository.findAllByCreatedDate(statDateModel.getShedLockInfo()) projectMetricsList.forEach { projectMetrics -> projectMetrics.repoMetrics.forEach { val value = if (metrics == StatMetrics.NUM) it.num else it.size From db58dc16b1520592dbde04728979a86e0b9298a2 Mon Sep 17 00:00:00 2001 From: zacyanliu Date: Mon, 23 Oct 2023 19:58:11 +0800 Subject: [PATCH 07/15] =?UTF-8?q?feat:=20=E4=BB=A3=E7=A0=81=E8=B0=83?= =?UTF-8?q?=E6=95=B4=20#1314?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../kotlin/com/tencent/bkrepo/opdata/model/StatDateModel.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/model/StatDateModel.kt b/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/model/StatDateModel.kt index 4d035f82f1..6d53e3db80 100644 --- a/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/model/StatDateModel.kt +++ b/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/model/StatDateModel.kt @@ -44,7 +44,7 @@ class StatDateModel @Autowired constructor( ) { companion object { - private val JOB_NAME = "NodeStatCompositeMongoDbBatchJob" + private const val JOB_NAME = "NodeStatCompositeMongoDbBatchJob" } fun getShedLockInfo(id: String = JOB_NAME): LocalDateTime { From 9966d64cfd463e10a5027ce290e87932b2f82c76 Mon Sep 17 00:00:00 2001 From: zacYL Date: Tue, 24 Oct 2023 19:39:56 +0800 Subject: [PATCH 08/15] =?UTF-8?q?feat:=20=E6=95=B0=E6=8D=AE=E6=9F=A5?= =?UTF-8?q?=E8=AF=A2=E6=97=B6=E9=97=B4=E8=B0=83=E6=95=B4=20#1314?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../bkrepo/opdata/model/StatDateModel.kt | 35 +++++++++++-------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/model/StatDateModel.kt b/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/model/StatDateModel.kt index 6d53e3db80..3b732cb6aa 100644 --- a/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/model/StatDateModel.kt +++ b/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/model/StatDateModel.kt @@ -33,7 +33,6 @@ import org.springframework.beans.factory.annotation.Autowired import org.springframework.data.mongodb.core.MongoTemplate import org.springframework.data.mongodb.core.query.Criteria import org.springframework.data.mongodb.core.query.Query -import org.springframework.data.mongodb.core.query.isEqualTo import org.springframework.stereotype.Service import java.time.LocalDate import java.time.LocalDateTime @@ -44,32 +43,40 @@ class StatDateModel @Autowired constructor( ) { companion object { - private const val JOB_NAME = "NodeStatCompositeMongoDbBatchJob" + private val JOB_NAMES = listOf("NodeStatCompositeMongoDbBatchJob", "ProjectRepoMetricsStatJob") } - fun getShedLockInfo(id: String = JOB_NAME): LocalDateTime { + fun getShedLockInfo(ids: List = JOB_NAMES): LocalDateTime { return try { - val query = Query(Criteria.where(ID).isEqualTo(id)) + val query = Query(Criteria.where(ID).`in`(ids)) val result = mongoTemplate.find(query, ShedlockInfo::class.java, SHED_LOCK_COLLECTION_NAME) - if (result.isEmpty()) { - LocalDate.now().minusDays(1).atStartOfDay() - } else { - getLockedAtDate(result.first()) - } + getLockedAtDate(result) } catch (e: Exception) { LocalDate.now().minusDays(1).atStartOfDay() } } - private fun getLockedAtDate(lockInfo: ShedlockInfo): LocalDateTime { - with(lockInfo) { - return if (lockUntil!!.isBefore(LocalDateTime.now())) { - lockedAt!!.toLocalDate().atStartOfDay() + private fun getLockedAtDate(lockInfos: List): LocalDateTime { + if (lockInfos.isEmpty()) { + return LocalDate.now().minusDays(1).atStartOfDay() + } + var statDateTime: LocalDateTime? = null + lockInfos.forEach { + val tempStatDate = if (it.lockUntil!!.isBefore(LocalDateTime.now())) { + it.lockedAt!!.toLocalDate().atStartOfDay() + } else { + it.lockedAt!!.toLocalDate().minusDays(1).atStartOfDay() + } + if (statDateTime == null) { + statDateTime = tempStatDate } else { - lockedAt!!.toLocalDate().minusDays(1).atStartOfDay() + if (statDateTime!!.isBefore(tempStatDate)) { + statDateTime = tempStatDate + } } } + return statDateTime!! } data class ShedlockInfo( From b7b913af176157611db57d6ccc003c8b57cf85f3 Mon Sep 17 00:00:00 2001 From: zacYL Date: Tue, 24 Oct 2023 19:41:07 +0800 Subject: [PATCH 09/15] =?UTF-8?q?feat:=20=E5=A2=9E=E5=8A=A0=E9=A1=B9?= =?UTF-8?q?=E7=9B=AE=E4=B8=AD=E9=95=9C=E5=83=8F=E4=BB=93=E5=BA=93=E5=92=8C?= =?UTF-8?q?=20generic=20=E4=BB=93=E5=BA=93=E6=80=BB=E5=A4=A7=E5=B0=8F?= =?UTF-8?q?=E6=95=B0=E6=8D=AE=E6=9F=A5=E8=AF=A2=E9=80=BB=E8=BE=91=20#1314?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../tencent/bkrepo/opdata/pojo/RepoMetrics.kt | 4 +- .../bkrepo/opdata/pojo/enums/Metrics.kt | 2 + .../handler/impl/DockerRepoSizeInProject.kt | 69 +++++++++++++++++++ .../handler/impl/GenericRepoSizeInProject.kt | 69 +++++++++++++++++++ 4 files changed, 143 insertions(+), 1 deletion(-) create mode 100644 src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/handler/impl/DockerRepoSizeInProject.kt create mode 100644 src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/handler/impl/GenericRepoSizeInProject.kt diff --git a/src/backend/opdata/api-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/pojo/RepoMetrics.kt b/src/backend/opdata/api-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/pojo/RepoMetrics.kt index 7b61bace74..25d2571d50 100644 --- a/src/backend/opdata/api-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/pojo/RepoMetrics.kt +++ b/src/backend/opdata/api-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/pojo/RepoMetrics.kt @@ -41,5 +41,7 @@ data class RepoMetrics( @ApiModelProperty("size") var size: Long, @ApiModelProperty("num") - val num: Long + val num: Long, + @ApiModelProperty("type") + val type: String? = null ) diff --git a/src/backend/opdata/api-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/pojo/enums/Metrics.kt b/src/backend/opdata/api-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/pojo/enums/Metrics.kt index 57faaf21c2..40893d6f30 100644 --- a/src/backend/opdata/api-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/pojo/enums/Metrics.kt +++ b/src/backend/opdata/api-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/pojo/enums/Metrics.kt @@ -50,4 +50,6 @@ enum class Metrics { NODECOLLECTION, REPONODENUM, REPONODESIZE, + GENERICREPOSIZEINPROJECT, + DOCKERREPOSIZEINPROJECT, } diff --git a/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/handler/impl/DockerRepoSizeInProject.kt b/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/handler/impl/DockerRepoSizeInProject.kt new file mode 100644 index 0000000000..2f9c6d240b --- /dev/null +++ b/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/handler/impl/DockerRepoSizeInProject.kt @@ -0,0 +1,69 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.opdata.handler.impl + +import com.tencent.bkrepo.opdata.constant.TO_GIGABYTE +import com.tencent.bkrepo.opdata.handler.QueryHandler +import com.tencent.bkrepo.opdata.model.StatDateModel +import com.tencent.bkrepo.opdata.pojo.Target +import com.tencent.bkrepo.opdata.pojo.enums.Metrics +import com.tencent.bkrepo.opdata.repository.ProjectMetricsRepository +import org.springframework.stereotype.Component + +/** + * 项目中镜像仓库总大小 + */ +@Component +class DockerRepoSizeInProject( + private val projectMetricsRepository: ProjectMetricsRepository, + private val statDateModel: StatDateModel +) : QueryHandler { + + override val metric: Metrics get() = Metrics.DOCKERREPOSIZEINPROJECT + + override fun handle(target: Target, result: MutableList): Any { + val projects = projectMetricsRepository.findAllByCreatedDate(statDateModel.getShedLockInfo()) + val tmpMap = HashMap() + projects.forEach { tP -> + val projectId = tP.projectId + var dockerSize = 0L + tP.repoMetrics.filter { !it.type.isNullOrEmpty()&& it.type in DOCKER_TYPES }.forEach {repo -> + dockerSize += repo.size + } + val gbSize = dockerSize / TO_GIGABYTE + if (gbSize != 0L) { + tmpMap[projectId] = gbSize + } + } + return convToDisplayData(tmpMap, result) + } + + companion object { + private val DOCKER_TYPES = listOf("DOCKER", "OCI") + } +} \ No newline at end of file diff --git a/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/handler/impl/GenericRepoSizeInProject.kt b/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/handler/impl/GenericRepoSizeInProject.kt new file mode 100644 index 0000000000..2632e56e76 --- /dev/null +++ b/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/handler/impl/GenericRepoSizeInProject.kt @@ -0,0 +1,69 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.opdata.handler.impl + +import com.tencent.bkrepo.opdata.constant.TO_GIGABYTE +import com.tencent.bkrepo.opdata.handler.QueryHandler +import com.tencent.bkrepo.opdata.model.StatDateModel +import com.tencent.bkrepo.opdata.pojo.Target +import com.tencent.bkrepo.opdata.pojo.enums.Metrics +import com.tencent.bkrepo.opdata.repository.ProjectMetricsRepository +import org.springframework.stereotype.Component + +/** + * 项目中generic仓库总大小 + */ +@Component +class GenericRepoSizeInProject( + private val projectMetricsRepository: ProjectMetricsRepository, + private val statDateModel: StatDateModel +) : QueryHandler { + + override val metric: Metrics get() = Metrics.GENERICREPOSIZEINPROJECT + + override fun handle(target: Target, result: MutableList): Any { + val projects = projectMetricsRepository.findAllByCreatedDate(statDateModel.getShedLockInfo()) + val tmpMap = HashMap() + projects.forEach { tP -> + val projectId = tP.projectId + var dockerSize = 0L + tP.repoMetrics.filter { !it.type.isNullOrEmpty()&& it.type == GENERIC_TYPE }.forEach {repo -> + dockerSize += repo.size + } + val gbSize = dockerSize / TO_GIGABYTE + if (gbSize != 0L) { + tmpMap[projectId] = gbSize + } + } + return convToDisplayData(tmpMap, result) + } + + companion object { + private const val GENERIC_TYPE = "GENERIC" + } +} \ No newline at end of file From 12bbc53ded1e1ebe02ae561067511610497f2fb1 Mon Sep 17 00:00:00 2001 From: zacYL Date: Tue, 24 Oct 2023 20:10:44 +0800 Subject: [PATCH 10/15] =?UTF-8?q?feat:=20=E6=95=B0=E6=8D=AE=E6=9F=A5?= =?UTF-8?q?=E8=AF=A2=E6=8E=A5=E5=8F=A3=E5=A2=9E=E5=8A=A0=E4=BB=93=E5=BA=93?= =?UTF-8?q?=E7=B1=BB=E5=9E=8B=20#1314?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../opdata/pojo/ProjectMetricsRequest.kt | 3 +- .../opdata/service/ProjectMetricsService.kt | 33 +++++++++++++++---- 2 files changed, 28 insertions(+), 8 deletions(-) diff --git a/src/backend/opdata/api-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/pojo/ProjectMetricsRequest.kt b/src/backend/opdata/api-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/pojo/ProjectMetricsRequest.kt index 26a474f178..514f8701e1 100644 --- a/src/backend/opdata/api-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/pojo/ProjectMetricsRequest.kt +++ b/src/backend/opdata/api-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/pojo/ProjectMetricsRequest.kt @@ -30,5 +30,6 @@ package com.tencent.bkrepo.opdata.pojo data class ProjectMetricsRequest( val limitSize: Long = 1024 * 1024 * 1024L, val minusDay: Long = 1, - val default: Boolean = true + val default: Boolean = true, + val type: String? = null ) diff --git a/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/service/ProjectMetricsService.kt b/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/service/ProjectMetricsService.kt index 52fd0dc340..7e964e0e45 100644 --- a/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/service/ProjectMetricsService.kt +++ b/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/service/ProjectMetricsService.kt @@ -92,13 +92,32 @@ class ProjectMetricsService ( val queryResult = projectMetricsRepository.findAllByCreatedDate(createdDate) val result = mutableListOf() queryResult.forEach { - if (it.capSize >= metricsRequest.limitSize) { - result.add(ProjectMetrics( - projectId = it.projectId, - capSize = it.capSize / TO_GIGABYTE, - nodeNum = it.nodeNum, - createdDate = it.createdDate - )) + if (metricsRequest.type.isNullOrEmpty()) { + if (it.capSize >= metricsRequest.limitSize) { + result.add(ProjectMetrics( + projectId = it.projectId, + capSize = it.capSize / TO_GIGABYTE, + nodeNum = it.nodeNum, + createdDate = it.createdDate + )) + } + } else { + var sizeOfRepoType: Long = 0 + var nodeNumOfRepoType: Long = 0 + it.repoMetrics.forEach { repo -> + if (repo.type == metricsRequest.type) { + sizeOfRepoType += repo.size + nodeNumOfRepoType += repo.num + } + } + if (sizeOfRepoType >= metricsRequest.limitSize) { + result.add(ProjectMetrics( + projectId = it.projectId, + capSize = sizeOfRepoType / TO_GIGABYTE, + nodeNum = nodeNumOfRepoType, + createdDate = it.createdDate + )) + } } } return result.sortedByDescending { it.capSize } From 1f5a67a8df7b3ab73bbb28e315db7f4124978f2c Mon Sep 17 00:00:00 2001 From: zacYL Date: Tue, 24 Oct 2023 20:47:25 +0800 Subject: [PATCH 11/15] =?UTF-8?q?feat:=20=E4=BB=A3=E7=A0=81=E8=B0=83?= =?UTF-8?q?=E6=95=B4=20#1314?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../opdata/service/ProjectMetricsService.kt | 29 ++++++++----------- 1 file changed, 12 insertions(+), 17 deletions(-) diff --git a/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/service/ProjectMetricsService.kt b/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/service/ProjectMetricsService.kt index 7e964e0e45..b9cacd2711 100644 --- a/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/service/ProjectMetricsService.kt +++ b/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/service/ProjectMetricsService.kt @@ -92,15 +92,8 @@ class ProjectMetricsService ( val queryResult = projectMetricsRepository.findAllByCreatedDate(createdDate) val result = mutableListOf() queryResult.forEach { - if (metricsRequest.type.isNullOrEmpty()) { - if (it.capSize >= metricsRequest.limitSize) { - result.add(ProjectMetrics( - projectId = it.projectId, - capSize = it.capSize / TO_GIGABYTE, - nodeNum = it.nodeNum, - createdDate = it.createdDate - )) - } + val (sizeOfRepoType, nodeNumOfRepoType) = if (metricsRequest.type.isNullOrEmpty()) { + Pair(it.capSize , it.nodeNum) } else { var sizeOfRepoType: Long = 0 var nodeNumOfRepoType: Long = 0 @@ -110,16 +103,18 @@ class ProjectMetricsService ( nodeNumOfRepoType += repo.num } } - if (sizeOfRepoType >= metricsRequest.limitSize) { - result.add(ProjectMetrics( - projectId = it.projectId, - capSize = sizeOfRepoType / TO_GIGABYTE, - nodeNum = nodeNumOfRepoType, - createdDate = it.createdDate - )) - } + Pair(sizeOfRepoType, nodeNumOfRepoType) + } + if (sizeOfRepoType >= metricsRequest.limitSize) { + result.add(ProjectMetrics( + projectId = it.projectId, + capSize = sizeOfRepoType / TO_GIGABYTE, + nodeNum = nodeNumOfRepoType, + createdDate = it.createdDate + )) } } return result.sortedByDescending { it.capSize } } + } \ No newline at end of file From a40c1b2ef4ad4fcb871d532bce5094658b9d4ed3 Mon Sep 17 00:00:00 2001 From: zacYL Date: Tue, 24 Oct 2023 21:50:51 +0800 Subject: [PATCH 12/15] =?UTF-8?q?feat:=20=E4=BB=A3=E7=A0=81=E8=B0=83?= =?UTF-8?q?=E6=95=B4=20#1314?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../opdata/service/ProjectMetricsService.kt | 30 +++++++++++-------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/service/ProjectMetricsService.kt b/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/service/ProjectMetricsService.kt index b9cacd2711..5a59b7b3e8 100644 --- a/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/service/ProjectMetricsService.kt +++ b/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/service/ProjectMetricsService.kt @@ -92,19 +92,7 @@ class ProjectMetricsService ( val queryResult = projectMetricsRepository.findAllByCreatedDate(createdDate) val result = mutableListOf() queryResult.forEach { - val (sizeOfRepoType, nodeNumOfRepoType) = if (metricsRequest.type.isNullOrEmpty()) { - Pair(it.capSize , it.nodeNum) - } else { - var sizeOfRepoType: Long = 0 - var nodeNumOfRepoType: Long = 0 - it.repoMetrics.forEach { repo -> - if (repo.type == metricsRequest.type) { - sizeOfRepoType += repo.size - nodeNumOfRepoType += repo.num - } - } - Pair(sizeOfRepoType, nodeNumOfRepoType) - } + val (sizeOfRepoType, nodeNumOfRepoType) = getSizeAndNodeNum(metricsRequest.type, it) if (sizeOfRepoType >= metricsRequest.limitSize) { result.add(ProjectMetrics( projectId = it.projectId, @@ -117,4 +105,20 @@ class ProjectMetricsService ( return result.sortedByDescending { it.capSize } } + + private fun getSizeAndNodeNum(type: String?, projectMetrics: TProjectMetrics): Pair { + return if (type.isNullOrEmpty()) { + Pair(projectMetrics.capSize , projectMetrics.nodeNum) + } else { + var sizeOfRepoType: Long = 0 + var nodeNumOfRepoType: Long = 0 + projectMetrics.repoMetrics.forEach { repo -> + if (repo.type == type) { + sizeOfRepoType += repo.size + nodeNumOfRepoType += repo.num + } + } + Pair(sizeOfRepoType, nodeNumOfRepoType) + } + } } \ No newline at end of file From 458f3b02fce83ab4b5307bf888ddbfa4653881a3 Mon Sep 17 00:00:00 2001 From: zacYL Date: Wed, 25 Oct 2023 15:51:47 +0800 Subject: [PATCH 13/15] =?UTF-8?q?feat:=20=E4=BB=A3=E7=A0=81=E8=B0=83?= =?UTF-8?q?=E6=95=B4=20#1314?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../handler/impl/DockerRepoSizeInProject.kt | 31 +++------- .../handler/impl/GenericRepoSizeInProject.kt | 31 +++------- .../opdata/handler/impl/RepoSizeInProject.kt | 62 +++++++++++++++++++ 3 files changed, 76 insertions(+), 48 deletions(-) create mode 100644 src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/handler/impl/RepoSizeInProject.kt diff --git a/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/handler/impl/DockerRepoSizeInProject.kt b/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/handler/impl/DockerRepoSizeInProject.kt index 2f9c6d240b..cd5abb82d9 100644 --- a/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/handler/impl/DockerRepoSizeInProject.kt +++ b/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/handler/impl/DockerRepoSizeInProject.kt @@ -27,10 +27,7 @@ package com.tencent.bkrepo.opdata.handler.impl -import com.tencent.bkrepo.opdata.constant.TO_GIGABYTE -import com.tencent.bkrepo.opdata.handler.QueryHandler import com.tencent.bkrepo.opdata.model.StatDateModel -import com.tencent.bkrepo.opdata.pojo.Target import com.tencent.bkrepo.opdata.pojo.enums.Metrics import com.tencent.bkrepo.opdata.repository.ProjectMetricsRepository import org.springframework.stereotype.Component @@ -40,28 +37,14 @@ import org.springframework.stereotype.Component */ @Component class DockerRepoSizeInProject( - private val projectMetricsRepository: ProjectMetricsRepository, - private val statDateModel: StatDateModel -) : QueryHandler { + projectMetricsRepository: ProjectMetricsRepository, + statDateModel: StatDateModel +) : RepoSizeInProject(projectMetricsRepository, statDateModel) { - override val metric: Metrics get() = Metrics.DOCKERREPOSIZEINPROJECT - - override fun handle(target: Target, result: MutableList): Any { - val projects = projectMetricsRepository.findAllByCreatedDate(statDateModel.getShedLockInfo()) - val tmpMap = HashMap() - projects.forEach { tP -> - val projectId = tP.projectId - var dockerSize = 0L - tP.repoMetrics.filter { !it.type.isNullOrEmpty()&& it.type in DOCKER_TYPES }.forEach {repo -> - dockerSize += repo.size - } - val gbSize = dockerSize / TO_GIGABYTE - if (gbSize != 0L) { - tmpMap[projectId] = gbSize - } - } - return convToDisplayData(tmpMap, result) - } + override val metric: Metrics + get() = Metrics.DOCKERREPOSIZEINPROJECT + override val repoType: List + get() = DOCKER_TYPES companion object { private val DOCKER_TYPES = listOf("DOCKER", "OCI") diff --git a/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/handler/impl/GenericRepoSizeInProject.kt b/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/handler/impl/GenericRepoSizeInProject.kt index 2632e56e76..d40b92fce7 100644 --- a/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/handler/impl/GenericRepoSizeInProject.kt +++ b/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/handler/impl/GenericRepoSizeInProject.kt @@ -27,10 +27,7 @@ package com.tencent.bkrepo.opdata.handler.impl -import com.tencent.bkrepo.opdata.constant.TO_GIGABYTE -import com.tencent.bkrepo.opdata.handler.QueryHandler import com.tencent.bkrepo.opdata.model.StatDateModel -import com.tencent.bkrepo.opdata.pojo.Target import com.tencent.bkrepo.opdata.pojo.enums.Metrics import com.tencent.bkrepo.opdata.repository.ProjectMetricsRepository import org.springframework.stereotype.Component @@ -40,28 +37,14 @@ import org.springframework.stereotype.Component */ @Component class GenericRepoSizeInProject( - private val projectMetricsRepository: ProjectMetricsRepository, - private val statDateModel: StatDateModel -) : QueryHandler { + projectMetricsRepository: ProjectMetricsRepository, + statDateModel: StatDateModel +) : RepoSizeInProject(projectMetricsRepository, statDateModel) { - override val metric: Metrics get() = Metrics.GENERICREPOSIZEINPROJECT - - override fun handle(target: Target, result: MutableList): Any { - val projects = projectMetricsRepository.findAllByCreatedDate(statDateModel.getShedLockInfo()) - val tmpMap = HashMap() - projects.forEach { tP -> - val projectId = tP.projectId - var dockerSize = 0L - tP.repoMetrics.filter { !it.type.isNullOrEmpty()&& it.type == GENERIC_TYPE }.forEach {repo -> - dockerSize += repo.size - } - val gbSize = dockerSize / TO_GIGABYTE - if (gbSize != 0L) { - tmpMap[projectId] = gbSize - } - } - return convToDisplayData(tmpMap, result) - } + override val metric: Metrics + get() = Metrics.GENERICREPOSIZEINPROJECT + override val repoType: List + get() = listOf(GENERIC_TYPE) companion object { private const val GENERIC_TYPE = "GENERIC" diff --git a/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/handler/impl/RepoSizeInProject.kt b/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/handler/impl/RepoSizeInProject.kt new file mode 100644 index 0000000000..e86e101ed4 --- /dev/null +++ b/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/handler/impl/RepoSizeInProject.kt @@ -0,0 +1,62 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.opdata.handler.impl + +import com.tencent.bkrepo.opdata.constant.TO_GIGABYTE +import com.tencent.bkrepo.opdata.handler.QueryHandler +import com.tencent.bkrepo.opdata.model.StatDateModel +import com.tencent.bkrepo.opdata.pojo.Target +import com.tencent.bkrepo.opdata.pojo.enums.Metrics +import com.tencent.bkrepo.opdata.repository.ProjectMetricsRepository + +open class RepoSizeInProject( + private val projectMetricsRepository: ProjectMetricsRepository, + private val statDateModel: StatDateModel +) : QueryHandler { + + override val metric: Metrics get() = Metrics.DEFAULT + + open val repoType: List = emptyList() + + override fun handle(target: Target, result: MutableList): Any { + val projects = projectMetricsRepository.findAllByCreatedDate(statDateModel.getShedLockInfo()) + val tmpMap = HashMap() + projects.forEach { tP -> + val projectId = tP.projectId + var repoSize = 0L + tP.repoMetrics.filter { !it.type.isNullOrEmpty()&& it.type in repoType }.forEach {repo -> + repoSize += repo.size + } + val gbSize = repoSize / TO_GIGABYTE + if (gbSize != 0L) { + tmpMap[projectId] = gbSize + } + } + return convToDisplayData(tmpMap, result) + } +} \ No newline at end of file From 5bbdeb65e32b8d541a490a70cb6d1acc265eb4a1 Mon Sep 17 00:00:00 2001 From: zacYL Date: Thu, 26 Oct 2023 10:40:03 +0800 Subject: [PATCH 14/15] =?UTF-8?q?feat:=20=E5=A2=9E=E5=8A=A0docker=E5=92=8C?= =?UTF-8?q?=20generic=20=E4=BB=93=E5=BA=93=E6=80=BB=E5=A4=A7=E5=B0=8F?= =?UTF-8?q?=E6=95=B0=E6=8D=AE=20#1314?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../bkrepo/opdata/constant/Constants.kt | 2 + .../bkrepo/opdata/pojo/enums/Metrics.kt | 2 + .../opdata/handler/impl/BaseCapSizeHandler.kt | 71 +++++++++++++++++++ .../opdata/handler/impl/CapSizeHandler.kt | 25 +------ .../handler/impl/DockerCapSizeHandler.kt | 49 +++++++++++++ .../handler/impl/GenericCapSizeHandler.kt | 48 +++++++++++++ .../bkrepo/opdata/model/StatDateModel.kt | 2 +- 7 files changed, 176 insertions(+), 23 deletions(-) create mode 100644 src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/handler/impl/BaseCapSizeHandler.kt create mode 100644 src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/handler/impl/DockerCapSizeHandler.kt create mode 100644 src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/handler/impl/GenericCapSizeHandler.kt diff --git a/src/backend/opdata/api-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/constant/Constants.kt b/src/backend/opdata/api-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/constant/Constants.kt index f79125559c..b183fbce3d 100644 --- a/src/backend/opdata/api-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/constant/Constants.kt +++ b/src/backend/opdata/api-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/constant/Constants.kt @@ -52,6 +52,8 @@ const val OPDATA_PIPELINE_SIZE = "pipelineSize" const val OPDATA_PROJECT_NUM = "ProjectNum" const val OPDATA_CAP_SIZE = "CapSize" +const val OPDATA_DOCKER_CAP_SIZE = "DockerCapSize" +const val OPDATA_GENERIC_CAP_SIZE = "GenericCapSize" const val OPDATA_NODE_NUM = "NodeNum" const val OPDATA_PROJECT = "project" diff --git a/src/backend/opdata/api-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/pojo/enums/Metrics.kt b/src/backend/opdata/api-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/pojo/enums/Metrics.kt index 40893d6f30..5a1937f554 100644 --- a/src/backend/opdata/api-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/pojo/enums/Metrics.kt +++ b/src/backend/opdata/api-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/pojo/enums/Metrics.kt @@ -52,4 +52,6 @@ enum class Metrics { REPONODESIZE, GENERICREPOSIZEINPROJECT, DOCKERREPOSIZEINPROJECT, + DOCKERCAPSIZE, + GENERICCAPSIZE, } diff --git a/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/handler/impl/BaseCapSizeHandler.kt b/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/handler/impl/BaseCapSizeHandler.kt new file mode 100644 index 0000000000..fc221591f6 --- /dev/null +++ b/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/handler/impl/BaseCapSizeHandler.kt @@ -0,0 +1,71 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.opdata.handler.impl + +import com.tencent.bkrepo.opdata.constant.OPDATA_CAP_SIZE +import com.tencent.bkrepo.opdata.constant.OPDATA_GRAFANA_NUMBER +import com.tencent.bkrepo.opdata.constant.TO_GIGABYTE +import com.tencent.bkrepo.opdata.handler.QueryHandler +import com.tencent.bkrepo.opdata.model.StatDateModel +import com.tencent.bkrepo.opdata.pojo.Columns +import com.tencent.bkrepo.opdata.pojo.QueryResult +import com.tencent.bkrepo.opdata.pojo.Target +import com.tencent.bkrepo.opdata.pojo.enums.Metrics +import com.tencent.bkrepo.opdata.repository.ProjectMetricsRepository + +open class BaseCapSizeHandler( + private val projectMetricsRepository: ProjectMetricsRepository, + private val statDateModel: StatDateModel +) : QueryHandler { + + override val metric: Metrics get() = Metrics.DEFAULT + + open val repoType: List = emptyList() + + open val columnsText: String = OPDATA_CAP_SIZE + + override fun handle(target: Target, result: MutableList) { + var size = 0L + val projects = projectMetricsRepository.findAllByCreatedDate(statDateModel.getShedLockInfo()) + if (repoType.isEmpty()) { + projects.forEach { + size += it.capSize + } + } else { + projects.forEach { project -> + project.repoMetrics.filter { !it.type.isNullOrEmpty() && it.type in repoType }.forEach { repo -> + size += repo.size + } + } + } + val column = Columns(columnsText, OPDATA_GRAFANA_NUMBER) + val row = listOf(size / TO_GIGABYTE) + val data = QueryResult(listOf(column), listOf(row), target.type) + result.add(data) + } +} diff --git a/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/handler/impl/CapSizeHandler.kt b/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/handler/impl/CapSizeHandler.kt index bde2fc95f1..e04cc75a63 100644 --- a/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/handler/impl/CapSizeHandler.kt +++ b/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/handler/impl/CapSizeHandler.kt @@ -31,14 +31,7 @@ package com.tencent.bkrepo.opdata.handler.impl -import com.tencent.bkrepo.opdata.constant.OPDATA_CAP_SIZE -import com.tencent.bkrepo.opdata.constant.OPDATA_GRAFANA_NUMBER -import com.tencent.bkrepo.opdata.constant.TO_GIGABYTE -import com.tencent.bkrepo.opdata.handler.QueryHandler import com.tencent.bkrepo.opdata.model.StatDateModel -import com.tencent.bkrepo.opdata.pojo.Columns -import com.tencent.bkrepo.opdata.pojo.QueryResult -import com.tencent.bkrepo.opdata.pojo.Target import com.tencent.bkrepo.opdata.pojo.enums.Metrics import com.tencent.bkrepo.opdata.repository.ProjectMetricsRepository import org.springframework.stereotype.Component @@ -48,21 +41,9 @@ import org.springframework.stereotype.Component */ @Component class CapSizeHandler( - private val projectMetricsRepository: ProjectMetricsRepository, - private val statDateModel: StatDateModel -) : QueryHandler { + projectMetricsRepository: ProjectMetricsRepository, + statDateModel: StatDateModel +) : BaseCapSizeHandler(projectMetricsRepository, statDateModel) { override val metric: Metrics get() = Metrics.CAPSIZE - - override fun handle(target: Target, result: MutableList) { - var size = 0L - val projects = projectMetricsRepository.findAllByCreatedDate(statDateModel.getShedLockInfo()) - projects.forEach { - size += it.capSize - } - val column = Columns(OPDATA_CAP_SIZE, OPDATA_GRAFANA_NUMBER) - val row = listOf(size / TO_GIGABYTE) - val data = QueryResult(listOf(column), listOf(row), target.type) - result.add(data) - } } diff --git a/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/handler/impl/DockerCapSizeHandler.kt b/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/handler/impl/DockerCapSizeHandler.kt new file mode 100644 index 0000000000..07e5c57b68 --- /dev/null +++ b/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/handler/impl/DockerCapSizeHandler.kt @@ -0,0 +1,49 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.opdata.handler.impl + +import com.tencent.bkrepo.opdata.constant.OPDATA_DOCKER_CAP_SIZE +import com.tencent.bkrepo.opdata.model.StatDateModel +import com.tencent.bkrepo.opdata.pojo.enums.Metrics +import com.tencent.bkrepo.opdata.repository.ProjectMetricsRepository +import org.springframework.stereotype.Component + +/** + * DOCKER 仓库总容量统计 + */ +@Component +class DockerCapSizeHandler( + projectMetricsRepository: ProjectMetricsRepository, + statDateModel: StatDateModel +) : BaseCapSizeHandler(projectMetricsRepository, statDateModel) { + + override val metric: Metrics get() = Metrics.DOCKERCAPSIZE + + override val columnsText: String = OPDATA_DOCKER_CAP_SIZE + +} \ No newline at end of file diff --git a/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/handler/impl/GenericCapSizeHandler.kt b/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/handler/impl/GenericCapSizeHandler.kt new file mode 100644 index 0000000000..a0aa8148cf --- /dev/null +++ b/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/handler/impl/GenericCapSizeHandler.kt @@ -0,0 +1,48 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.opdata.handler.impl + +import com.tencent.bkrepo.opdata.constant.OPDATA_GENERIC_CAP_SIZE +import com.tencent.bkrepo.opdata.model.StatDateModel +import com.tencent.bkrepo.opdata.pojo.enums.Metrics +import com.tencent.bkrepo.opdata.repository.ProjectMetricsRepository +import org.springframework.stereotype.Component + +/** + * GENERIC 仓库总容量统计 + */ +@Component +class GenericCapSizeHandler( + projectMetricsRepository: ProjectMetricsRepository, + statDateModel: StatDateModel +) : BaseCapSizeHandler(projectMetricsRepository, statDateModel) { + + override val metric: Metrics get() = Metrics.GENERICCAPSIZE + + override val columnsText: String = OPDATA_GENERIC_CAP_SIZE +} \ No newline at end of file diff --git a/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/model/StatDateModel.kt b/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/model/StatDateModel.kt index 3b732cb6aa..3dbdd1f467 100644 --- a/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/model/StatDateModel.kt +++ b/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/model/StatDateModel.kt @@ -43,7 +43,7 @@ class StatDateModel @Autowired constructor( ) { companion object { - private val JOB_NAMES = listOf("NodeStatCompositeMongoDbBatchJob", "ProjectRepoMetricsStatJob") + private val JOB_NAMES = listOf("ProjectRepoMetricsStatJob") } fun getShedLockInfo(ids: List = JOB_NAMES): LocalDateTime { From 968ec3b0c084a7f390ef006ec6e118c6922c8650 Mon Sep 17 00:00:00 2001 From: zacYL Date: Thu, 26 Oct 2023 12:03:32 +0800 Subject: [PATCH 15/15] =?UTF-8?q?feat:=20=E9=87=8D=E5=86=99=E4=BB=93?= =?UTF-8?q?=E5=BA=93=E7=B1=BB=E5=9E=8B=20#1314?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../bkrepo/opdata/handler/impl/DockerCapSizeHandler.kt | 6 ++++++ .../bkrepo/opdata/handler/impl/GenericCapSizeHandler.kt | 7 +++++++ 2 files changed, 13 insertions(+) diff --git a/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/handler/impl/DockerCapSizeHandler.kt b/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/handler/impl/DockerCapSizeHandler.kt index 07e5c57b68..6036fe14c4 100644 --- a/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/handler/impl/DockerCapSizeHandler.kt +++ b/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/handler/impl/DockerCapSizeHandler.kt @@ -46,4 +46,10 @@ class DockerCapSizeHandler( override val columnsText: String = OPDATA_DOCKER_CAP_SIZE + override val repoType: List + get() = DOCKER_TYPES + + companion object { + private val DOCKER_TYPES = listOf("DOCKER", "OCI") + } } \ No newline at end of file diff --git a/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/handler/impl/GenericCapSizeHandler.kt b/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/handler/impl/GenericCapSizeHandler.kt index a0aa8148cf..a9956cc107 100644 --- a/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/handler/impl/GenericCapSizeHandler.kt +++ b/src/backend/opdata/biz-opdata/src/main/kotlin/com/tencent/bkrepo/opdata/handler/impl/GenericCapSizeHandler.kt @@ -45,4 +45,11 @@ class GenericCapSizeHandler( override val metric: Metrics get() = Metrics.GENERICCAPSIZE override val columnsText: String = OPDATA_GENERIC_CAP_SIZE + + override val repoType: List + get() = listOf(GENERIC_TYPE) + + companion object { + private const val GENERIC_TYPE = "GENERIC" + } } \ No newline at end of file