From d951a9b054042e93f2ef9e463caa833d9d5e22b5 Mon Sep 17 00:00:00 2001 From: zacyanliu Date: Tue, 26 Sep 2023 15:43:58 +0800 Subject: [PATCH 1/4] =?UTF-8?q?feat:=20=E5=A2=9E=E5=8A=A0=E7=A9=BA?= =?UTF-8?q?=E7=9B=AE=E5=BD=95=E6=B8=85=E7=90=86job=20#1209?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../batch/context/EmptyFolderChildContext.kt | 45 ++++ .../job/batch/node/EmptyFolderCleanupJob.kt | 195 ++++++++++++++++++ .../job/batch/node/FolderStatChildJob.kt | 53 +---- .../node/NodeStatCompositeMongoDbBatchJob.kt | 3 +- .../bkrepo/job/batch/utils/FolderUtils.kt | 79 +++++++ .../com/tencent/bkrepo/job/pojo/FolderInfo.kt | 34 +++ 6 files changed, 359 insertions(+), 50 deletions(-) create mode 100644 src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/context/EmptyFolderChildContext.kt create mode 100644 src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/node/EmptyFolderCleanupJob.kt create mode 100644 src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/utils/FolderUtils.kt create mode 100644 src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/pojo/FolderInfo.kt diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/context/EmptyFolderChildContext.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/context/EmptyFolderChildContext.kt new file mode 100644 index 0000000000..dad4ba1518 --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/context/EmptyFolderChildContext.kt @@ -0,0 +1,45 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.job.batch.context + +import com.tencent.bkrepo.job.batch.base.ChildJobContext +import com.tencent.bkrepo.job.batch.base.JobContext +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.atomic.LongAdder + +class EmptyFolderChildContext( + parentContent: JobContext, + // 用于内存缓存下存储目录统计信息 + var folders: ConcurrentHashMap = ConcurrentHashMap() +): ChildJobContext(parentContent) { + + data class FolderMetricsInfo( + var id: String? = null, + var nodeNum: LongAdder = LongAdder() + ) +} diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/node/EmptyFolderCleanupJob.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/node/EmptyFolderCleanupJob.kt new file mode 100644 index 0000000000..45efaa28f8 --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/node/EmptyFolderCleanupJob.kt @@ -0,0 +1,195 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.job.batch.node + +import com.tencent.bkrepo.common.api.constant.StringPool +import com.tencent.bkrepo.common.artifact.path.PathUtils +import com.tencent.bkrepo.common.mongo.constant.ID +import com.tencent.bkrepo.common.service.log.LoggerHolder +import com.tencent.bkrepo.job.DELETED_DATE +import com.tencent.bkrepo.job.FOLDER +import com.tencent.bkrepo.job.FULLPATH +import com.tencent.bkrepo.job.LAST_MODIFIED_DATE +import com.tencent.bkrepo.job.PROJECT +import com.tencent.bkrepo.job.REPO +import com.tencent.bkrepo.job.batch.base.ChildJobContext +import com.tencent.bkrepo.job.batch.base.ChildMongoDbBatchJob +import com.tencent.bkrepo.job.batch.base.JobContext +import com.tencent.bkrepo.job.batch.context.EmptyFolderChildContext +import com.tencent.bkrepo.job.batch.utils.FolderUtils.buildCacheKey +import com.tencent.bkrepo.job.batch.utils.FolderUtils.extractFolderInfoFromCacheKey +import com.tencent.bkrepo.job.config.properties.CompositeJobProperties +import org.bson.types.ObjectId +import org.springframework.data.mongodb.core.MongoTemplate +import org.springframework.data.mongodb.core.query.Criteria +import org.springframework.data.mongodb.core.query.Query +import org.springframework.data.mongodb.core.query.Update +import org.springframework.data.mongodb.core.query.isEqualTo +import java.time.LocalDateTime + +/** + * 空目录清理job + */ +class EmptyFolderCleanupJob( + val properties: CompositeJobProperties, + private val mongoTemplate: MongoTemplate, +) : ChildMongoDbBatchJob(properties) { + override fun onParentJobStart(context: ChildJobContext) { + logger.info("start to run emptyFolderCleanupJob") + } + + override fun run(row: NodeStatCompositeMongoDbBatchJob.Node, collectionName: String, context: JobContext) { + require(context is EmptyFolderChildContext) + if (row.deleted != null) return + if (row.folder) { + val folderKey = buildCacheKey( + collectionName = collectionName, projectId = row.projectId, + repoName = row.repoName, fullPath = row.fullPath + ) + val folderMetric = context.folders.getOrPut(folderKey) { + EmptyFolderChildContext.FolderMetricsInfo(id = row.id) + } + folderMetric.id = row.id + } else { + val ancestorFolder = PathUtils.resolveAncestor(row.fullPath) + for (folder in ancestorFolder) { + if (folder == PathUtils.ROOT) continue + val tempFullPath = PathUtils.toFullPath(folder) + val folderKey = buildCacheKey( + collectionName = collectionName, projectId = row.projectId, + repoName = row.repoName, fullPath = tempFullPath + ) + + val folderMetric = context.folders.getOrPut(folderKey) { EmptyFolderChildContext.FolderMetricsInfo() } + folderMetric.nodeNum.increment() + } + } + } + + override fun onParentJobFinished(context: ChildJobContext) { + logger.info("emptyFolderCleanupJob finished") + } + + + override fun createChildJobContext(parentJobContext: JobContext): ChildJobContext { + return EmptyFolderChildContext(parentJobContext) + } + + override fun onRunCollectionFinished(collectionName: String, context: JobContext) { + require(context is EmptyFolderChildContext) + super.onRunCollectionFinished(collectionName, context) + filterEmptyFolders(collectionName, context) + } + + + private fun filterEmptyFolders( + collectionName: String, + context: EmptyFolderChildContext + ) { + logger.info("will filter empty folder in table $collectionName") + if (context.folders.isEmpty()) return + val prefix = buildCacheKey(collectionName = collectionName, projectId = StringPool.EMPTY) + for(entry in context.folders) { + if (!entry.key.startsWith(prefix) || entry.value.nodeNum.toLong() > 0) continue + extractFolderInfoFromCacheKey(entry.key)?.let { + if (emptyFolderDoubleCheck( + projectId = it.projectId, + repoName = it.repoName, + path = it.fullPath, + collectionName = collectionName + )) { + logger.info("will delete empty folder ${it.fullPath} in repo ${it.projectId}|${it.repoName}") + deleteEmptyFolders(entry.value.id, collectionName) + } + } + } + clearCollectionContextCache(collectionName, context) + } + + /** + * 再次确认过滤出的空目录下是否包含文件 + */ + private fun emptyFolderDoubleCheck( + projectId: String, + repoName: String, + path: String, + collectionName: String + ): Boolean { + val criteria = Criteria.where(PROJECT).isEqualTo(projectId) + .and(REPO).isEqualTo(repoName) + .and(DELETED_DATE).isEqualTo(null) + .and(FULLPATH).regex("^${PathUtils.escapeRegex(path)}") + .and(FOLDER).isEqualTo(false) + + val query = Query(criteria).withHint(FULL_PATH_IDX) + val result = mongoTemplate.find(query, Map::class.java, collectionName) + return result.isNullOrEmpty() + } + + + /** + * 删除空目录 + */ + private fun deleteEmptyFolders( + objectId: String?, + collectionName: String + ) { + if (objectId.isNullOrEmpty()) return + val query = Query( + Criteria.where(ID).isEqualTo(ObjectId(objectId)) + .and(FOLDER).isEqualTo(true) + ) + val deleteTime = LocalDateTime.now() + val update = Update() + .set(LAST_MODIFIED_DATE, deleteTime) + .set(DELETED_DATE, deleteTime) + mongoTemplate.updateFirst(query, update, collectionName) + } + + /** + * 清除collection在context中对应的缓存记录 + */ + private fun clearCollectionContextCache( + collectionName: String, + context: EmptyFolderChildContext + ) { + val prefix = buildCacheKey(collectionName = collectionName, projectId = StringPool.EMPTY) + for(entry in context.folders) { + if (entry.key.contains(prefix)) + context.folders.remove(entry.key) + } + } + + + + companion object { + private val logger = LoggerHolder.jobLogger + const val FULL_PATH_IDX = "projectId_repoName_fullPath_idx" + + } +} diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/node/FolderStatChildJob.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/node/FolderStatChildJob.kt index 79ccd023f5..daa15caefd 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/node/FolderStatChildJob.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/node/FolderStatChildJob.kt @@ -43,8 +43,11 @@ import com.tencent.bkrepo.job.batch.base.ChildJobContext import com.tencent.bkrepo.job.batch.base.ChildMongoDbBatchJob import com.tencent.bkrepo.job.batch.base.JobContext import com.tencent.bkrepo.job.batch.context.FolderChildContext +import com.tencent.bkrepo.job.batch.utils.FolderUtils.buildCacheKey +import com.tencent.bkrepo.job.batch.utils.FolderUtils.extractFolderInfoFromCacheKey import com.tencent.bkrepo.job.config.properties.CompositeJobProperties import com.tencent.bkrepo.job.config.properties.NodeStatCompositeMongoDbBatchJobProperties +import com.tencent.bkrepo.job.pojo.FolderInfo import org.springframework.data.mongodb.core.BulkOperations.BulkMode import org.springframework.data.mongodb.core.MongoTemplate import org.springframework.data.mongodb.core.query.Criteria @@ -460,33 +463,6 @@ class FolderStatChildJob( } - /** - * 生成缓存key - */ - private fun buildCacheKey( - projectId: String, - repoName: String? = null, - fullPath: String? = null, - collectionName: String? = null, - tag: String? = null, - ): String { - return StringBuilder().apply { - collectionName?.let { - this.append(it).append(StringPool.COLON) - } - this.append(projectId) - repoName?.let { - this.append(StringPool.COLON).append(repoName) - } - fullPath?.let { - this.append(StringPool.COLON).append(fullPath) - } - tag?.let { - this.append(StringPool.COLON).append(tag) - } - }.toString() - } - private fun extractFolderInfoFromRedisKey(key: String): FolderInfo? { val values = key.split(StringPool.COLON) return try { @@ -502,21 +478,7 @@ class FolderStatChildJob( - /** - * 从缓存key中解析出目录信息 - */ - private fun extractFolderInfoFromCacheKey(key: String): FolderInfo? { - val values = key.split(StringPool.COLON) - return try { - FolderInfo( - projectId = values[1], - repoName = values[2], - fullPath = values[3] - ) - } catch (e: Exception) { - null - } - } + /** * 从缓存key中解析出collectionName @@ -530,13 +492,6 @@ class FolderStatChildJob( } } - - data class FolderInfo( - var projectId: String, - var repoName: String, - var fullPath: String - ) - data class StatInfo( var size: Long, var nodeNum: Long diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/node/NodeStatCompositeMongoDbBatchJob.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/node/NodeStatCompositeMongoDbBatchJob.kt index 8bba4274f2..ead91088f5 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/node/NodeStatCompositeMongoDbBatchJob.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/node/NodeStatCompositeMongoDbBatchJob.kt @@ -33,7 +33,8 @@ class NodeStatCompositeMongoDbBatchJob( override fun createChildJobs(): List> { return listOf( ProjectRepoStatChildJob(properties, mongoTemplate), - FolderStatChildJob(properties, mongoTemplate, redisTemplate) + FolderStatChildJob(properties, mongoTemplate, redisTemplate), + EmptyFolderCleanupJob(properties, mongoTemplate) ) } diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/utils/FolderUtils.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/utils/FolderUtils.kt new file mode 100644 index 0000000000..a133ccc938 --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/utils/FolderUtils.kt @@ -0,0 +1,79 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.job.batch.utils + +import com.tencent.bkrepo.common.api.constant.StringPool +import com.tencent.bkrepo.job.pojo.FolderInfo + +object FolderUtils { + + + /** + * 生成缓存key + */ + fun buildCacheKey( + projectId: String, + repoName: String? = null, + fullPath: String? = null, + collectionName: String? = null, + tag: String? = null, + ): String { + return StringBuilder().apply { + collectionName?.let { + this.append(it).append(StringPool.COLON) + } + this.append(projectId) + repoName?.let { + this.append(StringPool.COLON).append(repoName) + } + fullPath?.let { + this.append(StringPool.COLON).append(fullPath) + } + tag?.let { + this.append(StringPool.COLON).append(tag) + } + }.toString() + } + + /** + * 从缓存key中解析出节点信息 + */ + fun extractFolderInfoFromCacheKey(key: String): FolderInfo? { + val values = key.split(StringPool.COLON) + return try { + FolderInfo( + projectId = values[1], + repoName = values[2], + fullPath = values[3] + ) + } catch (e: Exception) { + null + } + } + +} \ No newline at end of file diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/pojo/FolderInfo.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/pojo/FolderInfo.kt new file mode 100644 index 0000000000..2d628b2b33 --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/pojo/FolderInfo.kt @@ -0,0 +1,34 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.job.pojo + +data class FolderInfo( + var projectId: String, + var repoName: String, + var fullPath: String +) From 74c70720d4b355c0e52be307ca51ab563169a476 Mon Sep 17 00:00:00 2001 From: zacyanliu Date: Tue, 26 Sep 2023 17:18:16 +0800 Subject: [PATCH 2/4] =?UTF-8?q?feat:=20=E7=9B=AE=E5=BD=95=E4=B8=8B?= =?UTF-8?q?=E8=8A=82=E7=82=B9=E5=88=A4=E6=96=AD=E8=B0=83=E6=95=B4=20#1209?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/tencent/bkrepo/job/batch/node/EmptyFolderCleanupJob.kt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/node/EmptyFolderCleanupJob.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/node/EmptyFolderCleanupJob.kt index 45efaa28f8..79ad054eae 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/node/EmptyFolderCleanupJob.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/node/EmptyFolderCleanupJob.kt @@ -140,10 +140,11 @@ class EmptyFolderCleanupJob( path: String, collectionName: String ): Boolean { + val nodePath = PathUtils.toPath(path) val criteria = Criteria.where(PROJECT).isEqualTo(projectId) .and(REPO).isEqualTo(repoName) .and(DELETED_DATE).isEqualTo(null) - .and(FULLPATH).regex("^${PathUtils.escapeRegex(path)}") + .and(FULLPATH).regex("^${PathUtils.escapeRegex(nodePath)}") .and(FOLDER).isEqualTo(false) val query = Query(criteria).withHint(FULL_PATH_IDX) From 40ad075f24667000a5b976982f05a20ed36207a8 Mon Sep 17 00:00:00 2001 From: zacyanliu Date: Tue, 10 Oct 2023 10:49:04 +0800 Subject: [PATCH 3/4] =?UTF-8?q?feat:=20=E6=B8=85=E7=90=86=E7=A9=BA?= =?UTF-8?q?=E7=9B=AE=E5=BD=95=E6=97=B6=E6=8C=87=E5=AE=9A=E5=8F=AA=E6=B8=85?= =?UTF-8?q?=E7=90=86=E9=BB=98=E8=AE=A4=E7=9A=844=E4=B8=AA=E4=BB=93?= =?UTF-8?q?=E5=BA=93=20#1209?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../bkrepo/common/artifact/constant/ArtifactConstants.kt | 5 +++++ .../tencent/bkrepo/job/batch/node/EmptyFolderCleanupJob.kt | 6 ++++++ 2 files changed, 11 insertions(+) diff --git a/src/backend/common/common-artifact/artifact-api/src/main/kotlin/com/tencent/bkrepo/common/artifact/constant/ArtifactConstants.kt b/src/backend/common/common-artifact/artifact-api/src/main/kotlin/com/tencent/bkrepo/common/artifact/constant/ArtifactConstants.kt index dc42d579cc..5a5774b684 100644 --- a/src/backend/common/common-artifact/artifact-api/src/main/kotlin/com/tencent/bkrepo/common/artifact/constant/ArtifactConstants.kt +++ b/src/backend/common/common-artifact/artifact-api/src/main/kotlin/com/tencent/bkrepo/common/artifact/constant/ArtifactConstants.kt @@ -112,6 +112,11 @@ const val X_CHECKSUM_SHA256 = "X-Checksum-Sha256" */ const val PIPELINE = "pipeline" +/** + * 自定义仓库 + */ +const val CUSTOM = "custom" + /** * 报告仓库 */ diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/node/EmptyFolderCleanupJob.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/node/EmptyFolderCleanupJob.kt index 79ad054eae..6216d9dd48 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/node/EmptyFolderCleanupJob.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/node/EmptyFolderCleanupJob.kt @@ -28,6 +28,10 @@ package com.tencent.bkrepo.job.batch.node import com.tencent.bkrepo.common.api.constant.StringPool +import com.tencent.bkrepo.common.artifact.constant.CUSTOM +import com.tencent.bkrepo.common.artifact.constant.LOG +import com.tencent.bkrepo.common.artifact.constant.PIPELINE +import com.tencent.bkrepo.common.artifact.constant.REPORT import com.tencent.bkrepo.common.artifact.path.PathUtils import com.tencent.bkrepo.common.mongo.constant.ID import com.tencent.bkrepo.common.service.log.LoggerHolder @@ -66,6 +70,7 @@ class EmptyFolderCleanupJob( override fun run(row: NodeStatCompositeMongoDbBatchJob.Node, collectionName: String, context: JobContext) { require(context is EmptyFolderChildContext) if (row.deleted != null) return + if (row.repoName !in TARGET_REPO_LIST) return if (row.folder) { val folderKey = buildCacheKey( collectionName = collectionName, projectId = row.projectId, @@ -191,6 +196,7 @@ class EmptyFolderCleanupJob( companion object { private val logger = LoggerHolder.jobLogger const val FULL_PATH_IDX = "projectId_repoName_fullPath_idx" + private val TARGET_REPO_LIST = listOf(REPORT, LOG, PIPELINE, CUSTOM) } } From d38ab22502b841dedca7a550559ef58514b50a0a Mon Sep 17 00:00:00 2001 From: zacyanliu Date: Tue, 10 Oct 2023 11:56:09 +0800 Subject: [PATCH 4/4] =?UTF-8?q?feat:=20=E7=BB=9F=E8=AE=A1=E6=B8=85?= =?UTF-8?q?=E7=90=86=E7=9A=84=E7=A9=BA=E7=9B=AE=E5=BD=95=E4=B8=AA=E6=95=B0?= =?UTF-8?q?=20#1209?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../bkrepo/job/batch/context/EmptyFolderChildContext.kt | 4 +++- .../tencent/bkrepo/job/batch/node/EmptyFolderCleanupJob.kt | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/context/EmptyFolderChildContext.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/context/EmptyFolderChildContext.kt index dad4ba1518..c5a6968ea8 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/context/EmptyFolderChildContext.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/context/EmptyFolderChildContext.kt @@ -35,7 +35,9 @@ import java.util.concurrent.atomic.LongAdder class EmptyFolderChildContext( parentContent: JobContext, // 用于内存缓存下存储目录统计信息 - var folders: ConcurrentHashMap = ConcurrentHashMap() + var folders: ConcurrentHashMap = ConcurrentHashMap(), + // 总共删除的路径个数 + var totalDeletedNum: LongAdder = LongAdder() ): ChildJobContext(parentContent) { data class FolderMetricsInfo( diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/node/EmptyFolderCleanupJob.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/node/EmptyFolderCleanupJob.kt index 6216d9dd48..8ca8aa3ac4 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/node/EmptyFolderCleanupJob.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/node/EmptyFolderCleanupJob.kt @@ -97,7 +97,8 @@ class EmptyFolderCleanupJob( } override fun onParentJobFinished(context: ChildJobContext) { - logger.info("emptyFolderCleanupJob finished") + require(context is EmptyFolderChildContext) + logger.info("emptyFolderCleanupJob finished, deleted empty folder num: ${context.totalDeletedNum}") } @@ -129,6 +130,7 @@ class EmptyFolderCleanupJob( collectionName = collectionName )) { logger.info("will delete empty folder ${it.fullPath} in repo ${it.projectId}|${it.repoName}") + context.totalDeletedNum.increment() deleteEmptyFolders(entry.value.id, collectionName) } }