From 5f5b2af4596b55d82f34285ce163db2ffee27070 Mon Sep 17 00:00:00 2001 From: zacYL <100330102+zacYL@users.noreply.github.com> Date: Wed, 1 Nov 2023 09:55:57 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E6=B7=BB=E5=8A=A0=E5=AE=9A=E6=97=B6?= =?UTF-8?q?=E4=BB=BB=E5=8A=A1=E7=94=A8=E4=BA=8E=E5=88=A0=E9=99=A4=E5=B7=B2?= =?UTF-8?q?=E7=BB=8F=E5=88=86=E5=8F=91=E5=AE=8C=E6=88=90=E7=9A=84=E9=95=9C?= =?UTF-8?q?=E5=83=8F=20#1336=20(#1349)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: 添加定时任务用于删除已经分发完成的镜像 #1336 * feat: blob路径更新job锁时间调整 #1336 * feat: 问题修复 #1336 * feat: 增加配置 #1336 --- .../batch/DistributedDockerImageCleanupJob.kt | 167 ++++++++++++++++++ .../bkrepo/job/batch/OciBlobNodeRefreshJob.kt | 3 + ...tributedDockerImageCleanupJobProperties.kt | 39 ++++ .../com/tencent/bkrepo/oci/api/OciClient.kt | 9 + .../service/OciPackageController.kt | 11 ++ 5 files changed, 229 insertions(+) create mode 100644 src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/DistributedDockerImageCleanupJob.kt create mode 100644 src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/config/properties/DistributedDockerImageCleanupJobProperties.kt diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/DistributedDockerImageCleanupJob.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/DistributedDockerImageCleanupJob.kt new file mode 100644 index 0000000000..ad67bf82fb --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/DistributedDockerImageCleanupJob.kt @@ -0,0 +1,167 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.job.batch + +import com.tencent.bkrepo.common.artifact.constant.REPO_NAME +import com.tencent.bkrepo.common.artifact.constant.SCAN_STATUS +import com.tencent.bkrepo.common.service.log.LoggerHolder +import com.tencent.bkrepo.job.TYPE +import com.tencent.bkrepo.job.batch.base.DefaultContextMongoDbJob +import com.tencent.bkrepo.job.batch.base.JobContext +import com.tencent.bkrepo.job.config.properties.DistributedDockerImageCleanupJobProperties +import com.tencent.bkrepo.job.exception.JobExecuteException +import com.tencent.bkrepo.oci.api.OciClient +import org.springframework.boot.context.properties.EnableConfigurationProperties +import org.springframework.data.mongodb.core.MongoTemplate +import org.springframework.data.mongodb.core.find +import org.springframework.data.mongodb.core.query.Criteria +import org.springframework.data.mongodb.core.query.Query +import org.springframework.data.mongodb.core.query.isEqualTo +import org.springframework.stereotype.Component +import java.time.Duration +import java.time.LocalDateTime + +/** + * 清理镜像仓库下存储的已经分发的镜像 + */ +@Component +@EnableConfigurationProperties(DistributedDockerImageCleanupJobProperties::class) +class DistributedDockerImageCleanupJob( + private val properties: DistributedDockerImageCleanupJobProperties, + private val mongoTemplate: MongoTemplate, + private val ociClient: OciClient +) : DefaultContextMongoDbJob(properties) { + + + override fun entityClass(): Class { + return PackageData::class.java + } + + override fun collectionNames(): List { + return listOf(PACKAGE_COLLECTION_NAME) + } + + override fun buildQuery(): Query { + return Query( + Criteria.where(TYPE).`in`(properties.repositoryTypes). + and(REPO_NAME).isEqualTo(DISTRIBUTION_IMAGE_REPO) + ) + } + + override fun getLockAtMostFor(): Duration = Duration.ofDays(3) + + + override fun run(row: PackageData, collectionName: String, context: JobContext) { + try { + val query = Query(Criteria(PACKAGE_ID).isEqualTo(row.id)) + val versionList = mongoTemplate.find( + query, PACKAGE_VERSION_NAME + ) + if (versionList.isEmpty()) return + for (versionInfo in versionList) { + if (!filterDistributedAndScannedImage(versionInfo)) continue + logger.info( + "Preparing to send image delete request for package ${row.name}|${versionInfo.name}" + + " in repo ${row.projectId}|${row.repoName}." + ) + ociClient.deleteVersion(row.projectId, row.repoName, row.name, versionInfo.name) + } + } catch (e: Exception) { + throw JobExecuteException( + "Failed to send image delete request for package ${row.name}" + + " in repo ${row.projectId}|${row.repoName}, error: ${e.message}", e + ) + } + } + + + /** + * 如果未开启扫描， 标记为分发并且分发已经结束的镜像直接删除 + * 如果开启扫描， 删除已扫描完的被标记为分发已结束的镜像 + */ + private fun filterDistributedAndScannedImage(versionData: PackageVersionData): Boolean { + with(versionData) { + // 避免误删，只删除最后更新时间是前一天的镜像 + if (versionData.lastModifiedDate.isAfter(LocalDateTime.now().minusDays(properties.keepDays))) return false + if (metadata.isEmpty()) return false + val enableDistribution = metadata.firstOrNull { it[METADATA_KEY] == DISTRIBUTION_METADATA_KEY } + ?.get(METADATA_VALUE) as? Boolean ?: return false + // 如果未标记为开启镜像分发，则保留 + if (!enableDistribution) return false + val enableImageScan = metadata.firstOrNull { it[METADATA_KEY] == IMAGE_SCAN_METADATA_KEY } + ?.get(METADATA_VALUE) as? Boolean + val scanStatus = metadata.firstOrNull { it[METADATA_KEY] == SCAN_STATUS } + ?.get(METADATA_VALUE) as? String? + val distributionStatus = metadata.firstOrNull { it[METADATA_KEY] == DISTRIBUTION_STATUS_METADATA_KEY } + ?.get(METADATA_VALUE) as? String? + // 镜像分发未结束，不进行删除 + if (distributionStatus.isNullOrEmpty() || distributionStatus != DISTRIBUTION_FINISH_STATUS) return false + // 不开启镜像扫描，删除 + if ((enableImageScan == null || !enableImageScan)) return true + if (!scanStatus.isNullOrEmpty() && scanStatus !in SCAN_RUNNING_STATUS) return true + return false + } + } + + data class PackageData(private val map: Map) { + val id: String by map + val repoName: String by map + val projectId: String by map + val name: String by map + val key: String by map + val type: String by map + } + + data class PackageVersionData( + val name: String, + val metadata: List>, + val lastModifiedDate: LocalDateTime + ) + + + + override fun mapToEntity(row: Map): PackageData { + return PackageData(row) + } + + companion object { + private val logger = LoggerHolder.jobLogger + const val PACKAGE_COLLECTION_NAME = "package" + private const val PACKAGE_VERSION_NAME = "package_version" + private const val DISTRIBUTION_METADATA_KEY = "enableDistribution" + private const val IMAGE_SCAN_METADATA_KEY = "enableImageScan" + private const val DISTRIBUTION_STATUS_METADATA_KEY = "distributionStatus" + private const val METADATA_KEY = "key" + private const val METADATA_VALUE = "value" + private const val PACKAGE_ID = "packageId" + private val SCAN_RUNNING_STATUS = listOf("INIT", "RUNNING") + private const val DISTRIBUTION_FINISH_STATUS = "finished" + private const val DISTRIBUTION_IMAGE_REPO = "image" + + } +} \ No newline at end of file diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/OciBlobNodeRefreshJob.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/OciBlobNodeRefreshJob.kt index 68a139804a..fa9f65497c 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/OciBlobNodeRefreshJob.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/OciBlobNodeRefreshJob.kt @@ -41,6 +41,7 @@ import org.springframework.data.mongodb.core.query.Criteria import org.springframework.data.mongodb.core.query.Query import org.springframework.data.mongodb.core.query.isEqualTo import org.springframework.stereotype.Component +import java.time.Duration /** * 用于将存储在blobs目录下的公共blob节点全部迁移到对应版本目录下， @@ -74,6 +75,8 @@ class OciBlobNodeRefreshJob( ) } + override fun getLockAtMostFor(): Duration = Duration.ofDays(7) + override fun run(row: PackageData, collectionName: String, context: JobContext) { with(row) { try { diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/config/properties/DistributedDockerImageCleanupJobProperties.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/config/properties/DistributedDockerImageCleanupJobProperties.kt new file mode 100644 index 0000000000..6846aa65e9 --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/config/properties/DistributedDockerImageCleanupJobProperties.kt @@ -0,0 +1,39 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.job.config.properties + +import org.springframework.boot.context.properties.ConfigurationProperties + +@ConfigurationProperties(value = "job.distributed-docker-image-cleanup") +data class DistributedDockerImageCleanupJobProperties( + override var enabled: Boolean = false, + var repositoryTypes: List = listOf("OCI", "DOCKER"), + // 保留分发的镜像天数 + var keepDays: Long = 1, + override var cron: String = "0 0 5 * * ?" +): MongodbJobProperties() diff --git a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/api/OciClient.kt b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/api/OciClient.kt index b911c0df8a..a04271a647 100644 --- a/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/api/OciClient.kt +++ b/src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/api/OciClient.kt @@ -77,4 +77,13 @@ interface OciClient { @PathVariable repoName: String, @RequestParam packageName: String ): Response + + @ApiOperation("删除仓库下的包版本") + @DeleteMapping("version/delete/{projectId}/{repoName}") + fun deleteVersion( + @PathVariable projectId: String, + @PathVariable repoName: String, + @RequestParam packageName: String, + @RequestParam version: String + ): Response } diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/controller/service/OciPackageController.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/controller/service/OciPackageController.kt index 8aaaccd54c..44fdf39ee7 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/controller/service/OciPackageController.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/controller/service/OciPackageController.kt @@ -35,9 +35,11 @@ import com.tencent.bkrepo.oci.api.OciClient import com.tencent.bkrepo.oci.dao.OciReplicationRecordDao import com.tencent.bkrepo.oci.listener.base.EventExecutor import com.tencent.bkrepo.oci.model.TOciReplicationRecord +import com.tencent.bkrepo.oci.pojo.artifact.OciDeleteArtifactInfo import com.tencent.bkrepo.oci.pojo.artifact.OciManifestArtifactInfo import com.tencent.bkrepo.oci.pojo.third.OciReplicationRecordInfo import com.tencent.bkrepo.oci.service.OciOperationService +import com.tencent.bkrepo.repository.constant.SYSTEM_USER import org.springframework.data.mongodb.core.query.Criteria import org.springframework.data.mongodb.core.query.Query import org.springframework.web.bind.annotation.RestController @@ -97,4 +99,13 @@ class OciPackageController( operationService.deleteBlobsFolderAfterRefreshed(projectId, repoName, packageName) return ResponseBuilder.success() } + + override fun deleteVersion( + projectId: String, repoName: String, + packageName: String, version: String + ): Response { + val artifactInfo = OciDeleteArtifactInfo(projectId, repoName, packageName, version) + operationService.deleteVersion(SYSTEM_USER, artifactInfo) + return ResponseBuilder.success() + } }