feat: 添加定时任务用于删除已经分发完成的镜像 #1336 (#1349)

* feat: 添加定时任务用于删除已经分发完成的镜像 #1336

* feat: blob路径更新job锁时间调整 #1336

* feat: 问题修复 #1336

* feat: 增加配置 #1336

src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/DistributedDockerImageCleanupJob.kt

@@ -0,0 +1,167 @@
+/*
+ * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available.
+ *
+ * Copyright (C) 2022 THL A29 Limited, a Tencent company.  All rights reserved.
+ *
+ * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license.
+ *
+ * A copy of the MIT License is included in this file.
+ *
+ *
+ * Terms of the MIT License:
+ * ---------------------------------------------------
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+ * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all copies or substantial portions of
+ * the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT
+ * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
+ * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package com.tencent.bkrepo.job.batch
+
+import com.tencent.bkrepo.common.artifact.constant.REPO_NAME
+import com.tencent.bkrepo.common.artifact.constant.SCAN_STATUS
+import com.tencent.bkrepo.common.service.log.LoggerHolder
+import com.tencent.bkrepo.job.TYPE
+import com.tencent.bkrepo.job.batch.base.DefaultContextMongoDbJob
+import com.tencent.bkrepo.job.batch.base.JobContext
+import com.tencent.bkrepo.job.config.properties.DistributedDockerImageCleanupJobProperties
+import com.tencent.bkrepo.job.exception.JobExecuteException
+import com.tencent.bkrepo.oci.api.OciClient
+import org.springframework.boot.context.properties.EnableConfigurationProperties
+import org.springframework.data.mongodb.core.MongoTemplate
+import org.springframework.data.mongodb.core.find
+import org.springframework.data.mongodb.core.query.Criteria
+import org.springframework.data.mongodb.core.query.Query
+import org.springframework.data.mongodb.core.query.isEqualTo
+import org.springframework.stereotype.Component
+import java.time.Duration
+import java.time.LocalDateTime
+
+/**
+ * 清理镜像仓库下存储的已经分发的镜像
+ */
+@Component
+@EnableConfigurationProperties(DistributedDockerImageCleanupJobProperties::class)
+class DistributedDockerImageCleanupJob(
+    private val properties: DistributedDockerImageCleanupJobProperties,
+    private val mongoTemplate: MongoTemplate,
+    private val ociClient: OciClient
+) : DefaultContextMongoDbJob<DistributedDockerImageCleanupJob.PackageData>(properties) {
+
+
+    override fun entityClass(): Class<PackageData> {
+        return PackageData::class.java
+    }
+
+    override fun collectionNames(): List<String> {
+        return listOf(PACKAGE_COLLECTION_NAME)
+    }
+
+    override fun buildQuery(): Query {
+        return Query(
+            Criteria.where(TYPE).`in`(properties.repositoryTypes).
+                and(REPO_NAME).isEqualTo(DISTRIBUTION_IMAGE_REPO)
+        )
+    }
+
+    override fun getLockAtMostFor(): Duration = Duration.ofDays(3)
+
+
+    override fun run(row: PackageData, collectionName: String, context: JobContext) {
+        try {
+            val query = Query(Criteria(PACKAGE_ID).isEqualTo(row.id))
+            val versionList = mongoTemplate.find<PackageVersionData>(
+                query, PACKAGE_VERSION_NAME
+            )
+            if (versionList.isEmpty()) return
+            for (versionInfo in versionList) {
+                if (!filterDistributedAndScannedImage(versionInfo)) continue
+                logger.info(
+                    "Preparing to send image delete request for package ${row.name}|${versionInfo.name}" +
+                        " in repo ${row.projectId}|${row.repoName}."
+                )
+                ociClient.deleteVersion(row.projectId, row.repoName, row.name, versionInfo.name)
+            }
+        } catch (e: Exception) {
+            throw JobExecuteException(
+                "Failed to send  image delete request for package ${row.name}" +
+                    " in repo ${row.projectId}|${row.repoName}, error: ${e.message}", e
+            )
+        }
+    }
+
+
+    /**
+     * 如果未开启扫描， 标记为分发并且分发已经结束的镜像直接删除
+     * 如果开启扫描， 删除已扫描完的被标记为分发已结束的镜像
+     */
+    private fun filterDistributedAndScannedImage(versionData: PackageVersionData): Boolean {
+        with(versionData) {
+            // 避免误删，只删除最后更新时间是前一天的镜像
+            if (versionData.lastModifiedDate.isAfter(LocalDateTime.now().minusDays(properties.keepDays))) return false
+            if (metadata.isEmpty()) return false
+            val enableDistribution = metadata.firstOrNull { it[METADATA_KEY] == DISTRIBUTION_METADATA_KEY }
+                ?.get(METADATA_VALUE) as? Boolean ?: return false
+            // 如果未标记为开启镜像分发，则保留
+            if (!enableDistribution)  return false
+            val enableImageScan =  metadata.firstOrNull { it[METADATA_KEY] == IMAGE_SCAN_METADATA_KEY }
+                ?.get(METADATA_VALUE) as? Boolean
+            val scanStatus =  metadata.firstOrNull { it[METADATA_KEY] == SCAN_STATUS }
+                ?.get(METADATA_VALUE) as? String?
+            val distributionStatus = metadata.firstOrNull { it[METADATA_KEY] == DISTRIBUTION_STATUS_METADATA_KEY }
+                ?.get(METADATA_VALUE) as? String?
+            // 镜像分发未结束，不进行删除
+            if (distributionStatus.isNullOrEmpty() || distributionStatus != DISTRIBUTION_FINISH_STATUS) return false
+            // 不开启镜像扫描，删除
+            if ((enableImageScan == null || !enableImageScan)) return true
+            if (!scanStatus.isNullOrEmpty()  && scanStatus !in SCAN_RUNNING_STATUS) return true
+            return false
+        }
+    }
+
+    data class PackageData(private val map: Map<String, Any?>) {
+        val id: String by map
+        val repoName: String by map
+        val projectId: String by map
+        val name: String by map
+        val key: String by map
+        val type: String by map
+    }
+
+    data class PackageVersionData(
+        val name: String,
+        val metadata: List<Map<String, Any>>,
+        val lastModifiedDate: LocalDateTime
+    )
+
+
+
+    override fun mapToEntity(row: Map<String, Any?>): PackageData {
+        return PackageData(row)
+    }
+
+    companion object {
+        private val logger = LoggerHolder.jobLogger
+        const val PACKAGE_COLLECTION_NAME = "package"
+        private const val PACKAGE_VERSION_NAME = "package_version"
+        private const val DISTRIBUTION_METADATA_KEY = "enableDistribution"
+        private const val IMAGE_SCAN_METADATA_KEY = "enableImageScan"
+        private const val DISTRIBUTION_STATUS_METADATA_KEY = "distributionStatus"
+        private const val METADATA_KEY = "key"
+        private const val METADATA_VALUE = "value"
+        private const val PACKAGE_ID = "packageId"
+        private val SCAN_RUNNING_STATUS = listOf("INIT", "RUNNING")
+        private const val DISTRIBUTION_FINISH_STATUS = "finished"
+        private const val DISTRIBUTION_IMAGE_REPO = "image"
+
+    }
+}

src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/OciBlobNodeRefreshJob.kt

@@ -41,6 +41,7 @@ import org.springframework.data.mongodb.core.query.Criteria
 import org.springframework.data.mongodb.core.query.Query
 import org.springframework.data.mongodb.core.query.isEqualTo
 import org.springframework.stereotype.Component
+import java.time.Duration
 
 /**
  * 用于将存储在blobs目录下的公共blob节点全部迁移到对应版本目录下，
@@ -74,6 +75,8 @@ class OciBlobNodeRefreshJob(
         )
     }
 
+    override fun getLockAtMostFor(): Duration = Duration.ofDays(7)
+
     override fun run(row: PackageData, collectionName: String, context: JobContext) {
         with(row) {
             try {

src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/config/properties/DistributedDockerImageCleanupJobProperties.kt

@@ -0,0 +1,39 @@
+/*
+ * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available.
+ *
+ * Copyright (C) 2022 THL A29 Limited, a Tencent company.  All rights reserved.
+ *
+ * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license.
+ *
+ * A copy of the MIT License is included in this file.
+ *
+ *
+ * Terms of the MIT License:
+ * ---------------------------------------------------
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+ * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all copies or substantial portions of
+ * the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT
+ * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
+ * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package com.tencent.bkrepo.job.config.properties
+
+import org.springframework.boot.context.properties.ConfigurationProperties
+
+@ConfigurationProperties(value = "job.distributed-docker-image-cleanup")
+data class DistributedDockerImageCleanupJobProperties(
+    override var enabled: Boolean = false,
+    var repositoryTypes: List<String> = listOf("OCI", "DOCKER"),
+    // 保留分发的镜像天数
+    var keepDays: Long = 1,
+    override var cron: String = "0 0 5 * * ?"
+): MongodbJobProperties()

src/backend/oci/api-oci/src/main/kotlin/com/tencent/bkrepo/oci/api/OciClient.kt

@@ -77,4 +77,13 @@ interface OciClient {
         @PathVariable repoName: String,
         @RequestParam packageName: String
     ): Response<Void>
+
+    @ApiOperation("删除仓库下的包版本")
+    @DeleteMapping("version/delete/{projectId}/{repoName}")
+    fun deleteVersion(
+        @PathVariable projectId: String,
+        @PathVariable repoName: String,
+        @RequestParam packageName: String,
+        @RequestParam version: String
+    ): Response<Void>
 }

src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/controller/service/OciPackageController.kt

@@ -35,9 +35,11 @@ import com.tencent.bkrepo.oci.api.OciClient
 import com.tencent.bkrepo.oci.dao.OciReplicationRecordDao
 import com.tencent.bkrepo.oci.listener.base.EventExecutor
 import com.tencent.bkrepo.oci.model.TOciReplicationRecord
+import com.tencent.bkrepo.oci.pojo.artifact.OciDeleteArtifactInfo
 import com.tencent.bkrepo.oci.pojo.artifact.OciManifestArtifactInfo
 import com.tencent.bkrepo.oci.pojo.third.OciReplicationRecordInfo
 import com.tencent.bkrepo.oci.service.OciOperationService
+import com.tencent.bkrepo.repository.constant.SYSTEM_USER
 import org.springframework.data.mongodb.core.query.Criteria
 import org.springframework.data.mongodb.core.query.Query
 import org.springframework.web.bind.annotation.RestController
@@ -97,4 +99,13 @@ class OciPackageController(
         operationService.deleteBlobsFolderAfterRefreshed(projectId, repoName, packageName)
         return ResponseBuilder.success()
     }
+
+    override fun deleteVersion(
+        projectId: String, repoName: String,
+        packageName: String, version: String
+    ): Response<Void> {
+        val artifactInfo = OciDeleteArtifactInfo(projectId, repoName, packageName, version)
+        operationService.deleteVersion(SYSTEM_USER, artifactInfo)
+        return ResponseBuilder.success()
+    }
 }