feat: 增加目录统计 #1012

src/backend/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/utils/EasyExcelUtils.kt

@@ -1,3 +1,30 @@
+/*
+ * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available.
+ *
+ * Copyright (C) 2022 THL A29 Limited, a Tencent company.  All rights reserved.
+ *
+ * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license.
+ *
+ * A copy of the MIT License is included in this file.
+ *
+ *
+ * Terms of the MIT License:
+ * ---------------------------------------------------
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+ * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all copies or substantial portions of
+ * the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT
+ * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
+ * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
 package com.tencent.bkrepo.analyst.utils
 
 import com.alibaba.excel.EasyExcel

src/backend/common/common-artifact/artifact-api/src/main/kotlin/com/tencent/bkrepo/common/artifact/constant/ArtifactConstants.kt

@@ -112,6 +112,12 @@ const val PIPELINE = "pipeline"
  */
 const val REPORT = "report"
 
+
+/**
+ * 日志仓库
+ */
+const val LOG = "log"
+
 /**
  * 文件访问请求是否为直接下载
  */

src/backend/common/common-artifact/artifact-api/src/main/kotlin/com/tencent/bkrepo/common/artifact/path/PathUtils.kt

@@ -207,6 +207,19 @@ object PathUtils {
         return result
     }
 
+    /**
+     * 获取当前路径的所有上级目录列表
+     */
+    fun resolveAncestorFolder(fullPath: String): List<String> {
+        return resolveAncestor(fullPath).map {
+            if (it != ROOT) {
+                it.removeSuffix(StringPool.SLASH)
+            } else {
+                it
+            }
+        }
+    }
+
     /**
      * 根据fullPath解析文件名称，返回格式abc.txt
      *

src/backend/job/biz-job/build.gradle.kts

@@ -31,6 +31,7 @@ dependencies {
     implementation(project(":common:common-security"))
     implementation(project(":common:common-storage:storage-service"))
     implementation(project(":common:common-stream"))
+    implementation(project(":common:common-redis"))
     implementation(project(":repository:api-repository"))
     implementation(project(":helm:api-helm"))
     implementation(project(":oci:api-oci"))

src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/Constants.kt

@@ -51,6 +51,14 @@ const val CATEGORY = "category"
 const val CREATED_DATE = "createdDate"
 const val LAST_MODIFIED_DATE = "lastModifiedDate"
 const val DELETED_DATE = "deleted"
+const val FULLPATH = "fullPath"
+const val PATH = "path"
+
+/**
+ * 缓存类型
+ */
+const val REDIS_CACHE_TYPE = "redis"
+const val MEMORY_CACHE_TYPE = "memory"
 
 /**
  * metrics

src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/base/CompositeMongoDbBatchJob.kt

@@ -44,6 +44,12 @@ abstract class CompositeMongoDbBatchJob<T>(
         return CompositeJobContext(enabledChildJobs)
     }
 
+    override fun onRunCollectionFinished(collectionName: String, context: CompositeJobContext<T>) {
+        context.childJobs.forEach {
+            logException { it.onRunCollectionFinished(collectionName, context.childContext(it.getJobName())) }
+        }
+    }
+
     protected abstract fun createChildJobs(): List<ChildMongoDbBatchJob<T>>
 
     @Suppress("TooGenericExceptionCaught")

src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/base/MongoDbBatchJob.kt

@@ -76,6 +76,12 @@ abstract class MongoDbBatchJob<Entity, Context : JobContext>(
 
     abstract fun entityClass(): Class<Entity>
 
+
+    /**
+     * 表执行结束回调
+     * */
+    open fun onRunCollectionFinished(collectionName: String, context: Context) {}
+
     private val batchSize: Int
         get() = properties.batchSize
 
@@ -162,6 +168,7 @@ abstract class MongoDbBatchJob<Entity, Context : JobContext>(
             } while (querySize == pageSize && isRunning())
         }.apply {
             val elapsedTime = HumanReadable.time(this)
+            onRunCollectionFinished(collectionName, context)
             logger.info("Job[${getJobName()}]: collection $collectionName run completed,sum [$sum] elapse $elapsedTime")
         }
     }

src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/context/FolderChildContext.kt

@@ -0,0 +1,52 @@
+/*
+ * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available.
+ *
+ * Copyright (C) 2022 THL A29 Limited, a Tencent company.  All rights reserved.
+ *
+ * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license.
+ *
+ * A copy of the MIT License is included in this file.
+ *
+ *
+ * Terms of the MIT License:
+ * ---------------------------------------------------
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+ * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all copies or substantial portions of
+ * the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT
+ * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
+ * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package com.tencent.bkrepo.job.batch.context
+
+import com.tencent.bkrepo.job.MEMORY_CACHE_TYPE
+import com.tencent.bkrepo.job.batch.base.ChildJobContext
+import com.tencent.bkrepo.job.batch.base.JobContext
+import java.util.concurrent.ConcurrentHashMap
+import java.util.concurrent.atomic.LongAdder
+
+class FolderChildContext(
+    parentContent: JobContext,
+    // 是否执行任务
+    var runFlag: Boolean = false,
+    // 缓存类型redis和内存：数据量级大的建议使用redis
+    var cacheType: String = MEMORY_CACHE_TYPE,
+    // 表对应项目记录： 主要用于redis缓存生成key使用
+    var projectMap: ConcurrentHashMap<String, MutableSet<String>> = ConcurrentHashMap(),
+    // 用于内存缓存下存储目录统计信息
+    var folderCache: ConcurrentHashMap<String, FolderMetrics> = ConcurrentHashMap()
+) : ChildJobContext(parentContent) {
+
+    data class FolderMetrics(
+        var nodeNum: LongAdder = LongAdder(),
+        var capSize: LongAdder = LongAdder()
+    )
+}

src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/context/ProjectRepoChildContext.kt

@@ -41,7 +41,9 @@ class ProjectRepoChildContext(
                 .storageCredentials
                 ?.key ?: "default"
             val repo = repoMetrics.getOrPut(row.repoName) { RepoMetrics(row.repoName, credentialsKey) }
-            repo.size.add(row.size)
+            if (!row.folder) {
+                repo.size.add(row.size)
+            }
             repo.num.increment()
             repo.addFolderMetrics(row)
             repo.addExtensionMetrics(row)
@@ -94,7 +96,9 @@ class ProjectRepoChildContext(
             }
             val metric = folderMetrics.getOrPut(firstLevelPath) { FolderMetric(firstLevelPath) }
             metric.nodeNum.increment()
-            metric.capSize.add(row.size)
+            if (!row.folder) {
+                metric.capSize.add(row.size)
+            }
         }
 
         fun addExtensionMetrics(row: NodeStatCompositeMongoDbBatchJob.Node) {