From 8a72f36bb414c281f17f66b0dba2c9d533f4fdfe Mon Sep 17 00:00:00 2001 From: zacYL Date: Thu, 21 Sep 2023 21:04:55 +0800 Subject: [PATCH 1/8] =?UTF-8?q?feat:=20=E7=9B=91=E5=90=AC=E8=8A=82?= =?UTF-8?q?=E7=82=B9=E9=87=8D=E5=91=BD=E5=90=8D=E4=BA=8B=E4=BB=B6=EF=BC=8C?= =?UTF-8?q?=E5=B9=B6=E6=9B=B4=E6=96=B0=E5=AF=B9=E5=BA=94=E7=9B=AE=E5=BD=95?= =?UTF-8?q?=E4=BF=A1=E6=81=AF#1195?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../listener/NodeModifyEventListener.kt | 66 +++++++++++++------ 1 file changed, 47 insertions(+), 19 deletions(-) diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/listener/NodeModifyEventListener.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/listener/NodeModifyEventListener.kt index 05e9f26645..a91f21bd21 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/listener/NodeModifyEventListener.kt +++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/listener/NodeModifyEventListener.kt @@ -38,6 +38,7 @@ import com.tencent.bkrepo.common.artifact.event.node.NodeCopiedEvent import com.tencent.bkrepo.common.artifact.event.node.NodeCreatedEvent import com.tencent.bkrepo.common.artifact.event.node.NodeDeletedEvent import com.tencent.bkrepo.common.artifact.event.node.NodeMovedEvent +import com.tencent.bkrepo.common.artifact.event.node.NodeRenamedEvent import com.tencent.bkrepo.common.artifact.path.PathUtils import com.tencent.bkrepo.repository.dao.NodeDao import com.tencent.bkrepo.repository.model.TNode @@ -86,7 +87,8 @@ class NodeModifyEventListener( EventType.NODE_COPIED, EventType.NODE_CREATED, EventType.NODE_DELETED, - EventType.NODE_MOVED + EventType.NODE_MOVED, + EventType.NODE_RENAMED ) @@ -179,6 +181,18 @@ class NodeModifyEventListener( ) modifiedNodeList.add(createdNode) } + EventType.NODE_RENAMED -> { + require(event is NodeRenamedEvent) + // 节点重命名逻辑和其他操作不同，它会对旧节点下的目录删除，然后新建，但是对于非目录节点是进行更新动作，而不是删除再新建 + // 节点重命名操作只需要更新该节点下的子目录的统计信息，不需要更新其上层目录统计信息 + val renamedNode = ModifiedNodeInfo( + projectId = event.projectId, + repoName = event.repoName, + fullPath = event.newFullPath, + includePrefix = event.newFullPath + ) + modifiedNodeList.add(renamedNode) + } else -> throw UnsupportedOperationException() } modifiedNodeList.forEach { @@ -206,7 +220,8 @@ class NodeModifyEventListener( findAndCacheSubFolders( artifactInfo = artifactInfo, deleted = node.nodeInfo.deleted, - deletedFlag = modifiedNode.deleted + deletedFlag = modifiedNode.deleted, + includePrefix = modifiedNode.includePrefix ) } else { updateCache( @@ -214,40 +229,51 @@ class NodeModifyEventListener( repoName = artifactInfo.repoName, fullPath = artifactInfo.getArtifactFullPath(), size = node.size, - deleted = modifiedNode.deleted + deleted = modifiedNode.deleted, + includePrefix = modifiedNode.includePrefix ) } } + + /** + * 更新缓存 + * 当要更新包含该文件所有的目录的缓存记录时includePrefix为空 + * 当只需要更新特定目录前缀目录的缓存记录时设置includePrefix + */ private fun updateCache( projectId: String, repoName: String, fullPath: String, size: Long, - deleted: Boolean = false + deleted: Boolean = false, + includePrefix: String? = null ) { // 更新当前节点所有上级目录统计信息 - PathUtils.resolveAncestorFolder(fullPath).forEach{ - if (it != PathUtils.ROOT) { - val key = Triple(projectId, repoName, it) - var (cachedSize, nodeNum) = cache.getIfPresent(key) ?: Pair(0L, 0L) - if (deleted) { - cachedSize -= size - nodeNum -= 1 - } else { - cachedSize += size - nodeNum += 1 - } - cache.put(key, Pair(cachedSize, nodeNum)) + val folderPaths = PathUtils.resolveAncestorFolder(fullPath) + for (it in folderPaths) { + if (it == PathUtils.ROOT) continue + // 当只需要更新特定目录前缀目录的缓存记录时设置folderPrefix + if (!includePrefix.isNullOrEmpty() && !it.startsWith(includePrefix)) continue + val key = Triple(projectId, repoName, it) + var (cachedSize, nodeNum) = cache.getIfPresent(key) ?: Pair(0L, 0L) + if (deleted) { + cachedSize -= size + nodeNum -= 1 + } else { + cachedSize += size + nodeNum += 1 } + cache.put(key, Pair(cachedSize, nodeNum)) } } private fun findAndCacheSubFolders( artifactInfo: ArtifactInfo, deleted: String? = null, - deletedFlag: Boolean = false + deletedFlag: Boolean = false, + includePrefix: String? = null ) { findAllNodesUnderFolder( artifactInfo.projectId, @@ -260,7 +286,8 @@ class NodeModifyEventListener( repoName = artifactInfo.repoName, fullPath = it.fullPath.getFolderPath(), size = it.size, - deleted = deletedFlag + deleted = deletedFlag, + includePrefix = includePrefix ) } } @@ -319,7 +346,8 @@ class NodeModifyEventListener( var projectId: String, var repoName: String, var fullPath: String, - var deleted: Boolean = false + var deleted: Boolean = false, + var includePrefix: String? = null ) companion object { From 2621b043a697666b2fe4febc6c95e187bec1014a Mon Sep 17 00:00:00 2001 From: zacYL Date: Thu, 21 Sep 2023 21:08:16 +0800 Subject: [PATCH 2/8] =?UTF-8?q?feat:=20=E7=BC=A9=E7=9F=AD=E7=BC=93?= =?UTF-8?q?=E5=AD=98=E5=86=99=E5=85=A5db=E6=97=B6=E9=97=B4=E9=97=B4?= =?UTF-8?q?=E9=9A=94#1195?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../bkrepo/repository/listener/NodeModifyEventListener.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/listener/NodeModifyEventListener.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/listener/NodeModifyEventListener.kt index a91f21bd21..1698edf059 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/listener/NodeModifyEventListener.kt +++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/listener/NodeModifyEventListener.kt @@ -352,7 +352,7 @@ class NodeModifyEventListener( companion object { private val logger = LoggerFactory.getLogger(NodeModifyEventListener::class.java) - private const val FIXED_DELAY = 30000L + private const val FIXED_DELAY = 10000L private val IGNORE_PROJECT_PREFIX_LIST = listOf("CODE_", "CLOSED_SOURCE_", "git_") private val IGNORE_REPO_LIST = listOf(REPORT, LOG) } From 582bd57b66b9eca4b46e83c675ace6a9aa07f03c Mon Sep 17 00:00:00 2001 From: zacyanliu Date: Thu, 21 Sep 2023 22:39:09 +0800 Subject: [PATCH 3/8] =?UTF-8?q?feat:=20=E4=BF=AE=E5=A4=8D=E6=96=87?= =?UTF-8?q?=E4=BB=B6=E8=8A=82=E7=82=B9=E4=B8=8A=E7=BA=A7=E7=9B=AE=E5=BD=95?= =?UTF-8?q?=E6=B2=A1=E6=9C=89=E6=9B=B4=E6=96=B0=20#1195?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../bkrepo/repository/listener/NodeModifyEventListener.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/listener/NodeModifyEventListener.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/listener/NodeModifyEventListener.kt index 1698edf059..f345f4e83f 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/listener/NodeModifyEventListener.kt +++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/listener/NodeModifyEventListener.kt @@ -284,7 +284,7 @@ class NodeModifyEventListener( updateCache( projectId = artifactInfo.projectId, repoName = artifactInfo.repoName, - fullPath = it.fullPath.getFolderPath(), + fullPath = it.fullPath, size = it.size, deleted = deletedFlag, includePrefix = includePrefix From 18a0b249929afbed8dee1a4e488fad7ff2b1b4a0 Mon Sep 17 00:00:00 2001 From: zacyanliu Date: Fri, 22 Sep 2023 11:12:10 +0800 Subject: [PATCH 4/8] =?UTF-8?q?feat:=20=E8=8A=82=E7=82=B9=E5=A4=8D?= =?UTF-8?q?=E5=88=B6=E5=92=8C=E7=A7=BB=E5=8A=A8=E6=97=B6=E7=9B=AE=E5=BD=95?= =?UTF-8?q?=E4=BC=9A=E4=BF=9D=E7=95=99=E5=8E=9F=E6=9C=89=E6=95=B0=E6=8D=AE?= =?UTF-8?q?=E5=AF=BC=E8=87=B4=E7=BB=9F=E8=AE=A1=E5=BC=82=E5=B8=B8=20#1195?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../bkrepo/repository/service/node/impl/NodeMoveCopySupport.kt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeMoveCopySupport.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeMoveCopySupport.kt index db2b0f6d5a..56fc80036f 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeMoveCopySupport.kt +++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeMoveCopySupport.kt @@ -184,6 +184,8 @@ open class NodeMoveCopySupport( path = dstPath, name = dstName, fullPath = dstFullPath, + size = if (node.folder) 0 else node.size, + nodeNum = if (node.folder) null else node.nodeNum, lastModifiedBy = operator, lastModifiedDate = LocalDateTime.now() ) From 3c173f241dce22d12c73450862f5adfd8a4017ae Mon Sep 17 00:00:00 2001 From: zacyanliu Date: Fri, 22 Sep 2023 12:00:31 +0800 Subject: [PATCH 5/8] =?UTF-8?q?feat:=20=E8=B0=83=E6=95=B4=E5=88=9B?= =?UTF-8?q?=E5=BB=BA=E8=8A=82=E7=82=B9=E4=BA=8B=E4=BB=B6=E5=8F=91=E9=80=81?= =?UTF-8?q?=E6=97=B6=E9=97=B4=EF=BC=8C=E9=81=BF=E5=85=8D=E8=B6=85=E6=97=B6?= =?UTF-8?q?=E5=9B=9E=E6=BB=9A=E5=90=8E=E5=87=BA=E7=8E=B0=E5=BC=82=E5=B8=B8?= =?UTF-8?q?=20#1195?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../repository/service/node/impl/NodeBaseService.kt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeBaseService.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeBaseService.kt index 536a49b659..98b58c2b75 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeBaseService.kt +++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeBaseService.kt @@ -199,10 +199,6 @@ abstract class NodeBaseService( deletedTime: LocalDateTime? ) { with(node) { - if (isGenericRepo(repo)) { - publishEvent(buildCreatedEvent(node)) - } - reportNode2Bkbase(node) val createEnd = System.currentTimeMillis() val timeout = createEnd - createStart > repositoryProperties.nodeCreateTimeout if (timeout) { @@ -210,6 +206,10 @@ abstract class NodeBaseService( rollbackCreate(parents, node, deletedTime) throw ErrorCodeException(ArtifactMessageCode.NODE_CREATE_TIMEOUT, fullPath) } + if (isGenericRepo(repo)) { + publishEvent(buildCreatedEvent(node)) + } + reportNode2Bkbase(node) } } From bbf39a3688bb8d8e5f99fd7b011ca6945e76922f Mon Sep 17 00:00:00 2001 From: zacyanliu Date: Fri, 22 Sep 2023 12:27:15 +0800 Subject: [PATCH 6/8] =?UTF-8?q?feat:=20=E5=BC=82=E6=AD=A5=E5=8E=BB?= =?UTF-8?q?=E6=9B=B4=E6=96=B0=E7=9B=AE=E5=BD=95=E4=BF=A1=E6=81=AF=20#1195?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../bkrepo/repository/listener/NodeModifyEventListener.kt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/listener/NodeModifyEventListener.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/listener/NodeModifyEventListener.kt index f345f4e83f..9d18974b00 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/listener/NodeModifyEventListener.kt +++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/listener/NodeModifyEventListener.kt @@ -49,6 +49,7 @@ import org.springframework.data.mongodb.core.query.Query import org.springframework.data.mongodb.core.query.and import org.springframework.data.mongodb.core.query.isEqualTo import org.springframework.data.mongodb.core.query.where +import org.springframework.scheduling.annotation.Async import org.springframework.scheduling.annotation.Scheduled import org.springframework.stereotype.Component import java.time.LocalDateTime @@ -92,6 +93,7 @@ class NodeModifyEventListener( ) + @Async @EventListener(ArtifactEvent::class) fun handle(event: ArtifactEvent) { if (!acceptTypes.contains(event.type)) { From 66ffbe4279265e8b7cec49b3a48ea2b6507ec45e Mon Sep 17 00:00:00 2001 From: zacyanliu Date: Mon, 25 Sep 2023 19:38:14 +0800 Subject: [PATCH 7/8] =?UTF-8?q?feat:=20=E8=8A=82=E7=82=B9copy=E5=92=8Cmove?= =?UTF-8?q?=E9=80=BB=E8=BE=91=E8=B0=83=E6=95=B4=20#1195?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../listener/NodeModifyEventListener.kt | 122 +++++++++++++----- .../service/node/impl/NodeMoveCopySupport.kt | 7 +- 2 files changed, 93 insertions(+), 36 deletions(-) diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/listener/NodeModifyEventListener.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/listener/NodeModifyEventListener.kt index 9d18974b00..e311389a71 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/listener/NodeModifyEventListener.kt +++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/listener/NodeModifyEventListener.kt @@ -40,6 +40,7 @@ import com.tencent.bkrepo.common.artifact.event.node.NodeDeletedEvent import com.tencent.bkrepo.common.artifact.event.node.NodeMovedEvent import com.tencent.bkrepo.common.artifact.event.node.NodeRenamedEvent import com.tencent.bkrepo.common.artifact.path.PathUtils +import com.tencent.bkrepo.common.artifact.path.PathUtils.combinePath import com.tencent.bkrepo.repository.dao.NodeDao import com.tencent.bkrepo.repository.model.TNode import com.tencent.bkrepo.repository.service.node.NodeService @@ -137,23 +138,6 @@ class NodeModifyEventListener( logger.info("event type ${event.type}") val modifiedNodeList = mutableListOf() when (event.type) { - EventType.NODE_MOVED -> { - require(event is NodeMovedEvent) - val dstFullPath = buildDstFullPath(event.dstFullPath, event.resourceKey) - val createdNode = ModifiedNodeInfo( - projectId = event.dstProjectId, - repoName = event.dstRepoName, - fullPath = dstFullPath - ) - val deletedNode = ModifiedNodeInfo( - projectId = event.projectId, - repoName = event.repoName, - fullPath = event.resourceKey, - deleted = true - ) - modifiedNodeList.add(createdNode) - modifiedNodeList.add(deletedNode) - } EventType.NODE_DELETED -> { require(event is NodeDeletedEvent) val deletedNode = ModifiedNodeInfo( @@ -164,16 +148,6 @@ class NodeModifyEventListener( ) modifiedNodeList.add(deletedNode) } - EventType.NODE_COPIED -> { - require(event is NodeCopiedEvent) - val dstFullPath = buildDstFullPath(event.dstFullPath, event.resourceKey) - val createdNode = ModifiedNodeInfo( - projectId = event.dstProjectId, - repoName = event.dstRepoName, - fullPath = dstFullPath - ) - modifiedNodeList.add(createdNode) - } EventType.NODE_CREATED -> { require(event is NodeCreatedEvent) val createdNode = ModifiedNodeInfo( @@ -195,6 +169,43 @@ class NodeModifyEventListener( ) modifiedNodeList.add(renamedNode) } + EventType.NODE_MOVED -> { + require(event is NodeMovedEvent) + // 1 move空目录，2 move到已存在目录, 3 move到新目录 4 同路径，跳过 5 src为dst目录下的子节点，跳过 + // 针对1 2 两种情况，需要判断原目录中的节点，然后再进行目标目录的统计信息更新 + val dstFullPath = buildDstFullPath(event.dstFullPath, event.resourceKey) + val createdNode = ModifiedNodeInfo( + projectId = event.dstProjectId, + repoName = event.dstRepoName, + fullPath = dstFullPath, + srcProjectId = event.projectId, + srcRepoName = event.repoName, + srcFullPath = event.resourceKey + ) + val deletedNode = ModifiedNodeInfo( + projectId = event.projectId, + repoName = event.repoName, + fullPath = event.resourceKey, + deleted = true + ) + modifiedNodeList.add(createdNode) + modifiedNodeList.add(deletedNode) + } + EventType.NODE_COPIED -> { + require(event is NodeCopiedEvent) + // 1 copy空目录， 2 copy到已存在目录，3 copy到新目录 4 同路径，跳过 5 src为dst目录下的子节点，跳过 + // 针对1 2 两种情况，需要判断原目录中的节点，然后再进行目标目录的统计信息更新 + val dstFullPath = buildDstFullPath(event.dstFullPath, event.resourceKey) + val createdNode = ModifiedNodeInfo( + projectId = event.dstProjectId, + repoName = event.dstRepoName, + fullPath = dstFullPath, + srcProjectId = event.projectId, + srcRepoName = event.repoName, + srcFullPath = event.resourceKey, + ) + modifiedNodeList.add(createdNode) + } else -> throw UnsupportedOperationException() } modifiedNodeList.forEach { @@ -219,11 +230,13 @@ class NodeModifyEventListener( logger.info("start to stat modified node size with fullPath ${node.fullPath}" + " in repo ${node.projectId}|${node.repoName}") if (node.folder) { + val sourceNodes = filterSourceNodesFromMoveOrCopy(modifiedNode) findAndCacheSubFolders( artifactInfo = artifactInfo, deleted = node.nodeInfo.deleted, deletedFlag = modifiedNode.deleted, - includePrefix = modifiedNode.includePrefix + includePrefix = modifiedNode.includePrefix, + sourceNodes = sourceNodes ) } else { updateCache( @@ -237,7 +250,6 @@ class NodeModifyEventListener( } } - /** * 更新缓存 * 当要更新包含该文件所有的目录的缓存记录时includePrefix为空 @@ -254,10 +266,10 @@ class NodeModifyEventListener( // 更新当前节点所有上级目录统计信息 val folderPaths = PathUtils.resolveAncestorFolder(fullPath) - for (it in folderPaths) { - if (it == PathUtils.ROOT) continue + folderPaths.forEach { it -> + if (it == PathUtils.ROOT) return@forEach // 当只需要更新特定目录前缀目录的缓存记录时设置folderPrefix - if (!includePrefix.isNullOrEmpty() && !it.startsWith(includePrefix)) continue + if (!includePrefix.isNullOrEmpty() && !it.startsWith(includePrefix)) return@forEach val key = Triple(projectId, repoName, it) var (cachedSize, nodeNum) = cache.getIfPresent(key) ?: Pair(0L, 0L) if (deleted) { @@ -275,7 +287,8 @@ class NodeModifyEventListener( artifactInfo: ArtifactInfo, deleted: String? = null, deletedFlag: Boolean = false, - includePrefix: String? = null + includePrefix: String? = null, + sourceNodes: List = emptyList() ) { findAllNodesUnderFolder( artifactInfo.projectId, @@ -283,6 +296,7 @@ class NodeModifyEventListener( artifactInfo.getArtifactFullPath(), deleted = deleted ).forEach { + if (sourceNodes.isNotEmpty() && !sourceNodes.contains(it.fullPath)) return@forEach updateCache( projectId = artifactInfo.projectId, repoName = artifactInfo.repoName, @@ -307,6 +321,41 @@ class NodeModifyEventListener( } + /** + * 针对move/copy情况下目标节点是目录的情况下，过滤出变更的节点信息 + * 可能情况：1 源节点为空目录 2 目标节点为已存在的目录，其下可能已经包含文件 + */ + private fun filterSourceNodesFromMoveOrCopy(modifiedNode: ModifiedNodeInfo): List { + if (modifiedNode.srcFullPath.isNullOrEmpty()) return emptyList() + val artifactInfo = ArtifactInfo( + projectId = modifiedNode.srcProjectId!!, + repoName = modifiedNode.srcRepoName!!, + artifactUri = modifiedNode.srcFullPath!! + ) + val sourceNodes = mutableListOf() + val node = if (modifiedNode.deleted) { + nodeService.getDeletedNodeDetail(artifactInfo).firstOrNull() ?: return emptyList() + } else { + // 查询节点信息，当节点新增，然后删除后可能会找不到节点 + nodeService.getNodeDetail(artifactInfo) + ?: nodeService.getDeletedNodeDetail(artifactInfo).firstOrNull() ?: return emptyList() + } + val path = PathUtils.resolveParent(modifiedNode.srcFullPath!!) + if (node.folder) { + findAllNodesUnderFolder( + artifactInfo.projectId, + artifactInfo.repoName, + artifactInfo.getArtifactFullPath() + ).map { + sourceNodes.add(combinePath(modifiedNode.fullPath, it.fullPath.removePrefix(path))) + } + } else { + sourceNodes.add(combinePath(modifiedNode.fullPath, node.fullPath.removePrefix(path))) + } + return sourceNodes + } + + /** * 查询目录下的节点，排除path为"/"的节点 */ @@ -349,7 +398,12 @@ class NodeModifyEventListener( var repoName: String, var fullPath: String, var deleted: Boolean = false, - var includePrefix: String? = null + // 针对重命名去过滤上层目录 + var includePrefix: String? = null, + // 针对move/copy 目标节点是目录的情况下去判断来源节点信息 + var srcProjectId: String? = null, + var srcRepoName: String? = null, + var srcFullPath: String? = null ) companion object { diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeMoveCopySupport.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeMoveCopySupport.kt index 56fc80036f..5b628ed35f 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeMoveCopySupport.kt +++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeMoveCopySupport.kt @@ -73,13 +73,11 @@ open class NodeMoveCopySupport( override fun moveNode(moveRequest: NodeMoveCopyRequest) { moveCopy(moveRequest, true) - publishEvent(NodeEventFactory.buildMovedEvent(moveRequest)) logger.info("Move node success: [$moveRequest]") } override fun copyNode(copyRequest: NodeMoveCopyRequest) { moveCopy(copyRequest, false) - publishEvent(NodeEventFactory.buildCopiedEvent(copyRequest)) logger.info("Copy node success: [$copyRequest]") } @@ -97,6 +95,11 @@ open class NodeMoveCopySupport( } else { moveCopyFile(this) } + if (move) { + publishEvent(NodeEventFactory.buildMovedEvent(request)) + } else { + publishEvent(NodeEventFactory.buildCopiedEvent(request)) + } } } From 5ac8afe42c0567f5d757622cb23697c608542cad Mon Sep 17 00:00:00 2001 From: zacyanliu Date: Mon, 25 Sep 2023 21:01:03 +0800 Subject: [PATCH 8/8] =?UTF-8?q?feat:=20=E8=8A=82=E7=82=B9copy=E5=92=8Cmove?= =?UTF-8?q?=E9=80=BB=E8=BE=91=E8=B0=83=E6=95=B4=20#1195?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../listener/NodeModifyEventListener.kt | 46 ++++++++----------- 1 file changed, 19 insertions(+), 27 deletions(-) diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/listener/NodeModifyEventListener.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/listener/NodeModifyEventListener.kt index e311389a71..8d1627af31 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/listener/NodeModifyEventListener.kt +++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/listener/NodeModifyEventListener.kt @@ -40,7 +40,7 @@ import com.tencent.bkrepo.common.artifact.event.node.NodeDeletedEvent import com.tencent.bkrepo.common.artifact.event.node.NodeMovedEvent import com.tencent.bkrepo.common.artifact.event.node.NodeRenamedEvent import com.tencent.bkrepo.common.artifact.path.PathUtils -import com.tencent.bkrepo.common.artifact.path.PathUtils.combinePath +import com.tencent.bkrepo.common.artifact.path.PathUtils.combineFullPath import com.tencent.bkrepo.repository.dao.NodeDao import com.tencent.bkrepo.repository.model.TNode import com.tencent.bkrepo.repository.service.node.NodeService @@ -173,14 +173,14 @@ class NodeModifyEventListener( require(event is NodeMovedEvent) // 1 move空目录，2 move到已存在目录, 3 move到新目录 4 同路径，跳过 5 src为dst目录下的子节点，跳过 // 针对1 2 两种情况，需要判断原目录中的节点，然后再进行目标目录的统计信息更新 - val dstFullPath = buildDstFullPath(event.dstFullPath, event.resourceKey) val createdNode = ModifiedNodeInfo( projectId = event.dstProjectId, repoName = event.dstRepoName, - fullPath = dstFullPath, + fullPath = event.dstFullPath, srcProjectId = event.projectId, srcRepoName = event.repoName, - srcFullPath = event.resourceKey + srcFullPath = event.resourceKey, + srcDeleted = true ) val deletedNode = ModifiedNodeInfo( projectId = event.projectId, @@ -195,11 +195,10 @@ class NodeModifyEventListener( require(event is NodeCopiedEvent) // 1 copy空目录， 2 copy到已存在目录，3 copy到新目录 4 同路径，跳过 5 src为dst目录下的子节点，跳过 // 针对1 2 两种情况，需要判断原目录中的节点，然后再进行目标目录的统计信息更新 - val dstFullPath = buildDstFullPath(event.dstFullPath, event.resourceKey) val createdNode = ModifiedNodeInfo( projectId = event.dstProjectId, repoName = event.dstRepoName, - fullPath = dstFullPath, + fullPath = event.dstFullPath, srcProjectId = event.projectId, srcRepoName = event.repoName, srcFullPath = event.resourceKey, @@ -231,6 +230,9 @@ class NodeModifyEventListener( " in repo ${node.projectId}|${node.repoName}") if (node.folder) { val sourceNodes = filterSourceNodesFromMoveOrCopy(modifiedNode) + logger.info("the size of node ${modifiedNode.srcFullPath} is ${sourceNodes?.size}" + + " in repo ${modifiedNode.srcProjectId}|${modifiedNode.srcRepoName}") + if (sourceNodes != null && sourceNodes.isEmpty()) return findAndCacheSubFolders( artifactInfo = artifactInfo, deleted = node.nodeInfo.deleted, @@ -288,7 +290,7 @@ class NodeModifyEventListener( deleted: String? = null, deletedFlag: Boolean = false, includePrefix: String? = null, - sourceNodes: List = emptyList() + sourceNodes: List? = null ) { findAllNodesUnderFolder( artifactInfo.projectId, @@ -296,7 +298,7 @@ class NodeModifyEventListener( artifactInfo.getArtifactFullPath(), deleted = deleted ).forEach { - if (sourceNodes.isNotEmpty() && !sourceNodes.contains(it.fullPath)) return@forEach + if (!sourceNodes.isNullOrEmpty() && !sourceNodes.contains(it.fullPath)) return@forEach updateCache( projectId = artifactInfo.projectId, repoName = artifactInfo.repoName, @@ -325,15 +327,15 @@ class NodeModifyEventListener( * 针对move/copy情况下目标节点是目录的情况下，过滤出变更的节点信息 * 可能情况：1 源节点为空目录 2 目标节点为已存在的目录，其下可能已经包含文件 */ - private fun filterSourceNodesFromMoveOrCopy(modifiedNode: ModifiedNodeInfo): List { - if (modifiedNode.srcFullPath.isNullOrEmpty()) return emptyList() + private fun filterSourceNodesFromMoveOrCopy(modifiedNode: ModifiedNodeInfo): List? { + if (modifiedNode.srcFullPath.isNullOrEmpty()) return null val artifactInfo = ArtifactInfo( projectId = modifiedNode.srcProjectId!!, repoName = modifiedNode.srcRepoName!!, artifactUri = modifiedNode.srcFullPath!! ) val sourceNodes = mutableListOf() - val node = if (modifiedNode.deleted) { + val node = if (modifiedNode.srcDeleted) { nodeService.getDeletedNodeDetail(artifactInfo).firstOrNull() ?: return emptyList() } else { // 查询节点信息，当节点新增，然后删除后可能会找不到节点 @@ -345,12 +347,13 @@ class NodeModifyEventListener( findAllNodesUnderFolder( artifactInfo.projectId, artifactInfo.repoName, - artifactInfo.getArtifactFullPath() + artifactInfo.getArtifactFullPath(), + node.nodeInfo.deleted ).map { - sourceNodes.add(combinePath(modifiedNode.fullPath, it.fullPath.removePrefix(path))) + sourceNodes.add(combineFullPath(modifiedNode.fullPath, it.fullPath.removePrefix(path))) } } else { - sourceNodes.add(combinePath(modifiedNode.fullPath, node.fullPath.removePrefix(path))) + sourceNodes.add(combineFullPath(modifiedNode.fullPath, node.fullPath.removePrefix(path))) } return sourceNodes } @@ -381,18 +384,6 @@ class NodeModifyEventListener( return Query(criteria).withHint(TNode.FULL_PATH_IDX) } - - private fun buildDstFullPath(dstFullPath: String, srcFullPath: String): String { - val path = PathUtils.toPath(dstFullPath) - val name = PathUtils.resolveName(srcFullPath) - return PathUtils.combineFullPath(path, name) - } - - private fun String.getFolderPath(): String { - val path = PathUtils.resolveParent(this) - return PathUtils.normalizeFullPath(path) - } - private data class ModifiedNodeInfo( var projectId: String, var repoName: String, @@ -403,7 +394,8 @@ class NodeModifyEventListener( // 针对move/copy 目标节点是目录的情况下去判断来源节点信息 var srcProjectId: String? = null, var srcRepoName: String? = null, - var srcFullPath: String? = null + var srcFullPath: String? = null, + var srcDeleted: Boolean = false ) companion object {