From 46dedce0af7fbe823a3b7361c6d1d09b886d94f5 Mon Sep 17 00:00:00 2001 From: irwinsun Date: Mon, 21 Jun 2021 12:21:50 +0800 Subject: [PATCH 01/20] =?UTF-8?q?bug:=20assembly=20=E5=90=AF=E5=8A=A8?= =?UTF-8?q?=E5=A4=B1=E8=B4=A5[1.5.4]=20#4475=20=E6=B6=88=E9=99=A4=E9=87=8D?= =?UTF-8?q?=E5=A4=8D=E5=91=BD=E5=90=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../PipelineBuildHistoryDataClearJob.kt | 26 +++++++++---------- .../{ProjectDao.kt => ProjectMiscDao.kt} | 2 +- ...rocessService.kt => ProcessMiscService.kt} | 2 +- ...rojectService.kt => ProjectMiscService.kt} | 12 ++++----- 4 files changed, 21 insertions(+), 21 deletions(-) rename src/backend/ci/core/misc/biz-misc/src/main/kotlin/com/tencent/devops/misc/dao/project/{ProjectDao.kt => ProjectMiscDao.kt} (99%) rename src/backend/ci/core/misc/biz-misc/src/main/kotlin/com/tencent/devops/misc/service/process/{ProcessService.kt => ProcessMiscService.kt} (98%) rename src/backend/ci/core/misc/biz-misc/src/main/kotlin/com/tencent/devops/misc/service/project/{ProjectService.kt => ProjectMiscService.kt} (88%) diff --git a/src/backend/ci/core/misc/biz-misc/src/main/kotlin/com/tencent/devops/misc/cron/process/PipelineBuildHistoryDataClearJob.kt b/src/backend/ci/core/misc/biz-misc/src/main/kotlin/com/tencent/devops/misc/cron/process/PipelineBuildHistoryDataClearJob.kt index 92aca98e549..e396a8bd671 100644 --- a/src/backend/ci/core/misc/biz-misc/src/main/kotlin/com/tencent/devops/misc/cron/process/PipelineBuildHistoryDataClearJob.kt +++ b/src/backend/ci/core/misc/biz-misc/src/main/kotlin/com/tencent/devops/misc/cron/process/PipelineBuildHistoryDataClearJob.kt @@ -35,9 +35,9 @@ import com.tencent.devops.misc.service.artifactory.ArtifactoryDataClearService import com.tencent.devops.misc.service.dispatch.DispatchDataClearService import com.tencent.devops.misc.service.plugin.PluginDataClearService import com.tencent.devops.misc.service.process.ProcessDataClearService -import com.tencent.devops.misc.service.process.ProcessService +import com.tencent.devops.misc.service.process.ProcessMiscService import com.tencent.devops.misc.service.project.ProjectDataClearConfigFactory -import com.tencent.devops.misc.service.project.ProjectService +import com.tencent.devops.misc.service.project.ProjectMiscService import com.tencent.devops.misc.service.quality.QualityDataClearService import com.tencent.devops.misc.service.repository.RepositoryDataClearService import org.slf4j.LoggerFactory @@ -52,8 +52,8 @@ import java.time.LocalDateTime class PipelineBuildHistoryDataClearJob @Autowired constructor( private val redisOperation: RedisOperation, private val miscBuildDataClearConfig: MiscBuildDataClearConfig, - private val projectService: ProjectService, - private val processService: ProcessService, + private val projectMiscService: ProjectMiscService, + private val processMiscService: ProcessMiscService, private val processDataClearService: ProcessDataClearService, private val repositoryDataClearService: RepositoryDataClearService, private val dispatchDataClearService: DispatchDataClearService, @@ -97,9 +97,9 @@ class PipelineBuildHistoryDataClearJob @Autowired constructor( } var handleProjectPrimaryId = redisOperation.get(PIPELINE_BUILD_HISTORY_DATA_CLEAR_PROJECT_ID_KEY)?.toLong() if (handleProjectPrimaryId == null) { - handleProjectPrimaryId = projectService.getMinId(projectIdList) ?: 0L + handleProjectPrimaryId = projectMiscService.getMinId(projectIdList) ?: 0L } else { - val maxProjectPrimaryId = projectService.getMaxId(projectIdList) ?: 0L + val maxProjectPrimaryId = projectMiscService.getMaxId(projectIdList) ?: 0L if (handleProjectPrimaryId >= maxProjectPrimaryId) { // 已经清理完全部项目的流水线的过期构建记录，再重新开始清理 redisOperation.delete(PIPELINE_BUILD_HISTORY_DATA_CLEAR_PROJECT_ID_KEY) @@ -111,9 +111,9 @@ class PipelineBuildHistoryDataClearJob @Autowired constructor( var maxHandleProjectPrimaryId = handleProjectPrimaryId ?: 0L val projectInfoList = if (projectIdListConfig.isNullOrBlank()) { maxHandleProjectPrimaryId = handleProjectPrimaryId + maxEveryProjectHandleNum - projectService.getProjectInfoList(minId = handleProjectPrimaryId, maxId = maxHandleProjectPrimaryId) + projectMiscService.getProjectInfoList(minId = handleProjectPrimaryId, maxId = maxHandleProjectPrimaryId) } else { - projectService.getProjectInfoList(projectIdList = projectIdList) + projectMiscService.getProjectInfoList(projectIdList = projectIdList) } // 根据项目依次查询T_PIPELINE_INFO表中的流水线数据处理 projectInfoList?.forEach { projectInfo -> @@ -126,11 +126,11 @@ class PipelineBuildHistoryDataClearJob @Autowired constructor( maxHandleProjectPrimaryId = projectPrimaryId } val projectId = projectInfo.projectId - val pipelineIdList = processService.getPipelineIdListByProjectId(projectId) + val pipelineIdList = processMiscService.getPipelineIdListByProjectId(projectId) val deletePipelineIdList = if (pipelineIdList.isNullOrEmpty()) { null } else { - processService.getClearDeletePipelineIdList( + processMiscService.getClearDeletePipelineIdList( projectId = projectId, pipelineIdList = pipelineIdList, gapDays = deletedPipelineStoreDays @@ -175,7 +175,7 @@ class PipelineBuildHistoryDataClearJob @Autowired constructor( maxStartTime = projectDataClearConfig.maxStartTime ) // 判断构建记录是否超过系统展示的最大数量，如果超过则需清理超量的数据 - val maxPipelineBuildNum = processService.getMaxPipelineBuildNum(projectId, pipelineId) + val maxPipelineBuildNum = processMiscService.getMaxPipelineBuildNum(projectId, pipelineId) val maxKeepNum = projectDataClearConfig.maxKeepNum val maxBuildNum = maxPipelineBuildNum - maxKeepNum if (maxBuildNum > 0) { @@ -207,11 +207,11 @@ class PipelineBuildHistoryDataClearJob @Autowired constructor( maxBuildNum: Int? = null, maxStartTime: LocalDateTime? = null ) { - val totalBuildCount = processService.getTotalBuildCount(pipelineId, maxBuildNum, maxStartTime) + val totalBuildCount = processMiscService.getTotalBuildCount(pipelineId, maxBuildNum, maxStartTime) logger.info("pipelineBuildHistoryDataClear|$projectId|$pipelineId|totalBuildCount=$totalBuildCount") var totalHandleNum = 0 while (totalHandleNum < totalBuildCount) { - val pipelineHistoryBuildIdList = processService.getHistoryBuildIdList( + val pipelineHistoryBuildIdList = processMiscService.getHistoryBuildIdList( pipelineId = pipelineId, totalHandleNum = totalHandleNum, handlePageSize = PIPELINE_BUILD_HISTORY_PAGE_SIZE, diff --git a/src/backend/ci/core/misc/biz-misc/src/main/kotlin/com/tencent/devops/misc/dao/project/ProjectDao.kt b/src/backend/ci/core/misc/biz-misc/src/main/kotlin/com/tencent/devops/misc/dao/project/ProjectMiscDao.kt similarity index 99% rename from src/backend/ci/core/misc/biz-misc/src/main/kotlin/com/tencent/devops/misc/dao/project/ProjectDao.kt rename to src/backend/ci/core/misc/biz-misc/src/main/kotlin/com/tencent/devops/misc/dao/project/ProjectMiscDao.kt index d1c352aca3e..3744b5ee69c 100644 --- a/src/backend/ci/core/misc/biz-misc/src/main/kotlin/com/tencent/devops/misc/dao/project/ProjectDao.kt +++ b/src/backend/ci/core/misc/biz-misc/src/main/kotlin/com/tencent/devops/misc/dao/project/ProjectMiscDao.kt @@ -36,7 +36,7 @@ import org.jooq.impl.DSL import org.springframework.stereotype.Repository @Repository -class ProjectDao { +class ProjectMiscDao { fun getMinId( dslContext: DSLContext, diff --git a/src/backend/ci/core/misc/biz-misc/src/main/kotlin/com/tencent/devops/misc/service/process/ProcessService.kt b/src/backend/ci/core/misc/biz-misc/src/main/kotlin/com/tencent/devops/misc/service/process/ProcessMiscService.kt similarity index 98% rename from src/backend/ci/core/misc/biz-misc/src/main/kotlin/com/tencent/devops/misc/service/process/ProcessService.kt rename to src/backend/ci/core/misc/biz-misc/src/main/kotlin/com/tencent/devops/misc/service/process/ProcessMiscService.kt index 275a0d32ff2..67c741c3e8e 100644 --- a/src/backend/ci/core/misc/biz-misc/src/main/kotlin/com/tencent/devops/misc/service/process/ProcessService.kt +++ b/src/backend/ci/core/misc/biz-misc/src/main/kotlin/com/tencent/devops/misc/service/process/ProcessMiscService.kt @@ -36,7 +36,7 @@ import org.springframework.stereotype.Service import java.time.LocalDateTime @Service -class ProcessService @Autowired constructor( +class ProcessMiscService @Autowired constructor( private val dslContext: DSLContext, private val processDao: ProcessDao ) { diff --git a/src/backend/ci/core/misc/biz-misc/src/main/kotlin/com/tencent/devops/misc/service/project/ProjectService.kt b/src/backend/ci/core/misc/biz-misc/src/main/kotlin/com/tencent/devops/misc/service/project/ProjectMiscService.kt similarity index 88% rename from src/backend/ci/core/misc/biz-misc/src/main/kotlin/com/tencent/devops/misc/service/project/ProjectService.kt rename to src/backend/ci/core/misc/biz-misc/src/main/kotlin/com/tencent/devops/misc/service/project/ProjectMiscService.kt index 050848bff98..4db6e3cc51e 100644 --- a/src/backend/ci/core/misc/biz-misc/src/main/kotlin/com/tencent/devops/misc/service/project/ProjectService.kt +++ b/src/backend/ci/core/misc/biz-misc/src/main/kotlin/com/tencent/devops/misc/service/project/ProjectMiscService.kt @@ -27,28 +27,28 @@ package com.tencent.devops.misc.service.project -import com.tencent.devops.misc.dao.project.ProjectDao +import com.tencent.devops.misc.dao.project.ProjectMiscDao import com.tencent.devops.misc.pojo.project.ProjectInfo import org.jooq.DSLContext import org.springframework.beans.factory.annotation.Autowired import org.springframework.stereotype.Service @Service -class ProjectService @Autowired constructor( +class ProjectMiscService @Autowired constructor( private val dslContext: DSLContext, - private val projectDao: ProjectDao + private val projectMiscDao: ProjectMiscDao ) { fun getMinId( projectIdList: List? = null ): Long? { - return projectDao.getMinId(dslContext, projectIdList) + return projectMiscDao.getMinId(dslContext, projectIdList) } fun getMaxId( projectIdList: List? = null ): Long? { - return projectDao.getMaxId(dslContext, projectIdList) + return projectMiscDao.getMaxId(dslContext, projectIdList) } fun getProjectInfoList( @@ -56,7 +56,7 @@ class ProjectService @Autowired constructor( minId: Long? = null, maxId: Long? = null ): List? { - val projectInfoRecords = projectDao.getProjectInfoList( + val projectInfoRecords = projectMiscDao.getProjectInfoList( dslContext = dslContext, projectIdList = projectIdList, minId = minId, From 5a5a0fd08aaf30818ccb154f288f2d220cce69ae Mon Sep 17 00:00:00 2001 From: irwinsun Date: Mon, 21 Jun 2021 14:26:37 +0800 Subject: [PATCH 02/20] =?UTF-8?q?bug:=20assembly=20=E5=90=AF=E5=8A=A8?= =?UTF-8?q?=E5=A4=B1=E8=B4=A5[1.5.4]=20#4475=20=E6=B6=88=E9=99=A4=E9=87=8D?= =?UTF-8?q?=E5=A4=8D=E5=91=BD=E5=90=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../environment/agent/AgentGrayUtils.kt | 18 ++- .../dispatch/dao/PipelineDockerTemplateDao.kt | 94 ------------ .../BuildLessAgentShutdownListener.kt | 64 -------- .../listener/BuildLessAgentStartupListener.kt | 65 -------- .../PipelineBuildLessDispatchService.kt | 140 ------------------ .../dispatch/service/StoreImageService.kt | 86 ----------- .../service/ThirdPartyAgentService.kt | 10 +- .../service/dispatcher/BuildLessDispatcher.kt | 40 ----- .../agent/ThirdPartyAgentDispatcher.kt | 8 +- ...s.kt => ThirdPartyAgentBuildRedisUtils.kt} | 78 +++------- ...kt => OpThirdPartyAgentUpgradeResource.kt} | 2 +- ...> OpThirdPartyAgentUpgradeResourceImpl.kt} | 6 +- .../{ScmService.kt => ScmCheckService.kt} | 4 +- .../plugin/service/git/CodeWebhookService.kt | 14 +- .../{BuildBean.kt => PipelineBuildBean.kt} | 4 +- 15 files changed, 55 insertions(+), 578 deletions(-) delete mode 100644 src/backend/ci/core/dispatch/biz-dispatch/src/main/kotlin/com/tencent/devops/dispatch/dao/PipelineDockerTemplateDao.kt delete mode 100644 src/backend/ci/core/dispatch/biz-dispatch/src/main/kotlin/com/tencent/devops/dispatch/listener/BuildLessAgentShutdownListener.kt delete mode 100644 src/backend/ci/core/dispatch/biz-dispatch/src/main/kotlin/com/tencent/devops/dispatch/listener/BuildLessAgentStartupListener.kt delete mode 100644 src/backend/ci/core/dispatch/biz-dispatch/src/main/kotlin/com/tencent/devops/dispatch/service/PipelineBuildLessDispatchService.kt delete mode 100644 src/backend/ci/core/dispatch/biz-dispatch/src/main/kotlin/com/tencent/devops/dispatch/service/StoreImageService.kt delete mode 100644 src/backend/ci/core/dispatch/biz-dispatch/src/main/kotlin/com/tencent/devops/dispatch/service/dispatcher/BuildLessDispatcher.kt rename src/backend/ci/core/dispatch/biz-dispatch/src/main/kotlin/com/tencent/devops/dispatch/utils/redis/{RedisUtils.kt => ThirdPartyAgentBuildRedisUtils.kt} (52%) rename src/backend/ci/core/misc/api-misc/src/main/kotlin/com/tencent/devops/misc/api/{OpThirdPartyAgentResource.kt => OpThirdPartyAgentUpgradeResource.kt} (98%) rename src/backend/ci/core/misc/biz-misc/src/main/kotlin/com/tencent/devops/misc/resources/{OpThirdPartyAgentResourceImpl.kt => OpThirdPartyAgentUpgradeResourceImpl.kt} (92%) rename src/backend/ci/core/plugin/biz-plugin/src/main/kotlin/com/tencent/devops/plugin/service/{ScmService.kt => ScmCheckService.kt} (98%) rename src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/jmx/pipeline/{BuildBean.kt => PipelineBuildBean.kt} (96%) diff --git a/src/backend/ci/core/common/common-environment-thirdpartyagent/src/main/kotlin/com/tencent/devops/common/environment/agent/AgentGrayUtils.kt b/src/backend/ci/core/common/common-environment-thirdpartyagent/src/main/kotlin/com/tencent/devops/common/environment/agent/AgentGrayUtils.kt index 74c5dd7aa0a..25575dbc8c9 100644 --- a/src/backend/ci/core/common/common-environment-thirdpartyagent/src/main/kotlin/com/tencent/devops/common/environment/agent/AgentGrayUtils.kt +++ b/src/backend/ci/core/common/common-environment-thirdpartyagent/src/main/kotlin/com/tencent/devops/common/environment/agent/AgentGrayUtils.kt @@ -54,6 +54,8 @@ class AgentGrayUtils constructor( private const val DEFAULT_FILE_GATEWAY_KEY = "environment:thirdparty:default_file_gateway" private const val USE_DEFAULT_GATEWAY_KEY = "environment:thirdparty:use_default_gateway" private const val USE_DEFAULT_FILE_GATEWAY_KEY = "environment:thirdparty:use_default_file_gateway" + private const val PARALLEL_UPGRADE_COUNT = "environment.thirdparty.agent.parallel.upgrade.count" + private const val DEFAULT_PARALLEL_UPGRADE_COUNT = 50 } fun checkForceUpgrade(agentHashId: String): Boolean { @@ -75,7 +77,7 @@ class AgentGrayUtils constructor( fun getAllForceUpgradeAgents(): List { return (redisOperation.getSetMembers(FORCE_UPGRADE_AGENT_SET_KEY) - ?: setOf()).filter { !it.isBlank() }.map { it.toLong() } + ?: setOf()).filter { it.isNotBlank() }.map { it.toLong() } } fun cleanAllForceUpgradeAgents() { @@ -88,7 +90,7 @@ class AgentGrayUtils constructor( fun checkLockUpgrade(agentHashId: String): Boolean { val agentId = HashUtil.decodeIdToLong(agentHashId) return (redisOperation.getSetMembers(LOCK_UPGRADE_AGENT_SET_KEY) - ?: setOf()).filter { !it.isBlank() }.contains(agentId.toString()) + ?: setOf()).filter { it.isNotBlank() }.contains(agentId.toString()) } fun setLockUpgradeAgents(agentIds: List) { @@ -105,7 +107,7 @@ class AgentGrayUtils constructor( fun getAllLockUpgradeAgents(): List { return (redisOperation.getSetMembers(LOCK_UPGRADE_AGENT_SET_KEY) - ?: setOf()).filter { !it.isBlank() }.map { it.toLong() } + ?: setOf()).filter { it.isNotBlank() }.map { it.toLong() } } fun cleanAllLockUpgradeAgents() { @@ -138,7 +140,7 @@ class AgentGrayUtils constructor( fun getCanUpgradeAgents(): List { return (redisOperation.getSetMembers(getCanUpgradeAgentSetKey()) - ?: setOf()).filter { !it.isBlank() }.map { it.toLong() } + ?: setOf()).filter { it.isNotBlank() }.map { it.toLong() } } fun getAgentMasterVersionKey(): String { @@ -180,4 +182,12 @@ class AgentGrayUtils constructor( fun useDefaultFileGateway(): Boolean { return redisOperation.get(USE_DEFAULT_FILE_GATEWAY_KEY) == "true" } + + fun setMaxParallelUpgradeCount(count: Int) { + redisOperation.set(PARALLEL_UPGRADE_COUNT, count.toString()) + } + + fun getMaxParallelUpgradeCount(): Int { + return redisOperation.get(PARALLEL_UPGRADE_COUNT)?.toInt() ?: DEFAULT_PARALLEL_UPGRADE_COUNT + } } diff --git a/src/backend/ci/core/dispatch/biz-dispatch/src/main/kotlin/com/tencent/devops/dispatch/dao/PipelineDockerTemplateDao.kt b/src/backend/ci/core/dispatch/biz-dispatch/src/main/kotlin/com/tencent/devops/dispatch/dao/PipelineDockerTemplateDao.kt deleted file mode 100644 index a602b3d5d93..00000000000 --- a/src/backend/ci/core/dispatch/biz-dispatch/src/main/kotlin/com/tencent/devops/dispatch/dao/PipelineDockerTemplateDao.kt +++ /dev/null @@ -1,94 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. - * - * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. - * - * A copy of the MIT License is included in this file. - * - * - * Terms of the MIT License: - * --------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to - * permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT - * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN - * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, - * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE - * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ - -package com.tencent.devops.dispatch.dao - -import com.tencent.devops.model.dispatch.tables.TDispatchPipelineDockerTemplate -import com.tencent.devops.model.dispatch.tables.records.TDispatchPipelineDockerTemplateRecord -import org.jooq.DSLContext -import org.springframework.stereotype.Repository -import java.time.LocalDateTime - -@Repository@Suppress("ALL") -class PipelineDockerTemplateDao { - - fun updateTemplate( - dslContext: DSLContext, - versionId: Int, - showVersionId: Int, - showVersionName: String, - deploymentId: Int, - deploymentName: String, - ccAppId: Long, - bcsProjectId: String, - clusterId: String - ) { - with(TDispatchPipelineDockerTemplate.T_DISPATCH_PIPELINE_DOCKER_TEMPLATE) { - dslContext.insertInto(this, - VERSION_ID, - SHOW_VERSION_ID, - SHOW_VERSION_NAME, - DEPLOYMENT_ID, - DEPLOYMENT_NAME, - CC_APP_ID, - BCS_PROJECT_ID, - CLUSTER_ID, - CREATED_TIME) - .values( - versionId, - showVersionId, - showVersionName, - deploymentId, - deploymentName, - ccAppId, - bcsProjectId, - clusterId, - LocalDateTime.now() - ) - .execute() - } - } - - fun getTemplate(dslContext: DSLContext): TDispatchPipelineDockerTemplateRecord? { - with(TDispatchPipelineDockerTemplate.T_DISPATCH_PIPELINE_DOCKER_TEMPLATE) { - return dslContext.selectFrom(this) - .orderBy(ID.desc()) - .limit(1) - .fetchOne() - } - } - - fun getTemplateById(dslContext: DSLContext, id: Int): TDispatchPipelineDockerTemplateRecord? { - with(TDispatchPipelineDockerTemplate.T_DISPATCH_PIPELINE_DOCKER_TEMPLATE) { - return dslContext.selectFrom(this) - .where(ID.eq(id)) - .orderBy(ID.desc()) - .limit(1) - .fetchOne() - } - } -} diff --git a/src/backend/ci/core/dispatch/biz-dispatch/src/main/kotlin/com/tencent/devops/dispatch/listener/BuildLessAgentShutdownListener.kt b/src/backend/ci/core/dispatch/biz-dispatch/src/main/kotlin/com/tencent/devops/dispatch/listener/BuildLessAgentShutdownListener.kt deleted file mode 100644 index 45a50badb29..00000000000 --- a/src/backend/ci/core/dispatch/biz-dispatch/src/main/kotlin/com/tencent/devops/dispatch/listener/BuildLessAgentShutdownListener.kt +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. - * - * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. - * - * A copy of the MIT License is included in this file. - * - * - * Terms of the MIT License: - * --------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to - * permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT - * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN - * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, - * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE - * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ - -package com.tencent.devops.dispatch.listener - -import com.tencent.devops.dispatch.service.PipelineBuildLessDispatchService -import com.tencent.devops.process.pojo.mq.PipelineBuildLessShutdownDispatchEvent -import org.slf4j.LoggerFactory -import org.springframework.beans.factory.annotation.Autowired -import org.springframework.stereotype.Service - -@Service -class BuildLessAgentShutdownListener @Autowired -constructor(private val pipelineDispatchService: PipelineBuildLessDispatchService) { - -/* @RabbitListener( - bindings = [(QueueBinding( - key = [MQ.ROUTE_BUILD_LESS_AGENT_SHUTDOWN_DISPATCH], value = Queue( - value = MQ.QUEUE_BUILD_LESS_AGENT_SHUTDOWN_DISPATCH, durable = "true" - ), - exchange = Exchange( - value = MQ.EXCHANGE_BUILD_LESS_AGENT_LISTENER_DIRECT, - durable = "true", - delayed = "true", - type = ExchangeTypes.DIRECT - ) - ))] - )*/ - fun listenAgentStartUpEvent(pipelineBuildLessDockerAgentShutdownEvent: PipelineBuildLessShutdownDispatchEvent) { - try { - pipelineDispatchService.shutdown(pipelineBuildLessDockerAgentShutdownEvent) - } catch (ignored: Throwable) { - logger.error("Fail to start the pipe build($pipelineBuildLessDockerAgentShutdownEvent)", ignored) - } - } - - companion object { - private val logger = LoggerFactory.getLogger(BuildLessAgentShutdownListener::class.java) - } -} diff --git a/src/backend/ci/core/dispatch/biz-dispatch/src/main/kotlin/com/tencent/devops/dispatch/listener/BuildLessAgentStartupListener.kt b/src/backend/ci/core/dispatch/biz-dispatch/src/main/kotlin/com/tencent/devops/dispatch/listener/BuildLessAgentStartupListener.kt deleted file mode 100644 index 20623f32fb0..00000000000 --- a/src/backend/ci/core/dispatch/biz-dispatch/src/main/kotlin/com/tencent/devops/dispatch/listener/BuildLessAgentStartupListener.kt +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. - * - * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. - * - * A copy of the MIT License is included in this file. - * - * - * Terms of the MIT License: - * --------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to - * permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT - * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN - * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, - * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE - * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ - -package com.tencent.devops.dispatch.listener - -import com.tencent.devops.dispatch.service.PipelineBuildLessDispatchService -import com.tencent.devops.process.pojo.mq.PipelineBuildLessStartupDispatchEvent -import org.slf4j.LoggerFactory -import org.springframework.beans.factory.annotation.Autowired -import org.springframework.stereotype.Service - -@Service -class BuildLessAgentStartupListener @Autowired -constructor(private val pipelineDispatchService: PipelineBuildLessDispatchService) { - -/* @RabbitListener( - bindings = [(QueueBinding( - key = [MQ.ROUTE_BUILD_LESS_AGENT_STARTUP_DISPATCH], value = Queue( - value = MQ.QUEUE_BUILD_LESS_AGENT_STARTUP_DISPATCH, durable = "true" - ), - exchange = Exchange( - value = MQ.EXCHANGE_BUILD_LESS_AGENT_LISTENER_DIRECT, - durable = "true", - delayed = "true", - type = ExchangeTypes.DIRECT - ) - ))] - )*/ - fun listenAgentStartUpEvent(event: PipelineBuildLessStartupDispatchEvent) { - try { - logger.info("start build less($event)") - pipelineDispatchService.startUpBuildLess(event) - } catch (ignored: Throwable) { - logger.error("Fail to start the pipe build($event)", ignored) - } - } - - companion object { - private val logger = LoggerFactory.getLogger(BuildLessAgentStartupListener::class.java) - } -} diff --git a/src/backend/ci/core/dispatch/biz-dispatch/src/main/kotlin/com/tencent/devops/dispatch/service/PipelineBuildLessDispatchService.kt b/src/backend/ci/core/dispatch/biz-dispatch/src/main/kotlin/com/tencent/devops/dispatch/service/PipelineBuildLessDispatchService.kt deleted file mode 100644 index d9bb1236e7f..00000000000 --- a/src/backend/ci/core/dispatch/biz-dispatch/src/main/kotlin/com/tencent/devops/dispatch/service/PipelineBuildLessDispatchService.kt +++ /dev/null @@ -1,140 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. - * - * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. - * - * A copy of the MIT License is included in this file. - * - * - * Terms of the MIT License: - * --------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to - * permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT - * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN - * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, - * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE - * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ - -package com.tencent.devops.dispatch.service - -import com.tencent.devops.common.api.exception.InvalidParamException -import com.tencent.devops.common.client.Client -import com.tencent.devops.common.pipeline.enums.ChannelCode -import com.tencent.devops.common.service.utils.SpringContextUtil -import com.tencent.devops.dispatch.service.dispatcher.BuildLessDispatcher -import com.tencent.devops.common.log.utils.BuildLogPrinter -import com.tencent.devops.process.api.service.ServicePipelineResource -import com.tencent.devops.process.pojo.mq.PipelineBuildLessShutdownDispatchEvent -import com.tencent.devops.process.pojo.mq.PipelineBuildLessStartupDispatchEvent -import org.reflections.Reflections -import org.slf4j.LoggerFactory -import org.springframework.beans.factory.annotation.Autowired -import org.springframework.stereotype.Service - -@Service@Suppress("ALL") -class PipelineBuildLessDispatchService @Autowired constructor( - private val client: Client, - private val buildLogPrinter: BuildLogPrinter, - private val jobQuotaBusinessService: JobQuotaBusinessService -) { - - private var dispatchers: Set? = null - - private fun getDispatchers(): Set { - if (dispatchers == null) { - synchronized(this) { - if (dispatchers == null) { - val reflections = Reflections("com.tencent.devops.dispatch.service.dispatcher") - val dispatcherClasses = reflections.getSubTypesOf(BuildLessDispatcher::class.java) - if (dispatcherClasses == null || dispatcherClasses.isEmpty()) { - logger.error("The dispatcher is empty $dispatcherClasses") - throw InvalidParamException("Dispatcher is empty") - } - logger.info("Get the dispatch classes $dispatcherClasses") - dispatchers = dispatcherClasses.map { - SpringContextUtil.getBean(it) - }.toSet() - } - } - } - return dispatchers!! - } - - fun startUpBuildLess(pipelineBuildLessAgentStartupEvent: PipelineBuildLessStartupDispatchEvent) { - val pipelineId = pipelineBuildLessAgentStartupEvent.pipelineId - val buildId = pipelineBuildLessAgentStartupEvent.buildId - val vmSeqId = pipelineBuildLessAgentStartupEvent.vmSeqId - logger.info("[$buildId]|BUILD_LESS| pipelineId=$pipelineId, seq($vmSeqId)") - // Check if the pipeline is running - val record = client.get(ServicePipelineResource::class).isPipelineRunning( - pipelineBuildLessAgentStartupEvent.projectId, - buildId, - ChannelCode.valueOf(pipelineBuildLessAgentStartupEvent.channelCode) - ) - if (record.isNotOk() || record.data == null) { - logger.warn("[$buildId]|BUILD_LESS| Fail to check if pipeline is running because of ${record.message}") - return - } - - if (!record.data!!) { - logger.warn("[$buildId]|BUILD_LESS| The build is not running") - return - } - - if (pipelineBuildLessAgentStartupEvent.retryTime == 0) { - buildLogPrinter.addLine( - buildId, - "Prepare BuildLess Job(#$vmSeqId)...", - "", - pipelineBuildLessAgentStartupEvent.containerHashId, - pipelineBuildLessAgentStartupEvent.executeCount ?: 1 - ) - } - - val dispatchType = pipelineBuildLessAgentStartupEvent.dispatchType - logger.info("[$buildId]|BUILD_LESS| Get the dispatch $dispatchType") - - getDispatchers().forEach { - if (it.canDispatch(pipelineBuildLessAgentStartupEvent)) { -// if (!jobQuotaBusinessService.checkJobQuota(pipelineBuildLessAgentStartupEvent, buildLogPrinter)) { -// logger.error("[$buildId]|BUILD_LESS| AgentLess Job quota exceed quota.") -// return -// } - it.startUp(pipelineBuildLessAgentStartupEvent) -// // 到这里说明JOB已经启动成功，开始累加使用额度 -// jobQuotaBusinessService.insertRunningJob(projectId = pipelineBuildLessAgentStartupEvent.projectId, -// vmType = JobQuotaVmType.AGENTLESS, -// buildId = pipelineBuildLessAgentStartupEvent.buildId, -// vmSeqId = pipelineBuildLessAgentStartupEvent.vmSeqId) - return - } - } - throw InvalidParamException("Fail to find the right buildLessDispatcher for the build $dispatchType") - } - - fun shutdown(event: PipelineBuildLessShutdownDispatchEvent) { -// try { - logger.info("[${event.buildId}]| Start to finish the pipeline build($event)") - getDispatchers().forEach { - it.shutdown(event) - } -// } finally { - // 不管shutdown成功失败，都要回收配额；这里回收job，将自动累加agent执行时间 -// jobQuotaBusinessService.deleteRunningJob(event.projectId, event.buildId, event.vmSeqId) -// } - } - - companion object { - private val logger = LoggerFactory.getLogger(PipelineBuildLessDispatchService::class.java) - } -} diff --git a/src/backend/ci/core/dispatch/biz-dispatch/src/main/kotlin/com/tencent/devops/dispatch/service/StoreImageService.kt b/src/backend/ci/core/dispatch/biz-dispatch/src/main/kotlin/com/tencent/devops/dispatch/service/StoreImageService.kt deleted file mode 100644 index 1399036122a..00000000000 --- a/src/backend/ci/core/dispatch/biz-dispatch/src/main/kotlin/com/tencent/devops/dispatch/service/StoreImageService.kt +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. - * - * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. - * - * A copy of the MIT License is included in this file. - * - * - * Terms of the MIT License: - * --------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to - * permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT - * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN - * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, - * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE - * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ - -package com.tencent.devops.dispatch.service - -import com.tencent.devops.common.api.exception.InvalidParamException -import com.tencent.devops.common.client.Client -import com.tencent.devops.dispatch.exception.ImageNotInstalledException -import com.tencent.devops.store.api.image.service.ServiceStoreImageResource -import com.tencent.devops.store.constant.StoreMessageCode.USER_IMAGE_NOT_INSTALLED -import com.tencent.devops.store.pojo.image.response.ImageRepoInfo -import org.slf4j.LoggerFactory -import org.springframework.beans.factory.annotation.Autowired -import org.springframework.stereotype.Service - -/** - * @Description - * @Date 2019/10/17 - * @Version 1.0 - */ - -@Service@Suppress("ALL") -class StoreImageService @Autowired constructor( - private val client: Client -) { - - companion object { - private val logger = LoggerFactory.getLogger(StoreImageService::class.java) - } - - // 从镜像商店获取完整的镜像名称 - fun getImageRepoInfo( - userId: String, - projectId: String, - pipelineId: String?, - buildId: String?, - imageCode: String?, - imageVersion: String?, - defaultPrefix: String? - ): ImageRepoInfo { - logger.info("getImageRepoInfo:Input($userId,$projectId,$imageCode,$imageVersion)") - // 鉴权：安装了才有权限使用 - if (null == imageCode) { - throw InvalidParamException("Input:($userId,$projectId,$imageCode),imageCode is null") - } - val permissionResult = client.get(ServiceStoreImageResource::class).isInstalled(userId, projectId, imageCode) - if (permissionResult.isNotOk() || (!permissionResult.data!!)) { - throw ImageNotInstalledException("Input:($userId,$projectId,$imageCode)", USER_IMAGE_NOT_INSTALLED) - } - // 调商店接口获取镜像信息 - val result = client.get(ServiceStoreImageResource::class) - .getImageRepoInfoByCodeAndVersion( - userId = userId, - projectCode = projectId, - imageCode = imageCode, - imageVersion = imageVersion, - pipelineId = pipelineId, - buildId = buildId - ) - return result.data!! - } -} diff --git a/src/backend/ci/core/dispatch/biz-dispatch/src/main/kotlin/com/tencent/devops/dispatch/service/ThirdPartyAgentService.kt b/src/backend/ci/core/dispatch/biz-dispatch/src/main/kotlin/com/tencent/devops/dispatch/service/ThirdPartyAgentService.kt index 7c5b4cde036..302be291dc0 100644 --- a/src/backend/ci/core/dispatch/biz-dispatch/src/main/kotlin/com/tencent/devops/dispatch/service/ThirdPartyAgentService.kt +++ b/src/backend/ci/core/dispatch/biz-dispatch/src/main/kotlin/com/tencent/devops/dispatch/service/ThirdPartyAgentService.kt @@ -44,7 +44,7 @@ import com.tencent.devops.dispatch.pojo.thirdPartyAgent.AgentBuildInfo import com.tencent.devops.dispatch.pojo.thirdPartyAgent.ThirdPartyBuildInfo import com.tencent.devops.dispatch.pojo.thirdPartyAgent.ThirdPartyBuildWithStatus import com.tencent.devops.dispatch.utils.ThirdPartyAgentLock -import com.tencent.devops.dispatch.utils.redis.RedisUtils +import com.tencent.devops.dispatch.utils.redis.ThirdPartyAgentBuildRedisUtils import com.tencent.devops.environment.api.thirdPartyAgent.ServiceThirdPartyAgentResource import com.tencent.devops.model.dispatch.tables.records.TDispatchThirdpartyAgentBuildRecord import com.tencent.devops.process.api.service.ServiceBuildResource @@ -58,7 +58,7 @@ import javax.ws.rs.NotFoundException @Suppress("ALL") class ThirdPartyAgentService @Autowired constructor( private val dslContext: DSLContext, - private val redisUtils: RedisUtils, + private val thirdPartyAgentBuildRedisUtils: ThirdPartyAgentBuildRedisUtils, private val client: Client, private val redisOperation: RedisOperation, private val thirdPartyAgentBuildDao: ThirdPartyAgentBuildDao @@ -205,7 +205,7 @@ class ThirdPartyAgentService @Autowired constructor( return if (agentUpgradeResult.data != null && !agentUpgradeResult.data!!) { agentUpgradeResult } else { - redisUtils.setThirdPartyAgentUpgrading(projectId, agentId) + thirdPartyAgentBuildRedisUtils.setThirdPartyAgentUpgrading(projectId, agentId) AgentResult(AgentStatus.IMPORT_OK, true) } } catch (t: Throwable) { @@ -250,7 +250,7 @@ class ThirdPartyAgentService @Autowired constructor( ) throw NotFoundException("Fail to get the agent") } - redisUtils.thirdPartyAgentUpgradingDone(projectId, agentId) + thirdPartyAgentBuildRedisUtils.thirdPartyAgentUpgradingDone(projectId, agentId) return AgentResult(agentResult.agentStatus!!, true) } catch (ignored: Throwable) { logger.warn("Fail to finish upgrading", ignored) @@ -325,7 +325,7 @@ class ThirdPartyAgentService @Autowired constructor( logger.warn("Get the null third party agent(${record.agentId})") throw RemoteServiceException("Fail to get the third party agent") } - redisUtils.deleteThirdPartyBuild(agentResult.data!!.secretKey, record.agentId, record.buildId, record.vmSeqId) + thirdPartyAgentBuildRedisUtils.deleteThirdPartyBuild(agentResult.data!!.secretKey, record.agentId, record.buildId, record.vmSeqId) thirdPartyAgentBuildDao.updateStatus( dslContext, record.id, if (success) PipelineTaskStatus.DONE else PipelineTaskStatus.FAILURE diff --git a/src/backend/ci/core/dispatch/biz-dispatch/src/main/kotlin/com/tencent/devops/dispatch/service/dispatcher/BuildLessDispatcher.kt b/src/backend/ci/core/dispatch/biz-dispatch/src/main/kotlin/com/tencent/devops/dispatch/service/dispatcher/BuildLessDispatcher.kt deleted file mode 100644 index 29aace83940..00000000000 --- a/src/backend/ci/core/dispatch/biz-dispatch/src/main/kotlin/com/tencent/devops/dispatch/service/dispatcher/BuildLessDispatcher.kt +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. - * - * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. - * - * A copy of the MIT License is included in this file. - * - * - * Terms of the MIT License: - * --------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to - * permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT - * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN - * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, - * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE - * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ - -package com.tencent.devops.dispatch.service.dispatcher - -import com.tencent.devops.process.pojo.mq.PipelineBuildLessShutdownDispatchEvent -import com.tencent.devops.process.pojo.mq.PipelineBuildLessStartupDispatchEvent - -interface BuildLessDispatcher { - - fun canDispatch(event: PipelineBuildLessStartupDispatchEvent): Boolean - - fun startUp(event: PipelineBuildLessStartupDispatchEvent) - - fun shutdown(event: PipelineBuildLessShutdownDispatchEvent) -} diff --git a/src/backend/ci/core/dispatch/biz-dispatch/src/main/kotlin/com/tencent/devops/dispatch/service/dispatcher/agent/ThirdPartyAgentDispatcher.kt b/src/backend/ci/core/dispatch/biz-dispatch/src/main/kotlin/com/tencent/devops/dispatch/service/dispatcher/agent/ThirdPartyAgentDispatcher.kt index caa2b31b3fa..78e88ff51bb 100644 --- a/src/backend/ci/core/dispatch/biz-dispatch/src/main/kotlin/com/tencent/devops/dispatch/service/dispatcher/agent/ThirdPartyAgentDispatcher.kt +++ b/src/backend/ci/core/dispatch/biz-dispatch/src/main/kotlin/com/tencent/devops/dispatch/service/dispatcher/agent/ThirdPartyAgentDispatcher.kt @@ -44,7 +44,7 @@ import com.tencent.devops.dispatch.service.ThirdPartyAgentService import com.tencent.devops.dispatch.service.dispatcher.Dispatcher import com.tencent.devops.dispatch.utils.ThirdPartyAgentEnvLock import com.tencent.devops.dispatch.utils.ThirdPartyAgentLock -import com.tencent.devops.dispatch.utils.redis.RedisUtils +import com.tencent.devops.dispatch.utils.redis.ThirdPartyAgentBuildRedisUtils import com.tencent.devops.dispatch.utils.redis.ThirdPartyRedisBuild import com.tencent.devops.environment.api.thirdPartyAgent.ServiceThirdPartyAgentResource import com.tencent.devops.environment.pojo.thirdPartyAgent.ThirdPartyAgent @@ -63,7 +63,7 @@ class ThirdPartyAgentDispatcher @Autowired constructor( private val client: Client, private val redisOperation: RedisOperation, private val buildLogPrinter: BuildLogPrinter, - private val redisUtils: RedisUtils, + private val thirdPartyAgentBuildRedisUtils: ThirdPartyAgentBuildRedisUtils, private val pipelineEventDispatcher: PipelineEventDispatcher, private val thirdPartyAgentBuildService: ThirdPartyAgentService ) : Dispatcher { @@ -219,7 +219,7 @@ class ThirdPartyAgentDispatcher @Autowired constructor( try { if (redisLock.tryLock()) { - if (redisUtils.isThirdPartyAgentUpgrading(event.projectId, agentId)) { + if (thirdPartyAgentBuildRedisUtils.isThirdPartyAgentUpgrading(event.projectId, agentId)) { logger.warn("The agent($agentId) of project(${event.projectId}) is upgrading") buildLogPrinter.addLine( buildId = event.buildId, @@ -232,7 +232,7 @@ class ThirdPartyAgentDispatcher @Autowired constructor( return false } - redisUtils.setThirdPartyBuild( + thirdPartyAgentBuildRedisUtils.setThirdPartyBuild( agent.secretKey, ThirdPartyRedisBuild( projectId = event.projectId, diff --git a/src/backend/ci/core/dispatch/biz-dispatch/src/main/kotlin/com/tencent/devops/dispatch/utils/redis/RedisUtils.kt b/src/backend/ci/core/dispatch/biz-dispatch/src/main/kotlin/com/tencent/devops/dispatch/utils/redis/ThirdPartyAgentBuildRedisUtils.kt similarity index 52% rename from src/backend/ci/core/dispatch/biz-dispatch/src/main/kotlin/com/tencent/devops/dispatch/utils/redis/RedisUtils.kt rename to src/backend/ci/core/dispatch/biz-dispatch/src/main/kotlin/com/tencent/devops/dispatch/utils/redis/ThirdPartyAgentBuildRedisUtils.kt index 54fc6558ecd..26a3618a30e 100644 --- a/src/backend/ci/core/dispatch/biz-dispatch/src/main/kotlin/com/tencent/devops/dispatch/utils/redis/RedisUtils.kt +++ b/src/backend/ci/core/dispatch/biz-dispatch/src/main/kotlin/com/tencent/devops/dispatch/utils/redis/ThirdPartyAgentBuildRedisUtils.kt @@ -27,37 +27,16 @@ package com.tencent.devops.dispatch.utils.redis import com.fasterxml.jackson.databind.ObjectMapper -import com.tencent.devops.common.api.util.HashUtil -import com.tencent.devops.common.pipeline.utils.HeartBeatUtils import com.tencent.devops.common.redis.RedisOperation -import com.tencent.devops.dispatch.pojo.redis.RedisBuild -import org.slf4j.LoggerFactory import org.springframework.beans.factory.annotation.Autowired import org.springframework.stereotype.Component -@Component@Suppress("ALL") -class RedisUtils @Autowired constructor( +@Component +class ThirdPartyAgentBuildRedisUtils @Autowired constructor( private val redisOperation: RedisOperation, private val objectMapper: ObjectMapper ) { - fun setRedisBuild(ip: String, redisBuild: RedisBuild) { - redisOperation.set(ip, objectMapper.writeValueAsString(redisBuild)) - } - - fun getRedisBuild(ip: String): RedisBuild? { - val build = redisOperation.get(ip) ?: return null - try { - return objectMapper.readValue(build, RedisBuild::class.java) - } catch (t: Throwable) { - logger.warn("Fail to covert the redis build to object($build)", t) - } - return null - } - - fun deleteRedisBuild(ip: String) = - redisOperation.delete(ip) - fun setThirdPartyBuild(secretKey: String, redisBuild: ThirdPartyRedisBuild) { redisOperation.set( thirdPartyBuildKey( @@ -70,62 +49,39 @@ class RedisUtils @Autowired constructor( ) } - fun setDockerBuild(id: Long, secretKey: String, redisBuild: RedisBuild) = - redisOperation.set(dockerBuildKey(id, secretKey), objectMapper.writeValueAsString(redisBuild)) - - fun setDockerBuildLastHost(pipelineId: String, vmSeqId: String, hostIp: String) = - redisOperation.set(dockerBuildLastHostKey(pipelineId, vmSeqId), hostIp) - fun getDockerBuildLastHost(pipelineId: String, vmSeqId: String) = - redisOperation.get(dockerBuildLastHostKey(pipelineId, vmSeqId)) - fun deleteDockerBuildLastHost(pipelineId: String, vmSeqId: String) = - redisOperation.delete(dockerBuildLastHostKey(pipelineId, vmSeqId)) - - fun deleteDockerBuild(id: Long, secretKey: String) = - redisOperation.delete(dockerBuildKey(id, secretKey)) - - fun deleteHeartBeat(buildId: String, vmSeqId: String) = - redisOperation.delete(HeartBeatUtils.genHeartBeatKey(buildId, vmSeqId)) - fun deleteThirdPartyBuild(secretKey: String, agentId: String, buildId: String, vmSeqId: String) = - redisOperation.delete(thirdPartyBuildKey(secretKey, agentId, buildId, vmSeqId)) + redisOperation.delete( + key = thirdPartyBuildKey( + secretKey = secretKey, + agentId = agentId, + buildId = buildId, + vmSeqId = vmSeqId + ) + ) fun isThirdPartyAgentUpgrading(projectId: String, agentId: String): Boolean { return try { redisOperation.get(thirdPartyUpgradeKey(projectId, agentId)) == "true" - } catch (t: Throwable) { + } catch (ignored: Exception) { false } } fun setThirdPartyAgentUpgrading(projectId: String, agentId: String) { - redisOperation.set(thirdPartyUpgradeKey(projectId, agentId), "true", 60L) + redisOperation.set( + key = thirdPartyUpgradeKey(projectId = projectId, agentId = agentId), + value = "true", + expiredInSecond = 60L + ) } fun thirdPartyAgentUpgradingDone(projectId: String, agentId: String) { - redisOperation.delete(thirdPartyUpgradeKey(projectId, agentId)) + redisOperation.delete(key = thirdPartyUpgradeKey(projectId = projectId, agentId = agentId)) } - private fun dockerBuildKey(id: Long, secretKey: String) = - "docker_build_key_${HashUtil.encodeLongId(id)}_$secretKey" - - private fun dockerBuildLastHostKey(pipelineId: String, vmSeqId: String) = - "dispatch_docker_build_last_host_key_${pipelineId}_$vmSeqId" - private fun thirdPartyBuildKey(secretKey: String, agentId: String, buildId: String, vmSeqId: String) = "third_party_agent_${secretKey}_${agentId}_${buildId}_$vmSeqId" private fun thirdPartyUpgradeKey(projectId: String, agentId: String) = "third_party_agent_upgrade_${projectId}_$agentId" - - fun setRedisDebugMsg(pipelineId: String, vmSeqId: String, msg: String) { - redisOperation.set("docker_debug_msg_key_${pipelineId}_$vmSeqId", msg, 3600L) - } - - fun getRedisDebugMsg(pipelineId: String, vmSeqId: String): String? { - return redisOperation.get("docker_debug_msg_key_${pipelineId}_$vmSeqId") - } - - companion object { - private val logger = LoggerFactory.getLogger(RedisUtils::class.java) - } } diff --git a/src/backend/ci/core/misc/api-misc/src/main/kotlin/com/tencent/devops/misc/api/OpThirdPartyAgentResource.kt b/src/backend/ci/core/misc/api-misc/src/main/kotlin/com/tencent/devops/misc/api/OpThirdPartyAgentUpgradeResource.kt similarity index 98% rename from src/backend/ci/core/misc/api-misc/src/main/kotlin/com/tencent/devops/misc/api/OpThirdPartyAgentResource.kt rename to src/backend/ci/core/misc/api-misc/src/main/kotlin/com/tencent/devops/misc/api/OpThirdPartyAgentUpgradeResource.kt index 5451e9ef02f..5eb2e9dc1b3 100644 --- a/src/backend/ci/core/misc/api-misc/src/main/kotlin/com/tencent/devops/misc/api/OpThirdPartyAgentResource.kt +++ b/src/backend/ci/core/misc/api-misc/src/main/kotlin/com/tencent/devops/misc/api/OpThirdPartyAgentUpgradeResource.kt @@ -41,7 +41,7 @@ import javax.ws.rs.core.MediaType @Path("/op/thirdPartyAgent") @Produces(MediaType.APPLICATION_JSON) @Consumes(MediaType.APPLICATION_JSON) -interface OpThirdPartyAgentResource { +interface OpThirdPartyAgentUpgradeResource { @ApiOperation("设置agent最大并发升级数量") @POST diff --git a/src/backend/ci/core/misc/biz-misc/src/main/kotlin/com/tencent/devops/misc/resources/OpThirdPartyAgentResourceImpl.kt b/src/backend/ci/core/misc/biz-misc/src/main/kotlin/com/tencent/devops/misc/resources/OpThirdPartyAgentUpgradeResourceImpl.kt similarity index 92% rename from src/backend/ci/core/misc/biz-misc/src/main/kotlin/com/tencent/devops/misc/resources/OpThirdPartyAgentResourceImpl.kt rename to src/backend/ci/core/misc/biz-misc/src/main/kotlin/com/tencent/devops/misc/resources/OpThirdPartyAgentUpgradeResourceImpl.kt index 5b67c079487..eeaaf8fed87 100644 --- a/src/backend/ci/core/misc/biz-misc/src/main/kotlin/com/tencent/devops/misc/resources/OpThirdPartyAgentResourceImpl.kt +++ b/src/backend/ci/core/misc/biz-misc/src/main/kotlin/com/tencent/devops/misc/resources/OpThirdPartyAgentUpgradeResourceImpl.kt @@ -29,7 +29,7 @@ package com.tencent.devops.misc.resources import com.tencent.devops.common.api.pojo.Result import com.tencent.devops.common.web.RestResource -import com.tencent.devops.misc.api.OpThirdPartyAgentResource +import com.tencent.devops.misc.api.OpThirdPartyAgentUpgradeResource import com.tencent.devops.misc.service.environment.AgentUpgradeService import org.springframework.beans.factory.annotation.Autowired @@ -38,9 +38,9 @@ import org.springframework.beans.factory.annotation.Autowired * 2018/5/9 */ @RestResource -class OpThirdPartyAgentResourceImpl @Autowired constructor( +class OpThirdPartyAgentUpgradeResourceImpl @Autowired constructor( private val upgradeService: AgentUpgradeService -) : OpThirdPartyAgentResource { +) : OpThirdPartyAgentUpgradeResource { override fun setMaxParallelUpgradeCount(maxParallelUpgradeCount: Int): Result { upgradeService.setMaxParallelUpgradeCount(maxParallelUpgradeCount) diff --git a/src/backend/ci/core/plugin/biz-plugin/src/main/kotlin/com/tencent/devops/plugin/service/ScmService.kt b/src/backend/ci/core/plugin/biz-plugin/src/main/kotlin/com/tencent/devops/plugin/service/ScmCheckService.kt similarity index 98% rename from src/backend/ci/core/plugin/biz-plugin/src/main/kotlin/com/tencent/devops/plugin/service/ScmService.kt rename to src/backend/ci/core/plugin/biz-plugin/src/main/kotlin/com/tencent/devops/plugin/service/ScmCheckService.kt index 996d1b99743..ae69f59da6f 100644 --- a/src/backend/ci/core/plugin/biz-plugin/src/main/kotlin/com/tencent/devops/plugin/service/ScmService.kt +++ b/src/backend/ci/core/plugin/biz-plugin/src/main/kotlin/com/tencent/devops/plugin/service/ScmCheckService.kt @@ -59,8 +59,8 @@ import java.net.URLEncoder import java.util.Base64 @Service@Suppress("ALL") -class ScmService @Autowired constructor(private val client: Client) { - private val logger = LoggerFactory.getLogger(ScmService::class.java) +class ScmCheckService @Autowired constructor(private val client: Client) { + private val logger = LoggerFactory.getLogger(ScmCheckService::class.java) fun addGitCommitCheck( event: GitCommitCheckEvent, diff --git a/src/backend/ci/core/plugin/biz-plugin/src/main/kotlin/com/tencent/devops/plugin/service/git/CodeWebhookService.kt b/src/backend/ci/core/plugin/biz-plugin/src/main/kotlin/com/tencent/devops/plugin/service/git/CodeWebhookService.kt index 7150696ddc0..b5044f26ece 100644 --- a/src/backend/ci/core/plugin/biz-plugin/src/main/kotlin/com/tencent/devops/plugin/service/git/CodeWebhookService.kt +++ b/src/backend/ci/core/plugin/biz-plugin/src/main/kotlin/com/tencent/devops/plugin/service/git/CodeWebhookService.kt @@ -49,7 +49,7 @@ import com.tencent.devops.plugin.api.pojo.GithubPrEvent import com.tencent.devops.plugin.api.pojo.PluginGitCheck import com.tencent.devops.plugin.dao.PluginGitCheckDao import com.tencent.devops.plugin.dao.PluginGithubCheckDao -import com.tencent.devops.plugin.service.ScmService +import com.tencent.devops.plugin.service.ScmCheckService import com.tencent.devops.process.api.service.ServiceBuildResource import com.tencent.devops.process.utils.PIPELINE_BUILD_NUM import com.tencent.devops.process.utils.PIPELINE_WEBHOOK_BLOCK @@ -86,7 +86,7 @@ class CodeWebhookService @Autowired constructor( private val pluginGithubCheckDao: PluginGithubCheckDao, private val redisOperation: RedisOperation, private val pipelineEventDispatcher: PipelineEventDispatcher, - private val scmService: ScmService, + private val scmCheckService: ScmCheckService, private val gitWebhookUnlockService: GitWebhookUnlockService ) { @@ -401,7 +401,7 @@ class CodeWebhookService @Autowired constructor( ) if (record == null) { - scmService.addGitCommitCheck( + scmCheckService.addGitCommitCheck( event = event, targetUrl = targetUrl, context = context, @@ -420,7 +420,7 @@ class CodeWebhookService @Autowired constructor( ) } else { if (buildNum.toInt() >= record.buildNumber) { - scmService.addGitCommitCheck( + scmCheckService.addGitCommitCheck( event = event, targetUrl = targetUrl, context = record.context ?: pipelineName, @@ -551,7 +551,7 @@ class CodeWebhookService @Autowired constructor( val record = pluginGithubCheckDao.getOrNull(dslContext, pipelineId, repositoryConfig, commitId) if (record == null) { - val result = scmService.addGithubCheckRuns( + val result = scmCheckService.addGithubCheckRuns( projectId = projectId, repositoryConfig = repositoryConfig, name = name, @@ -578,7 +578,7 @@ class CodeWebhookService @Autowired constructor( if (buildNum.toInt() >= record.buildNumber) { // 如果重试或者reopen，需要将状态重新置为in_progress val checkRunId = if (conclusion == null) { - val result = scmService.addGithubCheckRuns( + val result = scmCheckService.addGithubCheckRuns( projectId = projectId, repositoryConfig = repositoryConfig, name = record.checkRunName ?: "$pipelineName #$buildNum", @@ -596,7 +596,7 @@ class CodeWebhookService @Autowired constructor( ) result.id } else { - scmService.updateGithubCheckRuns( + scmCheckService.updateGithubCheckRuns( checkRunId = record.checkRunId, projectId = projectId, repositoryConfig = repositoryConfig, diff --git a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/jmx/pipeline/BuildBean.kt b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/jmx/pipeline/PipelineBuildBean.kt similarity index 96% rename from src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/jmx/pipeline/BuildBean.kt rename to src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/jmx/pipeline/PipelineBuildBean.kt index 141895f9e38..4979c214d77 100644 --- a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/jmx/pipeline/BuildBean.kt +++ b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/jmx/pipeline/PipelineBuildBean.kt @@ -41,7 +41,7 @@ import java.util.concurrent.atomic.AtomicInteger objectName = "com.tencent.devops.process:type=builds", description = "build jmx metrics" ) -class BuildBean @Autowired constructor( +class PipelineBuildBean @Autowired constructor( private val pipelineRuntimeService: PipelineRuntimeService ) { @@ -119,6 +119,6 @@ class BuildBean @Autowired constructor( } companion object { - private val logger = LoggerFactory.getLogger(BuildBean::class.java) + private val logger = LoggerFactory.getLogger(PipelineBuildBean::class.java) } } From 6f34cbd6e3d502a510f63db68c9a0b8551c471fd Mon Sep 17 00:00:00 2001 From: irwinsun Date: Mon, 21 Jun 2021 16:32:20 +0800 Subject: [PATCH 03/20] =?UTF-8?q?bug:=20assembly=20=E5=90=AF=E5=8A=A8?= =?UTF-8?q?=E5=A4=B1=E8=B4=A5[1.5.4]=20#4475=20=E5=90=8C=E6=AD=A5=E9=85=8D?= =?UTF-8?q?=E7=BD=AE=E9=A1=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/main/resources/application.yml | 4 +- .../#etc#ci#application-assembly.yml | 176 +++++++++++++++--- .../#etc#ci#application-repository.yml | 2 +- .../templates/#etc#ci#application-ticket.yml | 2 + 4 files changed, 154 insertions(+), 30 deletions(-) diff --git a/src/backend/ci/boot-assembly/src/main/resources/application.yml b/src/backend/ci/boot-assembly/src/main/resources/application.yml index 5a5e94c11e4..399e609f2ca 100644 --- a/src/backend/ci/boot-assembly/src/main/resources/application.yml +++ b/src/backend/ci/boot-assembly/src/main/resources/application.yml @@ -1,4 +1,6 @@ spring: + main: + allow-bean-definition-overriding: true application: name: bk-ci desc: DevOps BK-CI Service @@ -17,4 +19,4 @@ aes: aesMock: "******" server: - port: 21900 \ No newline at end of file + port: 21900 diff --git a/support-files/templates/#etc#ci#application-assembly.yml b/support-files/templates/#etc#ci#application-assembly.yml index 7d3ff8e1d6a..5cb956c7533 100644 --- a/support-files/templates/#etc#ci#application-assembly.yml +++ b/support-files/templates/#etc#ci#application-assembly.yml @@ -4,6 +4,38 @@ spring: url: jdbc:mysql://__BK_CI_MYSQL_ADDR__/devops_ci_process?useSSL=false&autoReconnect=true&serverTimezone=GMT%2B8&useUnicode=true&characterEncoding=utf8&allowMultiQueries=true&sessionVariables=sql_mode=%27STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION%27 username: __BK_CI_MYSQL_USER__ password: __BK_CI_MYSQL_PASSWORD__ + process: + url: jdbc:mysql://__BK_CI_MYSQL_ADDR__/devops_ci_process?useSSL=false&autoReconnect=true&serverTimezone=GMT%2B8&useUnicode=true&characterEncoding=utf8&allowMultiQueries=true&sessionVariables=sql_mode=%27STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION%27 + username: __BK_CI_MYSQL_USER__ + password: __BK_CI_MYSQL_PASSWORD__ + project: + url: jdbc:mysql://__BK_CI_MYSQL_ADDR__/devops_ci_project?useSSL=false&autoReconnect=true&serverTimezone=GMT%2B8&useUnicode=true&characterEncoding=utf8&allowMultiQueries=true&sessionVariables=sql_mode=%27STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION%27 + username: __BK_CI_MYSQL_USER__ + password: __BK_CI_MYSQL_PASSWORD__ + repository: + url: jdbc:mysql://__BK_CI_MYSQL_ADDR__/devops_ci_repository?useSSL=false&autoReconnect=true&serverTimezone=GMT%2B8&useUnicode=true&characterEncoding=utf8&allowMultiQueries=true&sessionVariables=sql_mode=%27STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION%27 + username: __BK_CI_MYSQL_USER__ + password: __BK_CI_MYSQL_PASSWORD__ + dispatch: + url: jdbc:mysql://__BK_CI_MYSQL_ADDR__/devops_ci_dispatch?useSSL=false&autoReconnect=true&serverTimezone=GMT%2B8&useUnicode=true&characterEncoding=utf8&allowMultiQueries=true&sessionVariables=sql_mode=%27STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION%27 + username: __BK_CI_MYSQL_USER__ + password: __BK_CI_MYSQL_PASSWORD__ + plugin: + url: jdbc:mysql://__BK_CI_MYSQL_ADDR__/devops_ci_plugin?useSSL=false&autoReconnect=true&serverTimezone=GMT%2B8&useUnicode=true&characterEncoding=utf8&allowMultiQueries=true&sessionVariables=sql_mode=%27STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION%27 + username: __BK_CI_MYSQL_USER__ + password: __BK_CI_MYSQL_PASSWORD__ + quality: + url: jdbc:mysql://__BK_CI_MYSQL_ADDR__/devops_ci_quality?useSSL=false&autoReconnect=true&serverTimezone=GMT%2B8&useUnicode=true&characterEncoding=utf8&allowMultiQueries=true&sessionVariables=sql_mode=%27STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION%27 + username: __BK_CI_MYSQL_USER__ + password: __BK_CI_MYSQL_PASSWORD__ + artifactory: + url: jdbc:mysql://__BK_CI_MYSQL_ADDR__/devops_ci_artifactory?useSSL=false&autoReconnect=true&serverTimezone=GMT%2B8&useUnicode=true&characterEncoding=utf8&allowMultiQueries=true&sessionVariables=sql_mode=%27STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION%27 + username: __BK_CI_MYSQL_USER__ + password: __BK_CI_MYSQL_PASSWORD__ + environment: + url: jdbc:mysql://__BK_CI_MYSQL_ADDR__/devops_ci_environment?useSSL=false&autoReconnect=true&serverTimezone=GMT%2B8&useUnicode=true&characterEncoding=utf8&allowMultiQueries=true&sessionVariables=sql_mode=%27STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION%27 + username: __BK_CI_MYSQL_USER__ + password: __BK_CI_MYSQL_PASSWORD__ server: port: __BK_CI_ASSEMBLY_API_PORT__ @@ -13,7 +45,7 @@ server: # 如果微服务是部署多机节点，则以下目录需要做成分布式高可用的，比如NFS，CephFS挂载 # 保证多节点都能读取到 artifactory: - archiveLocalBasePath: __BK_CI_DATA_DIR__/artifactory + archiveLocalBasePath: __BK_CI_DATA_DIR__/artifactory # 如果多节点部署, 需要使用共享存储. fileTask: savedir: __BK_CI_DATA_DIR__/artifactory-filetask/ # 不一定共享, 但是建议准备较大容量. file: @@ -60,8 +92,9 @@ influxdb: password: __BK_CI_INFLUXDB_PASSWORD__ #### DockerHost + dockerhost: - mode: docker_build + mode: docker_no_build # docker client 配置 dockerCli: @@ -123,6 +156,11 @@ dockerCli: clearLocalImageCron: 0 0 2 * * ? localImageCacheDays: 7 + # 无编译构建容器配置 + memoryLimitBytes: 2147483648 + cpuPeriod: 50000 + cpuQuota: 50000 + #### image image: @@ -137,22 +175,56 @@ image: #### log -elasticsearch: - ip: __BK_CI_ES_REST_ADDR__ # 今后只使用REST client. - port: __BK_CI_ES_REST_PORT__ - cluster: __BK_CI_ES_CLUSTER_NAME__ - name: DEVOPS -# ES集群如果不要求账号密码认证，则可以去掉以下2个ES的访问账号密码配置项或留空 - username: __BK_CI_ES_USER__ - password: __BK_CI_ES_PASSWORD__ -# ES集群如果要求HTTPS协议请求，则需要需要打开以下5个配置项， 设置keystore文件和truststore文件以及密码 -# https: true -# keystore: -# filePath: -# password: -# truststore: -# filePath: -# password: +log: + storage: + type: __BK_CI_LOG_STORAGE_TYPE__ # 日志存储方式 lucene/elasticsearch + closeInDay: __BK_CI_LOG_CLOSE_DAY__ # 索引自动关闭时间，留空则为永不关闭 + deleteInDay: __BK_CI_LOG_DELETE_DAY__ # 索引自动删除时间，留空则为永不删除 + # 通过本地文件系统进行存储的必要配置 + lucene: + dataDirectory: __BK_CI_LUCENE_DATA_DIR__ # 建立lucene索引的根目录 + indexMaxSize: __BK_CI_LUCENE_INDEX_MAX_SIZE__ # 单个构建的最大日志行数，建议设在100万内 + # 通过Elasticsearch服务进行存储的必要配置 + elasticsearch: + ip: __BK_CI_ES_REST_ADDR__ # 今后只使用REST client. + port: __BK_CI_ES_REST_PORT__ + cluster: __BK_CI_ES_CLUSTER_NAME__ + name: DEVOPS + # ES集群如果不要求账号密码认证，则可以去掉以下2个ES的访问账号密码配置项或留空 + username: __BK_CI_ES_USER__ + password: __BK_CI_ES_PASSWORD__ + # ES集群如果要求HTTPS协议请求，则需要需要打开以下5个配置项， 设置keystore文件和truststore文件以及密码 +# https: true +# keystore: +# filePath: +# password: +# truststore: +# filePath: +# password: + +#### misc + +build: + atomMonitorData: + report: + switch: false + maxMonitorDataSize: 1677216 + data: + clear: + switch: false + maxEveryProjectHandleNum: 5 + monthRange: -1 + maxKeepNum: 10000 + codeccDayRange: -14 + codeccMaxKeepNum: 14 + otherMonthRange: -1 + otherMaxKeepNum: 500 + +#### openapi +# 是否开启apifilter和aspect功能 +api: + gateway: + auth: false #### plugin @@ -210,17 +282,22 @@ scm: redirectUrl: __BK_CI_PUBLIC_URL__/console/codelib appUrl: https://github.com/apps/__BK_CI_REPOSITORY_GITHUB_APP__ signSecret: __BK_CI_REPOSITORY_GITHUB_SIGN_SECRET__ - + tGit: + apiUrl: https://git.tencent.com/api/v3 + tGitHookUrl: __BK_CI_PUBLIC_URL__/ms/process/api/external/scm/codetgit/commit + hookSecret: + enableHookSecret: false #### store store: commentNotifyAdmin: admin profileUrlPrefix: __BK_CI_STORE_USER_AVATARS_URL__ - atomDetailBaseUrl: __BK_CI_PUBLIC_URL__/console/store/atomStore/detail/atom/ - templateDetailBaseUrl: __BK_CI_PUBLIC_URL__/console/store/atomStore/detail/template/ + atomDetailBaseUrl: /console/store/atomStore/detail/atom/ + templateDetailBaseUrl: /console/store/atomStore/detail/template/ artifactoryServiceUrlPrefix: __BK_CI_PUBLIC_URL__/ms/artifactory/api - ideAtomDetailBaseUrl: __BK_CI_PUBLIC_URL__/console/store/atomStore/detail/ide/ - imageDetailBaseUrl: __BK_CI_PUBLIC_URL__/console/store/atomStore/detail/image/ + ideAtomDetailBaseUrl: /console/store/atomStore/detail/ide/ + imageDetailBaseUrl: /console/store/atomStore/detail/image/ + serviceDetailBaseUrl: /console/store/atomStore/detail/service/ baseImageDocsLink: /console/store/atomStore/detail/image/ imageAdminUsers: admin buildResultBaseUrl: __BK_CI_PUBLIC_URL__/console/pipeline @@ -246,6 +323,8 @@ aes: aesKey: "J%k^yO{?vt}3tXpG" aesMock: "******" +statistics: + timeSpanMonth: -3 #### ticket @@ -278,8 +357,49 @@ queueConcurrency: task: __BK_CI_PROCESS_EVENT_CONCURRENT__ # Task消息并发处理数量 buildFinish: __BK_CI_PROCESS_EVENT_CONCURRENT__ # 构建结束消息并发处理数量 -build: - atomMonitorData: - report: - switch: false - maxMonitorDataSize: 1677216 +# 流水线模板配置 +template: + instanceListUrl: "__BK_CI_PUBLIC_URL__/console/store/pipeline/{0}/template/{0}/instance" + +# 流水线相关配置 +pipeline: + setting: + common: + maxModelSize: 16777215 + maxStageNum: 20 + stage: + maxJobNum: 20 + job: + maxTaskNum: 50 + task: + maxInputNum: 100 + maxOutputNum: 100 + inputComponent: + input: + size: 1024 + textarea: + size: 16384 + codeEditor: + size: 16384 + default: + size: 1024 + multiple: + member: "selector,select-input,devops-select,atom-checkbox-list,staff-input,company-staff-input,parameter,dynamic-parameter" + size: 4000 + outputComponent: + default: + size: 4000 + editPath: "console/pipeline/{0}/{1}/edit" + atom: + maxRelQueryNum: 2000 # 最大查询数量 + maxRelQueryRangeTime: 30 # 最大查询时间跨度 + version: + max_keep_num: 50 + spec_channels: "AM,CODECC,GCLOUD,GIT,GONGFENGSCAN,CODECC_EE" + spec_channel_max_keep_num: 2 + +#### sign +# 签名服务临时解压缩目录 +bkci: + sign: + tmpDir: "/tmp/enterprise_sign_tmp/" diff --git a/support-files/templates/#etc#ci#application-repository.yml b/support-files/templates/#etc#ci#application-repository.yml index e7bb27fc8ac..577855bde36 100644 --- a/support-files/templates/#etc#ci#application-repository.yml +++ b/support-files/templates/#etc#ci#application-repository.yml @@ -18,7 +18,7 @@ scm: #svn open api url，如果不需要则放着即可 svn: apiKey: __BK_CI_REPOSITORY_SVN_API_KEY__ - apiUrl: __BK_CI_REPOSITORY_SVN_API_URL__ # 结尾一般为 /rest + apiUrl: __BK_CI_REPOSITORY_SVN_API_URL__ # 结尾一般为 /rest webhookApiUrl: __BK_CI_REPOSITORY_SVN_WEBHOOK_URL__ # 一般为 SVN_API_URL/webhooks svnHookUrl: __BK_CI_PUBLIC_URL__/ms/process/api/external/scm/codesvn/commit # git 仓库配置，如果不需要则放着即可 diff --git a/support-files/templates/#etc#ci#application-ticket.yml b/support-files/templates/#etc#ci#application-ticket.yml index 466428455d9..1290abe9aa3 100644 --- a/support-files/templates/#etc#ci#application-ticket.yml +++ b/support-files/templates/#etc#ci#application-ticket.yml @@ -11,8 +11,10 @@ server: cert: + # 部署前修改好，后续升级如果再修改，会导致历史数据读取不了，所以如果修改需要先解密重新入库 aes-key: "gHi(xG9Af)jEvCx&" credential: mixer: "******" + # 部署前修改好，后续升级如果再修改，会导致历史数据读取不了，所以如果修改需要先解密重新入库 aes-key: "G/I%yP{?ST}2TXPg" From e71dc4d3b9e1f5296ac1708111984f054144a1dd Mon Sep 17 00:00:00 2001 From: irwinsun Date: Mon, 21 Jun 2021 16:43:50 +0800 Subject: [PATCH 04/20] =?UTF-8?q?bug:=20assembly=20=E5=90=AF=E5=8A=A8?= =?UTF-8?q?=E5=A4=B1=E8=B4=A5[1.5.4]=20#4475=20=E5=A4=9A=E6=95=B0=E6=8D=AE?= =?UTF-8?q?=E6=BA=90=E6=97=B6=E4=B8=8D=E6=8C=87=E5=AE=9A=E9=BB=98=E8=AE=A4?= =?UTF-8?q?=EF=BC=8C=E9=80=9A=E8=BF=87=E5=90=8D=E7=A7=B0=E6=9D=A5=E8=AF=86?= =?UTF-8?q?=E5=88=AB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../kotlin/com/tencent/devops/misc/config/DataSourceConfig.kt | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/backend/ci/core/misc/biz-misc/src/main/kotlin/com/tencent/devops/misc/config/DataSourceConfig.kt b/src/backend/ci/core/misc/biz-misc/src/main/kotlin/com/tencent/devops/misc/config/DataSourceConfig.kt index e17fa518c32..db2c6e3f999 100644 --- a/src/backend/ci/core/misc/biz-misc/src/main/kotlin/com/tencent/devops/misc/config/DataSourceConfig.kt +++ b/src/backend/ci/core/misc/biz-misc/src/main/kotlin/com/tencent/devops/misc/config/DataSourceConfig.kt @@ -33,7 +33,6 @@ import org.springframework.beans.factory.annotation.Value import org.springframework.boot.autoconfigure.AutoConfigureOrder import org.springframework.context.annotation.Bean import org.springframework.context.annotation.Configuration -import org.springframework.context.annotation.Primary import org.springframework.core.Ordered import org.springframework.transaction.annotation.EnableTransactionManagement import javax.sql.DataSource @@ -48,7 +47,6 @@ import javax.sql.DataSource class DataSourceConfig { @Bean - @Primary fun processDataSource( @Value("\${spring.datasource.process.url}") datasourceUrl: String, From 312fbfd6ad4ce8352b8fd3b027adc761c99760b6 Mon Sep 17 00:00:00 2001 From: irwinsun Date: Mon, 21 Jun 2021 16:52:59 +0800 Subject: [PATCH 05/20] =?UTF-8?q?bug:=20assembly=20=E5=90=AF=E5=8A=A8?= =?UTF-8?q?=E5=A4=B1=E8=B4=A5[1.5.4]=20#4475=20=E5=8D=87=E7=BA=A71.5.5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/backend/ci/build.gradle.kts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend/ci/build.gradle.kts b/src/backend/ci/build.gradle.kts index 2e4e8463841..601c875383f 100644 --- a/src/backend/ci/build.gradle.kts +++ b/src/backend/ci/build.gradle.kts @@ -9,7 +9,7 @@ allprojects { group = "com.tencent.bk.devops.ci" // 版本 - version = "1.6.0" + version = "1.5.5" val devopsVersion = System.getProperty("ci_version") if (devopsVersion != null) { version = devopsVersion From 0672e65bbd4546cc7ba59fe94ea7e85fceea2780 Mon Sep 17 00:00:00 2001 From: irwinsun Date: Mon, 21 Jun 2021 17:35:54 +0800 Subject: [PATCH 06/20] =?UTF-8?q?bug:=20assembly=20=E5=90=AF=E5=8A=A8?= =?UTF-8?q?=E5=A4=B1=E8=B4=A5[1.5.4]=20#4475=20=E9=87=8D=E5=90=8D=E5=A4=84?= =?UTF-8?q?=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ...dCodeccResource.kt => BuildCodeccToolResource.kt} | 12 ++++++------ ...esourceImpl.kt => BuildCodeccToolResourceImpl.kt} | 6 +++--- ...onfiguration.kt => LogPermissionConfiguration.kt} | 2 +- .../log/service/impl/V3LogPermissionService.kt | 4 ++-- .../codecc/resources/BuildCodeccResourceImpl.kt | 8 ++++---- ...aderService.kt => CodeccToolDownloaderService.kt} | 2 +- ...ourceImpl.kt => ServiceStoreImageResourceImpl.kt} | 2 +- 7 files changed, 18 insertions(+), 18 deletions(-) rename src/backend/ci/core/dispatch/api-dispatch/src/main/kotlin/com/tencent/devops/dispatch/api/{BuildCodeccResource.kt => BuildCodeccToolResource.kt} (91%) rename src/backend/ci/core/dispatch/biz-dispatch/src/main/kotlin/com/tencent/devops/dispatch/controller/{BuildCodeccResourceImpl.kt => BuildCodeccToolResourceImpl.kt} (94%) rename src/backend/ci/core/log/biz-log-sample/src/main/kotlin/com/tencent/devops/log/config/{LogConfiguration.kt => LogPermissionConfiguration.kt} (99%) rename src/backend/ci/core/plugin/codecc-plugin/biz-codecc/src/main/kotlin/com/tencent/devops/plugin/codecc/service/{CodeccDownloaderService.kt => CodeccToolDownloaderService.kt} (99%) rename src/backend/ci/core/store/biz-store-image/src/main/kotlin/com/tencent/devops/store/resources/image/service/{ServiceImageResourceImpl.kt => ServiceStoreImageResourceImpl.kt} (98%) diff --git a/src/backend/ci/core/dispatch/api-dispatch/src/main/kotlin/com/tencent/devops/dispatch/api/BuildCodeccResource.kt b/src/backend/ci/core/dispatch/api-dispatch/src/main/kotlin/com/tencent/devops/dispatch/api/BuildCodeccToolResource.kt similarity index 91% rename from src/backend/ci/core/dispatch/api-dispatch/src/main/kotlin/com/tencent/devops/dispatch/api/BuildCodeccResource.kt rename to src/backend/ci/core/dispatch/api-dispatch/src/main/kotlin/com/tencent/devops/dispatch/api/BuildCodeccToolResource.kt index 63ee2834b2e..8c50f880c1a 100644 --- a/src/backend/ci/core/dispatch/api-dispatch/src/main/kotlin/com/tencent/devops/dispatch/api/BuildCodeccResource.kt +++ b/src/backend/ci/core/dispatch/api-dispatch/src/main/kotlin/com/tencent/devops/dispatch/api/BuildCodeccToolResource.kt @@ -36,24 +36,24 @@ import io.swagger.annotations.ApiResponses import javax.ws.rs.Consumes import javax.ws.rs.GET import javax.ws.rs.Path +import javax.ws.rs.PathParam import javax.ws.rs.Produces import javax.ws.rs.QueryParam -import javax.ws.rs.PathParam import javax.ws.rs.core.MediaType import javax.ws.rs.core.Response @Api(tags = ["BUILD_CODECC"], description = "构建-CODECC相关资源") -@Path("/build/codecc") +@Path("/build/codecc/") @Produces(MediaType.APPLICATION_JSON) @Consumes(MediaType.APPLICATION_JSON) -interface BuildCodeccResource { +interface BuildCodeccToolResource { @ApiOperation("下载toolName对应的tool") @GET @Path("/{toolName}") @Produces(MediaType.APPLICATION_OCTET_STREAM) @ApiResponses( - ApiResponse(code = 304, message = "tool已是最新，无需下载") + ApiResponse(code = 304, message = "tool已是最新，无需下载") ) fun downloadTool( @ApiParam("工具类型", required = true) @@ -75,7 +75,7 @@ interface BuildCodeccResource { @Path("/coverity/script") @Produces(MediaType.APPLICATION_OCTET_STREAM) @ApiResponses( - ApiResponse(code = 304, message = "本地的coverity script已是最新，无需下载") + ApiResponse(code = 304, message = "本地的coverity script已是最新，无需下载") ) fun downloadCovScript( @ApiParam("系统类型", required = true) @@ -91,7 +91,7 @@ interface BuildCodeccResource { @Path("/tools/script") @Produces(MediaType.APPLICATION_OCTET_STREAM) @ApiResponses( - ApiResponse(code = 304, message = "本地的多工具script已是最新，无需下载") + ApiResponse(code = 304, message = "本地的多工具script已是最新，无需下载") ) fun downloadToolsScript( @ApiParam("系统类型", required = true) diff --git a/src/backend/ci/core/dispatch/biz-dispatch/src/main/kotlin/com/tencent/devops/dispatch/controller/BuildCodeccResourceImpl.kt b/src/backend/ci/core/dispatch/biz-dispatch/src/main/kotlin/com/tencent/devops/dispatch/controller/BuildCodeccToolResourceImpl.kt similarity index 94% rename from src/backend/ci/core/dispatch/biz-dispatch/src/main/kotlin/com/tencent/devops/dispatch/controller/BuildCodeccResourceImpl.kt rename to src/backend/ci/core/dispatch/biz-dispatch/src/main/kotlin/com/tencent/devops/dispatch/controller/BuildCodeccToolResourceImpl.kt index 909ba4885fd..86a042b0dcf 100644 --- a/src/backend/ci/core/dispatch/biz-dispatch/src/main/kotlin/com/tencent/devops/dispatch/controller/BuildCodeccResourceImpl.kt +++ b/src/backend/ci/core/dispatch/biz-dispatch/src/main/kotlin/com/tencent/devops/dispatch/controller/BuildCodeccToolResourceImpl.kt @@ -29,15 +29,15 @@ package com.tencent.devops.dispatch.controller import com.tencent.devops.common.api.enums.OSType import com.tencent.devops.common.web.RestResource -import com.tencent.devops.dispatch.api.BuildCodeccResource +import com.tencent.devops.dispatch.api.BuildCodeccToolResource import com.tencent.devops.dispatch.service.CodeccDownloaderService import org.springframework.beans.factory.annotation.Autowired import javax.ws.rs.core.Response @RestResource -class BuildCodeccResourceImpl @Autowired constructor( +class BuildCodeccToolResourceImpl @Autowired constructor( private val codeccDownloaderService: CodeccDownloaderService -) : BuildCodeccResource { +) : BuildCodeccToolResource { override fun downloadTool(toolName: String, osType: OSType, fileMd5: String, is32Bit: Boolean?): Response { return codeccDownloaderService.downloadTool(toolName, osType, fileMd5, is32Bit) } diff --git a/src/backend/ci/core/log/biz-log-sample/src/main/kotlin/com/tencent/devops/log/config/LogConfiguration.kt b/src/backend/ci/core/log/biz-log-sample/src/main/kotlin/com/tencent/devops/log/config/LogPermissionConfiguration.kt similarity index 99% rename from src/backend/ci/core/log/biz-log-sample/src/main/kotlin/com/tencent/devops/log/config/LogConfiguration.kt rename to src/backend/ci/core/log/biz-log-sample/src/main/kotlin/com/tencent/devops/log/config/LogPermissionConfiguration.kt index b9dc0b4970d..6561286631a 100644 --- a/src/backend/ci/core/log/biz-log-sample/src/main/kotlin/com/tencent/devops/log/config/LogConfiguration.kt +++ b/src/backend/ci/core/log/biz-log-sample/src/main/kotlin/com/tencent/devops/log/config/LogPermissionConfiguration.kt @@ -46,7 +46,7 @@ import org.springframework.core.Ordered @Configuration @ConditionalOnWebApplication @AutoConfigureOrder(Ordered.LOWEST_PRECEDENCE) -class LogConfiguration { +class LogPermissionConfiguration { @Bean @ConditionalOnProperty(prefix = "auth", name = ["idProvider"], havingValue = "bk_login") diff --git a/src/backend/ci/core/log/biz-log-sample/src/main/kotlin/com/tencent/devops/log/service/impl/V3LogPermissionService.kt b/src/backend/ci/core/log/biz-log-sample/src/main/kotlin/com/tencent/devops/log/service/impl/V3LogPermissionService.kt index 4ddd5d161ad..85d78c00717 100644 --- a/src/backend/ci/core/log/biz-log-sample/src/main/kotlin/com/tencent/devops/log/service/impl/V3LogPermissionService.kt +++ b/src/backend/ci/core/log/biz-log-sample/src/main/kotlin/com/tencent/devops/log/service/impl/V3LogPermissionService.kt @@ -49,7 +49,7 @@ class V3LogPermissionService @Autowired constructor( projectCode: String, pipelineId: String, userId: String, - authPermission: AuthPermission? + permission: AuthPermission? ): Boolean { logger.info("checkPipelinePermission only check action project[$projectCode]") if (isProjectOwner(projectCode, userId)) { @@ -62,7 +62,7 @@ class V3LogPermissionService @Autowired constructor( resourceType = AuthResourceType.PIPELINE_DEFAULT, projectCode = projectCode, resourceCode = pipelineId, - permission = authPermission ?: AuthPermission.VIEW, + permission = permission ?: AuthPermission.VIEW, relationResourceType = null ) } diff --git a/src/backend/ci/core/plugin/codecc-plugin/biz-codecc/src/main/kotlin/com/tencent/devops/plugin/codecc/resources/BuildCodeccResourceImpl.kt b/src/backend/ci/core/plugin/codecc-plugin/biz-codecc/src/main/kotlin/com/tencent/devops/plugin/codecc/resources/BuildCodeccResourceImpl.kt index d889fd4e1e3..c9f6d42fefd 100644 --- a/src/backend/ci/core/plugin/codecc-plugin/biz-codecc/src/main/kotlin/com/tencent/devops/plugin/codecc/resources/BuildCodeccResourceImpl.kt +++ b/src/backend/ci/core/plugin/codecc-plugin/biz-codecc/src/main/kotlin/com/tencent/devops/plugin/codecc/resources/BuildCodeccResourceImpl.kt @@ -33,7 +33,7 @@ import com.tencent.devops.common.web.RestResource import com.tencent.devops.plugin.api.BuildCodeccResource import com.tencent.devops.plugin.codecc.config.CodeccScriptConfig import com.tencent.devops.plugin.codecc.pojo.CodeccCallback -import com.tencent.devops.plugin.codecc.service.CodeccDownloaderService +import com.tencent.devops.plugin.codecc.service.CodeccToolDownloaderService import com.tencent.devops.plugin.codecc.service.CodeccService import org.springframework.beans.factory.annotation.Autowired import javax.ws.rs.core.Response @@ -41,15 +41,15 @@ import javax.ws.rs.core.Response @RestResource class BuildCodeccResourceImpl @Autowired constructor( private val codeccService: CodeccService, - private val codeccDownloaderService: CodeccDownloaderService + private val codeccToolDownloaderService: CodeccToolDownloaderService ) : BuildCodeccResource { override fun downloadTool(toolName: String, osType: OSType, fileMd5: String, is32Bit: Boolean?): Response { - return codeccDownloaderService.downloadTool(toolName, osType, fileMd5, is32Bit) + return codeccToolDownloaderService.downloadTool(toolName, osType, fileMd5, is32Bit) } override fun downloadToolsScript(osType: OSType, fileMd5: String): Response { - return codeccDownloaderService.downloadToolsScript(osType, fileMd5) + return codeccToolDownloaderService.downloadToolsScript(osType, fileMd5) } override fun queryCodeccTaskDetailUrl(projectId: String, pipelineId: String, buildId: String): String { diff --git a/src/backend/ci/core/plugin/codecc-plugin/biz-codecc/src/main/kotlin/com/tencent/devops/plugin/codecc/service/CodeccDownloaderService.kt b/src/backend/ci/core/plugin/codecc-plugin/biz-codecc/src/main/kotlin/com/tencent/devops/plugin/codecc/service/CodeccToolDownloaderService.kt similarity index 99% rename from src/backend/ci/core/plugin/codecc-plugin/biz-codecc/src/main/kotlin/com/tencent/devops/plugin/codecc/service/CodeccDownloaderService.kt rename to src/backend/ci/core/plugin/codecc-plugin/biz-codecc/src/main/kotlin/com/tencent/devops/plugin/codecc/service/CodeccToolDownloaderService.kt index 9adb17c6dd1..ce80ab892a4 100644 --- a/src/backend/ci/core/plugin/codecc-plugin/biz-codecc/src/main/kotlin/com/tencent/devops/plugin/codecc/service/CodeccDownloaderService.kt +++ b/src/backend/ci/core/plugin/codecc-plugin/biz-codecc/src/main/kotlin/com/tencent/devops/plugin/codecc/service/CodeccToolDownloaderService.kt @@ -41,7 +41,7 @@ import javax.ws.rs.core.Response import javax.ws.rs.core.StreamingOutput @Service -class CodeccDownloaderService { +class CodeccToolDownloaderService { @Value("\${plugin.codecc.path:#{null}}") private val codeccPath: String? = null diff --git a/src/backend/ci/core/store/biz-store-image/src/main/kotlin/com/tencent/devops/store/resources/image/service/ServiceImageResourceImpl.kt b/src/backend/ci/core/store/biz-store-image/src/main/kotlin/com/tencent/devops/store/resources/image/service/ServiceStoreImageResourceImpl.kt similarity index 98% rename from src/backend/ci/core/store/biz-store-image/src/main/kotlin/com/tencent/devops/store/resources/image/service/ServiceImageResourceImpl.kt rename to src/backend/ci/core/store/biz-store-image/src/main/kotlin/com/tencent/devops/store/resources/image/service/ServiceStoreImageResourceImpl.kt index 8c8b25c49e6..65e34c974e7 100644 --- a/src/backend/ci/core/store/biz-store-image/src/main/kotlin/com/tencent/devops/store/resources/image/service/ServiceImageResourceImpl.kt +++ b/src/backend/ci/core/store/biz-store-image/src/main/kotlin/com/tencent/devops/store/resources/image/service/ServiceStoreImageResourceImpl.kt @@ -37,7 +37,7 @@ import com.tencent.devops.store.service.image.ImageService import org.springframework.beans.factory.annotation.Autowired @RestResource -class ServiceImageResourceImpl @Autowired constructor( +class ServiceStoreImageResourceImpl @Autowired constructor( private val imageService: ImageService, private val imageFeatureService: ImageFeatureService, private val storeProjectService: StoreProjectService From f5c445023a2b85073e16f35d29d4bafde12a695c Mon Sep 17 00:00:00 2001 From: irwinsun Date: Mon, 21 Jun 2021 17:48:47 +0800 Subject: [PATCH 07/20] =?UTF-8?q?bug:=20assembly=20=E5=90=AF=E5=8A=A8?= =?UTF-8?q?=E5=A4=B1=E8=B4=A5[1.5.4]=20#4475=20=E9=87=8D=E5=90=8D=E5=A4=84?= =?UTF-8?q?=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../{GroupDao.kt => QualityNotifyGroupDao.kt} | 5 ++-- .../resources/UserGroupResourceImpl.kt | 18 +++++------ ...ervice.kt => QualityNotifyGroupService.kt} | 30 +++++++++---------- .../service/v2/QualityRuleCheckService.kt | 6 ++-- 4 files changed, 30 insertions(+), 29 deletions(-) rename src/backend/ci/core/quality/biz-quality/src/main/kotlin/com/tencent/devops/quality/dao/{GroupDao.kt => QualityNotifyGroupDao.kt} (99%) rename src/backend/ci/core/quality/biz-quality/src/main/kotlin/com/tencent/devops/quality/service/{GroupService.kt => QualityNotifyGroupService.kt} (90%) diff --git a/src/backend/ci/core/quality/biz-quality/src/main/kotlin/com/tencent/devops/quality/dao/GroupDao.kt b/src/backend/ci/core/quality/biz-quality/src/main/kotlin/com/tencent/devops/quality/dao/QualityNotifyGroupDao.kt similarity index 99% rename from src/backend/ci/core/quality/biz-quality/src/main/kotlin/com/tencent/devops/quality/dao/GroupDao.kt rename to src/backend/ci/core/quality/biz-quality/src/main/kotlin/com/tencent/devops/quality/dao/QualityNotifyGroupDao.kt index a3f8a71e368..b1c4b3bfd16 100644 --- a/src/backend/ci/core/quality/biz-quality/src/main/kotlin/com/tencent/devops/quality/dao/GroupDao.kt +++ b/src/backend/ci/core/quality/biz-quality/src/main/kotlin/com/tencent/devops/quality/dao/QualityNotifyGroupDao.kt @@ -35,8 +35,9 @@ import org.springframework.stereotype.Repository import java.time.LocalDateTime import javax.ws.rs.NotFoundException -@Repository@Suppress("ALL") -class GroupDao { +@Repository +@Suppress("ALL") +class QualityNotifyGroupDao { fun list(dslContext: DSLContext, projectId: String, offset: Int, limit: Int): Result { with(TGroup.T_GROUP) { diff --git a/src/backend/ci/core/quality/biz-quality/src/main/kotlin/com/tencent/devops/quality/resources/UserGroupResourceImpl.kt b/src/backend/ci/core/quality/biz-quality/src/main/kotlin/com/tencent/devops/quality/resources/UserGroupResourceImpl.kt index 5c64ebe5359..0a775ba1b2a 100644 --- a/src/backend/ci/core/quality/biz-quality/src/main/kotlin/com/tencent/devops/quality/resources/UserGroupResourceImpl.kt +++ b/src/backend/ci/core/quality/biz-quality/src/main/kotlin/com/tencent/devops/quality/resources/UserGroupResourceImpl.kt @@ -39,12 +39,12 @@ import com.tencent.devops.quality.pojo.GroupSummaryWithPermission import com.tencent.devops.quality.pojo.GroupUpdate import com.tencent.devops.quality.pojo.GroupUsers import com.tencent.devops.quality.pojo.ProjectGroupAndUsers -import com.tencent.devops.quality.service.GroupService +import com.tencent.devops.quality.service.QualityNotifyGroupService import org.springframework.beans.factory.annotation.Autowired @RestResource class UserGroupResourceImpl @Autowired constructor( - private val groupService: GroupService + private val qualityNotifyGroupService: QualityNotifyGroupService ) : UserGroupResource { override fun list( userId: String, @@ -56,40 +56,40 @@ class UserGroupResourceImpl @Autowired constructor( val pageNotNull = page ?: 0 val pageSizeNotNull = pageSize ?: 20 val limit = PageUtil.convertPageSizeToSQLLimit(pageNotNull, pageSizeNotNull) - val result = groupService.list(userId, projectId, limit.offset, limit.limit) + val result = qualityNotifyGroupService.list(userId, projectId, limit.offset, limit.limit) return Result(Page(pageNotNull, pageSizeNotNull, result.first, result.second)) } override fun projectGroupAndUsers(userId: String, projectId: String): Result> { checkParam(userId, projectId) - return Result(groupService.getProjectGroupAndUsers(userId, projectId)) + return Result(qualityNotifyGroupService.getProjectGroupAndUsers(userId, projectId)) } override fun create(userId: String, projectId: String, group: GroupCreate): Result { checkParam(userId, projectId) - groupService.create(userId, projectId, group) + qualityNotifyGroupService.create(userId, projectId, group) return Result(true) } override fun get(userId: String, projectId: String, groupHashId: String): Result { checkParam(userId, projectId, groupHashId) - return Result(groupService.get(userId, projectId, groupHashId)) + return Result(qualityNotifyGroupService.get(userId, projectId, groupHashId)) } override fun getUsers(userId: String, projectId: String, groupHashId: String): Result { checkParam(userId, projectId, groupHashId) - return Result(groupService.getUsers(userId, projectId, groupHashId)) + return Result(qualityNotifyGroupService.getUsers(userId, projectId, groupHashId)) } override fun edit(userId: String, projectId: String, groupHashId: String, group: GroupUpdate): Result { checkParam(userId, projectId, groupHashId) - groupService.edit(userId, projectId, groupHashId, group) + qualityNotifyGroupService.edit(userId, projectId, groupHashId, group) return Result(true) } override fun delete(userId: String, projectId: String, groupHashId: String): Result { checkParam(userId, projectId, groupHashId) - groupService.delete(userId, projectId, groupHashId) + qualityNotifyGroupService.delete(userId, projectId, groupHashId) return Result(true) } diff --git a/src/backend/ci/core/quality/biz-quality/src/main/kotlin/com/tencent/devops/quality/service/GroupService.kt b/src/backend/ci/core/quality/biz-quality/src/main/kotlin/com/tencent/devops/quality/service/QualityNotifyGroupService.kt similarity index 90% rename from src/backend/ci/core/quality/biz-quality/src/main/kotlin/com/tencent/devops/quality/service/GroupService.kt rename to src/backend/ci/core/quality/biz-quality/src/main/kotlin/com/tencent/devops/quality/service/QualityNotifyGroupService.kt index adc2486baf6..b01f588f441 100644 --- a/src/backend/ci/core/quality/biz-quality/src/main/kotlin/com/tencent/devops/quality/service/GroupService.kt +++ b/src/backend/ci/core/quality/biz-quality/src/main/kotlin/com/tencent/devops/quality/service/QualityNotifyGroupService.kt @@ -38,7 +38,7 @@ import com.tencent.devops.common.auth.api.AuthResourceType import com.tencent.devops.common.auth.code.QualityAuthServiceCode import com.tencent.devops.common.service.utils.MessageCodeUtil import com.tencent.devops.quality.constant.QualityMessageCode -import com.tencent.devops.quality.dao.GroupDao +import com.tencent.devops.quality.dao.QualityNotifyGroupDao import com.tencent.devops.quality.pojo.Group import com.tencent.devops.quality.pojo.GroupCreate import com.tencent.devops.quality.pojo.GroupPermission @@ -54,10 +54,10 @@ import java.util.regex.Pattern import javax.ws.rs.core.Response @Service -class GroupService @Autowired constructor( +class QualityNotifyGroupService @Autowired constructor( private val dslContext: DSLContext, private val objectMapper: ObjectMapper, - private val groupDao: GroupDao, + private val qualityNotifyGroupDao: QualityNotifyGroupDao, private val bkAuthProjectApi: AuthProjectApi, private val serviceCode: QualityAuthServiceCode, private val qualityPermissionService: QualityPermissionService @@ -73,9 +73,9 @@ class GroupService @Autowired constructor( authPermissions = setOf(AuthPermission.EDIT, AuthPermission.DELETE) ) - val count = groupDao.count(dslContext, projectId) + val count = qualityNotifyGroupDao.count(dslContext, projectId) val finalLimit = if (limit == -1) count.toInt() else limit - val list = groupDao.list(dslContext, projectId, offset, finalLimit).map { + val list = qualityNotifyGroupDao.list(dslContext, projectId, offset, finalLimit).map { val canEdit = groupPermissionListMap[AuthPermission.EDIT]!!.contains(it.id) val canDelete = groupPermissionListMap[AuthPermission.DELETE]!!.contains(it.id) GroupSummaryWithPermission( @@ -108,7 +108,7 @@ class GroupService @Autowired constructor( } fun create(userId: String, projectId: String, group: GroupCreate) { - if (groupDao.has(dslContext, projectId, group.name)) { + if (qualityNotifyGroupDao.has(dslContext, projectId, group.name)) { throw ErrorCodeException( statusCode = Response.Status.BAD_REQUEST.statusCode, errorCode = QualityMessageCode.USER_GROUP_IS_EXISTS, @@ -121,7 +121,7 @@ class GroupService @Autowired constructor( val outerUsersCount = outerUsers.filter { it.isNotBlank() && it.isNotEmpty() }.size val innerUsersCount = group.innerUsers.size - val groupId = groupDao.create( + val groupId = qualityNotifyGroupDao.create( dslContext = dslContext, projectId = projectId, name = group.name, @@ -142,7 +142,7 @@ class GroupService @Autowired constructor( fun serviceGet(groupHashId: String): Group { val groupId = HashUtil.decodeIdToLong(groupHashId) - val groupRecord = groupDao.get(dslContext, groupId) + val groupRecord = qualityNotifyGroupDao.get(dslContext, groupId) return Group( groupHashId = groupHashId, name = groupRecord.name, @@ -158,14 +158,14 @@ class GroupService @Autowired constructor( fun serviceGetUsers(groupHashId: String): GroupUsers { val groupId = HashUtil.decodeIdToLong(groupHashId) - val groupRecord = groupDao.get(dslContext, groupId) + val groupRecord = qualityNotifyGroupDao.get(dslContext, groupId) val innerUsers = objectMapper.readValue>(groupRecord.innerUsers) val outerUsers = regex.split(groupRecord.outerUsers).toSet() return GroupUsers(innerUsers, outerUsers) } fun serviceGetUsers(groupIdList: Collection): GroupUsers { - val result = groupDao.list(dslContext, groupIdList) + val result = qualityNotifyGroupDao.list(dslContext, groupIdList) val innerUsersSet = mutableSetOf() val outerUsersSet = mutableSetOf() @@ -187,7 +187,7 @@ class GroupService @Autowired constructor( authPermission = AuthPermission.EDIT, message = "用户没有用户组的编辑权限" ) - if (groupDao.getOrNull(dslContext, groupId) == null) { + if (qualityNotifyGroupDao.getOrNull(dslContext, groupId) == null) { throw ErrorCodeException( statusCode = Response.Status.NOT_FOUND.statusCode, errorCode = QualityMessageCode.USER_GROUP_NOT_EXISTS, @@ -195,7 +195,7 @@ class GroupService @Autowired constructor( params = arrayOf(groupHashId) ) } - if (groupDao.has(dslContext, projectId, group.name, groupId)) { + if (qualityNotifyGroupDao.has(dslContext, projectId, group.name, groupId)) { throw ErrorCodeException( statusCode = Response.Status.BAD_REQUEST.statusCode, errorCode = QualityMessageCode.USER_GROUP_IS_EXISTS, @@ -208,7 +208,7 @@ class GroupService @Autowired constructor( val outerUsersCount = outerUsers.filter { it.isNotBlank() && it.isNotEmpty() }.size val innerUsersCount = group.innerUsers.size - groupDao.update( + qualityNotifyGroupDao.update( dslContext = dslContext, id = groupId, name = group.name, @@ -233,10 +233,10 @@ class GroupService @Autowired constructor( ) qualityPermissionService.deleteGroupResource(projectId = projectId, groupId = groupId) - groupDao.delete(dslContext, groupId) + qualityNotifyGroupDao.delete(dslContext, groupId) } companion object { - private val logger = LoggerFactory.getLogger(GroupService::class.java) + private val logger = LoggerFactory.getLogger(QualityNotifyGroupService::class.java) } } diff --git a/src/backend/ci/core/quality/biz-quality/src/main/kotlin/com/tencent/devops/quality/service/v2/QualityRuleCheckService.kt b/src/backend/ci/core/quality/biz-quality/src/main/kotlin/com/tencent/devops/quality/service/v2/QualityRuleCheckService.kt index 411b379e778..0d49a98cd94 100644 --- a/src/backend/ci/core/quality/biz-quality/src/main/kotlin/com/tencent/devops/quality/service/v2/QualityRuleCheckService.kt +++ b/src/backend/ci/core/quality/biz-quality/src/main/kotlin/com/tencent/devops/quality/service/v2/QualityRuleCheckService.kt @@ -55,7 +55,7 @@ import com.tencent.devops.quality.pojo.RuleCheckResult import com.tencent.devops.quality.pojo.RuleCheckSingleResult import com.tencent.devops.quality.pojo.enum.RuleInterceptResult import com.tencent.devops.quality.pojo.enum.RuleOperation -import com.tencent.devops.quality.service.GroupService +import com.tencent.devops.quality.service.QualityNotifyGroupService import com.tencent.devops.quality.util.ThresholdOperationUtil import org.apache.commons.lang3.math.NumberUtils import org.slf4j.LoggerFactory @@ -72,7 +72,7 @@ import java.util.concurrent.Executors class QualityRuleCheckService @Autowired constructor( private val ruleService: QualityRuleService, private val qualityHisMetadataService: QualityHisMetadataService, - private val groupService: GroupService, + private val qualityNotifyGroupService: QualityNotifyGroupService, private val countService: QualityCountService, private val historyService: QualityHistoryService, private val controlPointService: QualityControlPointService, @@ -571,7 +571,7 @@ class QualityRuleCheckService @Autowired constructor( // 获取通知用户集合 val notifyUserSet = mutableSetOf() - val groupUsers = groupService.serviceGetUsers(endNotifyGroupList.map { it.toLong() }) + val groupUsers = qualityNotifyGroupService.serviceGetUsers(endNotifyGroupList.map { it.toLong() }) notifyUserSet.addAll(groupUsers.innerUsers) notifyUserSet.addAll(endNotifyUserList) From 35e7f0e62bc99561b19798926458dfcc097b3410 Mon Sep 17 00:00:00 2001 From: irwinsun Date: Mon, 21 Jun 2021 18:24:24 +0800 Subject: [PATCH 08/20] =?UTF-8?q?bug:=20assembly=20=E5=90=AF=E5=8A=A8?= =?UTF-8?q?=E5=A4=B1=E8=B4=A5[1.5.4]=20#4475=E6=AF=81=20v3=E6=9D=83?= =?UTF-8?q?=E9=99=90=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../tencent/devops/auth/service/IamService.kt | 41 ++++++++++--------- 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/src/backend/ci/core/auth/biz-auth-blueking/src/main/kotlin/com/tencent/devops/auth/service/IamService.kt b/src/backend/ci/core/auth/biz-auth-blueking/src/main/kotlin/com/tencent/devops/auth/service/IamService.kt index c334e8cb2bb..1ab618a2c63 100644 --- a/src/backend/ci/core/auth/biz-auth-blueking/src/main/kotlin/com/tencent/devops/auth/service/IamService.kt +++ b/src/backend/ci/core/auth/biz-auth-blueking/src/main/kotlin/com/tencent/devops/auth/service/IamService.kt @@ -44,7 +44,8 @@ import org.springframework.stereotype.Service @Service class IamService @Autowired constructor( val iamEsbService: IamEsbService, - val iamConfiguration: IamConfiguration + @Autowired(required = false) // v3 才会有 + val iamConfiguration: IamConfiguration? ) { fun getPermissionUrl(permissionUrlDTO: List): Result { logger.info("get permissionUrl permissionUrlDTO: $permissionUrlDTO") @@ -60,15 +61,15 @@ class IamService @Autowired constructor( if (instanceList == null || instanceList.isEmpty()) { relatedResourceTypes.add(RelatedResourceTypes( - system = iamConfiguration.systemId, - type = relatedResourceType, - instances = emptyList() + system = iamConfiguration!!.systemId, + type = relatedResourceType, + instances = emptyList() )) } else { relatedResourceTypes.add(RelatedResourceTypes( - system = iamConfiguration.systemId, - type = relatedResourceType, - instances = listOf(instanceList)) + system = iamConfiguration!!.systemId, + type = relatedResourceType, + instances = listOf(instanceList)) ) } @@ -76,26 +77,26 @@ class IamService @Autowired constructor( if (it.resourceId == AuthResourceType.PROJECT && it.actionId == AuthPermission.CREATE) { logger.info("projectCreate ${it.actionId} |${it.instanceId}| ${it.resourceId}") actions.add( - Action( - id = resourceType, - related_resource_types = emptyList() - ) + Action( + id = resourceType, + related_resource_types = emptyList() + ) ) } else { actions.add( - Action( - id = resourceType, - related_resource_types = relatedResourceTypes - ) + Action( + id = resourceType, + related_resource_types = relatedResourceTypes + ) ) } } val iamEsbReq = IamPermissionUrlReq( - system = iamConfiguration.systemId, - actions = actions, - bk_app_code = "", - bk_app_secret = "", - bk_username = "admin" + system = iamConfiguration!!.systemId, + actions = actions, + bk_app_code = "", + bk_app_secret = "", + bk_username = "admin" ) logger.info("get permissionUrl iamEsbReq: $iamEsbReq") return Result(iamEsbService.getPermissionUrl(iamEsbReq)) From 3d36c800a1a4612bacf19c01fe759f2473962c08 Mon Sep 17 00:00:00 2001 From: irwinsun Date: Mon, 21 Jun 2021 19:47:01 +0800 Subject: [PATCH 09/20] =?UTF-8?q?bug:=20assembly=20=E5=90=AF=E5=8A=A8?= =?UTF-8?q?=E5=A4=B1=E8=B4=A5[1.5.4]=20#4475=20=E9=85=8D=E7=BD=AE=E9=A1=B9?= =?UTF-8?q?=E7=BC=BA=E5=A4=B1=E5=A4=84=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/backend/ci/boot-assembly/build.gradle.kts | 20 ++++++++++++++----- .../websocket/boot-websocket/build.gradle.kts | 1 - .../#etc#ci#application-artifactory.yml | 1 + .../#etc#ci#application-assembly.yml | 1 + 4 files changed, 17 insertions(+), 6 deletions(-) diff --git a/src/backend/ci/boot-assembly/build.gradle.kts b/src/backend/ci/boot-assembly/build.gradle.kts index a469d506cc8..687a133c752 100644 --- a/src/backend/ci/boot-assembly/build.gradle.kts +++ b/src/backend/ci/boot-assembly/build.gradle.kts @@ -32,25 +32,35 @@ dependencies { testImplementation("junit:junit") testImplementation("org.springframework.boot:spring-boot-starter-test") + implementation(project(":core:common:common-auth:common-auth-mock")) + implementation(project(":core:common:common-auth:common-auth-blueking")) + implementation(project(":core:common:common-auth:common-auth-v3")) + implementation(project(":core:common:common-archive")) + implementation(project(":core:artifactory:biz-artifactory-store")) implementation(project(":core:artifactory:biz-artifactory-sample")) + implementation(project(":core:auth:biz-auth-blueking")) implementation(project(":core:dispatch:biz-dispatch")) - implementation(project(":core:dispatch-docker:biz-dispatch-docker")) + implementation(project(":core:dispatch-docker:biz-dispatch-docker-sample")) implementation(project(":core:environment:biz-environment-sample")) implementation(project(":core:image:biz-image")) - implementation(project(":core:log:biz-log")) - implementation(project(":core:misc:biz-misc")) + implementation(project(":core:log:biz-log-sample")) + implementation(project(":core:misc:biz-misc-sample")) implementation(project(":core:notify:biz-notify-blueking")) implementation(project(":core:openapi:biz-openapi")) implementation(project(":core:plugin:biz-plugin")) + implementation(project(":core:plugin:codecc-plugin:biz-codecc")) + implementation(project(":core:process:plugin-load")) + implementation(project(":core:process:plugin-trigger")) implementation(project(":core:process:biz-engine")) implementation(project(":core:process:biz-process-sample")) implementation(project(":core:project:biz-project-sample")) - implementation(project(":core:quality:biz-quality")) + implementation(project(":core:quality:biz-quality-sample")) implementation(project(":core:repository:biz-repository-sample")) implementation(project(":core:store:biz-store-sample")) + implementation(project(":core:store:biz-store-image-sample")) implementation(project(":core:ticket:biz-ticket-sample")) - implementation(project(":core:websocket:biz-websocket")) + implementation(project(":core:websocket:biz-websocket-blueking")) } plugins { diff --git a/src/backend/ci/core/websocket/boot-websocket/build.gradle.kts b/src/backend/ci/core/websocket/boot-websocket/build.gradle.kts index bc78c00d38a..ff7fa0b2665 100644 --- a/src/backend/ci/core/websocket/boot-websocket/build.gradle.kts +++ b/src/backend/ci/core/websocket/boot-websocket/build.gradle.kts @@ -30,7 +30,6 @@ dependencies { api(project(":core:websocket:biz-websocket-blueking")) } - plugins { `task-spring-boot-package` } diff --git a/support-files/templates/#etc#ci#application-artifactory.yml b/support-files/templates/#etc#ci#application-artifactory.yml index 46248c24867..636ad9460bf 100644 --- a/support-files/templates/#etc#ci#application-artifactory.yml +++ b/support-files/templates/#etc#ci#application-artifactory.yml @@ -15,6 +15,7 @@ server: # fileTask为文件托管任务配置，主要用于作品平台构件分发插件， # 可配置文件暂存目录、临时文件过期时间、数据库记录清理策略 artifactory: + realm: local # 如果使用蓝鲸制品库，则该值要修改为 bkrepo archiveLocalBasePath: __BK_CI_DATA_DIR__/artifactory # 如果多节点部署, 需要使用共享存储. fileTask: savedir: __BK_CI_DATA_DIR__/artifactory-filetask/ # 不一定共享, 但是建议准备较大容量. diff --git a/support-files/templates/#etc#ci#application-assembly.yml b/support-files/templates/#etc#ci#application-assembly.yml index 5cb956c7533..ab91ba2d1aa 100644 --- a/support-files/templates/#etc#ci#application-assembly.yml +++ b/support-files/templates/#etc#ci#application-assembly.yml @@ -45,6 +45,7 @@ server: # 如果微服务是部署多机节点，则以下目录需要做成分布式高可用的，比如NFS，CephFS挂载 # 保证多节点都能读取到 artifactory: + realm: local # 如果使用蓝鲸制品库，则该值要修改为 bkrepo archiveLocalBasePath: __BK_CI_DATA_DIR__/artifactory # 如果多节点部署, 需要使用共享存储. fileTask: savedir: __BK_CI_DATA_DIR__/artifactory-filetask/ # 不一定共享, 但是建议准备较大容量. From 46becf47f952658755f021b951d549f9bc847d2d Mon Sep 17 00:00:00 2001 From: irwinsun Date: Mon, 21 Jun 2021 20:52:00 +0800 Subject: [PATCH 10/20] =?UTF-8?q?bug:=20assembly=20=E5=90=AF=E5=8A=A8?= =?UTF-8?q?=E5=A4=B1=E8=B4=A5[1.5.4]=20#4475=20=E8=A7=A3=E5=86=B3=E5=A4=9A?= =?UTF-8?q?=E6=95=B0=E6=8D=AE=E6=BA=90=E9=97=AE=E9=A2=98=E5=AF=BC=E8=87=B4?= =?UTF-8?q?=E9=BB=98=E8=AE=A4=E6=95=B0=E6=8D=AE=E6=BA=90=E4=B8=A2=E5=A4=B1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../devops/misc/config/JooqConfiguration.kt | 22 +++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/src/backend/ci/core/misc/biz-misc/src/main/kotlin/com/tencent/devops/misc/config/JooqConfiguration.kt b/src/backend/ci/core/misc/biz-misc/src/main/kotlin/com/tencent/devops/misc/config/JooqConfiguration.kt index 368c65e42fd..156bc536d4b 100644 --- a/src/backend/ci/core/misc/biz-misc/src/main/kotlin/com/tencent/devops/misc/config/JooqConfiguration.kt +++ b/src/backend/ci/core/misc/biz-misc/src/main/kotlin/com/tencent/devops/misc/config/JooqConfiguration.kt @@ -34,6 +34,7 @@ import org.jooq.impl.DefaultConfiguration import org.slf4j.LoggerFactory import org.springframework.beans.factory.InjectionPoint import org.springframework.beans.factory.NoSuchBeanDefinitionException +import org.springframework.beans.factory.annotation.Autowired import org.springframework.beans.factory.annotation.Qualifier import org.springframework.beans.factory.config.ConfigurableBeanFactory import org.springframework.context.annotation.Bean @@ -72,13 +73,30 @@ class JooqConfiguration { val declaringClass: Class<*> = (annotatedElement as Constructor<*>).declaringClass val packageName = declaringClass.getPackage().name val matchResult = regex.find(packageName) - ?: throw NoSuchBeanDefinitionException("no jooq configuration") + ?: return defaultContext(configurationMap["defaultJooqConfiguration"]!!) val configuration = configurationMap["${matchResult.groupValues[1]}JooqConfiguration"] ?: throw NoSuchBeanDefinitionException("no ${matchResult.groupValues[1]}JooqConfiguration") LOG.info("dslContext_init|${matchResult.groupValues[1]}JooqConfiguration|${declaringClass.name}") return DSL.using(configuration) } - throw NoSuchBeanDefinitionException("no jooq configuration") + return defaultContext(configurationMap["defaultJooqConfiguration"]!!) + } + + private fun defaultContext(defaultConfiguration: DefaultConfiguration): DSLContext { + return DSL.using(defaultConfiguration) + } + + @Bean + @Autowired(required = false) + fun defaultJooqConfiguration( + @Qualifier("dataSource") + @Autowired(required = false) + dataSource: DataSource? + ): DefaultConfiguration { + val configuration = DefaultConfiguration() + configuration.set(SQLDialect.MYSQL) + configuration.set(dataSource) + return configuration } @Bean From 87d84565172c9dbb789681ace843bafa479fdc40 Mon Sep 17 00:00:00 2001 From: irwinsun Date: Tue, 22 Jun 2021 10:07:32 +0800 Subject: [PATCH 11/20] =?UTF-8?q?bug:=20assembly=20=E5=90=AF=E5=8A=A8?= =?UTF-8?q?=E5=A4=B1=E8=B4=A5[1.5.4]=20#4475=20=E8=A7=A3=E5=86=B3IoC?= =?UTF-8?q?=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../common/auth/BkAuthAutoConfiguration.kt | 5 +++ .../devops/common/auth/api/BkAuthUserApi.kt | 3 +- .../devops/common/auth/api/utils/AuthUtils.kt | 38 ++++++++----------- 3 files changed, 21 insertions(+), 25 deletions(-) diff --git a/src/backend/ci/core/common/common-auth/common-auth-blueking/src/main/kotlin/com/tencent/devops/common/auth/BkAuthAutoConfiguration.kt b/src/backend/ci/core/common/common-auth/common-auth-blueking/src/main/kotlin/com/tencent/devops/common/auth/BkAuthAutoConfiguration.kt index 586157439a4..7ce7e3f199b 100644 --- a/src/backend/ci/core/common/common-auth/common-auth-blueking/src/main/kotlin/com/tencent/devops/common/auth/BkAuthAutoConfiguration.kt +++ b/src/backend/ci/core/common/common-auth/common-auth-blueking/src/main/kotlin/com/tencent/devops/common/auth/BkAuthAutoConfiguration.kt @@ -33,6 +33,7 @@ import com.tencent.devops.common.auth.api.BkAuthProjectApi import com.tencent.devops.common.auth.api.BkAuthProperties import com.tencent.devops.common.auth.api.BkAuthResourceApi import com.tencent.devops.common.auth.api.BkAuthTokenApi +import com.tencent.devops.common.auth.api.BkAuthUserApi import com.tencent.devops.common.auth.api.utils.AuthUtils import com.tencent.devops.common.auth.code.BkArtifactoryAuthServiceCode import com.tencent.devops.common.auth.code.BkBcsAuthServiceCode @@ -75,6 +76,10 @@ class BkAuthAutoConfiguration { authProperties.bkloginUrl ) + @Bean + @Primary + fun bkAuthUserApi(objectMapper: ObjectMapper, authUtils: AuthUtils) = BkAuthUserApi(objectMapper, authUtils) + @Bean @Primary fun bkAuthTokenApi(bkAuthProperties: BkAuthProperties, objectMapper: ObjectMapper, redisOperation: RedisOperation) = diff --git a/src/backend/ci/core/common/common-auth/common-auth-blueking/src/main/kotlin/com/tencent/devops/common/auth/api/BkAuthUserApi.kt b/src/backend/ci/core/common/common-auth/common-auth-blueking/src/main/kotlin/com/tencent/devops/common/auth/api/BkAuthUserApi.kt index 5af27907d54..c513f5f3beb 100644 --- a/src/backend/ci/core/common/common-auth/common-auth-blueking/src/main/kotlin/com/tencent/devops/common/auth/api/BkAuthUserApi.kt +++ b/src/backend/ci/core/common/common-auth/common-auth-blueking/src/main/kotlin/com/tencent/devops/common/auth/api/BkAuthUserApi.kt @@ -37,9 +37,8 @@ import com.tencent.devops.common.auth.api.utils.AuthUtils import org.json.JSONObject import org.slf4j.LoggerFactory import org.springframework.beans.factory.annotation.Autowired -import org.springframework.stereotype.Component -@Component@Suppress("ALL") +@Suppress("ALL", "UNUSED") class BkAuthUserApi @Autowired constructor( private val objectMapper: ObjectMapper, private val authUtils: AuthUtils diff --git a/src/backend/ci/core/common/common-auth/common-auth-blueking/src/main/kotlin/com/tencent/devops/common/auth/api/utils/AuthUtils.kt b/src/backend/ci/core/common/common-auth/common-auth-blueking/src/main/kotlin/com/tencent/devops/common/auth/api/utils/AuthUtils.kt index 0a4eb664150..e2dda0db6e6 100644 --- a/src/backend/ci/core/common/common-auth/common-auth-blueking/src/main/kotlin/com/tencent/devops/common/auth/api/utils/AuthUtils.kt +++ b/src/backend/ci/core/common/common-auth/common-auth-blueking/src/main/kotlin/com/tencent/devops/common/auth/api/utils/AuthUtils.kt @@ -37,15 +37,10 @@ import org.json.JSONArray import org.json.JSONException import org.json.JSONObject import org.slf4j.LoggerFactory -import org.springframework.stereotype.Component import java.io.IOException -import java.util.Comparator import java.util.TreeMap -@Component -class AuthUtils constructor( - xBkAuthProperties: BkAuthProperties -) { +class AuthUtils constructor(xBkAuthProperties: BkAuthProperties) { private val logger = LoggerFactory.getLogger(AuthUtils::class.java) private val host = xBkAuthProperties.url!! @@ -54,7 +49,7 @@ class AuthUtils constructor( */ fun doAuthGetRequest(uri: String, params: Map?, xBkAppCode: String, xBkAppSecret: String): JSONObject { val url = this.getAuthRequestUrl(uri, params) - logger.debug("bkiam get request: $url") + logger.debug("bkiam get url: {}", url) val request = Request.Builder() .url(url) @@ -62,7 +57,7 @@ class AuthUtils constructor( .addHeader("X-BK-APP-SECRET", xBkAppSecret) .build() val result = this.doRequest(request) - logger.debug("bkiam get request result: $result") + logger.debug("bkiam get request result: {}", result) return result } @@ -72,8 +67,7 @@ class AuthUtils constructor( fun doAuthPostRequest(uri: String, jsonbody: JSONObject, bkAppCode: String, bkAppSecret: String): JSONObject { val body = RequestBody.create(MediaType.parse("application/json"), jsonbody.toString()) val url = this.getAuthRequestUrl(uri, null) - logger.debug("bkiam post url: $url") - logger.debug("bkiam post body: $jsonbody") + logger.debug("bkiam post url: {}, body: {}", url, jsonbody) val request = Request.Builder() .url(url) @@ -82,7 +76,7 @@ class AuthUtils constructor( .addHeader("X-BK-APP-SECRET", bkAppSecret) .build() val result = this.doRequest(request) - logger.debug("bkiam post request result: $result") + logger.debug("bkiam post request result: {}", result) return result } @@ -92,8 +86,7 @@ class AuthUtils constructor( fun doAuthPutRequest(uri: String, jsonbody: JSONObject, xBkAppCode: String, xBkAppSecret: String): JSONObject { val body = RequestBody.create(MediaType.parse("application/json"), jsonbody.toString()) val url = this.getAuthRequestUrl(uri, null) - logger.debug("bkiam put url: $url") - logger.debug("bkiam put body: $jsonbody") + logger.debug("bkiam put url: {}, body: {}", url, jsonbody) val request = Request.Builder() .url(url) @@ -102,7 +95,7 @@ class AuthUtils constructor( .addHeader("X-BK-APP-SECRET", xBkAppSecret) .build() val result = this.doRequest(request) - logger.debug("bkiam put request result: $result") + logger.debug("bkiam put request result: {}", result) return result } @@ -112,7 +105,7 @@ class AuthUtils constructor( fun doAuthDeleteRequest(uri: String, jsonbody: JSONObject, xBkAppCode: String, xBkAppSecret: String): JSONObject { val body = RequestBody.create(MediaType.parse("application/json"), jsonbody.toString()) val url = this.getAuthRequestUrl(uri, null) - logger.debug("bkiam delete request: $url") + logger.debug("bkiam delete url: {}", url) val request = Request.Builder() .url(url) @@ -121,7 +114,7 @@ class AuthUtils constructor( .addHeader("X-BK-APP-SECRET", xBkAppSecret) .build() val result = this.doRequest(request) - logger.debug("bkiam delete request result: $result") + logger.debug("bkiam delete request result: {}", result) return result } @@ -151,12 +144,11 @@ class AuthUtils constructor( val msg = jsonObject.getString("message") jsonObject.put("message", msg) jsonObject.put("code", responseCode) - logger.error("bkiam failed , message: $msg") - } catch (e: IOException) { - e.printStackTrace() - jsonObject.put("msg", "IO exception, network not ok") - } catch (e: JSONException) { - jsonObject.put("msg", "json parse error, bkiam response json format failed") + logger.warn("bkiam failed , message: $msg") + } catch (ioe: IOException) { + jsonObject.put("msg", "IO exception, network not ok: ${ioe.message}") + } catch (je: JSONException) { + jsonObject.put("msg", "json parse error, bkiam response json format failed: ${je.message}") } return jsonObject } @@ -181,7 +173,7 @@ class AuthUtils constructor( } else { notfirst = true } - sb.append(String.format("%s=%s", key, value.toString())) + sb.append("$key=$value") } return host + sb.toString() } From b20f30ec1a879fea683c3a2a074d1f3e9e8a4b7d Mon Sep 17 00:00:00 2001 From: irwinsun Date: Tue, 22 Jun 2021 11:04:46 +0800 Subject: [PATCH 12/20] =?UTF-8?q?bug:=20assembly=20=E5=90=AF=E5=8A=A8?= =?UTF-8?q?=E5=A4=B1=E8=B4=A5[1.5.4]=20#4475=20=E8=A7=A3=E5=86=B3=E9=BB=98?= =?UTF-8?q?=E8=AE=A4datasource=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/tencent/devops/AssemblyApplication.kt | 19 +++++++++++- .../devops/misc/config/JooqConfiguration.kt | 30 +++++-------------- 2 files changed, 25 insertions(+), 24 deletions(-) diff --git a/src/backend/ci/boot-assembly/src/main/kotlin/com/tencent/devops/AssemblyApplication.kt b/src/backend/ci/boot-assembly/src/main/kotlin/com/tencent/devops/AssemblyApplication.kt index 520e22bab2b..adcd3ea5002 100644 --- a/src/backend/ci/boot-assembly/src/main/kotlin/com/tencent/devops/AssemblyApplication.kt +++ b/src/backend/ci/boot-assembly/src/main/kotlin/com/tencent/devops/AssemblyApplication.kt @@ -29,11 +29,28 @@ package com.tencent.devops import com.tencent.devops.common.service.MicroService import com.tencent.devops.common.service.MicroServiceApplication +import org.jooq.SQLDialect +import org.jooq.impl.DefaultConfiguration +import org.springframework.beans.factory.annotation.Qualifier +import org.springframework.context.annotation.Bean import org.springframework.context.annotation.ComponentScan +import javax.sql.DataSource @MicroService @ComponentScan("com.tencent.devops") -class AssemblyApplication +class AssemblyApplication { + + @Bean + fun defaultJooqConfiguration( + @Qualifier("dataSource") + dataSource: DataSource + ): DefaultConfiguration { + val configuration = DefaultConfiguration() + configuration.set(SQLDialect.MYSQL) + configuration.set(dataSource) + return configuration + } +} fun main(args: Array) { MicroServiceApplication.run(AssemblyApplication::class, args) diff --git a/src/backend/ci/core/misc/biz-misc/src/main/kotlin/com/tencent/devops/misc/config/JooqConfiguration.kt b/src/backend/ci/core/misc/biz-misc/src/main/kotlin/com/tencent/devops/misc/config/JooqConfiguration.kt index 156bc536d4b..37916416e61 100644 --- a/src/backend/ci/core/misc/biz-misc/src/main/kotlin/com/tencent/devops/misc/config/JooqConfiguration.kt +++ b/src/backend/ci/core/misc/biz-misc/src/main/kotlin/com/tencent/devops/misc/config/JooqConfiguration.kt @@ -73,30 +73,14 @@ class JooqConfiguration { val declaringClass: Class<*> = (annotatedElement as Constructor<*>).declaringClass val packageName = declaringClass.getPackage().name val matchResult = regex.find(packageName) - ?: return defaultContext(configurationMap["defaultJooqConfiguration"]!!) - val configuration = configurationMap["${matchResult.groupValues[1]}JooqConfiguration"] - ?: throw NoSuchBeanDefinitionException("no ${matchResult.groupValues[1]}JooqConfiguration") - LOG.info("dslContext_init|${matchResult.groupValues[1]}JooqConfiguration|${declaringClass.name}") - return DSL.using(configuration) + if (matchResult != null) { + val configuration = configurationMap["${matchResult.groupValues[1]}JooqConfiguration"] + ?: throw NoSuchBeanDefinitionException("no ${matchResult.groupValues[1]}JooqConfiguration") + LOG.info("dslContext_init|${matchResult.groupValues[1]}JooqConfiguration|${declaringClass.name}") + return DSL.using(configuration) + } } - return defaultContext(configurationMap["defaultJooqConfiguration"]!!) - } - - private fun defaultContext(defaultConfiguration: DefaultConfiguration): DSLContext { - return DSL.using(defaultConfiguration) - } - - @Bean - @Autowired(required = false) - fun defaultJooqConfiguration( - @Qualifier("dataSource") - @Autowired(required = false) - dataSource: DataSource? - ): DefaultConfiguration { - val configuration = DefaultConfiguration() - configuration.set(SQLDialect.MYSQL) - configuration.set(dataSource) - return configuration + return DSL.using(configurationMap["defaultJooqConfiguration"]!!) } @Bean From e07fb2fffd6f773a93ca6cb75d791ef6fd5ef2a6 Mon Sep 17 00:00:00 2001 From: irwinsun Date: Tue, 22 Jun 2021 11:57:38 +0800 Subject: [PATCH 13/20] =?UTF-8?q?bug:=20assembly=20=E5=90=AF=E5=8A=A8?= =?UTF-8?q?=E5=A4=B1=E8=B4=A5[1.5.4]=20#4475=20=E8=A7=A3=E5=86=B3misc?= =?UTF-8?q?=E9=BB=98=E8=AE=A4datasource=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../tencent/devops/misc/MiscApplication.kt | 53 ++++++++++++++++++- 1 file changed, 52 insertions(+), 1 deletion(-) diff --git a/src/backend/ci/core/misc/boot-misc/src/main/kotlin/com/tencent/devops/misc/MiscApplication.kt b/src/backend/ci/core/misc/boot-misc/src/main/kotlin/com/tencent/devops/misc/MiscApplication.kt index 7401bded0d4..0ef3aff8253 100644 --- a/src/backend/ci/core/misc/boot-misc/src/main/kotlin/com/tencent/devops/misc/MiscApplication.kt +++ b/src/backend/ci/core/misc/boot-misc/src/main/kotlin/com/tencent/devops/misc/MiscApplication.kt @@ -27,11 +27,62 @@ package com.tencent.devops.misc +import com.mysql.jdbc.Driver import com.tencent.devops.common.service.MicroService import com.tencent.devops.common.service.MicroServiceApplication +import com.zaxxer.hikari.HikariDataSource +import org.jooq.SQLDialect +import org.jooq.impl.DefaultConfiguration +import org.springframework.beans.factory.annotation.Qualifier +import org.springframework.beans.factory.annotation.Value +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean +import org.springframework.context.annotation.Bean +import org.springframework.context.annotation.Primary +import javax.sql.DataSource @MicroService -class MiscApplication +class MiscApplication { + + @ConditionalOnMissingBean(name = ["dataSource"]) + class PrimaryDSConfiguration { + + @Bean + @Primary + fun dataSource( + @Value("\${spring.datasource.process.url}") + datasourceUrl: String, + @Value("\${spring.datasource.process.username}") + datasourceUsername: String, + @Value("\${spring.datasource.process.password}") + datasourcePassword: String, + @Value("\${spring.datasource.process.initSql:#{null}}") + datasourceInitSql: String? = null, + @Value("\${spring.datasource.process.leakDetectionThreshold:#{0}}") + datasouceLeakDetectionThreshold: Long = 0 + ): DataSource { + return HikariDataSource().apply { + poolName = "DBPool-Misc-Process" + jdbcUrl = datasourceUrl + username = datasourceUsername + password = datasourcePassword + driverClassName = Driver::class.java.name + minimumIdle = 1 + maximumPoolSize = 10 + idleTimeout = 60000 + connectionInitSql = datasourceInitSql + leakDetectionThreshold = datasouceLeakDetectionThreshold + } + } + + @Bean + fun defaultJooqConfiguration(@Qualifier("dataSource") dataSource: DataSource): DefaultConfiguration { + val configuration = DefaultConfiguration() + configuration.set(SQLDialect.MYSQL) + configuration.set(dataSource) + return configuration + } + } +} fun main(args: Array) { MicroServiceApplication.run(MiscApplication::class, args) From de17bebaac67f4d6fd06c56251b3064719c14e03 Mon Sep 17 00:00:00 2001 From: irwinsun Date: Tue, 22 Jun 2021 12:14:10 +0800 Subject: [PATCH 14/20] =?UTF-8?q?bug:=20assembly=20=E5=90=AF=E5=8A=A8?= =?UTF-8?q?=E5=A4=B1=E8=B4=A5[1.5.4]=20#4475=20=E8=A7=A3=E5=86=B3=E6=9C=8D?= =?UTF-8?q?=E5=8A=A1=E5=8F=91=E7=8E=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ci/boot-assembly/src/main/resources/application.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/backend/ci/boot-assembly/src/main/resources/application.yml b/src/backend/ci/boot-assembly/src/main/resources/application.yml index 399e609f2ca..5a50f6b6064 100644 --- a/src/backend/ci/boot-assembly/src/main/resources/application.yml +++ b/src/backend/ci/boot-assembly/src/main/resources/application.yml @@ -2,14 +2,14 @@ spring: main: allow-bean-definition-overriding: true application: - name: bk-ci + name: bk-ci${service-suffix} desc: DevOps BK-CI Service version: 0.0.1 packageName: com.tencent.devops cloud: consul: discovery: - service-name: bk-ci # single service 合并后的单体服务 + service-name: bk-ci${service-suffix} # single service 合并后的单体服务 file: allowUploadFileTypes: jpg,png,gif,swf,txt,doc,docx,xls,xlsx,ppt,pptx,pdf,svg,mp4,wmv,rm,rmvb,3gp,mov,m4v,avi,dat,mkv,flv,vob maxUploadFileSize: 52428800 From c3b7333fc0cd145df0a2cae6c7643a0fd9df692b Mon Sep 17 00:00:00 2001 From: irwinsun Date: Tue, 22 Jun 2021 12:19:44 +0800 Subject: [PATCH 15/20] =?UTF-8?q?bug:=20assembly=20=E5=90=AF=E5=8A=A8?= =?UTF-8?q?=E5=A4=B1=E8=B4=A5[1.5.4]=20#447?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/tencent/devops/dispatch/api/BuildCodeccToolResource.kt | 2 +- .../kotlin/com/tencent/devops/misc/config/JooqConfiguration.kt | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/backend/ci/core/dispatch/api-dispatch/src/main/kotlin/com/tencent/devops/dispatch/api/BuildCodeccToolResource.kt b/src/backend/ci/core/dispatch/api-dispatch/src/main/kotlin/com/tencent/devops/dispatch/api/BuildCodeccToolResource.kt index 8c50f880c1a..e2a9035d28f 100644 --- a/src/backend/ci/core/dispatch/api-dispatch/src/main/kotlin/com/tencent/devops/dispatch/api/BuildCodeccToolResource.kt +++ b/src/backend/ci/core/dispatch/api-dispatch/src/main/kotlin/com/tencent/devops/dispatch/api/BuildCodeccToolResource.kt @@ -43,7 +43,7 @@ import javax.ws.rs.core.MediaType import javax.ws.rs.core.Response @Api(tags = ["BUILD_CODECC"], description = "构建-CODECC相关资源") -@Path("/build/codecc/") +@Path("/build/codecc") @Produces(MediaType.APPLICATION_JSON) @Consumes(MediaType.APPLICATION_JSON) interface BuildCodeccToolResource { diff --git a/src/backend/ci/core/misc/biz-misc/src/main/kotlin/com/tencent/devops/misc/config/JooqConfiguration.kt b/src/backend/ci/core/misc/biz-misc/src/main/kotlin/com/tencent/devops/misc/config/JooqConfiguration.kt index 37916416e61..4b8a17c8e56 100644 --- a/src/backend/ci/core/misc/biz-misc/src/main/kotlin/com/tencent/devops/misc/config/JooqConfiguration.kt +++ b/src/backend/ci/core/misc/biz-misc/src/main/kotlin/com/tencent/devops/misc/config/JooqConfiguration.kt @@ -34,7 +34,6 @@ import org.jooq.impl.DefaultConfiguration import org.slf4j.LoggerFactory import org.springframework.beans.factory.InjectionPoint import org.springframework.beans.factory.NoSuchBeanDefinitionException -import org.springframework.beans.factory.annotation.Autowired import org.springframework.beans.factory.annotation.Qualifier import org.springframework.beans.factory.config.ConfigurableBeanFactory import org.springframework.context.annotation.Bean From 3ee8d93e8a622791a4515651137815ce67159bf1 Mon Sep 17 00:00:00 2001 From: irwinsun Date: Tue, 22 Jun 2021 21:51:41 +0800 Subject: [PATCH 16/20] =?UTF-8?q?bug:=20assembly=20=E5=90=AF=E5=8A=A8?= =?UTF-8?q?=E5=A4=B1=E8=B4=A5[1.5.4]=20#447=20=E8=A7=A3=E5=86=B3=E9=87=8D?= =?UTF-8?q?=E5=90=8D=E5=92=8C=E9=A9=B1=E5=8A=A8=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/tencent/devops/common/db/DBAutoConfiguration.kt | 2 +- .../com/tencent/devops/misc/config/DataSourceConfig.kt | 2 +- .../kotlin/com/tencent/devops/misc/MiscApplication.kt | 2 +- .../com/tencent/devops/quality/QualityConfiguration.kt | 2 +- .../quality/service/SampleQualityPermissionServiceImpl.kt | 8 +++----- 5 files changed, 7 insertions(+), 9 deletions(-) diff --git a/src/backend/ci/core/common/common-db/src/main/kotlin/com/tencent/devops/common/db/DBAutoConfiguration.kt b/src/backend/ci/core/common/common-db/src/main/kotlin/com/tencent/devops/common/db/DBAutoConfiguration.kt index 77bdbad246d..92b5b4e69ea 100644 --- a/src/backend/ci/core/common/common-db/src/main/kotlin/com/tencent/devops/common/db/DBAutoConfiguration.kt +++ b/src/backend/ci/core/common/common-db/src/main/kotlin/com/tencent/devops/common/db/DBAutoConfiguration.kt @@ -27,7 +27,7 @@ package com.tencent.devops.common.db -import com.mysql.jdbc.Driver +import com.mysql.cj.jdbc.Driver import com.zaxxer.hikari.HikariDataSource import org.springframework.beans.factory.annotation.Value import org.springframework.boot.autoconfigure.AutoConfigureBefore diff --git a/src/backend/ci/core/misc/biz-misc/src/main/kotlin/com/tencent/devops/misc/config/DataSourceConfig.kt b/src/backend/ci/core/misc/biz-misc/src/main/kotlin/com/tencent/devops/misc/config/DataSourceConfig.kt index db2c6e3f999..e572924cb47 100644 --- a/src/backend/ci/core/misc/biz-misc/src/main/kotlin/com/tencent/devops/misc/config/DataSourceConfig.kt +++ b/src/backend/ci/core/misc/biz-misc/src/main/kotlin/com/tencent/devops/misc/config/DataSourceConfig.kt @@ -27,7 +27,7 @@ package com.tencent.devops.misc.config -import com.mysql.jdbc.Driver +import com.mysql.cj.jdbc.Driver import com.zaxxer.hikari.HikariDataSource import org.springframework.beans.factory.annotation.Value import org.springframework.boot.autoconfigure.AutoConfigureOrder diff --git a/src/backend/ci/core/misc/boot-misc/src/main/kotlin/com/tencent/devops/misc/MiscApplication.kt b/src/backend/ci/core/misc/boot-misc/src/main/kotlin/com/tencent/devops/misc/MiscApplication.kt index 0ef3aff8253..292044b05a7 100644 --- a/src/backend/ci/core/misc/boot-misc/src/main/kotlin/com/tencent/devops/misc/MiscApplication.kt +++ b/src/backend/ci/core/misc/boot-misc/src/main/kotlin/com/tencent/devops/misc/MiscApplication.kt @@ -27,7 +27,7 @@ package com.tencent.devops.misc -import com.mysql.jdbc.Driver +import com.mysql.cj.jdbc.Driver import com.tencent.devops.common.service.MicroService import com.tencent.devops.common.service.MicroServiceApplication import com.zaxxer.hikari.HikariDataSource diff --git a/src/backend/ci/core/quality/biz-quality-sample/src/main/kotlin/com/tencent/devops/quality/QualityConfiguration.kt b/src/backend/ci/core/quality/biz-quality-sample/src/main/kotlin/com/tencent/devops/quality/QualityConfiguration.kt index 91af5dac1db..bb0ceaec611 100644 --- a/src/backend/ci/core/quality/biz-quality-sample/src/main/kotlin/com/tencent/devops/quality/QualityConfiguration.kt +++ b/src/backend/ci/core/quality/biz-quality-sample/src/main/kotlin/com/tencent/devops/quality/QualityConfiguration.kt @@ -47,7 +47,7 @@ class QualityConfiguration { @Bean @ConditionalOnProperty(prefix = "auth", name = ["idProvider"], havingValue = "sample") - fun sampleProjectPermissionService( + fun sampleQualityPermissionService( authPermissionApi: AuthPermissionApi, authResourceApi: AuthResourceApi, qualityAuthServiceCode: QualityAuthServiceCode diff --git a/src/backend/ci/core/quality/biz-quality-sample/src/main/kotlin/com/tencent/devops/quality/service/SampleQualityPermissionServiceImpl.kt b/src/backend/ci/core/quality/biz-quality-sample/src/main/kotlin/com/tencent/devops/quality/service/SampleQualityPermissionServiceImpl.kt index f53d43edc90..6687b0e263f 100644 --- a/src/backend/ci/core/quality/biz-quality-sample/src/main/kotlin/com/tencent/devops/quality/service/SampleQualityPermissionServiceImpl.kt +++ b/src/backend/ci/core/quality/biz-quality-sample/src/main/kotlin/com/tencent/devops/quality/service/SampleQualityPermissionServiceImpl.kt @@ -36,11 +36,9 @@ import com.tencent.devops.common.auth.api.AuthResourceApi import com.tencent.devops.common.auth.api.AuthResourceType import com.tencent.devops.common.auth.code.QualityAuthServiceCode import com.tencent.devops.common.service.utils.MessageCodeUtil -import org.springframework.beans.factory.annotation.Autowired -import org.springframework.stereotype.Service -@Service -class SampleQualityPermissionServiceImpl @Autowired constructor( +@Suppress("ALL") +class SampleQualityPermissionServiceImpl constructor( val authPermissionApi: AuthPermissionApi, val authResourceApi: AuthResourceApi, val qualityAuthServiceCode: QualityAuthServiceCode @@ -224,7 +222,7 @@ class SampleQualityPermissionServiceImpl @Autowired constructor( supplier = null ) val permissionRuleMap = mutableMapOf>() - permissionResourceMap.forEach { permission, list -> + permissionResourceMap.forEach { (permission, list) -> permissionRuleMap[permission] = list.map { HashUtil.decodeIdToLong(it) } } return permissionRuleMap From d01477c17bbe79b944766c78761eeef800ecbc2b Mon Sep 17 00:00:00 2001 From: irwinsun Date: Wed, 23 Jun 2021 11:58:07 +0800 Subject: [PATCH 17/20] =?UTF-8?q?bug:=20assembly=20=E5=90=AF=E5=8A=A8?= =?UTF-8?q?=E5=A4=B1=E8=B4=A5[1.5.4]=20#447=20=E7=A7=BB=E9=99=A4=E5=86=B2?= =?UTF-8?q?=E7=AA=81jar?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/backend/ci/boot-assembly/build.gradle.kts | 1 - src/backend/ci/core/common/common-client/build.gradle.kts | 4 +++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/backend/ci/boot-assembly/build.gradle.kts b/src/backend/ci/boot-assembly/build.gradle.kts index 687a133c752..2e68c884adc 100644 --- a/src/backend/ci/boot-assembly/build.gradle.kts +++ b/src/backend/ci/boot-assembly/build.gradle.kts @@ -49,7 +49,6 @@ dependencies { implementation(project(":core:notify:biz-notify-blueking")) implementation(project(":core:openapi:biz-openapi")) implementation(project(":core:plugin:biz-plugin")) - implementation(project(":core:plugin:codecc-plugin:biz-codecc")) implementation(project(":core:process:plugin-load")) implementation(project(":core:process:plugin-trigger")) implementation(project(":core:process:biz-engine")) diff --git a/src/backend/ci/core/common/common-client/build.gradle.kts b/src/backend/ci/core/common/common-client/build.gradle.kts index de40d58bc4f..206fa4c99e3 100644 --- a/src/backend/ci/core/common/common-client/build.gradle.kts +++ b/src/backend/ci/core/common/common-client/build.gradle.kts @@ -27,7 +27,9 @@ dependencies { api(project(":core:common:common-service")) api(project(":core:common:common-api")) - api("io.github.openfeign:feign-jaxrs") + api("io.github.openfeign:feign-jaxrs") { + exclude(group = "javax.ws.rs", module = "jsr311-api") + } api("io.github.openfeign:feign-okhttp") api("io.github.openfeign:feign-jackson") api("io.github.openfeign.form:feign-form") From 1ad76d1412ac7038cffdb2405fc2d328884b3f46 Mon Sep 17 00:00:00 2001 From: irwinsun Date: Wed, 23 Jun 2021 19:42:48 +0800 Subject: [PATCH 18/20] =?UTF-8?q?bug:=20assembly=20=E5=90=AF=E5=8A=A8?= =?UTF-8?q?=E5=A4=B1=E8=B4=A5[1.5.4]=20#447=20=E7=A7=BB=E9=99=A4=E8=87=AA?= =?UTF-8?q?=E5=8A=A8=E7=94=9F=E6=88=90=E9=A1=B9=E7=9B=AELogo=E5=8A=9F?= =?UTF-8?q?=E8=83=BD=EF=BC=8C=E5=8F=97OpenJDK=E5=BD=B1=E5=93=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../service/PipelineBuildDetailService.kt | 1 - .../service/SimpleProjectServiceImpl.kt | 9 +- .../service/impl/AbsProjectServiceImpl.kt | 142 +++++++++--------- 3 files changed, 73 insertions(+), 79 deletions(-) diff --git a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/service/PipelineBuildDetailService.kt b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/service/PipelineBuildDetailService.kt index fe49515620e..422804f270f 100644 --- a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/service/PipelineBuildDetailService.kt +++ b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/service/PipelineBuildDetailService.kt @@ -930,7 +930,6 @@ class PipelineBuildDetailService @Autowired constructor( watcher.start("model") val model = JsonUtil.to(record!!.model, Model::class.java) - Preconditions.checkArgument(model.stages.size > 1, "Trigger container only") watcher.start("updateModel") update(model, modelInterface) diff --git a/src/backend/ci/core/project/biz-project-sample/src/main/kotlin/com/tencent/devops/project/service/SimpleProjectServiceImpl.kt b/src/backend/ci/core/project/biz-project-sample/src/main/kotlin/com/tencent/devops/project/service/SimpleProjectServiceImpl.kt index 2ae63e7d5ac..fea641427df 100644 --- a/src/backend/ci/core/project/biz-project-sample/src/main/kotlin/com/tencent/devops/project/service/SimpleProjectServiceImpl.kt +++ b/src/backend/ci/core/project/biz-project-sample/src/main/kotlin/com/tencent/devops/project/service/SimpleProjectServiceImpl.kt @@ -57,7 +57,7 @@ import org.springframework.beans.factory.annotation.Autowired import org.springframework.stereotype.Service import java.io.File -@Suppress("ALL") +@Suppress("ALL", "UNUSED") @Service class SimpleProjectServiceImpl @Autowired constructor( projectPermissionService: ProjectPermissionService, @@ -139,7 +139,10 @@ class SimpleProjectServiceImpl @Autowired constructor( ) } - override fun organizationMarkUp(projectCreateInfo: ProjectCreateInfo, userDeptDetail: UserDeptDetail): ProjectCreateInfo { + override fun organizationMarkUp( + projectCreateInfo: ProjectCreateInfo, + userDeptDetail: UserDeptDetail + ): ProjectCreateInfo { return projectCreateInfo } @@ -168,6 +171,6 @@ class SimpleProjectServiceImpl @Autowired constructor( } companion object { - val logger = LoggerFactory.getLogger(this::class.java) + private val logger = LoggerFactory.getLogger(SimpleProjectServiceImpl::class.java) } } diff --git a/src/backend/ci/core/project/biz-project/src/main/kotlin/com/tencent/devops/project/service/impl/AbsProjectServiceImpl.kt b/src/backend/ci/core/project/biz-project/src/main/kotlin/com/tencent/devops/project/service/impl/AbsProjectServiceImpl.kt index 698d2d1454a..c0456117404 100644 --- a/src/backend/ci/core/project/biz-project/src/main/kotlin/com/tencent/devops/project/service/impl/AbsProjectServiceImpl.kt +++ b/src/backend/ci/core/project/biz-project/src/main/kotlin/com/tencent/devops/project/service/impl/AbsProjectServiceImpl.kt @@ -73,7 +73,6 @@ import org.springframework.beans.factory.annotation.Autowired import org.springframework.dao.DuplicateKeyException import java.io.File import java.io.InputStream -import java.util.ArrayList import java.util.regex.Pattern @Suppress("ALL") @@ -155,89 +154,79 @@ abstract class AbsProjectServiceImpl @Autowired constructor( validate(ProjectValidateType.english_name, projectCreateInfo.englishName) } - // 随机生成首字母图片 - val logoFile = drawFile(projectCreateInfo.englishName) + val userDeptDetail = getDeptInfo(userId) + var projectId = defaultProjectId try { - // 保存Logo文件 - val logoAddress = saveLogoAddress(userId, projectCreateInfo.englishName, logoFile) - val userDeptDetail = getDeptInfo(userId) - var projectId = defaultProjectId - try { - if (createExtInfo.needAuth!!) { - // 注册项目到权限中心 - projectId = projectPermissionService.createResources( - userId = userId, - accessToken = accessToken, - resourceRegisterInfo = ResourceRegisterInfo( - resourceCode = projectCreateInfo.englishName, - resourceName = projectCreateInfo.projectName - ), - userDeptDetail = userDeptDetail - ) - } - } catch (e: PermissionForbiddenException) { - throw e - } catch (e: Exception) { - logger.warn("权限中心创建项目信息： $projectCreateInfo", e) - throw OperationException(MessageCodeUtil.getCodeLanMessage(ProjectMessageCode.PEM_CREATE_FAIL)) - } - if (projectId.isNullOrEmpty()) { - projectId = UUIDUtil.generate() + if (createExtInfo.needAuth!!) { + // 注册项目到权限中心 + projectId = projectPermissionService.createResources( + userId = userId, + accessToken = accessToken, + resourceRegisterInfo = ResourceRegisterInfo( + resourceCode = projectCreateInfo.englishName, + resourceName = projectCreateInfo.projectName + ), + userDeptDetail = userDeptDetail + ) } + } catch (e: PermissionForbiddenException) { + throw e + } catch (e: Exception) { + logger.warn("权限中心创建项目信息： $projectCreateInfo", e) + throw OperationException(MessageCodeUtil.getCodeLanMessage(ProjectMessageCode.PEM_CREATE_FAIL)) + } + if (projectId.isNullOrEmpty()) { + projectId = UUIDUtil.generate() + } - try { - dslContext.transaction { configuration -> - val projectInfo = organizationMarkUp(projectCreateInfo, userDeptDetail) - val context = DSL.using(configuration) - projectDao.create( - dslContext = context, + try { + dslContext.transaction { configuration -> + val projectInfo = organizationMarkUp(projectCreateInfo, userDeptDetail) + val context = DSL.using(configuration) + projectDao.create( + dslContext = context, + userId = userId, + logoAddress = "", + projectCreateInfo = projectInfo, + userDeptDetail = userDeptDetail, + projectId = projectId, + channelCode = projectChannel + ) + + try { + createExtProjectInfo( userId = userId, - logoAddress = logoAddress, + projectId = projectId, + accessToken = accessToken, projectCreateInfo = projectInfo, - userDeptDetail = userDeptDetail, - projectId = projectId!!, - channelCode = projectChannel + createExtInfo = createExtInfo ) - - try { - createExtProjectInfo( - userId = userId, - projectId = projectId!!, - accessToken = accessToken, - projectCreateInfo = projectInfo, - createExtInfo = createExtInfo - ) - } catch (e: Exception) { - logger.warn("fail to create the project[$projectId] ext info $projectCreateInfo", e) - projectDao.delete(dslContext, projectId!!) - throw e - } - if (projectInfo.secrecy) { - redisOperation.addSetValue(SECRECY_PROJECT_REDIS_KEY, projectInfo.englishName) - } + } catch (e: Exception) { + logger.warn("fail to create the project[$projectId] ext info $projectCreateInfo", e) + projectDao.delete(dslContext, projectId) + throw e } - } catch (e: DuplicateKeyException) { - logger.warn("Duplicate project $projectCreateInfo", e) - if (createExtInfo.needAuth!!) { - deleteAuth(projectId!!, accessToken) + if (projectInfo.secrecy) { + redisOperation.addSetValue(SECRECY_PROJECT_REDIS_KEY, projectInfo.englishName) } - throw OperationException(MessageCodeUtil.getCodeLanMessage(ProjectMessageCode.PROJECT_NAME_EXIST)) - } catch (ignored: Throwable) { - logger.warn( - "Fail to create the project ($projectCreateInfo)", - ignored - ) - if (createExtInfo.needAuth!!) { - deleteAuth(projectId!!, accessToken) - } - throw ignored } - return projectId!! - } finally { - if (logoFile.exists()) { - logoFile.delete() + } catch (e: DuplicateKeyException) { + logger.warn("Duplicate project $projectCreateInfo", e) + if (createExtInfo.needAuth) { + deleteAuth(projectId, accessToken) } + throw OperationException(MessageCodeUtil.getCodeLanMessage(ProjectMessageCode.PROJECT_NAME_EXIST)) + } catch (ignored: Throwable) { + logger.warn( + "Fail to create the project ($projectCreateInfo)", + ignored + ) + if (createExtInfo.needAuth) { + deleteAuth(projectId, accessToken) + } + throw ignored } + return projectId } // 内部版独立实现 @@ -642,7 +631,7 @@ abstract class AbsProjectServiceImpl @Autowired constructor( createExtInfo: ProjectCreateExtInfo ) - abstract fun saveLogoAddress(userId: String, projectCode: String, file: File): String + abstract fun saveLogoAddress(userId: String, projectCode: String, logoFile: File): String abstract fun deleteAuth(projectId: String, accessToken: String?) @@ -652,7 +641,10 @@ abstract class AbsProjectServiceImpl @Autowired constructor( abstract fun drawFile(projectCode: String): File - abstract fun organizationMarkUp(projectCreateInfo: ProjectCreateInfo, userDeptDetail: UserDeptDetail): ProjectCreateInfo + abstract fun organizationMarkUp( + projectCreateInfo: ProjectCreateInfo, + userDeptDetail: UserDeptDetail + ): ProjectCreateInfo abstract fun modifyProjectAuthResource(projectCode: String, projectName: String) From 403f765cf19aaa009dc9693a203892eab6061293 Mon Sep 17 00:00:00 2001 From: irwinsun Date: Wed, 23 Jun 2021 19:43:12 +0800 Subject: [PATCH 19/20] =?UTF-8?q?bug:=20assembly=20=E5=90=AF=E5=8A=A8?= =?UTF-8?q?=E5=A4=B1=E8=B4=A5[1.5.4]=20#447=20=E8=A7=A3=E5=86=B3JSR311=20?= =?UTF-8?q?=E5=BD=B1=E5=93=8D=E5=AF=BC=E8=87=B4Jersey=E8=B5=84=E6=BA=90?= =?UTF-8?q?=E6=B3=A8=E5=86=8C=E4=B8=8D=E4=B8=8A?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/backend/ci/boot-assembly/build.gradle.kts | 4 ++++ src/backend/ci/core/common/common-service/build.gradle.kts | 2 -- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/backend/ci/boot-assembly/build.gradle.kts b/src/backend/ci/boot-assembly/build.gradle.kts index 2e68c884adc..c0a8488b790 100644 --- a/src/backend/ci/boot-assembly/build.gradle.kts +++ b/src/backend/ci/boot-assembly/build.gradle.kts @@ -65,3 +65,7 @@ dependencies { plugins { `task-spring-boot-package` } + +configurations.all { + exclude(group = "javax.ws.rs", module = "jsr311-api") +} diff --git a/src/backend/ci/core/common/common-service/build.gradle.kts b/src/backend/ci/core/common/common-service/build.gradle.kts index e2849a74906..f6e6c11f7a6 100644 --- a/src/backend/ci/core/common/common-service/build.gradle.kts +++ b/src/backend/ci/core/common/common-service/build.gradle.kts @@ -38,5 +38,3 @@ dependencies { testImplementation("org.springframework.boot:spring-boot-starter-test") } - - From 4fe3dc004e21fa57034286a59cbb875181911aca Mon Sep 17 00:00:00 2001 From: irwinsun Date: Wed, 23 Jun 2021 20:33:18 +0800 Subject: [PATCH 20/20] =?UTF-8?q?bug:=20assembly=20=E5=90=AF=E5=8A=A8?= =?UTF-8?q?=E5=A4=B1=E8=B4=A5[1.5.4]=20#447=20MQ=20Bean=E9=87=8D=E5=90=8D?= =?UTF-8?q?=E5=AF=BC=E8=87=B4=E6=97=A0=E6=B3=95=E5=AE=9E=E4=BE=8B=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../dispatch.sdk/config/MQConfiguration.kt | 146 ++++++++++-------- .../log/configuration/LogMQConfiguration.kt | 15 +- .../init/BuildEngineCoreBuildConfiguration.kt | 2 +- .../init/BuildEngineCoreStageConfiguration.kt | 2 +- 4 files changed, 89 insertions(+), 76 deletions(-) diff --git a/src/backend/ci/core/common/common-dispatch-sdk/src/main/kotlin/com/tencent/devops/common/dispatch.sdk/config/MQConfiguration.kt b/src/backend/ci/core/common/common-dispatch-sdk/src/main/kotlin/com/tencent/devops/common/dispatch.sdk/config/MQConfiguration.kt index ae991a1ceca..b10effb42eb 100644 --- a/src/backend/ci/core/common/common-dispatch-sdk/src/main/kotlin/com/tencent/devops/common/dispatch.sdk/config/MQConfiguration.kt +++ b/src/backend/ci/core/common/common-dispatch-sdk/src/main/kotlin/com/tencent/devops/common/dispatch.sdk/config/MQConfiguration.kt @@ -33,6 +33,7 @@ import com.tencent.devops.common.event.dispatcher.pipeline.mq.MQ import com.tencent.devops.common.event.dispatcher.pipeline.mq.MQ.EXCHANGE_AGENT_LISTENER_DIRECT import com.tencent.devops.common.event.dispatcher.pipeline.mq.MQ.ROUTE_AGENT_SHUTDOWN import com.tencent.devops.common.event.dispatcher.pipeline.mq.MQ.ROUTE_AGENT_STARTUP +import com.tencent.devops.common.event.dispatcher.pipeline.mq.Tools import org.slf4j.LoggerFactory import org.springframework.amqp.core.Binding import org.springframework.amqp.core.BindingBuilder @@ -50,6 +51,7 @@ import org.springframework.context.annotation.Bean import org.springframework.context.annotation.Configuration import org.springframework.core.Ordered +@Suppress("ALL") @Configuration @AutoConfigureOrder(Ordered.LOWEST_PRECEDENCE) class MQConfiguration @Autowired constructor() { @@ -70,36 +72,38 @@ class MQConfiguration @Autowired constructor() { } @Bean - fun pipelineBuildStartQueue(@Autowired buildListener: BuildListener) = + fun pipelineBuildDispatchStartQueue(@Autowired buildListener: BuildListener) = Queue(MQ.QUEUE_PIPELINE_BUILD_START_DISPATCHER + getStartQueue(buildListener)) @Bean - fun pipelineBuildStartQueueBind( - @Autowired pipelineBuildStartQueue: Queue, + fun pipelineBuildDispatchStartQueueBind( + @Autowired pipelineBuildDispatchStartQueue: Queue, @Autowired pipelineBuildStartFanoutExchange: FanoutExchange ): Binding { - return BindingBuilder.bind(pipelineBuildStartQueue).to(pipelineBuildStartFanoutExchange) + return BindingBuilder.bind(pipelineBuildDispatchStartQueue).to(pipelineBuildStartFanoutExchange) } @Bean - fun pipelineBuildStartListenerContainer( + fun pipelineBuildDispatchStartListenerContainer( @Autowired connectionFactory: ConnectionFactory, - @Autowired pipelineBuildStartQueue: Queue, + @Autowired pipelineBuildDispatchStartQueue: Queue, @Autowired rabbitAdmin: RabbitAdmin, @Autowired buildListener: BuildListener, @Autowired messageConverter: Jackson2JsonMessageConverter ): SimpleMessageListenerContainer { - val container = SimpleMessageListenerContainer(connectionFactory) - container.setQueueNames(pipelineBuildStartQueue.name) - container.setConcurrentConsumers(10) - container.setMaxConcurrentConsumers(10) - container.setAmqpAdmin(rabbitAdmin) - container.setPrefetchCount(1) - val adapter = MessageListenerAdapter(buildListener, buildListener::onPipelineStartup.name) adapter.setMessageConverter(messageConverter) - container.setMessageListener(adapter) - return container + return Tools.createSimpleMessageListenerContainerByAdapter( + connectionFactory = connectionFactory, + queue = pipelineBuildDispatchStartQueue, + rabbitAdmin = rabbitAdmin, + startConsumerMinInterval = 10000, + consecutiveActiveTrigger = 5, + concurrency = 10, + maxConcurrency = 10, + adapter = adapter, + prefetchCount = 1 + ) } /** @@ -113,36 +117,38 @@ class MQConfiguration @Autowired constructor() { } @Bean - fun pipelineBuildFinishQueue(@Autowired buildListener: BuildListener) = + fun pipelineBuildDispatchFinishQueue(@Autowired buildListener: BuildListener) = Queue(MQ.QUEUE_PIPELINE_BUILD_FINISH_DISPATCHER + getShutdownQueue(buildListener)) @Bean - fun pipelineBuildFinishQueueBind( - @Autowired pipelineBuildFinishQueue: Queue, + fun pipelineBuildDispatchFinishQueueBind( + @Autowired pipelineBuildDispatchFinishQueue: Queue, @Autowired pipelineBuildFinishFanoutExchange: FanoutExchange ): Binding { - return BindingBuilder.bind(pipelineBuildFinishQueue).to(pipelineBuildFinishFanoutExchange) + return BindingBuilder.bind(pipelineBuildDispatchFinishQueue).to(pipelineBuildFinishFanoutExchange) } @Bean - fun pipelineBuildFinishListenerContainer( + fun pipelineBuildDispatchFinishListenerContainer( @Autowired connectionFactory: ConnectionFactory, - @Autowired pipelineBuildFinishQueue: Queue, + @Autowired pipelineBuildDispatchFinishQueue: Queue, @Autowired rabbitAdmin: RabbitAdmin, @Autowired buildListener: BuildListener, @Autowired messageConverter: Jackson2JsonMessageConverter ): SimpleMessageListenerContainer { - val container = SimpleMessageListenerContainer(connectionFactory) - container.setQueueNames(pipelineBuildFinishQueue.name) - container.setConcurrentConsumers(10) - container.setMaxConcurrentConsumers(10) - container.setAmqpAdmin(rabbitAdmin) - container.setPrefetchCount(1) - val adapter = MessageListenerAdapter(buildListener, buildListener::onPipelineShutdown.name) adapter.setMessageConverter(messageConverter) - container.setMessageListener(adapter) - return container + return Tools.createSimpleMessageListenerContainerByAdapter( + connectionFactory = connectionFactory, + queue = pipelineBuildDispatchFinishQueue, + rabbitAdmin = rabbitAdmin, + startConsumerMinInterval = 10000, + consecutiveActiveTrigger = 5, + concurrency = 10, + maxConcurrency = 10, + adapter = adapter, + prefetchCount = 1 + ) } @Bean @@ -153,15 +159,16 @@ class MQConfiguration @Autowired constructor() { } @Bean - fun buildStartQueue( - @Autowired buildListener: BuildListener - ): Queue { + fun buildAgentStartQueue(@Autowired buildListener: BuildListener): Queue { return Queue(ROUTE_AGENT_STARTUP + getStartQueue(buildListener)) } @Bean - fun buildStartBind(@Autowired buildStartQueue: Queue, @Autowired exchange: DirectExchange): Binding { - return BindingBuilder.bind(buildStartQueue).to(exchange).with(buildStartQueue.name) + fun buildAgentStartQueueBind( + @Autowired buildAgentStartQueue: Queue, + @Autowired exchange: DirectExchange + ): Binding { + return BindingBuilder.bind(buildAgentStartQueue).to(exchange).with(buildAgentStartQueue.name) } @Bean @@ -170,55 +177,60 @@ class MQConfiguration @Autowired constructor() { @Bean fun startListener( @Autowired connectionFactory: ConnectionFactory, - @Autowired buildStartQueue: Queue, + @Autowired buildAgentStartQueue: Queue, @Autowired rabbitAdmin: RabbitAdmin, @Autowired buildListener: BuildListener, @Autowired messageConverter: Jackson2JsonMessageConverter ): SimpleMessageListenerContainer { - val container = SimpleMessageListenerContainer(connectionFactory) - container.setQueueNames(buildStartQueue.name) - container.setConcurrentConsumers(60) - container.setMaxConcurrentConsumers(100) - container.setAmqpAdmin(rabbitAdmin) - container.setMismatchedQueuesFatal(true) - container.setPrefetchCount(1) - val messageListenerAdapter = MessageListenerAdapter(buildListener, buildListener::handleStartMessage.name) - messageListenerAdapter.setMessageConverter(messageConverter) - container.setMessageListener(messageListenerAdapter) - logger.info("Start listener") - return container - } - - @Bean - fun buildShutdownQueue(@Autowired buildListener: BuildListener): Queue { + val adapter = MessageListenerAdapter(buildListener, buildListener::handleStartMessage.name) + adapter.setMessageConverter(messageConverter) + return Tools.createSimpleMessageListenerContainerByAdapter( + connectionFactory = connectionFactory, + queue = buildAgentStartQueue, + rabbitAdmin = rabbitAdmin, + startConsumerMinInterval = 10000, + consecutiveActiveTrigger = 5, + concurrency = 60, + maxConcurrency = 100, + adapter = adapter, + prefetchCount = 1 + ) + } + + @Bean + fun buildAgentShutdownQueue(@Autowired buildListener: BuildListener): Queue { return Queue(ROUTE_AGENT_SHUTDOWN + getShutdownQueue(buildListener)) } @Bean - fun buildShutdownBind(@Autowired buildShutdownQueue: Queue, @Autowired exchange: DirectExchange): Binding { - return BindingBuilder.bind(buildShutdownQueue).to(exchange).with(buildShutdownQueue.name) + fun buildAgentShutdownQueueBind( + @Autowired buildAgentShutdownQueue: Queue, + @Autowired exchange: DirectExchange + ): Binding { + return BindingBuilder.bind(buildAgentShutdownQueue).to(exchange).with(buildAgentShutdownQueue.name) } @Bean fun shutdownListener( @Autowired connectionFactory: ConnectionFactory, - @Autowired buildShutdownQueue: Queue, + @Autowired buildAgentShutdownQueue: Queue, @Autowired rabbitAdmin: RabbitAdmin, @Autowired buildListener: BuildListener, @Autowired messageConverter: Jackson2JsonMessageConverter ): SimpleMessageListenerContainer { - val container = SimpleMessageListenerContainer(connectionFactory) - container.setQueueNames(buildShutdownQueue.name) - container.setConcurrentConsumers(60) - container.setMaxConcurrentConsumers(100) - container.setAmqpAdmin(rabbitAdmin) - container.setMismatchedQueuesFatal(true) - container.setPrefetchCount(1) - val messageListenerAdapter = MessageListenerAdapter(buildListener, buildListener::handleShutdownMessage.name) - messageListenerAdapter.setMessageConverter(messageConverter) - container.setMessageListener(messageListenerAdapter) - logger.info("Start shutdown listener") - return container + val adapter = MessageListenerAdapter(buildListener, buildListener::handleShutdownMessage.name) + adapter.setMessageConverter(messageConverter) + return Tools.createSimpleMessageListenerContainerByAdapter( + connectionFactory = connectionFactory, + queue = buildAgentShutdownQueue, + rabbitAdmin = rabbitAdmin, + startConsumerMinInterval = 10000, + consecutiveActiveTrigger = 5, + concurrency = 60, + maxConcurrency = 100, + adapter = adapter, + prefetchCount = 1 + ) } companion object { diff --git a/src/backend/ci/core/log/biz-log/src/main/kotlin/com/tencent/devops/log/configuration/LogMQConfiguration.kt b/src/backend/ci/core/log/biz-log/src/main/kotlin/com/tencent/devops/log/configuration/LogMQConfiguration.kt index 8dcc7ac9670..7c10e11b8f8 100644 --- a/src/backend/ci/core/log/biz-log/src/main/kotlin/com/tencent/devops/log/configuration/LogMQConfiguration.kt +++ b/src/backend/ci/core/log/biz-log/src/main/kotlin/com/tencent/devops/log/configuration/LogMQConfiguration.kt @@ -63,6 +63,7 @@ import org.springframework.context.annotation.Bean import org.springframework.context.annotation.Configuration import org.springframework.core.Ordered +@Suppress("ALL") @Configuration @ConditionalOnWebApplication @AutoConfigureOrder(Ordered.LOWEST_PRECEDENCE) @@ -234,20 +235,20 @@ class LogMQConfiguration @Autowired constructor() { } @Bean - fun pipelineBuildFinishQueue() = Queue(MQ.QUEUE_PIPELINE_BUILD_FINISH_LOG) + fun pipelineBuildFinishLogQueue() = Queue(MQ.QUEUE_PIPELINE_BUILD_FINISH_LOG) @Bean - fun pipelineBuildFinishQueueBind( - @Autowired pipelineBuildFinishQueue: Queue, + fun pipelineBuildFinishLogQueueBind( + @Autowired pipelineBuildFinishLogQueue: Queue, @Autowired pipelineBuildFinishFanoutExchange: FanoutExchange ): Binding { - return BindingBuilder.bind(pipelineBuildFinishQueue).to(pipelineBuildFinishFanoutExchange) + return BindingBuilder.bind(pipelineBuildFinishLogQueue).to(pipelineBuildFinishFanoutExchange) } @Bean - fun pipelineBuildFinishListenerContainer( + fun pipelineBuildFinishLogListenerContainer( @Autowired connectionFactory: ConnectionFactory, - @Autowired pipelineBuildFinishQueue: Queue, + @Autowired pipelineBuildFinishLogQueue: Queue, @Autowired rabbitAdmin: RabbitAdmin, @Autowired logService: LogService, @Autowired messageConverter: Jackson2JsonMessageConverter @@ -256,7 +257,7 @@ class LogMQConfiguration @Autowired constructor() { adapter.setMessageConverter(messageConverter) return Tools.createSimpleMessageListenerContainerByAdapter( connectionFactory = connectionFactory, - queue = pipelineBuildFinishQueue, + queue = pipelineBuildFinishLogQueue, rabbitAdmin = rabbitAdmin, adapter = adapter, startConsumerMinInterval = 5000, diff --git a/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/init/BuildEngineCoreBuildConfiguration.kt b/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/init/BuildEngineCoreBuildConfiguration.kt index cfbfbda5f1d..9789f0fb068 100644 --- a/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/init/BuildEngineCoreBuildConfiguration.kt +++ b/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/init/BuildEngineCoreBuildConfiguration.kt @@ -74,7 +74,7 @@ class BuildEngineCoreBuildConfiguration { } @Bean - fun pipelineStageBuildStartListenerContainer( + fun pipelineBuildStartListenerContainer( @Autowired connectionFactory: ConnectionFactory, @Autowired pipelineBuildStartQueue: Queue, @Autowired rabbitAdmin: RabbitAdmin, diff --git a/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/init/BuildEngineCoreStageConfiguration.kt b/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/init/BuildEngineCoreStageConfiguration.kt index 6f0c78a1fbe..e42c0697106 100644 --- a/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/init/BuildEngineCoreStageConfiguration.kt +++ b/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/init/BuildEngineCoreStageConfiguration.kt @@ -68,7 +68,7 @@ class BuildEngineCoreStageConfiguration { } @Bean - fun pipelineStageBuildStageListenerContainer( + fun pipelineBuildStageListenerContainer( @Autowired connectionFactory: ConnectionFactory, @Autowired pipelineBuildStageQueue: Queue, @Autowired rabbitAdmin: RabbitAdmin,