From 710dd38ef007ab67839c7cd99fb5a6370eaf5007 Mon Sep 17 00:00:00 2001 From: kunlongli <16629885+cnlkl@users.noreply.github.com> Date: Wed, 18 Oct 2023 14:45:13 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E5=85=BC=E5=AE=B9UE4.26-4.27=20DDC=20#?= =?UTF-8?q?1256=20(#1264)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ddc/artifact/ReferenceArtifactInfo.kt | 4 + .../artifact/repository/DdcLocalRepository.kt | 131 ++++++++++++++++-- .../resolver/ReferenceArtifactInfoResolver.kt | 5 +- .../ddc/component/RefDownloadListener.kt | 2 +- .../bkrepo/ddc/controller/HealthController.kt | 6 +- .../controller/LegacyReferencesController.kt | 101 ++++++++++++++ .../bkrepo/ddc/metrics/DdcMeterBinder.kt | 46 ++++-- .../com/tencent/bkrepo/ddc/model/TDdcBlob.kt | 4 + .../tencent/bkrepo/ddc/model/TDdcLegacyRef.kt | 72 ++++++++++ .../com/tencent/bkrepo/ddc/model/TDdcRef.kt | 38 ++--- .../tencent/bkrepo/ddc/model/TDdcRefBase.kt | 50 +++++++ .../com/tencent/bkrepo/ddc/pojo/Blob.kt | 2 + .../com/tencent/bkrepo/ddc/pojo/RefId.kt | 1 + .../com/tencent/bkrepo/ddc/pojo/RefKey.kt | 37 ++++- .../com/tencent/bkrepo/ddc/pojo/Reference.kt | 16 +++ .../bkrepo/ddc/repository/BlobRepository.kt | 11 ++ .../ddc/repository/LegacyRefRepository.kt | 44 ++++++ .../ddc/repository/RefBaseRepository.kt | 71 ++++++++++ .../bkrepo/ddc/repository/RefRepository.kt | 34 +---- .../tencent/bkrepo/ddc/service/BlobService.kt | 8 +- .../ddc/service/ReferenceArtifactService.kt | 6 + .../bkrepo/ddc/service/ReferenceService.kt | 57 +++++++- .../job/batch/ddc/ExpiredDdcRefCleanupJob.kt | 5 +- 23 files changed, 661 insertions(+), 90 deletions(-) create mode 100644 src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/controller/LegacyReferencesController.kt create mode 100644 src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/model/TDdcLegacyRef.kt create mode 100644 src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/model/TDdcRefBase.kt create mode 100644 src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/repository/LegacyRefRepository.kt create mode 100644 src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/repository/RefBaseRepository.kt diff --git a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/artifact/ReferenceArtifactInfo.kt b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/artifact/ReferenceArtifactInfo.kt index c8f7832974..4d07a9b464 100644 --- a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/artifact/ReferenceArtifactInfo.kt +++ b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/artifact/ReferenceArtifactInfo.kt @@ -36,6 +36,10 @@ class ReferenceArtifactInfo( repoName: String, val bucket: String, val refKey: RefKey, + /** + * 是通过旧接口调用 + */ + var legacy: Boolean = false, var inlineBlobHash: String? = null, ) : ArtifactInfo(projectId, repoName, StringPool.EMPTY) { diff --git a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/artifact/repository/DdcLocalRepository.kt b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/artifact/repository/DdcLocalRepository.kt index f4dd68a024..cf9431761e 100644 --- a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/artifact/repository/DdcLocalRepository.kt +++ b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/artifact/repository/DdcLocalRepository.kt @@ -33,6 +33,8 @@ import com.tencent.bkrepo.common.api.exception.BadRequestException import com.tencent.bkrepo.common.api.exception.ErrorCodeException import com.tencent.bkrepo.common.api.message.CommonMessageCode import com.tencent.bkrepo.common.api.util.toJsonString +import com.tencent.bkrepo.common.artifact.api.ArtifactFile +import com.tencent.bkrepo.common.artifact.api.ArtifactInfo import com.tencent.bkrepo.common.artifact.message.ArtifactMessageCode import com.tencent.bkrepo.common.artifact.repository.context.ArtifactDownloadContext import com.tencent.bkrepo.common.artifact.repository.context.ArtifactUploadContext @@ -52,6 +54,7 @@ import com.tencent.bkrepo.ddc.exception.NotImplementedException import com.tencent.bkrepo.ddc.exception.ReferenceIsMissingBlobsException import com.tencent.bkrepo.ddc.metrics.DdcMeterBinder import com.tencent.bkrepo.ddc.pojo.Blob +import com.tencent.bkrepo.ddc.pojo.ContentHash import com.tencent.bkrepo.ddc.pojo.Reference import com.tencent.bkrepo.ddc.pojo.UploadCompressedBlobResponse import com.tencent.bkrepo.ddc.serialization.CbObject @@ -60,6 +63,7 @@ import com.tencent.bkrepo.ddc.service.ReferenceResolver import com.tencent.bkrepo.ddc.service.ReferenceService import com.tencent.bkrepo.ddc.utils.BlakeUtils.blake3 import com.tencent.bkrepo.ddc.utils.BlakeUtils.hex +import com.tencent.bkrepo.ddc.utils.DdcUtils import com.tencent.bkrepo.ddc.utils.MEDIA_TYPE_JUPITER_INLINED_PAYLOAD import com.tencent.bkrepo.ddc.utils.MEDIA_TYPE_UNREAL_COMPACT_BINARY import com.tencent.bkrepo.ddc.utils.MEDIA_TYPE_UNREAL_UNREAL_COMPRESSED_BUFFER @@ -88,7 +92,7 @@ class DdcLocalRepository( super.onUploadBefore(context) var uploadBlake3: String? = null val artifactInfo = context.artifactInfo - if (artifactInfo is ReferenceArtifactInfo) { + if (artifactInfo is ReferenceArtifactInfo && !artifactInfo.legacy) { uploadBlake3 = artifactInfo.inlineBlobHash!! } if (uploadBlake3 != null && uploadBlake3 != context.getStreamArtifactFile().blake3().hex()) { @@ -98,8 +102,11 @@ class DdcLocalRepository( override fun onUpload(context: ArtifactUploadContext) { val artifactInfo = context.artifactInfo - if (artifactInfo is ReferenceArtifactInfo) { - val startTime = System.nanoTime() + val startTime = System.nanoTime() + if (artifactInfo is ReferenceArtifactInfo && artifactInfo.legacy) { + onUploadLegacyReference(context) + ddcMeterBinder.legacyRefStoreTimer.record(System.nanoTime() - startTime, NANOSECONDS) + } else if (artifactInfo is ReferenceArtifactInfo) { onUploadReference(context) ddcMeterBinder.refStoreTimer.record(System.nanoTime() - startTime, NANOSECONDS) } else { @@ -117,7 +124,9 @@ class DdcLocalRepository( override fun onDownload(context: ArtifactDownloadContext): ArtifactResource? { val artifactInfo = context.artifactInfo - return if (artifactInfo is ReferenceArtifactInfo) { + return if (artifactInfo is ReferenceArtifactInfo && artifactInfo.legacy) { + onDownloadLegacyReference(context) + } else if (artifactInfo is ReferenceArtifactInfo) { onDownloadReference(context) } else { val startTime = System.nanoTime() @@ -139,6 +148,64 @@ class DdcLocalRepository( } } + private fun onUploadLegacyReference(context: ArtifactUploadContext) { + with(context) { + val artifactInfo = context.artifactInfo as ReferenceArtifactInfo + val sha1 = context.request.getHeader(HEADER_NAME_HASH_SHA1) + ?: throw BadRequestException( + CommonMessageCode.PARAMETER_INVALID, "Missing expected header $HEADER_NAME_HASH_SHA1" + ) + val artifactFileSha1 = getArtifactSha1() + if (sha1.toLowerCase() != artifactFileSha1) { + throw BadRequestException( + CommonMessageCode.PARAMETER_INVALID, + "Incorrect hash, got hash \"${sha1}\" " + + "but hash of content was determined to be \"${artifactFileSha1}\"" + ) + } + val blobIdByteArray = getStreamArtifactFile().blake3() + val blobId = blobIdByteArray.hex() + val blobIdContentHash = ContentHash(blobIdByteArray) + // 创建blob + val blobFullPath = "/${DdcUtils.DIR_BLOBS}/$blobId" + val createRequest = buildBlobNodeCreateRequest( + blobId, + blobId, + artifactInfo, + blobFullPath, + getArtifactFile(), + context.userId + ) + storageManager.storeArtifactFile(createRequest, getArtifactFile(), storageCredentials) + val blob = Blob( + projectId = projectId, + repoName = repoName, + sha256 = getArtifactSha256(), + fullPath = blobFullPath, + size = getArtifactFile().getSize(), + blobId = blobIdContentHash, + contentId = blobIdContentHash, + sha1 = artifactFileSha1, + ) + blobService.create(blob) + + // 创建Ref + val ref = referenceService.createLegacyReference( + Reference( + projectId = projectId, + repoName = repoName, + bucket = artifactInfo.bucket, + key = artifactInfo.refKey, + finalized = true, + blobId = blobIdContentHash, + legacy = true + ) + ) + + blobService.addRefToBlobs(ref, setOf(blobId)) + } + } + private fun onUploadReference(context: ArtifactUploadContext) { val contentType = context.request.contentType val artifactInfo = context.artifactInfo as ReferenceArtifactInfo @@ -187,7 +254,15 @@ class DdcLocalRepository( // TODO 改为读取流时直接计算blake3，避免重复读流 artifactInfo.compressedContentId = getStreamArtifactFile().blake3().hex() - storageManager.storeArtifactFile(buildBlobNodeCreateRequest(context), getArtifactFile(), storageCredentials) + val createRequest = buildBlobNodeCreateRequest( + artifactInfo.compressedContentId!!, + artifactInfo.contentId, + artifactInfo, + artifactInfo.getArtifactFullPath(), + getArtifactFile(), + userId + ) + storageManager.storeArtifactFile(createRequest, getArtifactFile(), storageCredentials) blobService.create(Blob.from(artifactInfo, getArtifactSha256(), getArtifactFile().getSize())) HttpContextHolder .getResponse() @@ -196,30 +271,61 @@ class DdcLocalRepository( } } - private fun buildBlobNodeCreateRequest(context: ArtifactUploadContext): NodeCreateRequest { - val artifactInfo = context.artifactInfo as CompressedBlobArtifactInfo + private fun buildBlobNodeCreateRequest( + blobId: String, + contentId: String, + artifactInfo: ArtifactInfo, + fullPath: String, + artifactFile: ArtifactFile, + userId: String + ): NodeCreateRequest { val metadata = ArrayList() metadata.add( MetadataModel( key = NODE_METADATA_KEY_BLOB_ID, - value = artifactInfo.compressedContentId!!, + value = blobId, system = true ) ) metadata.add( MetadataModel( key = NODE_METADATA_KEY_CONTENT_ID, - value = artifactInfo.contentId, + value = contentId, system = true ) ) - return buildNodeCreateRequest(context).copy( + return NodeCreateRequest( + projectId = artifactInfo.projectId, + repoName = artifactInfo.repoName, + folder = false, + fullPath = fullPath, + size = artifactFile.getSize(), + sha256 = artifactFile.getFileSha256(), + md5 = artifactFile.getFileMd5(), + operator = userId, overwrite = true, - nodeMetadata = metadata + nodeMetadata = metadata, ) } + private fun onDownloadLegacyReference(context: ArtifactDownloadContext): ArtifactResource? { + with(context) { + val startTime = System.nanoTime() + val artifactInfo = context.artifactInfo as ReferenceArtifactInfo + val ref = referenceService.getLegacyReference( + artifactInfo.projectId, artifactInfo.repoName, artifactInfo.bucket, artifactInfo.refKey.toString() + ) ?: return null + + refDownloadListener.onRefDownloaded(RefDownloadedEvent(ref, SecurityUtils.getUserId())) + val blob = blobService.findBlob(ref.projectId, ref.repoName, ref.blobId.toString()) ?: return null + response.addHeader(HEADER_NAME_HASH, ref.blobId.toString()) + response.addHeader(HEADER_NAME_HASH_SHA1, blob.sha1!!) + ddcMeterBinder.legacyRefLoadTimer.record(System.nanoTime() - startTime, NANOSECONDS) + return blobToArtifactResource(context, blob, MediaTypes.APPLICATION_OCTET_STREAM) + } + } + private fun onDownloadReference(context: ArtifactDownloadContext): ArtifactResource? { with(context) { val startTime = System.nanoTime() @@ -325,6 +431,7 @@ class DdcLocalRepository( val artifactInfo = context.artifactInfo as ReferenceArtifactInfo val blobs = refResolver.getReferencedBlobs(context.projectId, context.repoName, cb) if (blobs.size == 1) { + context.response.addHeader(HEADER_NAME_INLINE_PAYLOAD_HASH, blobs[0].toString()) blobToArtifactResource(context, blobs[0], responseType) } else if (blobs.isEmpty()) { null @@ -347,7 +454,6 @@ class DdcLocalRepository( responseType: String ): ArtifactResource? { with(context) { - response.addHeader(HEADER_NAME_INLINE_PAYLOAD_HASH, blob.toString()) return try { val blobInputStream = blobService.loadBlob(blob) ArtifactResource(blobInputStream, artifactInfo.getResponseName()).apply { contentType = responseType } @@ -376,6 +482,7 @@ class DdcLocalRepository( private val logger = LoggerFactory.getLogger(DdcLocalRepository::class.java) private val DATE_TIME_FORMATTER = DateTimeFormatter.ofPattern("MM/dd/yyyy hh:mm:ss") const val HEADER_NAME_HASH = "X-Jupiter-IoHash" + private const val HEADER_NAME_HASH_SHA1 = "X-Jupiter-Sha1" private const val HEADER_NAME_LAST_ACCESS = "X-Jupiter-LastAccess" private const val HEADER_NAME_INLINE_PAYLOAD_HASH = "X-Jupiter-InlinePayloadHash" } diff --git a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/artifact/resolver/ReferenceArtifactInfoResolver.kt b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/artifact/resolver/ReferenceArtifactInfoResolver.kt index 0dec39687d..911e93f1a5 100644 --- a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/artifact/resolver/ReferenceArtifactInfoResolver.kt +++ b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/artifact/resolver/ReferenceArtifactInfoResolver.kt @@ -32,6 +32,7 @@ import com.tencent.bkrepo.common.artifact.resolve.path.Resolver import com.tencent.bkrepo.ddc.artifact.ReferenceArtifactInfo import com.tencent.bkrepo.ddc.artifact.ReferenceArtifactInfo.Companion.PATH_VARIABLE_BUCKET import com.tencent.bkrepo.ddc.artifact.ReferenceArtifactInfo.Companion.PATH_VARIABLE_REF_ID +import com.tencent.bkrepo.ddc.controller.LegacyReferencesController.Companion.LEGACY_PREFIX import com.tencent.bkrepo.ddc.pojo.RefKey import org.springframework.stereotype.Component import org.springframework.web.servlet.HandlerMapping @@ -46,12 +47,14 @@ class ReferenceArtifactInfoResolver : ArtifactInfoResolver { artifactUri: String, request: HttpServletRequest ): ReferenceArtifactInfo { + val legacy = request.requestURI.startsWith(LEGACY_PREFIX) val attributes = request.getAttribute(HandlerMapping.URI_TEMPLATE_VARIABLES_ATTRIBUTE) as Map<*, *> return ReferenceArtifactInfo( projectId = projectId, repoName = repoName, bucket = attributes[PATH_VARIABLE_BUCKET].toString(), - refKey = RefKey.create(attributes[PATH_VARIABLE_REF_ID].toString()) + refKey = RefKey.create(attributes[PATH_VARIABLE_REF_ID].toString(), legacy), + legacy = legacy ) } } diff --git a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/component/RefDownloadListener.kt b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/component/RefDownloadListener.kt index 9d78c61425..84ada54a65 100644 --- a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/component/RefDownloadListener.kt +++ b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/component/RefDownloadListener.kt @@ -74,7 +74,7 @@ class RefDownloadListener( fun onRefDownloaded(event: RefDownloadedEvent) { with(event.ref) { - cache.put(RefId(projectId, repoName, bucket, key.toString()), LocalDateTime.now()) + cache.put(RefId(projectId, repoName, bucket, key.toString(), legacy), LocalDateTime.now()) } } diff --git a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/controller/HealthController.kt b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/controller/HealthController.kt index 3707ff7099..6acb762c7f 100644 --- a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/controller/HealthController.kt +++ b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/controller/HealthController.kt @@ -31,10 +31,14 @@ import com.tencent.bkrepo.common.api.constant.HttpStatus import com.tencent.bkrepo.common.service.util.HttpContextHolder import org.springframework.web.bind.annotation.GetMapping import org.springframework.web.bind.annotation.RestController +import com.tencent.bkrepo.ddc.controller.LegacyReferencesController.Companion.LEGACY_PREFIX @RestController class HealthController { - @GetMapping("/{projectId}/health/ready") + @GetMapping( + "/{projectId}/health/ready", + "$LEGACY_PREFIX{projectId}/health/ready" + ) fun ready() { HttpContextHolder.getResponse().apply { status = HttpStatus.OK.value diff --git a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/controller/LegacyReferencesController.kt b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/controller/LegacyReferencesController.kt new file mode 100644 index 0000000000..56c776d7db --- /dev/null +++ b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/controller/LegacyReferencesController.kt @@ -0,0 +1,101 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2023 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.ddc.controller + +import com.tencent.bkrepo.auth.pojo.enums.PermissionAction +import com.tencent.bkrepo.auth.pojo.enums.ResourceType +import com.tencent.bkrepo.common.artifact.api.ArtifactFile +import com.tencent.bkrepo.common.artifact.api.ArtifactPathVariable +import com.tencent.bkrepo.common.security.permission.Permission +import com.tencent.bkrepo.ddc.artifact.ReferenceArtifactInfo +import com.tencent.bkrepo.ddc.artifact.ReferenceArtifactInfo.Companion.PATH_VARIABLE_BUCKET +import com.tencent.bkrepo.ddc.artifact.ReferenceArtifactInfo.Companion.PATH_VARIABLE_REF_ID +import com.tencent.bkrepo.ddc.controller.LegacyReferencesController.Companion.LEGACY_PREFIX +import com.tencent.bkrepo.ddc.service.ReferenceArtifactService +import com.tencent.bkrepo.ddc.utils.MEDIA_TYPE_UNREAL_COMPACT_BINARY +import io.swagger.annotations.ApiOperation +import io.swagger.annotations.ApiParam +import org.springframework.http.MediaType +import org.springframework.web.bind.annotation.DeleteMapping +import org.springframework.web.bind.annotation.GetMapping +import org.springframework.web.bind.annotation.PutMapping +import org.springframework.web.bind.annotation.RequestMapping +import org.springframework.web.bind.annotation.RestController + +@RestController +@RequestMapping("$LEGACY_PREFIX{projectId}/api/v1/c") +class LegacyReferencesController( + private val referenceArtifactService: ReferenceArtifactService, +) { + @ApiOperation("获取ref") + @Permission(ResourceType.REPO, PermissionAction.READ) + @GetMapping( + "/ddc/{repoName}/{${PATH_VARIABLE_BUCKET}}/{${PATH_VARIABLE_REF_ID}}.raw", + "/ddc/{repoName}/{${PATH_VARIABLE_BUCKET}}/{${PATH_VARIABLE_REF_ID}}", + produces = [ + MediaType.APPLICATION_JSON_VALUE, + MediaType.APPLICATION_OCTET_STREAM_VALUE, + MEDIA_TYPE_UNREAL_COMPACT_BINARY + ] + ) + fun get( + @ApiParam(value = "ddc ref", required = true) + @ArtifactPathVariable + artifactInfo: ReferenceArtifactInfo + ) { + referenceArtifactService.downloadRef(artifactInfo) + } + + @PutMapping( + "/ddc/{repoName}/{${PATH_VARIABLE_BUCKET}}/{${PATH_VARIABLE_REF_ID}}", + consumes = [MediaType.APPLICATION_OCTET_STREAM_VALUE] + ) + @Permission(ResourceType.REPO, PermissionAction.WRITE) + fun put( + @ApiParam(value = "ddc ref", required = true) + @ArtifactPathVariable + artifactInfo: ReferenceArtifactInfo, + file: ArtifactFile + ) { + referenceArtifactService.createRef(artifactInfo, file) + } + + @DeleteMapping("/ddc/{repoName}/{${PATH_VARIABLE_BUCKET}}/{${PATH_VARIABLE_REF_ID}}") + @Permission(ResourceType.REPO, PermissionAction.WRITE) + fun delete( + @ApiParam(value = "ddc ref", required = true) + @ArtifactPathVariable + artifactInfo: ReferenceArtifactInfo + ) { + referenceArtifactService.deleteRef(artifactInfo) + } + + companion object { + const val LEGACY_PREFIX = "/legacy/" + } +} diff --git a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/metrics/DdcMeterBinder.kt b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/metrics/DdcMeterBinder.kt index 0a09156e05..ca82e52e8f 100644 --- a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/metrics/DdcMeterBinder.kt +++ b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/metrics/DdcMeterBinder.kt @@ -45,6 +45,11 @@ class DdcMeterBinder(private val registry: MeterRegistry) : MeterBinder { */ lateinit var refLoadTimer: Timer + /** + * legacy ref 加载耗时 + */ + lateinit var legacyRefLoadTimer: Timer + /** * blob 加载耗时 */ @@ -55,17 +60,38 @@ class DdcMeterBinder(private val registry: MeterRegistry) : MeterBinder { */ lateinit var refStoreTimer: Timer + /** + * legacy ref 加载耗时 + */ + lateinit var legacyRefStoreTimer: Timer + override fun bindTo(registry: MeterRegistry) { refInlineLoadTimer = Timer - .builder(DDC_REF) - .tag("type", "inline") - .tag("method", "load") + .builder(DDC_REF_LOAD) + .tag("inline", "true") + .tag("legacy", "false") .register(registry) refLoadTimer = Timer - .builder(DDC_REF) - .tag("type", "cb") - .tag("method", "load") + .builder(DDC_REF_LOAD) + .tag("inline", "false") + .tag("legacy", "false") + .register(registry) + + legacyRefLoadTimer = Timer + .builder(DDC_REF_LOAD) + .tag("inline", "false") + .tag("legacy", "true") + .register(registry) + + refStoreTimer = Timer + .builder(DDC_REF_STORE) + .tag("legacy", "false") + .register(registry) + + legacyRefStoreTimer = Timer + .builder(DDC_REF_STORE) + .tag("legacy", "true") .register(registry) blobLoadTimer = Timer @@ -73,11 +99,6 @@ class DdcMeterBinder(private val registry: MeterRegistry) : MeterBinder { .tag("type", "compressed") .tag("method", "load") .register(registry) - - refStoreTimer = Timer - .builder(DDC_REF) - .tag("method", "store") - .register(registry) } /** @@ -109,7 +130,8 @@ class DdcMeterBinder(private val registry: MeterRegistry) : MeterBinder { companion object { private const val DDC_REF_GETS = "ddc.ref.gets" - private const val DDC_REF = "ddc.ref" + private const val DDC_REF_LOAD = "ddc.ref.load" + private const val DDC_REF_STORE = "ddc.ref.store" private const val DDC_BLOB = "ddc.blob" } } diff --git a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/model/TDdcBlob.kt b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/model/TDdcBlob.kt index 7814df778c..e518b52cf3 100644 --- a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/model/TDdcBlob.kt +++ b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/model/TDdcBlob.kt @@ -72,6 +72,10 @@ data class TDdcBlob( * blob sha256 */ var sha256: String, + /** + * 仅再legacy ref引用的blob中存在值 + */ + var sha1: String? = null, /** * blob size */ diff --git a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/model/TDdcLegacyRef.kt b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/model/TDdcLegacyRef.kt new file mode 100644 index 0000000000..47990f8a74 --- /dev/null +++ b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/model/TDdcLegacyRef.kt @@ -0,0 +1,72 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2023 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.ddc.model + +import org.springframework.data.mongodb.core.index.CompoundIndex +import org.springframework.data.mongodb.core.index.CompoundIndexes +import org.springframework.data.mongodb.core.mapping.Document +import java.time.LocalDateTime + +/** + * DDC Ref,用于兼容UE4.26-4.27 + */ +@Document("ddc_legacy_ref") +@CompoundIndexes( + CompoundIndex( + name = "projectId_repoName_bucket_key_idx", + def = "{'projectId': 1, 'repoName': 1, 'bucket': 1, 'key': 1}", + unique = true, + background = true + ) +) +class TDdcLegacyRef( + createdBy: String, + createdDate: LocalDateTime, + lastModifiedBy: String, + lastModifiedDate: LocalDateTime, + lastAccessDate: LocalDateTime, + projectId: String, + repoName: String, + bucket: String, + key: String, + + /** + * 实际存放缓存的blob的id + */ + var contentHash: String, +): TDdcRefBase( + createdBy = createdBy, + createdDate = createdDate, + lastModifiedBy = lastModifiedBy, + lastModifiedDate = lastModifiedDate, + lastAccessDate = lastAccessDate, + projectId = projectId, + repoName = repoName, + bucket = bucket, + key = key +) diff --git a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/model/TDdcRef.kt b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/model/TDdcRef.kt index 70f2b09ac3..084bd58159 100644 --- a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/model/TDdcRef.kt +++ b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/model/TDdcRef.kt @@ -42,24 +42,16 @@ import java.time.LocalDateTime background = true ) ) -data class TDdcRef( - var id: String? = null, - var createdBy: String, - var createdDate: LocalDateTime, - var lastModifiedBy: String, - var lastModifiedDate: LocalDateTime, - var lastAccessDate: LocalDateTime, - - var projectId: String, - var repoName: String, - /** - * ref bucket - */ - var bucket: String, - /** - * ref key, blake3 hash - */ - var key: String, +class TDdcRef( + createdBy: String, + createdDate: LocalDateTime, + lastModifiedBy: String, + lastModifiedDate: LocalDateTime, + lastAccessDate: LocalDateTime, + projectId: String, + repoName: String, + bucket: String, + key: String, /** * 是否所有blob都上传完成 */ @@ -76,4 +68,14 @@ data class TDdcRef( * 过期时间 */ var expireDate: LocalDateTime? = null, +) : TDdcRefBase( + createdBy = createdBy, + createdDate = createdDate, + lastModifiedBy = lastModifiedBy, + lastModifiedDate = lastModifiedDate, + lastAccessDate = lastAccessDate, + projectId = projectId, + repoName = repoName, + bucket = bucket, + key = key ) diff --git a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/model/TDdcRefBase.kt b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/model/TDdcRefBase.kt new file mode 100644 index 0000000000..56bd1913be --- /dev/null +++ b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/model/TDdcRefBase.kt @@ -0,0 +1,50 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2023 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.ddc.model + +import java.time.LocalDateTime + +abstract class TDdcRefBase( + var id: String? = null, + var createdBy: String, + var createdDate: LocalDateTime, + var lastModifiedBy: String, + var lastModifiedDate: LocalDateTime, + var lastAccessDate: LocalDateTime, + + var projectId: String, + var repoName: String, + /** + * ref bucket + */ + var bucket: String, + /** + * ref key, blake3 hash + */ + var key: String, +) diff --git a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/pojo/Blob.kt b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/pojo/Blob.kt index 1387063a9b..4cea682882 100644 --- a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/pojo/Blob.kt +++ b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/pojo/Blob.kt @@ -40,6 +40,7 @@ data class Blob( val blobId: ContentHash, val contentId: ContentHash, val references: Set = emptySet(), + val sha1: String? = null, ) { companion object { fun from(blob: TDdcBlob) = with(blob) { @@ -52,6 +53,7 @@ data class Blob( blobId = ContentHash.fromHex(blobId), contentId = ContentHash.fromHex(contentId), references = references, + sha1 = sha1 ) } diff --git a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/pojo/RefId.kt b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/pojo/RefId.kt index 64f70e2ee5..c6694f7775 100644 --- a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/pojo/RefId.kt +++ b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/pojo/RefId.kt @@ -32,4 +32,5 @@ data class RefId( val repoName: String, val bucket: String, val key: String, + val legacy: Boolean = false, ) diff --git a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/pojo/RefKey.kt b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/pojo/RefKey.kt index 70028429e8..5b95e12fd4 100644 --- a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/pojo/RefKey.kt +++ b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/pojo/RefKey.kt @@ -28,16 +28,40 @@ package com.tencent.bkrepo.ddc.pojo import com.tencent.bkrepo.common.api.exception.BadRequestException -import com.tencent.bkrepo.common.api.message.CommonMessageCode +import com.tencent.bkrepo.common.api.message.CommonMessageCode.PARAMETER_INVALID -data class RefKey(private var text: String) { +data class RefKey( + private var text: String, + val legacy: Boolean = false +) { init { - text = text.toLowerCase() + if (legacy) { + checkLegacyKey(text) + } else { + checkKey(text) + text = text.toLowerCase() + } + } + + private fun checkKey(text: String) { if (text.length != 40) { - throw BadRequestException(CommonMessageCode.PARAMETER_INVALID, "IoHashKeys must be exactly 40 bytes.") + throw BadRequestException(PARAMETER_INVALID, "IoHashKeys must be exactly 40 bytes.") } if (text.any { !isValidCharacter(it) }) { - throw BadRequestException(CommonMessageCode.PARAMETER_INVALID, "${this.text} contains invalid character.") + throw BadRequestException(PARAMETER_INVALID, "${this.text} contains invalid character.") + } + } + + private fun checkLegacyKey(text: String) { + if (text.isEmpty()) { + throw BadRequestException(PARAMETER_INVALID, "Keys must have at least one character.") + } + + if (text.length > LEGACY_KEY_MAX_LENGTH) { + throw BadRequestException( + PARAMETER_INVALID, + "Keys may not be longer than $LEGACY_KEY_MAX_LENGTH characters" + ) } } @@ -46,6 +70,7 @@ data class RefKey(private var text: String) { private fun isValidCharacter(c: Char): Boolean = c in 'a'..'z' || c in '0'..'9' companion object { - fun create(s: String): RefKey = RefKey(s.toLowerCase()) + private const val LEGACY_KEY_MAX_LENGTH = 250 + fun create(s: String, legacy: Boolean = false): RefKey = RefKey(s, legacy) } } diff --git a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/pojo/Reference.kt b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/pojo/Reference.kt index d9596ee16c..137e7ec5f9 100644 --- a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/pojo/Reference.kt +++ b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/pojo/Reference.kt @@ -28,6 +28,7 @@ package com.tencent.bkrepo.ddc.pojo import com.tencent.bkrepo.ddc.artifact.ReferenceArtifactInfo +import com.tencent.bkrepo.ddc.model.TDdcLegacyRef import com.tencent.bkrepo.ddc.model.TDdcRef import java.time.LocalDateTime @@ -40,6 +41,7 @@ data class Reference( val lastAccessDate: LocalDateTime? = null, var blobId: ContentHash? = null, var inlineBlob: ByteArray? = null, + var legacy: Boolean = false, ) { fun fullPath() = "/$bucket/$key" @@ -59,6 +61,7 @@ data class Reference( if (other.inlineBlob == null) return false if (!inlineBlob.contentEquals(other.inlineBlob)) return false } else if (other.inlineBlob != null) return false + if (legacy != other.legacy) return false return true } @@ -71,6 +74,7 @@ data class Reference( result = 31 * result + blobId.hashCode() result = 31 * result + finalized.hashCode() result = 31 * result + (inlineBlob?.contentHashCode() ?: 0) + result = 31 * result + legacy.hashCode() return result } @@ -86,6 +90,18 @@ data class Reference( inlineBlob = ref.inlineBlob?.data ) + fun from(ref: TDdcLegacyRef) = Reference( + projectId = ref.projectId, + repoName = ref.repoName, + bucket = ref.bucket, + key = RefKey.create(ref.key, true), + lastAccessDate = ref.lastAccessDate, + blobId = ContentHash.fromHex(ref.contentHash), + finalized = true, + inlineBlob = null, + legacy = true + ) + fun from(artifactInfo: ReferenceArtifactInfo, payload: ByteArray) = with(artifactInfo) { Reference( projectId = projectId, diff --git a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/repository/BlobRepository.kt b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/repository/BlobRepository.kt index 9d4bcbc775..cab34fb23a 100644 --- a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/repository/BlobRepository.kt +++ b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/repository/BlobRepository.kt @@ -32,6 +32,7 @@ import com.tencent.bkrepo.common.mongo.dao.simple.SimpleMongoDao import com.tencent.bkrepo.ddc.model.TDdcBlob import org.springframework.data.domain.Sort import org.springframework.data.mongodb.core.FindAndReplaceOptions +import org.springframework.data.mongodb.core.query.Criteria import org.springframework.data.mongodb.core.query.Query import org.springframework.data.mongodb.core.query.Update import org.springframework.data.mongodb.core.query.inValues @@ -84,4 +85,14 @@ class BlobRepository : SimpleMongoDao() { val update = Update().addToSet(TDdcBlob::references.name, "ref/$bucket/$refKey") updateMulti(Query(criteria), update) } + + fun removeRefFromBlob(projectId: String, repoName: String, bucket: String, refKey: String) { + // 从blob ref列表中移除ref + val criteria = Criteria + .where(TDdcBlob::projectId.name).isEqualTo(projectId) + .and(TDdcBlob::repoName.name).isEqualTo(repoName) + .and(TDdcBlob::references.name).inValues("ref/${bucket}/${refKey}") + val update = Update().pull(TDdcBlob::references.name, refKey) + updateMulti(Query(criteria), update) + } } diff --git a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/repository/LegacyRefRepository.kt b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/repository/LegacyRefRepository.kt new file mode 100644 index 0000000000..a71d741ab8 --- /dev/null +++ b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/repository/LegacyRefRepository.kt @@ -0,0 +1,44 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2023 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.ddc.repository + +import com.tencent.bkrepo.ddc.model.TDdcLegacyRef +import org.springframework.data.mongodb.core.query.Query +import org.springframework.data.mongodb.core.query.isEqualTo +import org.springframework.stereotype.Repository + +@Repository +class LegacyRefRepository : RefBaseRepository() { + fun find(projectId: String, repoName: String, bucket: String, key: String): TDdcLegacyRef? { + val criteria = TDdcLegacyRef::projectId.isEqualTo(projectId) + .and(TDdcLegacyRef::repoName.name).isEqualTo(repoName) + .and(TDdcLegacyRef::bucket.name).isEqualTo(bucket) + .and(TDdcLegacyRef::key.name).isEqualTo(key) + return findOne(Query(criteria)) + } +} diff --git a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/repository/RefBaseRepository.kt b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/repository/RefBaseRepository.kt new file mode 100644 index 0000000000..a36eaa02a4 --- /dev/null +++ b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/repository/RefBaseRepository.kt @@ -0,0 +1,71 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2023 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.ddc.repository + +import com.mongodb.DuplicateKeyException +import com.mongodb.client.result.DeleteResult +import com.mongodb.client.result.UpdateResult +import com.tencent.bkrepo.common.mongo.dao.simple.SimpleMongoDao +import com.tencent.bkrepo.ddc.model.TDdcRefBase +import com.tencent.bkrepo.ddc.pojo.RefId +import org.springframework.data.mongodb.core.FindAndReplaceOptions +import org.springframework.data.mongodb.core.query.Criteria +import org.springframework.data.mongodb.core.query.Query +import org.springframework.data.mongodb.core.query.Update +import org.springframework.data.mongodb.core.query.isEqualTo +import java.time.LocalDateTime + +abstract class RefBaseRepository : SimpleMongoDao() { + fun updateLastAccess(refId: RefId, lastAccessDate: LocalDateTime): UpdateResult { + val criteria = refIdCriteria(refId.projectId, refId.repoName, refId.bucket, refId.key) + val update = Update.update(TDdcRefBase::lastAccessDate.name, lastAccessDate) + return updateFirst(Query(criteria), update) + } + + fun replace(ref: TDdcRefBase): E? { + val criteria = refIdCriteria(ref.projectId, ref.repoName, ref.bucket, ref.key) + val query = Query(criteria) + val options = FindAndReplaceOptions().upsert() + return try { + determineMongoTemplate().findAndReplace(query, ref, options) as E? + } catch (e: DuplicateKeyException) { + findOne(query) + } + } + + fun delete(projectId: String, repoName: String, bucket: String, key: String): DeleteResult { + val criteria = refIdCriteria(projectId, repoName, bucket, key) + return remove(Query(criteria)) + } + + private fun refIdCriteria(projectId: String, repoName: String, bucket: String, key: String): Criteria = + TDdcRefBase::projectId.isEqualTo(projectId) + .and(TDdcRefBase::repoName.name).isEqualTo(repoName) + .and(TDdcRefBase::bucket.name).isEqualTo(bucket) + .and(TDdcRefBase::key.name).isEqualTo(key) +} diff --git a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/repository/RefRepository.kt b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/repository/RefRepository.kt index a5f7ffaf2d..19474357f4 100644 --- a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/repository/RefRepository.kt +++ b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/repository/RefRepository.kt @@ -27,21 +27,15 @@ package com.tencent.bkrepo.ddc.repository -import com.mongodb.DuplicateKeyException import com.mongodb.client.result.UpdateResult -import com.tencent.bkrepo.common.mongo.dao.simple.SimpleMongoDao import com.tencent.bkrepo.ddc.model.TDdcRef -import com.tencent.bkrepo.ddc.pojo.RefId -import org.springframework.data.mongodb.core.FindAndReplaceOptions -import org.springframework.data.mongodb.core.query.Criteria import org.springframework.data.mongodb.core.query.Query import org.springframework.data.mongodb.core.query.Update import org.springframework.data.mongodb.core.query.isEqualTo import org.springframework.stereotype.Repository -import java.time.LocalDateTime @Repository -class RefRepository : SimpleMongoDao() { +class RefRepository : RefBaseRepository() { fun find(projectId: String, repoName: String, bucket: String, key: String, includePayload: Boolean): TDdcRef? { val criteria = TDdcRef::projectId.isEqualTo(projectId) .and(TDdcRef::repoName.name).isEqualTo(repoName) @@ -54,20 +48,6 @@ class RefRepository : SimpleMongoDao() { return findOne(query) } - fun replace(ref: TDdcRef): TDdcRef? { - val criteria = TDdcRef::projectId.isEqualTo(ref.projectId) - .and(TDdcRef::repoName.name).isEqualTo(ref.repoName) - .and(TDdcRef::bucket.name).isEqualTo(ref.bucket) - .and(TDdcRef::key.name).isEqualTo(ref.key) - val query = Query(criteria) - val options = FindAndReplaceOptions().upsert() - return try { - determineMongoTemplate().findAndReplace(query, ref, options) - } catch (e: DuplicateKeyException) { - findOne(query) - } - } - fun finalize(projectId: String, repoName: String, bucket: String, key: String): UpdateResult { val criteria = TDdcRef::projectId.isEqualTo(projectId) .and(TDdcRef::repoName.name).isEqualTo(repoName) @@ -76,16 +56,4 @@ class RefRepository : SimpleMongoDao() { val update = Update.update(TDdcRef::finalized.name, true) return updateFirst(Query(criteria), update) } - - fun updateLastAccess(refId: RefId, lastAccessDate: LocalDateTime): UpdateResult { - val criteria = refIdCriteria(refId.projectId, refId.repoName, refId.bucket, refId.key) - val update = Update.update(TDdcRef::lastAccessDate.name, lastAccessDate) - return updateFirst(Query(criteria), update) - } - - private fun refIdCriteria(projectId: String, repoName: String, bucket: String, key: String): Criteria = - TDdcRef::projectId.isEqualTo(projectId) - .and(TDdcRef::repoName.name).isEqualTo(repoName) - .and(TDdcRef::bucket.name).isEqualTo(bucket) - .and(TDdcRef::key.name).isEqualTo(key) } diff --git a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/service/BlobService.kt b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/service/BlobService.kt index 9b3da23096..f7bec5eaec 100644 --- a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/service/BlobService.kt +++ b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/service/BlobService.kt @@ -51,7 +51,6 @@ class BlobService( val userId = SecurityUtils.getUserId() val now = LocalDateTime.now() val tBlob = TDdcBlob( - id = null, createdBy = userId, createdDate = now, lastModifiedBy = userId, @@ -61,7 +60,8 @@ class BlobService( blobId = blobId.toString(), contentId = contentId.toString(), sha256 = sha256, - size = size + size = size, + sha1 = sha1, ) blobRepository.replace(tBlob) @@ -107,4 +107,8 @@ class BlobService( fun addRefToBlobs(ref: Reference, blobIds: Set) { blobRepository.addRefToBlob(ref.projectId, ref.repoName, ref.bucket, ref.key.toString(), blobIds) } + + fun removeRefFromBlobs(projectId: String, repoName: String, bucket: String, key: String) { + blobRepository.removeRefFromBlob(projectId, repoName, bucket, key) + } } diff --git a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/service/ReferenceArtifactService.kt b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/service/ReferenceArtifactService.kt index 2e2dc2cf0f..ea3cfaa12e 100644 --- a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/service/ReferenceArtifactService.kt +++ b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/service/ReferenceArtifactService.kt @@ -52,6 +52,12 @@ class ReferenceArtifactService( repository.upload(ArtifactUploadContext(file)) } + fun deleteRef(artifactInfo: ReferenceArtifactInfo) { + with(artifactInfo) { + referenceService.deleteReference(projectId, repoName, bucket, refKey.toString(), legacy) + } + } + fun finalize(artifactInfo: ReferenceArtifactInfo) { with(artifactInfo) { val ref = referenceService.getReference( diff --git a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/service/ReferenceService.kt b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/service/ReferenceService.kt index 40b26869d6..7e8b8b0baf 100644 --- a/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/service/ReferenceService.kt +++ b/src/backend/ddc/biz-ddc/src/main/kotlin/com/tencent/bkrepo/ddc/service/ReferenceService.kt @@ -34,11 +34,14 @@ import com.tencent.bkrepo.common.artifact.repository.context.ArtifactContextHold import com.tencent.bkrepo.common.security.util.SecurityUtils import com.tencent.bkrepo.ddc.config.DdcProperties import com.tencent.bkrepo.ddc.exception.ReferenceIsMissingBlobsException +import com.tencent.bkrepo.ddc.model.TDdcLegacyRef import com.tencent.bkrepo.ddc.model.TDdcRef import com.tencent.bkrepo.ddc.pojo.ContentHash import com.tencent.bkrepo.ddc.pojo.CreateRefResponse import com.tencent.bkrepo.ddc.pojo.RefId import com.tencent.bkrepo.ddc.pojo.Reference +import com.tencent.bkrepo.ddc.repository.LegacyRefRepository +import com.tencent.bkrepo.ddc.repository.RefBaseRepository import com.tencent.bkrepo.ddc.repository.RefRepository import com.tencent.bkrepo.ddc.serialization.CbObject import com.tencent.bkrepo.ddc.utils.hasAttachments @@ -55,6 +58,7 @@ class ReferenceService( private val blobService: BlobService, private val refResolver: ReferenceResolver, private val refRepository: RefRepository, + private val legacyRefRepository: LegacyRefRepository, private val nodeClient: NodeClient, private val storageManager: StorageManager, ) { @@ -68,7 +72,6 @@ class ReferenceService( val userId = SecurityUtils.getUserId() val now = LocalDateTime.now() val tRef = TDdcRef( - id = null, createdBy = userId, createdDate = now, lastModifiedBy = userId, @@ -87,6 +90,34 @@ class ReferenceService( return Reference.from(tRef) } + fun createLegacyReference(ref: Reference): Reference { + val userId = SecurityUtils.getUserId() + val now = LocalDateTime.now() + val tRef = TDdcLegacyRef( + createdBy = userId, + createdDate = now, + lastModifiedBy = userId, + lastModifiedDate = now, + lastAccessDate = now, + projectId = ref.projectId, + repoName = ref.repoName, + bucket = ref.bucket, + key = ref.key.toString(), + contentHash = ref.blobId!!.toString(), + ) + legacyRefRepository.replace(tRef) + return Reference.from(tRef) + } + + fun getLegacyReference( + projectId: String, + repoName: String, + bucket: String, + key: String, + ): Reference? { + return legacyRefRepository.find(projectId, repoName, bucket, key)?.let { Reference.from(it) } + } + fun getReference( projectId: String, repoName: String, @@ -118,6 +149,22 @@ class ReferenceService( } } + fun deleteReference( + projectId: String, + repoName: String, + bucket: String, + key: String, + legacy: Boolean = false + ) { + if (getRepository(legacy).delete(projectId, repoName, bucket, key).deletedCount == 0L) { + throw BadRequestException( + CommonMessageCode.PARAMETER_INVALID, + "Deleted 0 records, most likely the object did not exist" + ) + } + blobService.removeRefFromBlobs(projectId, repoName, bucket, key) + } + fun finalize(ref: Reference, payload: ByteArray): CreateRefResponse { val cbObject = CbObject(ByteBuffer.wrap(payload)) var missingBlobs = emptyList() @@ -138,7 +185,13 @@ class ReferenceService( } fun updateLastAccess(refId: RefId, lastAccessDate: LocalDateTime) { - refRepository.updateLastAccess(refId, lastAccessDate) + getRepository(refId.legacy).updateLastAccess(refId, lastAccessDate) + } + + private fun getRepository(legacy: Boolean): RefBaseRepository<*> = if (legacy) { + legacyRefRepository + } else { + refRepository } companion object { diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/ddc/ExpiredDdcRefCleanupJob.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/ddc/ExpiredDdcRefCleanupJob.kt index 488a7eb913..ff735ecef6 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/ddc/ExpiredDdcRefCleanupJob.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/ddc/ExpiredDdcRefCleanupJob.kt @@ -52,7 +52,7 @@ class ExpiredDdcRefCleanupJob( private val mongoTemplate: MongoTemplate, private val nodeClient: NodeClient, ) : DefaultContextMongoDbJob(properties) { - override fun collectionNames(): List = listOf(COLLECTION_NAME) + override fun collectionNames(): List = listOf(COLLECTION_NAME, COLLECTION_NAME_LEGACY) override fun getLockAtMostFor(): Duration = Duration.ofDays(1) @@ -79,7 +79,7 @@ class ExpiredDdcRefCleanupJob( override fun run(row: Ref, collectionName: String, context: JobContext) { // 清理过期ref mongoTemplate.remove(Query(Criteria.where(ID).isEqualTo(row.id)), collectionName) - if (row.inlineBlob == null) { + if (row.inlineBlob == null && collectionName == COLLECTION_NAME) { // inlineBlob为null时表示inlineBlob不存在数据库中而是单独存放于后端存储中，需要一并清理 nodeClient.deleteNode( NodeDeleteRequest(row.projectId, row.repoName, "/${row.bucket}/${row.key}", SYSTEM_USER) @@ -107,5 +107,6 @@ class ExpiredDdcRefCleanupJob( companion object { const val COLLECTION_NAME = "ddc_ref" + const val COLLECTION_NAME_LEGACY = "ddc_legacy_ref" } }