feat: 文件型配置支持文件内容编码选择 --story=120473383

cmd/api-server/service/config_import.go

@@ -33,6 +33,7 @@ import (
 	"github.com/go-chi/chi/v5"
 	"github.com/go-chi/render"
 	"github.com/panjf2000/ants/v2"
+	"github.com/saintfish/chardet"
 
 	"github.com/TencentBlueKing/bk-bscp/internal/dal/repository"
 	"github.com/TencentBlueKing/bk-bscp/internal/iam/auth"
@@ -64,8 +65,7 @@ type configImport struct {
 }
 
 // TemplateConfigFileImport Import template config file
-//
-//nolint:funlen
+// nolint:funlen
 func (c *configImport) TemplateConfigFileImport(w http.ResponseWriter, r *http.Request) {
 	kt := kit.MustGetKit(r.Context())
 
@@ -263,8 +263,7 @@ func (c *configImport) TemplateConfigFileImport(w http.ResponseWriter, r *http.R
 }
 
 // ConfigFileImport Import config file
-//
-//nolint:funlen
+// nolint:funlen
 func (c *configImport) ConfigFileImport(w http.ResponseWriter, r *http.Request) {
 
 	kt := kit.MustGetKit(r.Context())
@@ -595,7 +594,7 @@ func (c *configImport) fileScannerHasherUploader(kt *kit.Kit, path, rootDir stri
 	}
 
 	// NOCC:ineffassign/assign(设计如此)
-	fileType := ""
+	fileType, charest := "", ""
 	// Check if the file size is greater than 5MB (5 * 1024 * 1024 bytes)
 	if fileInfo.Size() > constant.MaxUploadTextFileSize {
 		fileType = string(table.Binary)
@@ -611,6 +610,8 @@ func (c *configImport) fileScannerHasherUploader(kt *kit.Kit, path, rootDir stri
 		_, _ = fileContent.Seek(0, 0)
 		_, _ = io.Copy(fileBuffer, fileContent)
 		fileType = detectFileType(fileBuffer.Bytes())
+		// 检测文件编码
+		charest = detectFileCharest(fileBuffer.Bytes())
 	}
 	_, _ = fileContent.Seek(0, 0)
 
@@ -644,6 +645,7 @@ func (c *configImport) fileScannerHasherUploader(kt *kit.Kit, path, rootDir stri
 	resp.ByteSize = uint64(fileInfo.Size())
 	resp.Sign = result.Sha256
 	resp.Md5 = repoRes.Md5
+	resp.Chartset = charest
 
 	return resp, nil
 }
@@ -695,6 +697,22 @@ func detectFileType(buf []byte) string {
 	return fileType
 }
 
+// 检测文件编码
+func detectFileCharest(buf []byte) string {
+	// 使用 chardet 检测字符集
+	detector := chardet.NewTextDetector()
+	results, err := detector.DetectAll(buf)
+	if err != nil {
+		return ""
+	}
+
+	// 返回置信度最高的结果
+	if len(results) > 0 {
+		return results[0].Charset
+	}
+	return "Unknown"
+}
+
 // 判断内容类型是否为文本类型
 func isTextType(contentType string) bool {
 	textTypes := []string{

cmd/config-server/service/config_item.go

@@ -68,6 +68,7 @@ func (s *Service) CreateConfigItem(ctx context.Context, req *pbcs.CreateConfigIt
 				UserGroup: req.UserGroup,
 				Privilege: req.Privilege,
 			},
+			Charset: req.Charset,
 		},
 		ContentSpec: &pbcontent.ContentSpec{
 			Signature: req.Sign,
@@ -122,6 +123,7 @@ func (s *Service) BatchUpsertConfigItems(ctx context.Context, req *pbcs.BatchUps
 					UserGroup: item.UserGroup,
 					Privilege: item.Privilege,
 				},
+				Charset: item.Charset,
 			},
 			ContentSpec: &pbcontent.ContentSpec{
 				Signature: item.Sign,
@@ -191,6 +193,7 @@ func (s *Service) UpdateConfigItem(ctx context.Context, req *pbcs.UpdateConfigIt
 				UserGroup: req.UserGroup,
 				Privilege: req.Privilege,
 			},
+			Charset: req.Charset,
 		},
 	}
 	_, err = s.client.DS.UpdateConfigItem(grpcKit.RpcCtx(), r)

cmd/config-server/service/template.go

@@ -89,6 +89,7 @@ func (s *Service) CreateTemplate(ctx context.Context, req *pbcs.CreateTemplateRe
 				ByteSize:  req.ByteSize,
 				Md5:       metadata.Md5,
 			},
+			Charset: req.Charset,
 		},
 		TemplateSetIds: req.TemplateSetIds,
 	}
@@ -535,6 +536,7 @@ func (s *Service) BatchUpsertTemplates(ctx context.Context, req *pbcs.BatchUpser
 						ByteSize:  item.ByteSize,
 						Md5:       item.Md5,
 					},
+					Charset: item.Charset,
 				},
 				Attachment: &pbtr.TemplateRevisionAttachment{
 					BizId:           req.BizId,

cmd/config-server/service/template_revision.go

@@ -69,6 +69,7 @@ func (s *Service) CreateTemplateRevision(ctx context.Context, req *pbcs.CreateTe
 				ByteSize:  req.ByteSize,
 				Md5:       metadata.Md5,
 			},
+			Charset: req.Charset,
 		},
 	}
 	rp, err := s.client.DS.CreateTemplateRevision(grpcKit.RpcCtx(), r)
@@ -254,6 +255,7 @@ func (s *Service) GetTemplateRevision(ctx context.Context, req *pbcs.GetTemplate
 			CreateAt:             tr.GetDetail().GetCreateAt(),
 			Md5:                  tr.GetDetail().GetMd5(),
 			IsLatest:             tr.GetDetail().GetIsLatest(),
+			Charset:              tr.GetDetail().GetCharset(),
 		},
 	}, nil
 }
@@ -299,6 +301,7 @@ func (s *Service) UpdateTemplateRevision(ctx context.Context, req *pbcs.UpdateTe
 				ByteSize:  req.ByteSize,
 				Md5:       metadata.Md5,
 			},
+			Charset: req.Charset,
 		},
 		TemplateRevisionId: req.GetTemplateRevisionId(),
 	}

cmd/data-service/db-migration/migrations/20241122104755_modif_config_items.go

@@ -0,0 +1,136 @@
+/*
+ * Tencent is pleased to support the open source community by making Blueking Container Service available.
+ * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved.
+ * Licensed under the MIT License (the "License"); you may not use this file except
+ * in compliance with the License. You may obtain a copy of the License at
+ * http://opensource.org/licenses/MIT
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+ * either express or implied. See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package migrations
+
+import (
+	"gorm.io/gorm"
+
+	"github.com/TencentBlueKing/bk-bcs/bcs-services/bcs-bscp/cmd/data-service/db-migration/migrator"
+)
+
+func init() {
+	// add current migration to migrator
+	migrator.GetMigrator().AddMigration(&migrator.Migration{
+		Version: "20241122104755",
+		Name:    "20241122104755_modif_config_items",
+		Mode:    migrator.GormMode,
+		Up:      mig20241122104755Up,
+		Down:    mig20241122104755Down,
+	})
+}
+
+// mig20241122104755Up for up migration
+func mig20241122104755Up(tx *gorm.DB) error {
+	// ConfigItems  : config_items
+	type ConfigItems struct {
+		Charset string `gorm:"column:charset;type:varchar(20);default:'';NOT NULL"`
+	}
+
+	// ReleasedConfigItems  : released_config_items
+	type ReleasedConfigItems struct {
+		Charset string `gorm:"column:charset;type:varchar(20);default:'';NOT NULL"`
+	}
+
+	// TemplateRevisions  : template_revisions
+	type TemplateRevisions struct {
+		Charset string `gorm:"column:charset;type:varchar(20);default:'';NOT NULL"`
+	}
+
+	// ReleasedAppTemplates  : released_app_templates
+	type ReleasedAppTemplates struct {
+		Charset string `gorm:"column:charset;type:varchar(20);default:'';NOT NULL"`
+	}
+
+	// ConfigItems add new column
+	if !tx.Migrator().HasColumn(&ConfigItems{}, "charset") {
+		if err := tx.Migrator().AddColumn(&ConfigItems{}, "charset"); err != nil {
+			return err
+		}
+	}
+
+	// ReleasedConfigItems add new column
+	if !tx.Migrator().HasColumn(&ReleasedConfigItems{}, "charset") {
+		if err := tx.Migrator().AddColumn(&ReleasedConfigItems{}, "charset"); err != nil {
+			return err
+		}
+	}
+
+	// TemplateRevisions add new column
+	if !tx.Migrator().HasColumn(&TemplateRevisions{}, "charset") {
+		if err := tx.Migrator().AddColumn(&TemplateRevisions{}, "charset"); err != nil {
+			return err
+		}
+	}
+
+	// ReleasedAppTemplates add new column
+	if !tx.Migrator().HasColumn(&ReleasedAppTemplates{}, "charset") {
+		if err := tx.Migrator().AddColumn(&ReleasedAppTemplates{}, "charset"); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// mig20241122104755Down for down migration
+func mig20241122104755Down(tx *gorm.DB) error {
+	// ConfigItems  : config_items
+	type ConfigItems struct {
+		Charset string `gorm:"column:charset;type:varchar(20);default:'';NOT NULL"`
+	}
+
+	// ReleasedConfigItems  : released_config_items
+	type ReleasedConfigItems struct {
+		Charset string `gorm:"column:charset;type:varchar(20);default:'';NOT NULL"`
+	}
+
+	// TemplateRevisions  : template_revisions
+	type TemplateRevisions struct {
+		Charset string `gorm:"column:charset;type:varchar(20);default:'';NOT NULL"`
+	}
+
+	// ReleasedAppTemplates  : released_app_templates
+	type ReleasedAppTemplates struct {
+		Charset string `gorm:"column:charset;type:varchar(20);default:'';NOT NULL"`
+	}
+
+	// ConfigItems drop column
+	if tx.Migrator().HasColumn(&ConfigItems{}, "charset") {
+		if err := tx.Migrator().DropColumn(&ConfigItems{}, "charset"); err != nil {
+			return err
+		}
+	}
+
+	// ReleasedConfigItems drop column
+	if tx.Migrator().HasColumn(&ReleasedConfigItems{}, "charset") {
+		if err := tx.Migrator().DropColumn(&ReleasedConfigItems{}, "charset"); err != nil {
+			return err
+		}
+	}
+
+	// TemplateRevisions drop column
+	if tx.Migrator().HasColumn(&TemplateRevisions{}, "charset") {
+		if err := tx.Migrator().DropColumn(&TemplateRevisions{}, "charset"); err != nil {
+			return err
+		}
+	}
+
+	// ReleasedAppTemplates drop column
+	if tx.Migrator().HasColumn(&ReleasedAppTemplates{}, "charset") {
+		if err := tx.Migrator().DropColumn(&ReleasedAppTemplates{}, "charset"); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

cmd/data-service/service/app_template_binding.go

@@ -168,8 +168,7 @@ func (s *Service) DeleteAppTemplateBinding(ctx context.Context, req *pbds.Delete
 }
 
 // ListAppBoundTmplRevisions list app bound template revisions.
-//
-//nolint:funlen,gocyclo
+// nolint:funlen,gocyclo
 func (s *Service) ListAppBoundTmplRevisions(ctx context.Context,
 	req *pbds.ListAppBoundTmplRevisionsReq) (*pbds.ListAppBoundTmplRevisionsResp, error) {
 
@@ -269,6 +268,7 @@ func (s *Service) ListAppBoundTmplRevisions(ctx context.Context,
 				ByteSize:             d.Spec.ContentSpec.ByteSize,
 				Creator:              d.Revision.Creator,
 				CreateAt:             d.Revision.CreatedAt.Format(time.RFC3339),
+				Charset:              string(d.Spec.Charset),
 			})
 		}
 	}

cmd/data-service/service/config_item.go

@@ -1556,6 +1556,7 @@ func (s *Service) handleNonTemplateConfig(grpcKit *kit.Kit, bizID, appID, otherA
 					UserGroup: v.ConfigItemSpec.Permission.UserGroup,
 					Privilege: v.ConfigItemSpec.Permission.Privilege,
 				},
+				Charset: string(v.ConfigItemSpec.Charset),
 			},
 			Variables: vars[path.Join(v.ConfigItemSpec.Path, v.ConfigItemSpec.Name)],
 			IsExist:   conflicts[path.Join(v.ConfigItemSpec.Path, v.ConfigItemSpec.Name)],

cmd/data-service/service/release.go

@@ -381,6 +381,7 @@ func (s *Service) createReleasedRenderedTemplateCIs(kt *kit.Kit, tx *gen.QueryTx
 				FileMode:   r.Spec.FileMode,
 				Memo:       r.Spec.RevisionMemo,
 				Permission: r.Spec.Permission,
+				Charset:    r.Spec.Charset,
 			},
 			Attachment: &table.ConfigItemAttachment{
 				BizID: kt.BizID,
@@ -462,6 +463,7 @@ func (s *Service) createReleasedRenderedCIs(kt *kit.Kit, tx *gen.QueryTx, releas
 					UserGroup: ci.Spec.Permission.UserGroup,
 					Privilege: ci.Spec.Permission.Privilege,
 				},
+				Charset: ci.Spec.ConfigItemSpec().Charset,
 			},
 			Attachment: &table.ConfigItemAttachment{
 				BizID: ci.Attachment.BizId,
@@ -543,6 +545,7 @@ func (s *Service) createReleasedAppTemplates(kt *kit.Kit, tx *gen.QueryTx, relea
 				OriginSignature:      r.Signature,
 				OriginByteSize:       r.ByteSize,
 				Md5:                  md5Map[r.TemplateRevisionId],
+				Charset:              r.Charset,
 			},
 			Attachment: &table.ReleasedAppTemplateAttachment{
 				BizID: kt.BizID,

cmd/data-service/service/template_revision.go

@@ -305,6 +305,7 @@ func (s *Service) GetTemplateRevision(ctx context.Context, req *pbds.GetTemplate
 			CreateAt:             revision.Revision.CreatedAt.Format(time.RFC3339),
 			Md5:                  revision.Spec.ContentSpec.Md5,
 			IsLatest:             isLatest,
+			Charset:              string(revision.Spec.Charset),
 		},
 	}, nil
 }