feat(redis): Redis备份支持配置Tag #7919

dbm-services/redis/db-tools/dbmon/config/config.go

@@ -29,7 +29,9 @@ type ConfServerItem struct {
 
 // ConfRedisFullBackup 全备配置
 type ConfRedisFullBackup struct {
-	ToBackupSystem   string `json:"to_backup_system" mapstructure:"to_backup_system"`
+	BackupFileTag  string `json:"backup_file_tag" mapstructure:"backup_file_tag"`
+	ToBackupSystem string `json:"to_backup_system" mapstructure:"to_backup_system"`
+
 	Cron             string `json:"cron" mapstructure:"cron"`
 	OldFileLeftDay   int    `json:"old_file_left_day" mapstructure:"old_file_left_day"`
 	TarSplit         bool   `json:"tar_split" mapstructure:"tar_split"`
@@ -38,7 +40,9 @@ type ConfRedisFullBackup struct {
 
 // ConfRedisBinlogBackup binlog备份配置
 type ConfRedisBinlogBackup struct {
+	BackupFileTag  string `json:"backup_file_tag" mapstructure:"backup_file_tag"`
 	ToBackupSystem string `json:"to_backup_system" mapstructure:"to_backup_system"`
+
 	Cron           string `json:"cron" mapstructure:"cron"`
 	OldFileLeftDay int    `json:"old_file_left_day" mapstructure:"old_file_left_day"`
 }

dbm-services/redis/db-tools/dbmon/pkg/consts/consts.go

@@ -144,7 +144,7 @@ const (
 
 	RedisFullBackupTAG    = "REDIS_FULL"
 	RedisBinlogTAG        = "REDIS_BINLOG"
-	RedisForeverBackupTAG = "DBFILE"
+	RedisForeverBackupTAG = "DBFILE3Y"
 
 	RedisFullBackupReportType   = "redis_fullbackup"
 	RedisBinlogBackupReportType = "redis_binlogbackup"

dbm-services/redis/db-tools/dbmon/pkg/redisbinlogbackup/job.go

@@ -76,9 +76,14 @@ func (job *Job) Run() {
 	}
 	defer job.closeDB()
 
+	// 设置默认值，兼容老配置文件
+	if job.Conf.RedisBinlogBackup.BackupFileTag == "" {
+		job.Conf.RedisBinlogBackup.BackupFileTag = consts.RedisBinlogTAG
+	}
+
 	// job.backupClient = backupsys.NewIBSBackupClient(consts.IBSBackupClient, consts.RedisBinlogTAG)
 	job.backupClient, job.Err = backupsys.NewCosBackupClient(consts.COSBackupClient,
-		consts.COSInfoFile, consts.RedisBinlogTAG, job.Conf.BackupClientStrorageType)
+		consts.COSInfoFile, job.Conf.RedisBinlogBackup.BackupFileTag, job.Conf.BackupClientStrorageType)
 	if job.Err != nil {
 		if strings.HasPrefix(job.Err.Error(), "backup_client path not found") {
 			mylog.Logger.Debug(fmt.Sprintf("backup_client path:%s not found", consts.COSBackupClient))
@@ -172,7 +177,7 @@ func (job *Job) createTasks() {
 				taskBackupDir, svrItem.ServerShards[instStr],
 				job.Conf.RedisBinlogBackup.OldFileLeftDay,
 				job.Reporter,
-				job.Conf.BackupClientStrorageType,
+				job.Conf.BackupClientStrorageType, job.Conf.RedisBinlogBackup.BackupFileTag,
 				job.sqdb)
 			if job.Err != nil {
 				return

dbm-services/redis/db-tools/dbmon/pkg/redisbinlogbackup/task.go

@@ -116,7 +116,7 @@ type Task struct {
 // NewBinlogBackupTask new binlog backup task
 func NewBinlogBackupTask(bkBizID string, bkCloudID int64, domain, ip string, port int,
 	password, toBackupSys, backupDir, shardValue string, oldFileLeftDay int,
-	reporter report.Reporter, storageType string,
+	reporter report.Reporter, storageType string, backupFileTag string,
 	sqdb *gorm.DB) (ret *Task, err error) {
 
 	timeZone, _ := time.Now().Local().Zone()
@@ -135,13 +135,13 @@ func NewBinlogBackupTask(bkBizID string, bkCloudID int64, domain, ip string, por
 		ServerIP:   ip,
 		ServerPort: port,
 		BackupDir:  backupDir,
-		BackupTag:  consts.RedisBinlogTAG,
+		BackupTag:  backupFileTag,
 		ShardValue: shardValue,
 		TimeZone:   timeZone,
 	}
 	// ret.backupClient = backupsys.NewIBSBackupClient(consts.IBSBackupClient, consts.RedisBinlogTAG)
 	ret.backupClient, err = backupsys.NewCosBackupClient(consts.COSBackupClient,
-		consts.COSInfoFile, consts.RedisBinlogTAG, storageType)
+		consts.COSInfoFile, backupFileTag, storageType)
 	if err != nil && strings.HasPrefix(err.Error(), "backup_client path not found") {
 		ret.backupClient = nil
 		err = nil

dbm-services/redis/db-tools/dbmon/pkg/redisfullbackup/backupjob.go

@@ -76,9 +76,14 @@ func (job *Job) Run() {
 	}
 	defer job.closeDB()
 
+	// 设置默认值，兼容老配置文件
+	if job.Conf.RedisFullBackup.BackupFileTag == "" {
+		job.Conf.RedisFullBackup.BackupFileTag = consts.RedisFullBackupTAG
+	}
+
 	// job.backupClient = backupsys.NewIBSBackupClient(consts.IBSBackupClient, consts.RedisFullBackupTAG)
 	job.backupClient, job.Err = backupsys.NewCosBackupClient(consts.COSBackupClient,
-		consts.COSInfoFile, consts.RedisFullBackupTAG, job.Conf.BackupClientStrorageType)
+		consts.COSInfoFile, job.Conf.RedisFullBackup.BackupFileTag, job.Conf.BackupClientStrorageType)
 	if job.Err != nil && !strings.HasPrefix(job.Err.Error(), "backup_client path not found") {
 		return
 	}
@@ -170,7 +175,7 @@ func (job *Job) createTasks() {
 				consts.NormalBackupType, svrItem.CacheBackupMode, job.RealBackupDir,
 				job.Conf.RedisFullBackup.TarSplit, job.Conf.RedisFullBackup.TarSplitPartSize,
 				svrItem.ServerShards[instStr], job.Reporter,
-				job.Conf.BackupClientStrorageType,
+				job.Conf.BackupClientStrorageType, job.Conf.RedisFullBackup.BackupFileTag,
 				job.sqdb)
 			if job.Err != nil {
 				return

dbm-services/redis/db-tools/dbmon/pkg/redisfullbackup/checkjob.go

@@ -68,7 +68,7 @@ func (job *CheckJob) Run() {
 
 	// job.backupClient = backupsys.NewIBSBackupClient(consts.IBSBackupClient, consts.RedisFullBackupTAG)
 	job.backupClient, job.Err = backupsys.NewCosBackupClient(consts.COSBackupClient,
-		consts.COSInfoFile, consts.RedisFullBackupTAG, job.Conf.BackupClientStrorageType)
+		consts.COSInfoFile, job.Conf.RedisFullBackup.BackupFileTag, job.Conf.BackupClientStrorageType)
 	if job.Err != nil && !strings.HasPrefix(job.Err.Error(), "backup_client path not found") {
 		return
 	}

dbm-services/redis/db-tools/dbmon/pkg/redisfullbackup/task.go

@@ -116,7 +116,7 @@ type BackupTask struct {
 func NewFullBackupTask(bkBizID string, bkCloudID int64, domain, ip string, port int, password,
 	toBackupSys, backupType, cacheBackupMode, backupDir string, tarSplit bool,
 	tarSplitSize, shardValue string,
-	reporter report.Reporter, storageType string,
+	reporter report.Reporter, storageType string, backupFileTag string,
 	sqdb *gorm.DB) (ret *BackupTask, err error) {
 	ret = &BackupTask{
 		Password:         password,
@@ -139,13 +139,13 @@ func NewFullBackupTask(bkBizID string, bkCloudID int64, domain, ip string, port
 		BackupDir:    backupDir,
 		BackupTaskID: "",
 		BackupMD5:    "",
-		BackupTag:    consts.RedisFullBackupTAG,
+		BackupTag:    backupFileTag,
 		TimeZone:     timeZone,
 		ShardValue:   shardValue,
 	}
 	// ret.backupClient = backupsys.NewIBSBackupClient(consts.IBSBackupClient, consts.RedisFullBackupTAG)
 	ret.backupClient, err = backupsys.NewCosBackupClient(consts.COSBackupClient,
-		consts.COSInfoFile, consts.RedisFullBackupTAG, storageType)
+		consts.COSInfoFile, backupFileTag, storageType)
 	if err != nil && strings.HasPrefix(err.Error(), "backup_client path not found") {
 		ret.backupClient = nil
 		err = nil

dbm-ui/backend/flow/engine/bamboo/scene/redis/dirty_machine_clear.py

@@ -12,7 +12,8 @@
 from typing import Dict, Optional
 
 from backend.configuration.constants import DBType
-from backend.db_meta.models import ProxyInstance, StorageInstance
+from backend.db_meta.enums import AccessLayer
+from backend.db_meta.models import Machine, ProxyInstance, StorageInstance
 from backend.flow.engine.bamboo.scene.common.builder import Builder
 from backend.flow.engine.bamboo.scene.common.get_file_list import GetFileList
 from backend.flow.engine.bamboo.scene.redis.atom_jobs import DirtyProxyMachineClear, DirtyRedisMachineClear
@@ -86,6 +87,9 @@ def dirty_machine_clear_flow(self):
             ]
         }
         """
+        # 加个前置检查，以免误删。
+        self.precheck_4_clean()
+
         redis_pipeline = Builder(root_id=self.root_id, data=self.data)
         trans_files = GetFileList(db_type=DBType.Redis)
         act_kwargs = ActKwargs()
@@ -102,10 +106,10 @@ def dirty_machine_clear_flow(self):
                 "only_clear_dbmeta": self.data.get("only_clear_dbmeta", False),
             }
             proxy_inst = ProxyInstance.objects.filter(
-                machine__ip=ip, machine__bk_cloud_id=self.data["bk_cloud_id"]
+                machine__ip=ip, machine__bk_cloud_id=self.data["bk_cloud_id"], bk_biz_id=self.data["bk_biz_id"]
             ).first()
             storage_inst = StorageInstance.objects.filter(
-                machine__ip=ip, machine__bk_cloud_id=self.data["bk_cloud_id"]
+                machine__ip=ip, machine__bk_cloud_id=self.data["bk_cloud_id"], bk_biz_id=self.data["bk_biz_id"]
             ).first()
             if proxy_inst:
                 sub_builder = DirtyProxyMachineClear(
@@ -120,3 +124,27 @@ def dirty_machine_clear_flow(self):
         if sub_pipelines:
             redis_pipeline.add_parallel_sub_pipeline(sub_flow_list=sub_pipelines)
         redis_pipeline.run_pipeline()
+
+    def precheck_4_clean(self):
+        """
+        元数据前置检查：
+        1. 不能属于任何集群
+        2. 必须传入正确的 bizID 和 IP
+        """
+        for ip in self.data["infos"]:
+            try:
+                host_obj = Machine.objects.get(
+                    ip=ip, bk_cloud_id=self.data["bk_cloud_id"], bk_biz_id=self.data["bk_biz_id"]
+                )
+                if host_obj.access_layer == AccessLayer.PROXY:
+                    for proxy_obj in ProxyInstance.objects.filter(machine=host_obj).all():
+                        if proxy_obj.cluster.count() > 0:
+                            raise Exception("proxy {} in cluster , can't do this .".format(proxy_obj))
+                elif host_obj.access_layer == AccessLayer.STORAGE:
+                    for inst_obj in StorageInstance.objects.filter(machine=host_obj).all():
+                        if inst_obj.cluster.count() > 0:
+                            raise Exception("storage {} in cluster , can't do this .".format(inst_obj))
+            except Machine.DoesNotExist:
+                logger.log.info("{} host does not exists , ignore ".format(ip))
+            finally:
+                raise Exception("unkown exception... bugs {}".format(ip))