From 9cde9667ae1d2741bcc778be5b4508e8bf131f21 Mon Sep 17 00:00:00 2001 From: xfwduke Date: Mon, 25 Nov 2024 12:08:50 +0800 Subject: [PATCH] =?UTF-8?q?fix(mysql):=20mysql=E5=A4=87=E4=BB=BD=E5=8D=95?= =?UTF-8?q?=E6=8D=AE=E8=B0=83=E6=95=B4v2=20#8142?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../subcmd/mysqlcmd/backup_database_table.go | 82 ------ .../internal/subcmd/mysqlcmd/backup_demand.go | 6 + .../mysql/backupdemand/backup_demand.go | 269 ++++++++++-------- ...ble_backup.py => mysql_db_table_backup.py} | 36 +-- ...ckup_flow.py => mysql_full_backup_flow.py} | 68 ++--- ..._db_table_backup.py => db_table_backup.py} | 30 +- ..._cluster_full_backup.py => full_backup.py} | 75 +++-- .../backend/flow/engine/controller/mysql.py | 15 +- .../backend/flow/engine/controller/spider.py | 4 +- .../utils/mysql/db_table_filter/filter.py | 82 +++--- .../flow/utils/mysql/db_table_filter/tools.py | 2 + .../flow/views/mysql_ha_db_table_backup.py | 2 +- .../builders/mysql/mysql_db_table_backup.py | 147 ++++++++++ .../builders/mysql/mysql_full_backup.py | 165 +++++++++++ .../ticket/builders/mysql/mysql_ha_backup.py | 58 ---- .../builders/mysql/mysql_ha_full_backup.py | 68 ----- .../builders/mysql/mysql_import_sqlfile.py | 2 +- .../builders/tendbcluster/db_table_backup.py | 182 ++++++++++++ .../builders/tendbcluster/full_backup.py | 178 ++++++++++++ .../builders/tendbcluster/tendb_backup.py | 51 ---- .../tendbcluster/tendb_full_backup.py | 70 ----- .../tendbcluster/tendb_import_sqlfile.py | 4 +- .../builders/tendbsingle/db_table_backup.py | 28 ++ .../ticket/builders/tendbsingle/fullbackup.py | 27 ++ dbm-ui/backend/ticket/constants.py | 8 +- 25 files changed, 1025 insertions(+), 634 deletions(-) delete mode 100644 dbm-services/mysql/db-tools/dbactuator/internal/subcmd/mysqlcmd/backup_database_table.go rename dbm-ui/backend/flow/engine/bamboo/scene/mysql/{mysql_ha_db_table_backup.py => mysql_db_table_backup.py} (81%) rename dbm-ui/backend/flow/engine/bamboo/scene/mysql/{mysql_ha_full_backup_flow.py => mysql_full_backup_flow.py} (66%) rename dbm-ui/backend/flow/engine/bamboo/scene/spider/{spider_cluster_db_table_backup.py => db_table_backup.py} (91%) rename dbm-ui/backend/flow/engine/bamboo/scene/spider/{spider_cluster_full_backup.py => full_backup.py} (85%) create mode 100644 dbm-ui/backend/ticket/builders/mysql/mysql_db_table_backup.py create mode 100644 dbm-ui/backend/ticket/builders/mysql/mysql_full_backup.py delete mode 100644 dbm-ui/backend/ticket/builders/mysql/mysql_ha_backup.py delete mode 100644 dbm-ui/backend/ticket/builders/mysql/mysql_ha_full_backup.py create mode 100644 dbm-ui/backend/ticket/builders/tendbcluster/db_table_backup.py create mode 100644 dbm-ui/backend/ticket/builders/tendbcluster/full_backup.py delete mode 100644 dbm-ui/backend/ticket/builders/tendbcluster/tendb_backup.py delete mode 100644 dbm-ui/backend/ticket/builders/tendbcluster/tendb_full_backup.py create mode 100644 dbm-ui/backend/ticket/builders/tendbsingle/db_table_backup.py create mode 100644 dbm-ui/backend/ticket/builders/tendbsingle/fullbackup.py diff --git a/dbm-services/mysql/db-tools/dbactuator/internal/subcmd/mysqlcmd/backup_database_table.go b/dbm-services/mysql/db-tools/dbactuator/internal/subcmd/mysqlcmd/backup_database_table.go deleted file mode 100644 index 02c739df97..0000000000 --- a/dbm-services/mysql/db-tools/dbactuator/internal/subcmd/mysqlcmd/backup_database_table.go +++ /dev/null @@ -1,82 +0,0 @@ -package mysqlcmd - -//import ( -// "fmt" -// -// "dbm-services/common/go-pubpkg/logger" -// "dbm-services/mysql/db-tools/dbactuator/internal/subcmd" -// "dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql" -// "dbm-services/mysql/db-tools/dbactuator/pkg/util" -// -// "github.com/spf13/cobra" -//) -// -//// BackupDatabaseTable TODO -//const BackupDatabaseTable = "backup-database-table" -// -//// BackupDatabaseTableAct TODO -//type BackupDatabaseTableAct struct { -// *subcmd.BaseOptions -// Payload mysql.BackupDatabaseTableComp -//} -// -//// NewBackupDatabaseTableCommand TODO -//func NewBackupDatabaseTableCommand() *cobra.Command { -// act := BackupDatabaseTableAct{ -// BaseOptions: subcmd.GBaseOptions, -// } -// cmd := &cobra.Command{ -// Use: BackupDatabaseTable, -// Short: "备库库表", -// Example: fmt.Sprintf(`dbactuator mysql %s %s`, BackupDatabaseTable, subcmd.CmdBaseExampleStr), -// Run: func(cmd *cobra.Command, args []string) { -// util.CheckErr(act.Validate()) -// util.CheckErr(act.Init()) -// util.CheckErr(act.Run()) -// }, -// } -// return cmd -//} -// -//// Validate TODO -//func (c *BackupDatabaseTableAct) Validate() (err error) { -// return c.BaseOptions.Validate() -//} -// -//// Init TODO -//func (c *BackupDatabaseTableAct) Init() (err error) { -// if err = c.DeserializeAndValidate(&c.Payload); err != nil { -// logger.Error("DeserializeAndValidate err %s", err.Error()) -// return err -// } -// return -//} -// -//// Run TODO -//func (c *BackupDatabaseTableAct) Run() (err error) { -// defer util.LoggerErrorStack(logger.Error, err) -// // subcmd.Steps 顺序执行，某个步骤error，剩下步骤不执行 -// steps := subcmd.Steps{ -// { -// FunName: "Precheck", -// Func: c.Payload.Precheck, -// }, -// { -// FunName: "CreateBackupConfigFile", -// Func: c.Payload.CreateBackupConfigFile, -// }, -// { -// FunName: "DoBackup", -// Func: c.Payload.DoBackup, -// }, -// { -// FunName: "OutputBackupInfo", -// Func: c.Payload.OutputBackupInfo, -// }, -// } -// if err = steps.Run(); err != nil { -// return err -// } -// logger.Info("备份成功") -// return nil -//} diff --git a/dbm-services/mysql/db-tools/dbactuator/internal/subcmd/mysqlcmd/backup_demand.go b/dbm-services/mysql/db-tools/dbactuator/internal/subcmd/mysqlcmd/backup_demand.go index 8ad5f7c87a..d540f4836f 100644 --- a/dbm-services/mysql/db-tools/dbactuator/internal/subcmd/mysqlcmd/backup_demand.go +++ b/dbm-services/mysql/db-tools/dbactuator/internal/subcmd/mysqlcmd/backup_demand.go @@ -52,6 +52,8 @@ func (d *BackupDemandAct) Init() (err error) { logger.Error("DeserializeAndValidate err %s", err.Error()) return err } + + d.Payload.GeneralParam = subcmd.GeneralRuntimeParam logger.Warn("params %+v", d.Payload.Params) return @@ -74,6 +76,10 @@ func (d *BackupDemandAct) Run() (err error) { FunName: "生成备份配置", Func: d.Payload.GenerateBackupConfig, }, + { + FunName: "终止残留备份进程", + Func: d.Payload.KillLegacyBackup, + }, { FunName: "执行备份", Func: d.Payload.DoBackup, diff --git a/dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/backupdemand/backup_demand.go b/dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/backupdemand/backup_demand.go index a708d6adc5..605f905994 100644 --- a/dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/backupdemand/backup_demand.go +++ b/dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/backupdemand/backup_demand.go @@ -24,7 +24,6 @@ import ( "github.com/pkg/errors" "dbm-services/common/go-pubpkg/cmutil" - "dbm-services/common/go-pubpkg/mysqlcomm" "dbm-services/mysql/db-tools/dbactuator/pkg/components" "dbm-services/mysql/db-tools/dbactuator/pkg/core/cst" "dbm-services/mysql/db-tools/dbactuator/pkg/tools" @@ -38,9 +37,10 @@ import ( ) type Component struct { - Params *Param `json:"extend"` - tools *tools.ToolSet - context `json:"-"` + GeneralParam *components.GeneralParam `json:"general"` + Params *Param `json:"extend"` + tools *tools.ToolSet + context `json:"-"` } type Param struct { @@ -62,14 +62,15 @@ type Param struct { } type context struct { - backupConfigPaths map[int]string - now time.Time - randString string + backupConfigPath string + now time.Time + randString string //resultReportPath string statusReportPath string reportPath string - backupPort []int // 当在 spider master备份时, 会有 [25000, 26000] 两个端口 + backupPort int // 当在 spider master备份时, 会有 [25000, 26000] 两个端口 backupDir string //只是兼容tbinlogdumper的备份日志输出，存储备份目录信息，没有任何处理逻辑 + db *native.DbWorker } type Report struct { @@ -90,126 +91,138 @@ func (c *Component) Init() (err error) { rand.Seed(c.now.UnixNano()) c.randString = fmt.Sprintf("%d%d", c.now.UnixNano(), rand.Intn(100)) - c.backupConfigPaths = make(map[int]string) + //c.backupConfigPath = make(map[int]string) - c.backupPort = append(c.backupPort, c.Params.Port) - c.backupConfigPaths[c.Params.Port] = filepath.Join( + //c.backupPort = append(c.backupPort, c.Params.Port) + c.backupPort = c.Params.Port + c.backupConfigPath = filepath.Join( cst.BK_PKG_INSTALL_PATH, fmt.Sprintf("dbactuator-%s", c.Params.BillId), fmt.Sprintf("dbbackup.%d.%s.ini", c.Params.Port, c.randString), ) - if c.Params.Role == cst.BackupRoleSpiderMaster { - tdbctlPort := mysqlcomm.GetTdbctlPortBySpider(c.Params.Port) - c.backupPort = append(c.backupPort, tdbctlPort) - - c.backupConfigPaths[tdbctlPort] = filepath.Join( - cst.BK_PKG_INSTALL_PATH, - fmt.Sprintf("dbactuator-%s", c.Params.BillId), - fmt.Sprintf("dbbackup.%d.%s.ini", tdbctlPort, c.randString), - ) + //if c.Params.Role == cst.BackupRoleSpiderMaster { + // tdbctlPort := mysqlcomm.GetTdbctlPortBySpider(c.Params.Port) + // c.backupPort = append(c.backupPort, tdbctlPort) + // + // c.backupConfigPath[tdbctlPort] = filepath.Join( + // cst.BK_PKG_INSTALL_PATH, + // fmt.Sprintf("dbactuator-%s", c.Params.BillId), + // fmt.Sprintf("dbbackup.%d.%s.ini", tdbctlPort, c.randString), + // ) + //} + + c.db, err = native.InsObject{ + Host: c.Params.Host, + Port: c.backupPort, + User: c.GeneralParam.RuntimeAccountParam.MonitorUser, + Pwd: c.GeneralParam.RuntimeAccountParam.MonitorPwd, + }.Conn() + if err != nil { + logger.Error("init db connection failed: %s", err.Error()) + return err } return nil } func (c *Component) GenerateBackupConfig() error { - for _, port := range c.backupPort { - dailyBackupConfigPath := filepath.Join( - cst.DbbackupGoInstallPath, - fmt.Sprintf("dbbackup.%d.ini", port), - ) - - dailyBackupConfigFile, err := ini.LoadSources(ini.LoadOptions{ - PreserveSurroundedQuote: true, - IgnoreInlineComment: true, - AllowBooleanKeys: true, - AllowShadows: true, - }, dailyBackupConfigPath) - if err != nil { - logger.Error("load %s failed: %s", dailyBackupConfigPath, err.Error()) - return err - } - - var backupConfig config.BackupConfig - err = dailyBackupConfigFile.MapTo(&backupConfig) - if err != nil { - logger.Error("map %s to struct failed: %s", dailyBackupConfigPath, err.Error()) - return err - } - - backupConfig.Public.BackupType = c.Params.BackupType - backupConfig.Public.BackupTimeOut = "" - backupConfig.Public.BillId = c.Params.BillId - backupConfig.Public.BackupId = c.Params.BackupId - backupConfig.Public.DataSchemaGrant = strings.Join(c.Params.BackupGSD, ",") - backupConfig.Public.ShardValue = c.Params.ShardID - if backupConfig.BackupClient.Enable && c.Params.BackupFileTag != "" { - backupConfig.BackupClient.FileTag = c.Params.BackupFileTag - } - - backupConfig.LogicalBackup.Regex = "" - if c.Params.BackupType == "logical" { - backupConfig.LogicalBackup.UseMysqldump = "auto" - ignoreDbs := slices.DeleteFunc(native.DBSys, func(s string) bool { - return s == "infodba_schema" - }) - ignoreDbs = append(ignoreDbs, c.Params.IgnoreDbs...) - backupConfig.LogicalBackup.Databases = strings.Join(c.Params.DbPatterns, ",") - backupConfig.LogicalBackup.ExcludeDatabases = strings.Join(ignoreDbs, ",") - backupConfig.LogicalBackup.Tables = strings.Join(c.Params.TablePatterns, ",") - backupConfig.LogicalBackup.ExcludeTables = strings.Join(c.Params.IgnoreTables, ",") - - tf, err := db_table_filter.NewFilter( - c.Params.DbPatterns, c.Params.TablePatterns, - ignoreDbs, c.Params.IgnoreTables, - ) - if err != nil { - logger.Error("create table filter failed: %s", err.Error()) - return err - } - backupConfig.LogicalBackup.Regex = tf.TableFilterRegex() - } + //for _, port := range c.backupPort { + dailyBackupConfigPath := filepath.Join( + cst.DbbackupGoInstallPath, + fmt.Sprintf("dbbackup.%d.ini", c.backupPort), + ) - if c.Params.CustomBackupDir != "" { - backupConfig.Public.BackupDir = filepath.Join( - backupConfig.Public.BackupDir, - fmt.Sprintf("%s_%s_%d_%s", - c.Params.CustomBackupDir, - c.now.Format("20060102150405"), - port, - c.randString)) + dailyBackupConfigFile, err := ini.LoadSources(ini.LoadOptions{ + PreserveSurroundedQuote: true, + IgnoreInlineComment: true, + AllowBooleanKeys: true, + AllowShadows: true, + }, dailyBackupConfigPath) + if err != nil { + logger.Error("load %s failed: %s", dailyBackupConfigPath, err.Error()) + return err + } - err := os.Mkdir(backupConfig.Public.BackupDir, 0755) - if err != nil { - logger.Error("mkdir %s failed: %s", backupConfig.Public.BackupDir, err.Error()) - return err - } + var backupConfig config.BackupConfig + err = dailyBackupConfigFile.MapTo(&backupConfig) + if err != nil { + logger.Error("map %s to struct failed: %s", dailyBackupConfigPath, err.Error()) + return err + } - } - // 增加为tbinlogdumper做库表备份的日志输出，保存流程上下文 - c.backupDir = backupConfig.Public.BackupDir + backupConfig.Public.BackupType = c.Params.BackupType + backupConfig.Public.BackupTimeOut = "" + backupConfig.Public.BillId = c.Params.BillId + backupConfig.Public.BackupId = c.Params.BackupId + backupConfig.Public.DataSchemaGrant = strings.Join(c.Params.BackupGSD, ",") + backupConfig.Public.ShardValue = c.Params.ShardID + if backupConfig.BackupClient.Enable && c.Params.BackupFileTag != "" { + backupConfig.BackupClient.FileTag = c.Params.BackupFileTag + } - backupConfigFile := ini.Empty() - err = backupConfigFile.ReflectFrom(&backupConfig) + backupConfig.LogicalBackup.Regex = "" + if c.Params.BackupType == "logical" { + backupConfig.LogicalBackup.UseMysqldump = "auto" + ignoreDbs := slices.DeleteFunc(native.DBSys, func(s string) bool { + return s == "infodba_schema" + }) + ignoreDbs = append(ignoreDbs, c.Params.IgnoreDbs...) + backupConfig.LogicalBackup.Databases = strings.Join(c.Params.DbPatterns, ",") + backupConfig.LogicalBackup.ExcludeDatabases = strings.Join(ignoreDbs, ",") + backupConfig.LogicalBackup.Tables = strings.Join(c.Params.TablePatterns, ",") + backupConfig.LogicalBackup.ExcludeTables = strings.Join(c.Params.IgnoreTables, ",") + + tf, err := db_table_filter.NewFilter( + c.Params.DbPatterns, c.Params.TablePatterns, + ignoreDbs, c.Params.IgnoreTables, + ) if err != nil { - logger.Error("reflect backup config failed: %s", err.Error()) + logger.Error("create table filter failed: %s", err.Error()) return err } + backupConfig.LogicalBackup.Regex = tf.TableFilterRegex() + } + + if c.Params.CustomBackupDir != "" { + backupConfig.Public.BackupDir = filepath.Join( + backupConfig.Public.BackupDir, + fmt.Sprintf("%s_%s_%d_%s", + c.Params.CustomBackupDir, + c.now.Format("20060102150405"), + c.backupPort, + c.randString)) - backupConfigPath := c.backupConfigPaths[port] - err = backupConfigFile.SaveTo(backupConfigPath) + err := os.Mkdir(backupConfig.Public.BackupDir, 0755) if err != nil { - logger.Error("write backup config to %s failed: %s", - backupConfigPath, err.Error()) + logger.Error("mkdir %s failed: %s", backupConfig.Public.BackupDir, err.Error()) return err } - c.statusReportPath = filepath.Join( - backupConfig.Public.StatusReportPath, - fmt.Sprintf("dbareport_status_%d.log", c.Params.Port), - ) } + // 增加为tbinlogdumper做库表备份的日志输出，保存流程上下文 + c.backupDir = backupConfig.Public.BackupDir + + backupConfigFile := ini.Empty() + err = backupConfigFile.ReflectFrom(&backupConfig) + if err != nil { + logger.Error("reflect backup config failed: %s", err.Error()) + return err + } + + //backupConfigPath := c.backupConfigPath[port] + err = backupConfigFile.SaveTo(c.backupConfigPath) + if err != nil { + logger.Error("write backup config to %s failed: %s", + c.backupConfigPath, err.Error()) + return err + } + + c.statusReportPath = filepath.Join( + backupConfig.Public.StatusReportPath, + fmt.Sprintf("dbareport_status_%d.log", c.Params.Port), + ) + //} return nil } @@ -219,22 +232,52 @@ func (c *Component) ActuatorWorkDir() string { return filepath.Dir(executable) } -func (c *Component) DoBackup() error { - for _, port := range c.backupPort { - backupConfigPath := c.backupConfigPaths[port] - cmdArgs := []string{c.tools.MustGet(tools.ToolDbbackupGo), "dumpbackup", "--config", backupConfigPath} - cmdArgs = append(cmdArgs, "--log-dir", filepath.Join(c.ActuatorWorkDir(), "logs")) - logger.Info("backup command: %s", strings.Join(cmdArgs, " ")) +// KillLegacyBackup +// 尽力而为尝试去结束残留的备份连接 +func (c *Component) KillLegacyBackup() error { + defer func() { + _ = c.db.Close + }() - _, errStr, err := cmutil.ExecCommand(false, "", - cmdArgs[0], cmdArgs[1:]...) + processList, err := c.db.SelectProcesslist([]string{c.GeneralParam.RuntimeAccountParam.DbBackupUser}) + if err != nil { + logger.Error("get process list failed: %s", err.Error()) + return err + } + + logger.Info("backup process %v found", processList) + if len(processList) == 0 { + return nil + } + for _, process := range processList { + _, err := c.db.Exec(`KILL ?`, process.ID) if err != nil { - logger.Error("execute %s failed: %s, msg:%s", cmdArgs, err.Error(), errStr) + logger.Error("kill %s failed: %s", process.ID, err.Error()) return err } - logger.Info("backup success with %s", backupConfigPath) + logger.Info("kill %v successfully", process) + } + + return nil +} + +func (c *Component) DoBackup() error { + //for _, port := range c.backupPort { + // backupConfigPath := c.backupConfigPath[c.b] + cmdArgs := []string{c.tools.MustGet(tools.ToolDbbackupGo), "dumpbackup", "--config", c.backupConfigPath} + cmdArgs = append(cmdArgs, "--log-dir", filepath.Join(c.ActuatorWorkDir(), "logs")) + logger.Info("backup command: %s", strings.Join(cmdArgs, " ")) + + _, errStr, err := cmutil.ExecCommand(false, "", + cmdArgs[0], cmdArgs[1:]...) + + if err != nil { + logger.Error("execute %s failed: %s, msg:%s", cmdArgs, err.Error(), errStr) + return err } + logger.Info("backup success with %s", c.backupConfigPath) + //} return nil } diff --git a/dbm-ui/backend/flow/engine/bamboo/scene/mysql/mysql_ha_db_table_backup.py b/dbm-ui/backend/flow/engine/bamboo/scene/mysql/mysql_db_table_backup.py similarity index 81% rename from dbm-ui/backend/flow/engine/bamboo/scene/mysql/mysql_ha_db_table_backup.py rename to dbm-ui/backend/flow/engine/bamboo/scene/mysql/mysql_db_table_backup.py index 4a077debb0..10dd25dc72 100644 --- a/dbm-ui/backend/flow/engine/bamboo/scene/mysql/mysql_ha_db_table_backup.py +++ b/dbm-ui/backend/flow/engine/bamboo/scene/mysql/mysql_db_table_backup.py @@ -8,18 +8,15 @@ an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ -import collections import logging import uuid from dataclasses import asdict from typing import Dict, Optional -from django.core.exceptions import ObjectDoesNotExist from django.utils.translation import ugettext as _ from backend.configuration.constants import DBType -from backend.db_meta.enums import ClusterType, InstanceInnerRole -from backend.db_meta.exceptions import ClusterNotExistException, DBMetaBaseException +from backend.db_meta.enums import ClusterType, InstanceInnerRole, InstanceStatus from backend.db_meta.models import Cluster from backend.flow.consts import DBA_SYSTEM_USER from backend.flow.engine.bamboo.scene.common.builder import Builder, SubBuilder @@ -42,7 +39,7 @@ logger = logging.getLogger("flow") -class MySQLHADBTableBackupFlow(object): +class MySQLDBTableBackupFlow(object): """ 支持跨云操作 """ @@ -58,7 +55,7 @@ def backup_flow(self): "uid": "2022051612120001", "created_by": "xxx", "bk_biz_id": "152", - "ticket_type": "MYSQL_HA_DB_TABLE_BACKUP", + "ticket_type": "MYSQL_DB_TABLE_BACKUP", "infos": [ { "cluster_id": int, @@ -73,31 +70,19 @@ def backup_flow(self): } 增加单据临时ADMIN账号的添加和删除逻辑 """ - cluster_ids = [job["cluster_id"] for job in self.data["infos"]] - dup_cluster_ids = [item for item, count in collections.Counter(cluster_ids).items() if count > 1] - if dup_cluster_ids: - raise DBMetaBaseException(message="duplicate clusters found: {}".format(dup_cluster_ids)) - backup_pipeline = Builder( - root_id=self.root_id, data=self.data, need_random_pass_cluster_ids=list(set(cluster_ids)) + root_id=self.root_id, data=self.data # , need_random_pass_cluster_ids=list(set(cluster_ids)) ) sub_pipes = [] for job in self.data["infos"]: - try: - cluster_obj = Cluster.objects.get( - pk=job["cluster_id"], bk_biz_id=self.data["bk_biz_id"], cluster_type=ClusterType.TenDBHA.value - ) - except ObjectDoesNotExist: - raise ClusterNotExistException( - cluster_type=ClusterType.TenDBHA.value, cluster_id=job["cluster_id"], immute_domain="" - ) + cluster_obj = Cluster.objects.get(pk=job["cluster_id"], bk_biz_id=self.data["bk_biz_id"]) - try: + if cluster_obj.cluster_type == ClusterType.TenDBHA: instance_obj = cluster_obj.storageinstance_set.get( - instance_inner_role=InstanceInnerRole.SLAVE.value, is_stand_by=True + instance_inner_role=InstanceInnerRole.SLAVE.value, is_stand_by=True, status=InstanceStatus.RUNNING ) - except ObjectDoesNotExist: - raise DBMetaBaseException(message=_("{} standby slave 不存在".format(cluster_obj.immute_domain))) + else: + instance_obj = cluster_obj.storageinstance_set.filter(status=InstanceStatus.RUNNING).first() sub_pipe = SubBuilder( root_id=self.root_id, @@ -152,7 +137,6 @@ def backup_flow(self): get_mysql_payload_func=MysqlActPayload.mysql_backup_demand_payload.__name__, ) ), - # write_payload_var="backup_report_response", ) sub_pipe.add_act( @@ -165,4 +149,4 @@ def backup_flow(self): backup_pipeline.add_parallel_sub_pipeline(sub_flow_list=sub_pipes) logger.info(_("构建库表备份流程成功")) - backup_pipeline.run_pipeline(init_trans_data_class=MySQLBackupDemandContext(), is_drop_random_user=True) + backup_pipeline.run_pipeline(init_trans_data_class=MySQLBackupDemandContext()) # , is_drop_random_user=True) diff --git a/dbm-ui/backend/flow/engine/bamboo/scene/mysql/mysql_ha_full_backup_flow.py b/dbm-ui/backend/flow/engine/bamboo/scene/mysql/mysql_full_backup_flow.py similarity index 66% rename from dbm-ui/backend/flow/engine/bamboo/scene/mysql/mysql_ha_full_backup_flow.py rename to dbm-ui/backend/flow/engine/bamboo/scene/mysql/mysql_full_backup_flow.py index 24df2cd91b..7bc0042d05 100644 --- a/dbm-ui/backend/flow/engine/bamboo/scene/mysql/mysql_ha_full_backup_flow.py +++ b/dbm-ui/backend/flow/engine/bamboo/scene/mysql/mysql_full_backup_flow.py @@ -13,17 +13,14 @@ from dataclasses import asdict from typing import Dict, Optional -from django.core.exceptions import ObjectDoesNotExist from django.utils.translation import ugettext as _ from backend.configuration.constants import DBType -from backend.db_meta.enums import ClusterType, InstanceInnerRole -from backend.db_meta.exceptions import ClusterNotExistException, DBMetaBaseException +from backend.db_meta.enums import ClusterType from backend.db_meta.models import Cluster from backend.flow.consts import DBA_SYSTEM_USER, LONG_JOB_TIMEOUT from backend.flow.engine.bamboo.scene.common.builder import Builder, SubBuilder from backend.flow.engine.bamboo.scene.common.get_file_list import GetFileList -from backend.flow.engine.exceptions import MySQLBackupLocalException from backend.flow.plugins.components.collections.mysql.exec_actuator_script import ExecuteDBActuatorScriptComponent from backend.flow.plugins.components.collections.mysql.mysql_link_backup_id_bill_id import ( MySQLLinkBackupIdBillIdComponent, @@ -36,7 +33,7 @@ logger = logging.getLogger("flow") -class MySQLHAFullBackupFlow(object): +class MySQLFullBackupFlow(object): """ mysql 库表备份流程 支持跨云管理 @@ -53,47 +50,33 @@ def full_backup_flow(self): "uid": "398346234", "created_type": "xxx", "bk_biz_id": "152", - "ticket_type": "MYSQL_HA_FULL_BACKUP", - "infos": { - "backup_type": enum of backend.flow.consts.MySQLBackupTypeEnum - "file_tag": enum of backend.flow.consts.MySQLBackupFileTagEnum - "clusters": [{"cluster_id":"", "backup_local": enum []}] - } + "ticket_type": "MYSQL_HA_FULL_BACKUP|MYSQL_SINGLE_FULL_BACKUP", + "backup_type": enum of backend.flow.consts.MySQLBackupTypeEnum + "file_tag": enum of backend.flow.consts.MySQLBackupFileTagEnum + "infos": [ + { + "cluster_id": int, + "backup_local": enum + } + ] } 增加单据临时ADMIN账号的添加和删除逻辑 """ - clusters = self.data["infos"]["clusters"] - cluster_ids = [cluster["cluster_id"] for cluster in clusters] - backup_pipeline = Builder( - root_id=self.root_id, data=self.data, need_random_pass_cluster_ids=list(set(cluster_ids)) + root_id=self.root_id, data=self.data # , need_random_pass_cluster_ids=list(set(cluster_ids)) ) sub_pipes = [] - for cluster in clusters: - cluster_id = cluster["cluster_id"] - backup_local = cluster["backup_local"] - - try: - cluster_obj = Cluster.objects.get( - pk=cluster_id, bk_biz_id=self.data["bk_biz_id"], cluster_type=ClusterType.TenDBHA.value - ) - except ObjectDoesNotExist: - raise ClusterNotExistException( - cluster_type=ClusterType.TenDBHA.value, cluster_id=cluster_id, immute_domain="" - ) - - if backup_local == InstanceInnerRole.MASTER.value: - backend_obj = cluster_obj.storageinstance_set.get(instance_inner_role=InstanceInnerRole.MASTER.value) - elif backup_local == InstanceInnerRole.SLAVE.value: - try: - backend_obj = cluster_obj.storageinstance_set.get( - instance_inner_role=InstanceInnerRole.SLAVE.value, is_stand_by=True - ) - except ObjectDoesNotExist: - raise DBMetaBaseException(message=_("{} standby slave 不存在".format(cluster_obj.immute_domain))) - else: - raise MySQLBackupLocalException(msg=_("不支持的备份位置 {}".format(backup_local))) + for job in self.data["infos"]: + cluster_id = job["cluster_id"] + backup_local = job["backup_local"] + + cluster_obj = Cluster.objects.get(pk=cluster_id, bk_biz_id=self.data["bk_biz_id"]) + + if cluster_obj.cluster_type == ClusterType.TenDBSingle: + backend_obj = cluster_obj.storageinstance_set.first() + else: # cluster_obj.cluster_type == ClusterType.TenDBHA: + backend_obj = cluster_obj.storageinstance_set.get(instance_inner_role=backup_local, is_stand_by=True) sub_pipe = SubBuilder( root_id=self.root_id, @@ -104,8 +87,8 @@ def full_backup_flow(self): "ticket_type": self.data["ticket_type"], "ip": backend_obj.machine.ip, "port": backend_obj.port, - "file_tag": self.data["infos"]["file_tag"], - "backup_type": self.data["infos"]["backup_type"], + "file_tag": self.data["file_tag"], + "backup_type": self.data["backup_type"], "backup_id": uuid.uuid1(), "backup_gsd": ["all"], "role": backend_obj.instance_role, @@ -136,7 +119,6 @@ def full_backup_flow(self): get_mysql_payload_func=MysqlActPayload.mysql_backup_demand_payload.__name__, ) ), - # write_payload_var="backup_report_response", ) sub_pipe.add_act( @@ -149,4 +131,4 @@ def full_backup_flow(self): backup_pipeline.add_parallel_sub_pipeline(sub_flow_list=sub_pipes) logger.info(_("构建全库备份流程成功")) - backup_pipeline.run_pipeline(init_trans_data_class=MySQLBackupDemandContext(), is_drop_random_user=True) + backup_pipeline.run_pipeline(init_trans_data_class=MySQLBackupDemandContext()) # , is_drop_random_user=True) diff --git a/dbm-ui/backend/flow/engine/bamboo/scene/spider/spider_cluster_db_table_backup.py b/dbm-ui/backend/flow/engine/bamboo/scene/spider/db_table_backup.py similarity index 91% rename from dbm-ui/backend/flow/engine/bamboo/scene/spider/spider_cluster_db_table_backup.py rename to dbm-ui/backend/flow/engine/bamboo/scene/spider/db_table_backup.py index f4a4350da6..d65a777c7d 100644 --- a/dbm-ui/backend/flow/engine/bamboo/scene/spider/spider_cluster_db_table_backup.py +++ b/dbm-ui/backend/flow/engine/bamboo/scene/spider/db_table_backup.py @@ -11,17 +11,15 @@ import copy import logging import uuid -from collections import Counter, defaultdict +from collections import defaultdict from dataclasses import asdict from typing import Dict, List, Optional -from django.core.exceptions import ObjectDoesNotExist from django.utils.translation import ugettext as _ from backend.configuration.constants import DBType from backend.constants import IP_PORT_DIVIDER from backend.db_meta.enums import ClusterType, InstanceInnerRole, TenDBClusterSpiderRole -from backend.db_meta.exceptions import ClusterNotExistException, DBMetaBaseException from backend.db_meta.models import Cluster, StorageInstanceTuple from backend.flow.consts import DBA_SYSTEM_USER from backend.flow.engine.bamboo.scene.common.builder import Builder, SubBuilder, SubProcess @@ -68,31 +66,19 @@ def backup_flow(self): "table_patterns": ["tb_role%", "tb_mail%", "*"], "ignore_tables": ["tb_role1", "tb_mail10"], }, - ... - ... ] } 增加单据临时ADMIN账号的添加和删除逻辑 """ - cluster_ids = [job["cluster_id"] for job in self.data["infos"]] - dup_cluster_ids = [item for item, count in Counter(cluster_ids).items() if count > 1] - if dup_cluster_ids: - raise DBMetaBaseException(message="duplicate clusters found: {}".format(dup_cluster_ids)) - backup_pipeline = Builder( - root_id=self.root_id, data=self.data, need_random_pass_cluster_ids=list(set(cluster_ids)) + root_id=self.root_id, data=self.data # , need_random_pass_cluster_ids=list(set(cluster_ids)) ) cluster_pipes = [] for job in self.data["infos"]: - try: - cluster_obj = Cluster.objects.get( - pk=job["cluster_id"], bk_biz_id=self.data["bk_biz_id"], cluster_type=ClusterType.TenDBCluster.value - ) - except ObjectDoesNotExist: - raise ClusterNotExistException( - cluster_type=ClusterType.TenDBCluster.value, cluster_id=job["cluster_id"], immute_domain="" - ) + cluster_obj = Cluster.objects.get( + pk=job["cluster_id"], bk_biz_id=self.data["bk_biz_id"], cluster_type=ClusterType.TenDBCluster.value + ) backup_id = uuid.uuid1() @@ -108,9 +94,9 @@ def backup_flow(self): }, ) - cluster_pipe.add_sub_pipeline( - sub_flow=self.backup_on_spider_ctl(backup_id=backup_id, job=job, cluster_obj=cluster_obj) - ) + # cluster_pipe.add_sub_pipeline( + # sub_flow=self.backup_on_spider_ctl(backup_id=backup_id, job=job, cluster_obj=cluster_obj) + # ) if job["backup_local"] == "remote": cluster_pipe.add_parallel_sub_pipeline( diff --git a/dbm-ui/backend/flow/engine/bamboo/scene/spider/spider_cluster_full_backup.py b/dbm-ui/backend/flow/engine/bamboo/scene/spider/full_backup.py similarity index 85% rename from dbm-ui/backend/flow/engine/bamboo/scene/spider/spider_cluster_full_backup.py rename to dbm-ui/backend/flow/engine/bamboo/scene/spider/full_backup.py index 2052c56c59..7819cf71e3 100644 --- a/dbm-ui/backend/flow/engine/bamboo/scene/spider/spider_cluster_full_backup.py +++ b/dbm-ui/backend/flow/engine/bamboo/scene/spider/full_backup.py @@ -15,18 +15,16 @@ from dataclasses import asdict from typing import Dict, List, Optional -from django.core.exceptions import ObjectDoesNotExist from django.utils.translation import ugettext as _ from backend.configuration.constants import DBType from backend.constants import IP_PORT_DIVIDER -from backend.db_meta.enums import ClusterType, InstanceInnerRole, TenDBClusterSpiderRole -from backend.db_meta.exceptions import ClusterNotExistException +from backend.db_meta.enums import InstanceInnerRole, TenDBClusterSpiderRole from backend.db_meta.models import Cluster, StorageInstanceTuple from backend.flow.consts import DBA_SYSTEM_USER, LONG_JOB_TIMEOUT from backend.flow.engine.bamboo.scene.common.builder import Builder, SubBuilder, SubProcess from backend.flow.engine.bamboo.scene.common.get_file_list import GetFileList -from backend.flow.engine.exceptions import IncompatibleBackupTypeAndLocal, MySQLBackupLocalException +from backend.flow.engine.exceptions import MySQLBackupLocalException from backend.flow.plugins.components.collections.mysql.exec_actuator_script import ExecuteDBActuatorScriptComponent from backend.flow.plugins.components.collections.mysql.mysql_link_backup_id_bill_id import ( MySQLLinkBackupIdBillIdComponent, @@ -51,49 +49,40 @@ def full_backup_flow(self): "created_type": "xxx", "bk_biz_id": "152", "ticket_type": "TENDBCLUSTER_FULL_BACKUP", - "infos": { - "backup_type": enum of backend.flow.consts.MySQLBackupTypeEnum, - "file_tag": enum of backend.flow.consts.MySQLBackupFileTagEnum, - “clusters": [ + "backup_type": enum of backend.flow.consts.MySQLBackupTypeEnum, + "file_tag": enum of backend.flow.consts.MySQLBackupFileTagEnum, + "infos": [ { "cluster_id": int, "backup_local": enum [backend.db_meta.enum.InstanceInnerRole, SPIDER_MNT], "spider_mnt_address": "x.x.x.x:y" # 如果 backup_local 是 spider_mnt }, ... - ], - } + ] } 增加单据临时ADMIN账号的添加和删除逻辑 """ - - clusters = self.data["infos"]["clusters"] - cluster_ids = [i["cluster_id"] for i in self.data["infos"]["clusters"]] - backup_pipeline = Builder( - root_id=self.root_id, data=self.data, need_random_pass_cluster_ids=list(set(cluster_ids)) + root_id=self.root_id, data=self.data # , need_random_pass_cluster_ids=list(set(cluster_ids)) ) cluster_pipes = [] - for cluster in clusters: - if ( - self.data["infos"]["backup_type"] == "physical" - and cluster["backup_local"] == TenDBClusterSpiderRole.SPIDER_MNT.value - ): - IncompatibleBackupTypeAndLocal( - backup_type=self.data["infos"]["backup_type"], backup_local=cluster["backup_local"] - ) - - try: - cluster_obj = Cluster.objects.get( - pk=cluster["cluster_id"], - bk_biz_id=self.data["bk_biz_id"], - cluster_type=ClusterType.TenDBCluster.value, - ) - except ObjectDoesNotExist: - raise ClusterNotExistException( - cluster_type=ClusterType.TenDBCluster.value, cluster_id=cluster["cluster_id"], immute_domain="" - ) + for job in self.data["infos"]: + cluster_id = job["cluster_id"] + backup_local = job["backup_local"] + + cluster_obj = Cluster.objects.get(pk=cluster_id, bk_biz_id=self.data["bk_biz_id"]) + + # try: + # cluster_obj = Cluster.objects.get( + # pk=cluster["cluster_id"], + # bk_biz_id=self.data["bk_biz_id"], + # cluster_type=ClusterType.TenDBCluster.value, + # ) + # except ObjectDoesNotExist: + # raise ClusterNotExistException( + # cluster_type=ClusterType.TenDBCluster.value, cluster_id=cluster["cluster_id"], immute_domain="" + # ) backup_id = uuid.uuid1() cluster_pipe = SubBuilder( @@ -109,26 +98,26 @@ def full_backup_flow(self): }, ) - cluster_pipe.add_sub_pipeline( - sub_flow=self.backup_on_spider_ctl(backup_id=backup_id, cluster_obj=cluster_obj) - ) + # cluster_pipe.add_sub_pipeline( + # sub_flow=self.backup_on_spider_ctl(backup_id=backup_id, cluster_obj=cluster_obj) + # ) - if cluster["backup_local"] == InstanceInnerRole.SLAVE.value: # "remote": + if backup_local == InstanceInnerRole.SLAVE.value: # "remote": cluster_pipe.add_parallel_sub_pipeline( sub_flow_list=self.backup_on_remote_slave(backup_id=backup_id, cluster_obj=cluster_obj) ) - elif cluster["backup_local"] == InstanceInnerRole.MASTER.value: + elif backup_local == InstanceInnerRole.MASTER.value: cluster_pipe.add_parallel_sub_pipeline( sub_flow_list=self.backup_on_remote_master(backup_id=backup_id, cluster_obj=cluster_obj) ) - elif cluster["backup_local"] == TenDBClusterSpiderRole.SPIDER_MNT.value: + elif backup_local == TenDBClusterSpiderRole.SPIDER_MNT.value: cluster_pipe.add_sub_pipeline( sub_flow=self.backup_on_spider_mnt( - backup_id=backup_id, cluster_obj=cluster_obj, spider_mnt_address=cluster["spider_mnt_address"] + backup_id=backup_id, cluster_obj=cluster_obj, spider_mnt_address=job["spider_mnt_address"] ) ) else: - raise MySQLBackupLocalException(msg=_("不支持的备份位置 {}".format(cluster["backup_local"]))) + raise MySQLBackupLocalException(msg=_("不支持的备份位置 {}".format(backup_local))) cluster_pipe.add_act( act_name=_("关联备份id"), @@ -140,7 +129,7 @@ def full_backup_flow(self): backup_pipeline.add_parallel_sub_pipeline(sub_flow_list=cluster_pipes) logger.info(_("构造全库备份流程成功")) - backup_pipeline.run_pipeline(init_trans_data_class=MySQLBackupDemandContext(), is_drop_random_user=True) + backup_pipeline.run_pipeline(init_trans_data_class=MySQLBackupDemandContext()) # , is_drop_random_user=True) def backup_on_spider_ctl(self, backup_id: uuid.UUID, cluster_obj: Cluster) -> SubProcess: ctl_primary_address = cluster_obj.tendbcluster_ctl_primary_address() diff --git a/dbm-ui/backend/flow/engine/controller/mysql.py b/dbm-ui/backend/flow/engine/controller/mysql.py index 57b4943875..8b5f875a10 100644 --- a/dbm-ui/backend/flow/engine/controller/mysql.py +++ b/dbm-ui/backend/flow/engine/controller/mysql.py @@ -20,15 +20,15 @@ from backend.flow.engine.bamboo.scene.mysql.mysql_authorize_rules import MySQLAuthorizeRulesFlows from backend.flow.engine.bamboo.scene.mysql.mysql_checksum import MysqlChecksumFlow from backend.flow.engine.bamboo.scene.mysql.mysql_data_migrate_flow import MysqlDataMigrateFlow +from backend.flow.engine.bamboo.scene.mysql.mysql_db_table_backup import MySQLDBTableBackupFlow from backend.flow.engine.bamboo.scene.mysql.mysql_edit_config_flow import MysqlEditConfigFlow from backend.flow.engine.bamboo.scene.mysql.mysql_fake_sql_semantic_check import MySQLFakeSemanticCheck from backend.flow.engine.bamboo.scene.mysql.mysql_flashback_flow import MysqlFlashbackFlow +from backend.flow.engine.bamboo.scene.mysql.mysql_full_backup_flow import MySQLFullBackupFlow from backend.flow.engine.bamboo.scene.mysql.mysql_ha_apply_flow import MySQLHAApplyFlow -from backend.flow.engine.bamboo.scene.mysql.mysql_ha_db_table_backup import MySQLHADBTableBackupFlow from backend.flow.engine.bamboo.scene.mysql.mysql_ha_destroy_flow import MySQLHADestroyFlow from backend.flow.engine.bamboo.scene.mysql.mysql_ha_disable_flow import MySQLHADisableFlow from backend.flow.engine.bamboo.scene.mysql.mysql_ha_enable_flow import MySQLHAEnableFlow -from backend.flow.engine.bamboo.scene.mysql.mysql_ha_full_backup_flow import MySQLHAFullBackupFlow from backend.flow.engine.bamboo.scene.mysql.mysql_ha_metadata_import import TenDBHAMetadataImportFlow from backend.flow.engine.bamboo.scene.mysql.mysql_ha_standardize_flow import MySQLHAStandardizeFlow from backend.flow.engine.bamboo.scene.mysql.mysql_ha_upgrade import ( @@ -410,19 +410,18 @@ def mysql_migrate_remote_scene(self): flow = MySQLMigrateClusterRemoteFlow(root_id=self.root_id, ticket_data=self.ticket_data) flow.migrate_cluster_flow() - def mysql_ha_db_table_backup_scene(self): + def mysql_db_table_backup_scene(self): """ - TenDBHA 库表备份 + MySQL 库表备份 ticket_data 参数结构样例 { "uid": "2022051612120001", "created_by": "xxx", "bk_biz_id": "152", - "ticket_type": "MYSQL_HA_DB_TABLE_BACKUP", + "ticket_type": "MYSQL_DB_TABLE_BACKUP", "infos": [ { "cluster_id": int, - "backup_on": str enum InstanceInnerRole "db_patterns": ["db1%", "db2%"], "ignore_dbs": ["db11", "db12", "db23"], "table_patterns": ["tb_role%", "tb_mail%", "*"], @@ -433,7 +432,7 @@ def mysql_ha_db_table_backup_scene(self): ] } """ - flow = MySQLHADBTableBackupFlow(root_id=self.root_id, data=self.ticket_data) + flow = MySQLDBTableBackupFlow(root_id=self.root_id, data=self.ticket_data) flow.backup_flow() def mysql_ha_switch_scene(self): @@ -502,7 +501,7 @@ def mysql_flashback_scene(self): flow.mysql_flashback_flow() def mysql_full_backup_scene(self): - flow = MySQLHAFullBackupFlow(root_id=self.root_id, data=self.ticket_data) + flow = MySQLFullBackupFlow(root_id=self.root_id, data=self.ticket_data) flow.full_backup_flow() def mysql_edit_config_scene(self): diff --git a/dbm-ui/backend/flow/engine/controller/spider.py b/dbm-ui/backend/flow/engine/controller/spider.py index f0bdffd8a1..5a1c4868aa 100644 --- a/dbm-ui/backend/flow/engine/controller/spider.py +++ b/dbm-ui/backend/flow/engine/controller/spider.py @@ -10,6 +10,8 @@ from backend.db_meta.enums import ClusterType from backend.flow.engine.bamboo.scene.spider.append_deploy_ctl_flow import AppendDeployCTLFlow +from backend.flow.engine.bamboo.scene.spider.db_table_backup import TenDBClusterDBTableBackupFlow +from backend.flow.engine.bamboo.scene.spider.full_backup import TenDBClusterFullBackupFlow from backend.flow.engine.bamboo.scene.spider.import_sqlfile_flow import ImportSQLFlow from backend.flow.engine.bamboo.scene.spider.remote_local_slave_recover import TenDBRemoteSlaveLocalRecoverFlow from backend.flow.engine.bamboo.scene.spider.remote_master_fail_over import RemoteMasterFailOverFlow @@ -19,13 +21,11 @@ from backend.flow.engine.bamboo.scene.spider.spider_add_mnt import TenDBClusterAddSpiderMNTFlow from backend.flow.engine.bamboo.scene.spider.spider_add_nodes import TenDBClusterAddNodesFlow from backend.flow.engine.bamboo.scene.spider.spider_checksum import SpiderChecksumFlow -from backend.flow.engine.bamboo.scene.spider.spider_cluster_db_table_backup import TenDBClusterDBTableBackupFlow from backend.flow.engine.bamboo.scene.spider.spider_cluster_deploy import TenDBClusterApplyFlow from backend.flow.engine.bamboo.scene.spider.spider_cluster_destroy import TenDBClusterDestroyFlow from backend.flow.engine.bamboo.scene.spider.spider_cluster_disable_deploy import SpiderClusterDisableFlow from backend.flow.engine.bamboo.scene.spider.spider_cluster_enable_deploy import SpiderClusterEnableFlow from backend.flow.engine.bamboo.scene.spider.spider_cluster_flashback import TenDBClusterFlashbackFlow -from backend.flow.engine.bamboo.scene.spider.spider_cluster_full_backup import TenDBClusterFullBackupFlow from backend.flow.engine.bamboo.scene.spider.spider_cluster_metadata_import_flow import SpiderClusterMetadataImportFlow from backend.flow.engine.bamboo.scene.spider.spider_cluster_rollback_flow import TenDBRollBackDataFlow from backend.flow.engine.bamboo.scene.spider.spider_cluster_standardize_flow import SpiderClusterStandardizeFlow diff --git a/dbm-ui/backend/flow/utils/mysql/db_table_filter/filter.py b/dbm-ui/backend/flow/utils/mysql/db_table_filter/filter.py index ce8d14875b..78a52b2b43 100644 --- a/dbm-ui/backend/flow/utils/mysql/db_table_filter/filter.py +++ b/dbm-ui/backend/flow/utils/mysql/db_table_filter/filter.py @@ -25,81 +25,81 @@ def __init__( exclude_db_patterns: List[str], exclude_table_patterns: List[str], ): - self.include_db_patterns = include_db_patterns - self.include_table_patterns = include_table_patterns - self.exclude_db_patterns = exclude_db_patterns - self.exclude_table_patterns = exclude_table_patterns + self.__include_db_patterns = include_db_patterns + self.__include_table_patterns = include_table_patterns + self.__exclude_db_patterns = exclude_db_patterns + self.__exclude_table_patterns = exclude_table_patterns - self.system_table_parts = [] - self.system_db_parts = [] + self.__system_table_parts = [] + self.__system_db_parts = [] - self._validate() - self._build_db_filter_regexp() - self._build_table_filter_regexp() + self.__validate() + self.__build_db_filter_regexp() + self.__build_table_filter_regexp() - def _validate(self): - if not self.include_db_patterns or not self.include_table_patterns: - raise DbTableFilterValidateException(msg=_("include patterns 不能为空")) + def __validate(self): + if not self.__include_db_patterns or not self.__include_table_patterns: + raise DbTableFilterValidateException(msg=_("include db/table patterns 不能为空")) # if not ( # (self.exclude_db_patterns and self.exclude_table_patterns) # or (not self.exclude_db_patterns and not self.exclude_table_patterns) # ): # raise DbTableFilterValidateException(msg=_("exclude patterns 要么同时为空, 要么都不为空")) - if "*" in self.exclude_db_patterns or "*" in self.exclude_table_patterns: + if "*" in self.__exclude_db_patterns or "*" in self.__exclude_table_patterns: raise DbTableFilterValidateException(msg=_("exclude patterns 不能包含 *")) for patterns in [ - self.include_db_patterns, - self.include_table_patterns, - self.exclude_db_patterns, - self.exclude_table_patterns, + self.__include_db_patterns, + self.__include_table_patterns, + self.__exclude_db_patterns, + self.__exclude_table_patterns, ]: glob_check(patterns) - def _build_db_filter_regexp(self): - include_parts = ["{}$".format(replace_glob(db)) for db in self.include_db_patterns] - exclude_parts = ["{}$".format(replace_glob(db)) for db in self.exclude_db_patterns] + self.system_db_parts + def __build_db_filter_regexp(self): + include_parts = ["{}$".format(replace_glob(db)) for db in self.__include_db_patterns] + exclude_parts = ["{}$".format(replace_glob(db)) for db in self.__exclude_db_patterns] + self.__system_db_parts - self.db_filter_include_regex = build_include_regexp(include_parts) - self.db_filter_exclude_regex = build_exclude_regexp(exclude_parts) + self.__db_filter_include_regex = build_include_regexp(include_parts) + self.__db_filter_exclude_regex = build_exclude_regexp(exclude_parts) - def _build_table_filter_regexp(self): + def __build_table_filter_regexp(self): include_parts = [ r"{}\.{}$".format(replace_glob(ele[0]), replace_glob(ele[1])) - for ele in itertools.product(self.include_db_patterns, self.include_table_patterns) + for ele in itertools.product(self.__include_db_patterns, self.__include_table_patterns) ] # 库排除 - exclude_parts = [r"{}\.{}$".format(replace_glob(edb), replace_glob("*")) for edb in self.exclude_db_patterns] + exclude_parts = [r"{}\.{}$".format(replace_glob(edb), replace_glob("*")) for edb in self.__exclude_db_patterns] # 表排除 exclude_parts += [ - r"{}\.{}$".format(replace_glob("*"), replace_glob(etb)) for etb in self.exclude_table_patterns + r"{}\.{}$".format(replace_glob("*"), replace_glob(etb)) for etb in self.__exclude_table_patterns ] - exclude_parts += self.system_db_parts + exclude_parts += self.__system_db_parts - self.table_filter_include_regex = build_include_regexp(include_parts) - self.table_filter_exclude_regex = build_exclude_regexp(exclude_parts) + self.__table_filter_include_regex = build_include_regexp(include_parts) + self.__table_filter_exclude_regex = build_exclude_regexp(exclude_parts) def table_filter_regexp(self) -> str: - return r"^{}{}".format(self.table_filter_include_regex, self.table_filter_exclude_regex) + return r"^{}{}".format(self.__table_filter_include_regex, self.__table_filter_exclude_regex) def db_filter_regexp(self) -> str: - return r"^{}{}".format(self.db_filter_include_regex, self.db_filter_exclude_regex) + return r"^{}{}".format(self.__db_filter_include_regex, self.__db_filter_exclude_regex) def table_filter_exclude_regexp_as_include(self) -> str: - return self.table_filter_exclude_regex.replace("!", "=", 1) + return self.__table_filter_exclude_regex.replace("!", "=", 1) def db_filter_exclude_regexp_as_include(self) -> str: - return self.db_filter_exclude_regex.replace("!", "=", 1) + return self.__db_filter_exclude_regex.replace("!", "=", 1) def inject_system_dbs(self, system_dbs: List[str]): - self.system_table_parts = [r"{}\..*$".format(replace_glob(sd)) for sd in system_dbs] - self.system_db_parts = [r"{}$".format(replace_glob(sd)) for sd in system_dbs] + self.__system_table_parts = [r"{}\..*$".format(replace_glob(sd)) for sd in system_dbs] + self.__system_db_parts = [r"{}$".format(replace_glob(sd)) for sd in system_dbs] - self._build_db_filter_regexp() - self._build_table_filter_regexp() + self.__build_db_filter_regexp() + self.__build_table_filter_regexp() def check_inclusion(self) -> Dict[str, List[Tuple[str, str]]]: """ @@ -108,8 +108,8 @@ def check_inclusion(self) -> Dict[str, List[Tuple[str, str]]]: 类似 [('p%', 'p2???'), ('p%', 'p211'), ('p%', 'p4'), ('p%', 'p5')] """ return { - "include-db": pattern_inclusion(self.include_db_patterns), - "exclude-db": pattern_inclusion(self.exclude_db_patterns), - "include-table": pattern_inclusion(self.include_table_patterns), - "exclude-table": pattern_inclusion(self.exclude_table_patterns), + "include-db": pattern_inclusion(self.__include_db_patterns), + "exclude-db": pattern_inclusion(self.__exclude_db_patterns), + "include-table": pattern_inclusion(self.__include_table_patterns), + "exclude-table": pattern_inclusion(self.__exclude_table_patterns), } diff --git a/dbm-ui/backend/flow/utils/mysql/db_table_filter/tools.py b/dbm-ui/backend/flow/utils/mysql/db_table_filter/tools.py index 5863d7a46e..d2e23820c6 100644 --- a/dbm-ui/backend/flow/utils/mysql/db_table_filter/tools.py +++ b/dbm-ui/backend/flow/utils/mysql/db_table_filter/tools.py @@ -54,6 +54,8 @@ def replace_glob(p: str) -> str: def _build_regexp(parts: List[str], template: str) -> str: if parts: return template.format("|".join(parts)) + else: + return "" def build_include_regexp(parts: List[str]) -> str: diff --git a/dbm-ui/backend/flow/views/mysql_ha_db_table_backup.py b/dbm-ui/backend/flow/views/mysql_ha_db_table_backup.py index 118d835852..a309f476f9 100644 --- a/dbm-ui/backend/flow/views/mysql_ha_db_table_backup.py +++ b/dbm-ui/backend/flow/views/mysql_ha_db_table_backup.py @@ -35,6 +35,6 @@ def post(request): logger.info("define root_id: {}".format(root_id)) c = MySQLController(root_id=root_id, ticket_data=request.data) - c.mysql_ha_db_table_backup_scene() + c.mysql_db_table_backup_scene() return Response({"root_id": root_id}) diff --git a/dbm-ui/backend/ticket/builders/mysql/mysql_db_table_backup.py b/dbm-ui/backend/ticket/builders/mysql/mysql_db_table_backup.py new file mode 100644 index 0000000000..04c9823c8c --- /dev/null +++ b/dbm-ui/backend/ticket/builders/mysql/mysql_db_table_backup.py @@ -0,0 +1,147 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. +Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + +import collections + +from django.utils.translation import ugettext_lazy as _ +from rest_framework import serializers + +from backend.db_meta.enums import ClusterType, InstanceInnerRole, InstanceStatus +from backend.db_meta.models import Cluster +from backend.flow.engine.controller.mysql import MySQLController +from backend.ticket import builders +from backend.ticket.builders.mysql.base import ( + BaseMySQLHATicketFlowBuilder, + DBTableField, + MySQLBaseOperateDetailSerializer, +) +from backend.ticket.constants import FlowRetryType, TicketType + + +class MySQLDBTableBackupDetailSerializer(MySQLBaseOperateDetailSerializer): + class DBTableBackupDataInfoSerializer(serializers.Serializer): + cluster_id = serializers.IntegerField(help_text=_("集群ID")) + db_patterns = serializers.ListField(help_text=_("匹配DB列表"), child=DBTableField(db_field=True)) + ignore_dbs = serializers.ListField(help_text=_("忽略DB列表"), child=DBTableField(db_field=True)) + table_patterns = serializers.ListField(help_text=_("匹配Table列表"), child=DBTableField()) + ignore_tables = serializers.ListField(help_text=_("忽略Table列表"), child=DBTableField()) + + infos = serializers.ListSerializer(help_text=_("备份信息列表"), child=DBTableBackupDataInfoSerializer()) + + def validate(self, attrs): + """验证库表数据库的数据""" + super().validate(attrs) + cluster_ids = [info["cluster_id"] for info in attrs["infos"]] + + errors = [] + + msg = self.__validate_cluster_id_unique(cluster_ids=cluster_ids) + if msg: + errors.append(msg) + + msg = self.__validate_cluster_type(cluster_ids=cluster_ids) + if msg: + errors.append(msg) + + msg = self.__validate_cluster_exists(cluster_ids=cluster_ids) + if msg: + errors.append(msg) + + msg = self.__validate_cluster_status(cluster_ids=cluster_ids) + if msg: + errors.append(msg) + + if errors: + raise serializers.ValidationError(errors) + + # 库表选择器校验 + super().validate_database_table_selector(attrs) + + return attrs + + @staticmethod + def __validate_cluster_id_unique(cluster_ids) -> str: + """ + 集群 id 不能重复出现 + """ + dup_cluster_ids = [cid for cid, cnt in collections.Counter(cluster_ids).items() if cnt > 1] + if dup_cluster_ids: + return _( + "重复输入集群: {}".format( + Cluster.objects.filter(pk__in=dup_cluster_ids).values_list("immute_domain", flat=True) + ) + ) + + @staticmethod + def __validate_cluster_type(cluster_ids) -> str: + """ + 集群类型不能混合 + """ + bad = [] + cluster_types = [] + for cluster_obj in Cluster.objects.filter(pk__in=cluster_ids): + if cluster_obj.cluster_type not in [ClusterType.TenDBHA, ClusterType.TenDBSingle]: + bad.append(str(_("不支持的集群类型 {} {}".format(cluster_obj.immute_domain, cluster_obj.cluster_type)))) + + cluster_types.append(cluster_obj.cluster_type) + + # if len(cluster_types) > 1: + # bad.append(_("集群类型混合输入")) + + if bad: + return ", ".join(bad) + + @staticmethod + def __validate_cluster_exists(cluster_ids) -> str: + """ + 集群 id 必须存在 + """ + exists_cluster_ids = list( + Cluster.objects.filter( + pk__in=cluster_ids, cluster_type__in=[ClusterType.TenDBHA, ClusterType.TenDBSingle] + ).values_list("id", flat=True) + ) + not_exists_cluster_ids = list(set(cluster_ids) - set(exists_cluster_ids)) + if not_exists_cluster_ids: + return _("cluster id: {} 不存在".format(cluster_ids)) + + @staticmethod + def __validate_cluster_status(cluster_ids) -> str: + """ + 库表备份强制在 slave 备份, 所以集群的 standby slave 必须正常 + """ + bad = [] + for cluster_id in cluster_ids: + cluster_obj = Cluster.objects.get(pk=cluster_id) + if ( + cluster_obj.cluster_type == ClusterType.TenDBHA + and not cluster_obj.storageinstance_set.filter( + status=InstanceStatus.RUNNING, is_stand_by=True, instance_inner_role=InstanceInnerRole.SLAVE + ).exists() + ): + bad.append(_("{} 缺少状态正常的 standby slave".format(cluster_obj.immute_domain))) + elif not cluster_obj.storageinstance_set.filter(status=InstanceStatus.RUNNING).exists(): + bad.append(_("{} 缺少状态正常的存储实例".format(cluster_obj.immute_domain))) + + if bad: + return _("{} 缺少状态正常的 standby slave".format(bad)) + + +class MySQLDBTableBackupFlowParamBuilder(builders.FlowParamBuilder): + controller = MySQLController.mysql_db_table_backup_scene + + +@builders.BuilderFactory.register(TicketType.MYSQL_HA_DB_TABLE_BACKUP) +class TenDBHADBTableBackupFlowBuilder(BaseMySQLHATicketFlowBuilder): + serializer = MySQLDBTableBackupDetailSerializer + inner_flow_builder = MySQLDBTableBackupFlowParamBuilder + inner_flow_name = _("TenDBHA 库表备份执行") + retry_type = FlowRetryType.MANUAL_RETRY diff --git a/dbm-ui/backend/ticket/builders/mysql/mysql_full_backup.py b/dbm-ui/backend/ticket/builders/mysql/mysql_full_backup.py new file mode 100644 index 0000000000..d359740967 --- /dev/null +++ b/dbm-ui/backend/ticket/builders/mysql/mysql_full_backup.py @@ -0,0 +1,165 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. +Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +import collections + +from django.utils.translation import ugettext_lazy as _ +from rest_framework import serializers + +from backend.db_meta.enums import ClusterType, InstanceInnerRole, InstanceStatus +from backend.db_meta.models import Cluster, StorageInstance +from backend.flow.consts import MySQLBackupFileTagEnum, MySQLBackupTypeEnum +from backend.flow.engine.controller.mysql import MySQLController +from backend.ticket import builders +from backend.ticket.builders.mysql.base import BaseMySQLHATicketFlowBuilder, MySQLBaseOperateDetailSerializer +from backend.ticket.constants import FlowRetryType, TicketType + + +class MySQLFullBackupDetailSerializer(MySQLBaseOperateDetailSerializer): + class MySQLFullBackupInfoSerializer(serializers.Serializer): + cluster_id = serializers.IntegerField(help_text=_("集群ID")) + backup_local = serializers.ChoiceField( + help_text=_("备份位置"), choices=InstanceInnerRole.get_choices(), default=InstanceInnerRole.SLAVE.value + ) + + backup_type = serializers.ChoiceField(help_text=_("备份类型"), choices=MySQLBackupTypeEnum.get_choices()) + file_tag = serializers.ChoiceField(help_text=_("备份文件tag"), choices=MySQLBackupFileTagEnum.get_choices()) + infos = serializers.ListSerializer(child=MySQLFullBackupInfoSerializer()) + + def validate(self, attrs): + cluster_ids = [info["cluster_id"] for info in attrs["infos"]] + + errors = [] + + msg = self.__validate_cluster_id_unique(cluster_ids=cluster_ids) + if msg: + errors.append(msg) + + msg = self.__validate_cluster_type(cluster_ids=cluster_ids) + if msg: + errors.append(msg) + + msg = self.__validate_backup_local(attrs=attrs) + if msg: + errors.append(msg) + + msg = self.__validate_cluster_status(attrs=attrs) + if msg: + errors.append(msg) + + if errors: + raise serializers.ValidationError(errors) + + return attrs + + @staticmethod + def __validate_cluster_id_unique(cluster_ids) -> str: + """ + 集群 id 不能重复出现 + """ + dup_cluster_ids = [cid for cid, cnt in collections.Counter(cluster_ids).items() if cnt > 1] + if dup_cluster_ids: + return _( + "重复输入集群: {}".format( + Cluster.objects.filter(pk__in=dup_cluster_ids).values_list("immute_domain", flat=True) + ) + ) + + @staticmethod + def __validate_cluster_type(cluster_ids) -> str: + """ + 集群类型不能混合 + """ + bad = [] + cluster_types = [] + for cluster_obj in Cluster.objects.filter(pk__in=cluster_ids): + if cluster_obj.cluster_type not in [ClusterType.TenDBHA, ClusterType.TenDBSingle]: + bad.append(str(_("不支持的集群类型 {} {}".format(cluster_obj.immute_domain, cluster_obj.cluster_type)))) + + cluster_types.append(cluster_obj.cluster_type) + + # if len(cluster_types) > 1: + # bad.append(_("集群类型混合输入")) + + if bad: + return ", ".join(bad) + + @staticmethod + def __validate_backup_local(attrs) -> str: + bad = [] + + for info in attrs["infos"]: + backup_local = info["backup_local"] + cluster_id = info["cluster_id"] + cluster_obj = Cluster.objects.get(pk=cluster_id) + + # 为了体验统一, single 也传入 master + # 后端得用 orphan + if cluster_obj.cluster_type == ClusterType.TenDBSingle and backup_local != "master": + bad.append(_("{} 备份位置只能是 {}".format(cluster_obj.immute_domain, "master"))) + elif cluster_obj.cluster_type == ClusterType.TenDBHA and backup_local not in [ + InstanceInnerRole.MASTER, + InstanceInnerRole.SLAVE, + ]: + bad.append( + str( + _( + "{} 备份位置只能是 {}".format( + cluster_obj.immute_domain, [InstanceInnerRole.MASTER, InstanceInnerRole.SLAVE] + ) + ) + ) + ) + + if bad: + return ", ".join(bad) + + @staticmethod + def __validate_cluster_status(attrs) -> str: + bad = [] + + for info in attrs["infos"]: + backup_local = info["backup_local"] + cluster_id = info["cluster_id"] + cluster_obj = Cluster.objects.get(pk=cluster_id) + + if cluster_obj.cluster_type == ClusterType.TenDBSingle: + backup_local = InstanceInnerRole.ORPHAN + + if not StorageInstance.objects.filter( + cluster=cluster_obj, instance_inner_role=backup_local, is_stand_by=True, status=InstanceStatus.RUNNING + ).exists(): + bad.append( + str( + _( + "{} 没找到正常的 {} 实例".format( + cluster_obj.immute_domain, + backup_local, + ) + ) + ) + ) + + if bad: + return ", ".join(bad) + + +class MySQLFullBackupFlowParamBuilder(builders.FlowParamBuilder): + """TenDB HA 备份执行单据参数""" + + controller = MySQLController.mysql_full_backup_scene + + +@builders.BuilderFactory.register(TicketType.MYSQL_HA_FULL_BACKUP) +class TenDBHAFullBackupFlowBuilder(BaseMySQLHATicketFlowBuilder): + serializer = MySQLFullBackupDetailSerializer + inner_flow_builder = MySQLFullBackupFlowParamBuilder + inner_flow_name = _("TenDBHA 全库备份执行") + retry_type = FlowRetryType.MANUAL_RETRY diff --git a/dbm-ui/backend/ticket/builders/mysql/mysql_ha_backup.py b/dbm-ui/backend/ticket/builders/mysql/mysql_ha_backup.py deleted file mode 100644 index 9a49e33e69..0000000000 --- a/dbm-ui/backend/ticket/builders/mysql/mysql_ha_backup.py +++ /dev/null @@ -1,58 +0,0 @@ -# -*- coding: utf-8 -*- -""" -TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. -Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. -Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. -You may obtain a copy of the License at https://opensource.org/licenses/MIT -Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on -an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the -specific language governing permissions and limitations under the License. -""" - -from django.utils.translation import ugettext_lazy as _ -from rest_framework import serializers - -from backend.flow.engine.controller.mysql import MySQLController -from backend.ticket import builders -from backend.ticket.builders.mysql.base import ( - BaseMySQLHATicketFlowBuilder, - DBTableField, - MySQLBaseOperateDetailSerializer, -) -from backend.ticket.constants import FlowRetryType, TicketType - - -class MySQLHaBackupDetailSerializer(MySQLBaseOperateDetailSerializer): - class BackupDataInfoSerializer(serializers.Serializer): - cluster_id = serializers.IntegerField(help_text=_("集群ID")) - db_patterns = serializers.ListField(help_text=_("匹配DB列表"), child=DBTableField(db_field=True)) - ignore_dbs = serializers.ListField(help_text=_("忽略DB列表"), child=DBTableField(db_field=True)) - table_patterns = serializers.ListField(help_text=_("匹配Table列表"), child=DBTableField()) - ignore_tables = serializers.ListField(help_text=_("忽略Table列表"), child=DBTableField()) - # 废弃backup_on参数，暂时不需要传递 - # backup_on = serializers.ChoiceField(choices=InstanceInnerRole.get_choices(), help_text=_("备份源")) - - infos = serializers.ListSerializer(help_text=_("备份信息列表"), child=BackupDataInfoSerializer()) - - def validate(self, attrs): - """验证库表数据库的数据""" - super().validate(attrs) - - # 库表选择器校验 - super().validate_database_table_selector(attrs) - - return attrs - - -class MySQLHaBackupFlowParamBuilder(builders.FlowParamBuilder): - """MySQL HA 备份执行单据参数""" - - controller = MySQLController.mysql_ha_db_table_backup_scene - - -@builders.BuilderFactory.register(TicketType.MYSQL_HA_DB_TABLE_BACKUP) -class MySQLHaBackupFlowBuilder(BaseMySQLHATicketFlowBuilder): - serializer = MySQLHaBackupDetailSerializer - inner_flow_builder = MySQLHaBackupFlowParamBuilder - inner_flow_name = _("库表备份执行") - retry_type = FlowRetryType.MANUAL_RETRY diff --git a/dbm-ui/backend/ticket/builders/mysql/mysql_ha_full_backup.py b/dbm-ui/backend/ticket/builders/mysql/mysql_ha_full_backup.py deleted file mode 100644 index 8776c658a6..0000000000 --- a/dbm-ui/backend/ticket/builders/mysql/mysql_ha_full_backup.py +++ /dev/null @@ -1,68 +0,0 @@ -# -*- coding: utf-8 -*- -""" -TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. -Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. -Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. -You may obtain a copy of the License at https://opensource.org/licenses/MIT -Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on -an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the -specific language governing permissions and limitations under the License. -""" - -from django.utils.translation import ugettext_lazy as _ -from rest_framework import serializers - -from backend.db_meta.enums import ClusterDBHAStatusFlags, InstanceInnerRole -from backend.db_meta.models import Cluster -from backend.flow.consts import MySQLBackupFileTagEnum, MySQLBackupTypeEnum -from backend.flow.engine.controller.mysql import MySQLController -from backend.ticket import builders -from backend.ticket.builders.common.base import fetch_cluster_ids -from backend.ticket.builders.mysql.base import BaseMySQLHATicketFlowBuilder, MySQLBaseOperateDetailSerializer -from backend.ticket.constants import FlowRetryType, TicketType - - -class MySQLHaFullBackupDetailSerializer(MySQLBaseOperateDetailSerializer): - class FullBackupDataInfoSerializer(serializers.Serializer): - class ClusterDetailSerializer(serializers.Serializer): - cluster_id = serializers.IntegerField(help_text=_("集群ID")) - backup_local = serializers.ChoiceField( - help_text=_("备份位置"), choices=InstanceInnerRole.get_choices(), default=InstanceInnerRole.SLAVE.value - ) - - # 废弃online，暂时不需要传递 - # online = serializers.BooleanField(help_text=_("是否在线备份"), required=False) - backup_type = serializers.ChoiceField(help_text=_("备份类型"), choices=MySQLBackupTypeEnum.get_choices()) - file_tag = serializers.ChoiceField(help_text=_("备份文件tag"), choices=MySQLBackupFileTagEnum.get_choices()) - clusters = serializers.ListSerializer(help_text=_("集群信息"), child=ClusterDetailSerializer()) - - infos = FullBackupDataInfoSerializer() - - def validate(self, attrs): - try: - self.validate_cluster_can_access(attrs) - except serializers.ValidationError as e: - clusters = Cluster.objects.filter(id__in=fetch_cluster_ids(details=attrs)) - id__cluster = {cluster.id: cluster for cluster in clusters} - # 如果备份位置选的是master，但是slave异常，则认为是可以的 - for info in attrs["infos"]["clusters"]: - if info["backup_local"] != InstanceInnerRole.MASTER: - raise serializers.ValidationError(e) - if id__cluster[info["cluster_id"]].status_flag & ClusterDBHAStatusFlags.BackendMasterUnavailable: - raise serializers.ValidationError(e) - - return attrs - - -class MySQLHaFullBackupFlowParamBuilder(builders.FlowParamBuilder): - """MySQL HA 备份执行单据参数""" - - controller = MySQLController.mysql_full_backup_scene - - -@builders.BuilderFactory.register(TicketType.MYSQL_HA_FULL_BACKUP) -class MySQLHaFullBackupFlowBuilder(BaseMySQLHATicketFlowBuilder): - serializer = MySQLHaFullBackupDetailSerializer - inner_flow_builder = MySQLHaFullBackupFlowParamBuilder - inner_flow_name = _("全库备份执行") - retry_type = FlowRetryType.MANUAL_RETRY diff --git a/dbm-ui/backend/ticket/builders/mysql/mysql_import_sqlfile.py b/dbm-ui/backend/ticket/builders/mysql/mysql_import_sqlfile.py index 9066919938..a052f4ee6e 100644 --- a/dbm-ui/backend/ticket/builders/mysql/mysql_import_sqlfile.py +++ b/dbm-ui/backend/ticket/builders/mysql/mysql_import_sqlfile.py @@ -85,7 +85,7 @@ def get_params(self): class MysqlSqlImportBackUpFlowParamBuilder(builders.FlowParamBuilder): - controller = MySQLController.mysql_ha_db_table_backup_scene + controller = MySQLController.mysql_db_table_backup_scene def format_ticket_data(self): backup_infos_list = [] diff --git a/dbm-ui/backend/ticket/builders/tendbcluster/db_table_backup.py b/dbm-ui/backend/ticket/builders/tendbcluster/db_table_backup.py new file mode 100644 index 0000000000..30a3bcd75e --- /dev/null +++ b/dbm-ui/backend/ticket/builders/tendbcluster/db_table_backup.py @@ -0,0 +1,182 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. +Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +import collections + +from django.utils.translation import gettext_lazy as _ +from rest_framework import serializers + +from backend.db_meta.enums import ClusterType, InstanceInnerRole, InstanceStatus, TenDBClusterSpiderRole +from backend.db_meta.models import Cluster +from backend.flow.engine.controller.spider import SpiderController +from backend.ticket import builders +from backend.ticket.builders.mysql.base import DBTableField +from backend.ticket.builders.tendbcluster.base import BaseTendbTicketFlowBuilder, TendbBaseOperateDetailSerializer +from backend.ticket.constants import TicketType + + +class TenDBClusterDBTableBackUpDetailSerializer(TendbBaseOperateDetailSerializer): + class TenDBClusterDBTableBackupInfoSerializer(serializers.Serializer): + cluster_id = serializers.IntegerField(help_text=_("集群ID")) + backup_local = serializers.CharField(help_text=_("备份位置")) + spider_mnt_address = serializers.CharField(help_text=_("运维节点地址"), required=False) + db_patterns = serializers.ListField(help_text=_("匹配DB列表"), child=DBTableField(db_field=True)) + ignore_dbs = serializers.ListField(help_text=_("忽略DB列表"), child=DBTableField(db_field=True)) + table_patterns = serializers.ListField(help_text=_("匹配Table列表"), child=DBTableField()) + ignore_tables = serializers.ListField(help_text=_("忽略Table列表"), child=DBTableField()) + + infos = serializers.ListSerializer(help_text=_("库表备份信息"), child=TenDBClusterDBTableBackupInfoSerializer()) + + @classmethod + def get_backup_local_params(cls, info): + """ + 对备份位置进行提取， + 两种情况：remote/spider_mnt::127.0.0.1 + """ + divider = "::" + if divider not in info["backup_local"]: + return info + + backup_local, spider_mnt_address = info["backup_local"].split(divider) + info["backup_local"] = backup_local + info["spider_mnt_address"] = spider_mnt_address + + return info + + def validate(self, attrs): + + for cluster_info in attrs["infos"]: + self.get_backup_local_params(cluster_info) + + # 集群不允许重复 + cluster_ids = [info["cluster_id"] for info in attrs["infos"]] + + errors = [] + + msg = self.__validate_cluster_id_unique(cluster_ids=cluster_ids) + if msg: + errors.append(msg) + + msg = self.__validate_cluster_type(cluster_ids=cluster_ids) + if msg: + errors.append(msg) + + msg = self.__validate_cluster_exists(cluster_ids=cluster_ids) + if msg: + errors.append(msg) + + msg = self.__validate_backup_local(attrs=attrs) + if msg: + errors.append(msg) + + msg = self.__validate_cluster_status(attrs=attrs) + if msg: + errors.append(msg) + + if errors: + raise serializers.ValidationError(errors) + + # 库表选择器校验 + super().validate_database_table_selector(attrs, role_key="backup_local") + return attrs + + @staticmethod + def __validate_cluster_id_unique(cluster_ids) -> str: + """ + 集群 id 不能重复出现 + """ + dup_cluster_ids = [cid for cid, cnt in collections.Counter(cluster_ids).items() if cnt > 1] + if dup_cluster_ids: + return _( + "重复输入集群: {}".format( + Cluster.objects.filter(pk__in=dup_cluster_ids).values_list("immute_domain", flat=True) + ) + ) + + @staticmethod + def __validate_cluster_type(cluster_ids) -> str: + """ + 集群类型不能混合 + """ + bad = list( + Cluster.objects.filter(pk__in=cluster_ids) + .exclude(cluster_type=ClusterType.TenDBCluster) + .values_list("immute_domain", flat=True) + ) + if bad: + return _("不支持的集群类型 {}".format(", ".join(bad))) + + @staticmethod + def __validate_cluster_exists(cluster_ids) -> str: + """ + 集群 id 必须存在 + """ + exists_cluster_ids = list( + Cluster.objects.filter(pk__in=cluster_ids, cluster_type=ClusterType.TenDBCluster).values_list( + "id", flat=True + ) + ) + not_exists_cluster_ids = list(set(cluster_ids) - set(exists_cluster_ids)) + if not_exists_cluster_ids: + return _("cluster id: {} 不存在".format(cluster_ids)) + + @staticmethod + def __validate_backup_local(attrs): + bad = [] + + for info in attrs["infos"]: + backup_local = info["backup_local"] + if backup_local not in ["remote", "spider_mnt"]: + bad.append(str(_("不支持的备份位置 {}".format(backup_local)))) + + if backup_local == "spider_mnt" and "spider_mnt_address" not in info: + bad.append(str(_("缺少 spider_mnt_address"))) + + if bad: + return ", ".join(list(set(bad))) + + @staticmethod + def __validate_cluster_status(attrs): + bad = [] + for info in attrs["infos"]: + cluster_id = info["cluster_id"] + backup_local = info["backup_local"] + cluster_obj = Cluster.objects.get(pk=cluster_id) + if ( + backup_local == "spider_mnt" + and not cluster_obj.proxyinstance_set.filter( + tendbclusterspiderext__spider_role=TenDBClusterSpiderRole.SPIDER_MNT, status=InstanceStatus.RUNNING + ).exists() + ): + bad.append(cluster_obj.immute_domain) + elif ( + backup_local == "remote" + and cluster_obj.storageinstance_set.filter( + instance_inner_role=InstanceInnerRole.SLAVE.value, + is_stand_by=True, + ) + .exclude(status=InstanceStatus.RUNNING) + .exists() + ): + bad.append(cluster_obj.immute_domain) + + if bad: + return _("集群状态异常: {}".format(", ".join(list(set(bad))))) + + +class TenDBClusterDBTableBackUpFlowParamBuilder(builders.FlowParamBuilder): + controller = SpiderController.database_table_backup + + +@builders.BuilderFactory.register(TicketType.TENDBCLUSTER_DB_TABLE_BACKUP) +class TendDBClusterDBTableBackUpFlowBuilder(BaseTendbTicketFlowBuilder): + serializer = TenDBClusterDBTableBackUpDetailSerializer + inner_flow_builder = TenDBClusterDBTableBackUpFlowParamBuilder + inner_flow_name = _("TenDB Cluster 库表备份") diff --git a/dbm-ui/backend/ticket/builders/tendbcluster/full_backup.py b/dbm-ui/backend/ticket/builders/tendbcluster/full_backup.py new file mode 100644 index 0000000000..aefddc41b0 --- /dev/null +++ b/dbm-ui/backend/ticket/builders/tendbcluster/full_backup.py @@ -0,0 +1,178 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. +Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +import collections + +from django.utils.translation import gettext_lazy as _ +from rest_framework import serializers + +from backend.db_meta.enums import ClusterType, InstanceInnerRole, InstanceStatus, TenDBClusterSpiderRole +from backend.db_meta.models import Cluster +from backend.flow.consts import MySQLBackupFileTagEnum, MySQLBackupTypeEnum +from backend.flow.engine.controller.spider import SpiderController +from backend.ticket import builders +from backend.ticket.builders.tendbcluster.base import BaseTendbTicketFlowBuilder, TendbBaseOperateDetailSerializer +from backend.ticket.constants import TicketType + + +class TenDBClusterFullBackUpDetailSerializer(TendbBaseOperateDetailSerializer): + class TenDBClusterFullBackupInfoSerializer(serializers.Serializer): + cluster_id = serializers.IntegerField(help_text=_("集群ID")) + backup_local = serializers.CharField(help_text=_("备份位置信息"), default=InstanceInnerRole.SLAVE) + spider_mnt_address = serializers.CharField(help_text=_("运维节点地址"), required=False) + + backup_type = serializers.ChoiceField(help_text=_("备份选项"), choices=MySQLBackupTypeEnum.get_choices()) + file_tag = serializers.ChoiceField(help_text=_("备份保存时间"), choices=MySQLBackupFileTagEnum.get_choices()) + + infos = serializers.ListSerializer(child=TenDBClusterFullBackupInfoSerializer()) + + @classmethod + def get_backup_local_params(cls, info): + """ + 对备份位置进行提取， + 两种情况：remote/spider_mnt::127.0.0.1 + """ + divider = "::" + if divider not in info["backup_local"]: + return info + + backup_local, spider_mnt_address = info["backup_local"].split(divider) + info["backup_local"] = backup_local + info["spider_mnt_address"] = spider_mnt_address + + return info + + def validate(self, attrs): + + for cluster_info in attrs["infos"]: + self.get_backup_local_params(cluster_info) + + cluster_ids = [info["cluster_id"] for info in attrs["infos"]] + + errors = [] + + msg = self.__validate_cluster_id_unique(cluster_ids=cluster_ids) + if msg: + errors.append(msg) + + msg = self.__validate_cluster_type(cluster_ids=cluster_ids) + if msg: + errors.append(msg) + + msg = self.__validate_cluster_exists(cluster_ids=cluster_ids) + if msg: + errors.append(msg) + + msg = self.__validate_backup_local(attrs=attrs) + if msg: + errors.append(msg) + + msg = self.__validate_cluster_status(attrs=attrs) + if msg: + errors.append(msg) + + if errors: + raise serializers.ValidationError(errors) + + return attrs + + @staticmethod + def __validate_cluster_id_unique(cluster_ids) -> str: + """ + 集群 id 不能重复出现 + """ + dup_cluster_ids = [cid for cid, cnt in collections.Counter(cluster_ids).items() if cnt > 1] + if dup_cluster_ids: + return _( + "重复输入集群: {}".format( + Cluster.objects.filter(pk__in=dup_cluster_ids).values_list("immute_domain", flat=True) + ) + ) + + @staticmethod + def __validate_cluster_type(cluster_ids) -> str: + """ + 集群类型不能混合 + """ + bad = list( + Cluster.objects.filter(pk__in=cluster_ids) + .exclude(cluster_type=ClusterType.TenDBCluster) + .values_list("immute_domain", flat=True) + ) + if bad: + return _("不支持的集群类型 {}".format(", ".join(bad))) + + @staticmethod + def __validate_cluster_exists(cluster_ids) -> str: + """ + 集群 id 必须存在 + """ + exists_cluster_ids = list( + Cluster.objects.filter(pk__in=cluster_ids, cluster_type=ClusterType.TenDBCluster).values_list( + "id", flat=True + ) + ) + not_exists_cluster_ids = list(set(cluster_ids) - set(exists_cluster_ids)) + if not_exists_cluster_ids: + return _("cluster id: {} 不存在".format(cluster_ids)) + + @staticmethod + def __validate_backup_local(attrs): + bad = [] + + for info in attrs["infos"]: + backup_local = info["backup_local"] + if backup_local not in ["master", "slave", "spider_mnt"]: + bad.append(str(_("不支持的备份位置 {}".format(backup_local)))) + + if backup_local == "spider_mnt" and "spider_mnt_address" not in info: + bad.append(str(_("缺少 spider_mnt_address"))) + + if bad: + return ", ".join(list(set(bad))) + + @staticmethod + def __validate_cluster_status(attrs): + bad = [] + for info in attrs["infos"]: + cluster_id = info["cluster_id"] + backup_local = info["backup_local"] + cluster_obj = Cluster.objects.get(pk=cluster_id) + if ( + backup_local == "spider_mnt" + and not cluster_obj.proxyinstance_set.filter( + tendbclusterspiderext__spider_role=TenDBClusterSpiderRole.SPIDER_MNT, status=InstanceStatus.RUNNING + ).exists() + ): + bad.append(cluster_obj.immute_domain) + elif ( + backup_local in ["remote", "slave"] + and cluster_obj.storageinstance_set.filter( + backup_local, + is_stand_by=True, + ) + .exclude(status=InstanceStatus.RUNNING) + .exists() + ): + bad.append(cluster_obj.immute_domain) + + if bad: + return _("集群状态异常: {}".format(", ".join(list(set(bad))))) + + +class TenDBClusterFullBackUpFlowParamBuilder(builders.FlowParamBuilder): + controller = SpiderController.full_backup + + +@builders.BuilderFactory.register(TicketType.TENDBCLUSTER_FULL_BACKUP) +class TenDBClusterFullBackUpFlowBuilder(BaseTendbTicketFlowBuilder): + serializer = TenDBClusterFullBackUpDetailSerializer + inner_flow_builder = TenDBClusterFullBackUpFlowParamBuilder + inner_flow_name = _("TenDB Cluster 全库备份") diff --git a/dbm-ui/backend/ticket/builders/tendbcluster/tendb_backup.py b/dbm-ui/backend/ticket/builders/tendbcluster/tendb_backup.py deleted file mode 100644 index 51cf63d13d..0000000000 --- a/dbm-ui/backend/ticket/builders/tendbcluster/tendb_backup.py +++ /dev/null @@ -1,51 +0,0 @@ -# -*- coding: utf-8 -*- -""" -TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. -Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. -Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. -You may obtain a copy of the License at https://opensource.org/licenses/MIT -Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on -an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the -specific language governing permissions and limitations under the License. -""" - -from django.utils.translation import gettext_lazy as _ -from rest_framework import serializers - -from backend.flow.engine.controller.spider import SpiderController -from backend.ticket import builders -from backend.ticket.builders.mysql.base import DBTableField -from backend.ticket.builders.tendbcluster.base import BaseTendbTicketFlowBuilder, TendbBaseOperateDetailSerializer -from backend.ticket.builders.tendbcluster.tendb_full_backup import TendbFullBackUpDetailSerializer -from backend.ticket.constants import TicketType - - -class TendbBackUpDetailSerializer(TendbBaseOperateDetailSerializer): - class TendbBackUpItemSerializer(serializers.Serializer): - cluster_id = serializers.IntegerField(help_text=_("集群ID")) - backup_local = serializers.CharField(help_text=_("备份位置")) - db_patterns = serializers.ListField(help_text=_("匹配DB列表"), child=DBTableField(db_field=True)) - ignore_dbs = serializers.ListField(help_text=_("忽略DB列表"), child=DBTableField(db_field=True)) - table_patterns = serializers.ListField(help_text=_("匹配Table列表"), child=DBTableField()) - ignore_tables = serializers.ListField(help_text=_("忽略Table列表"), child=DBTableField()) - - infos = serializers.ListSerializer(help_text=_("库表备份信息"), child=TendbBackUpItemSerializer()) - - def validate(self, attrs): - for info in attrs["infos"]: - TendbFullBackUpDetailSerializer.get_backup_local_params(info) - - # 库表选择器校验 - super().validate_database_table_selector(attrs, role_key="backup_local") - return attrs - - -class TendbBackUpFlowParamBuilder(builders.FlowParamBuilder): - controller = SpiderController.database_table_backup - - -@builders.BuilderFactory.register(TicketType.TENDBCLUSTER_DB_TABLE_BACKUP) -class TendbBackUpFlowBuilder(BaseTendbTicketFlowBuilder): - serializer = TendbBackUpDetailSerializer - inner_flow_builder = TendbBackUpFlowParamBuilder - inner_flow_name = _("TenDB Cluster 库表备份") diff --git a/dbm-ui/backend/ticket/builders/tendbcluster/tendb_full_backup.py b/dbm-ui/backend/ticket/builders/tendbcluster/tendb_full_backup.py deleted file mode 100644 index 8812ca8659..0000000000 --- a/dbm-ui/backend/ticket/builders/tendbcluster/tendb_full_backup.py +++ /dev/null @@ -1,70 +0,0 @@ -# -*- coding: utf-8 -*- -""" -TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. -Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. -Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. -You may obtain a copy of the License at https://opensource.org/licenses/MIT -Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on -an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the -specific language governing permissions and limitations under the License. -""" - -from django.utils.translation import gettext_lazy as _ -from rest_framework import serializers - -from backend.db_meta.enums import InstanceInnerRole -from backend.flow.consts import MySQLBackupFileTagEnum, MySQLBackupTypeEnum, TenDBBackUpLocation -from backend.flow.engine.controller.spider import SpiderController -from backend.ticket import builders -from backend.ticket.builders.tendbcluster.base import BaseTendbTicketFlowBuilder, TendbBaseOperateDetailSerializer -from backend.ticket.constants import TicketType - - -class TendbFullBackUpDetailSerializer(TendbBaseOperateDetailSerializer): - class FullBackUpItemSerializer(serializers.Serializer): - class FullBackUpClusterItemSerializer(serializers.Serializer): - cluster_id = serializers.IntegerField(help_text=_("集群ID")) - backup_local = serializers.CharField(help_text=_("备份位置信息"), default=InstanceInnerRole.SLAVE) - - backup_type = serializers.ChoiceField(help_text=_("备份选项"), choices=MySQLBackupTypeEnum.get_choices()) - file_tag = serializers.ChoiceField(help_text=_("备份保存时间"), choices=MySQLBackupFileTagEnum.get_choices()) - clusters = serializers.ListSerializer(help_text=_("集群备份信息"), child=FullBackUpClusterItemSerializer()) - - infos = FullBackUpItemSerializer() - - @classmethod - def get_backup_local_params(cls, info): - """ - 对备份位置进行提取， - 两种情况：remote/spider_mnt::127.0.0.1 - """ - divider = "::" - if divider not in info["backup_local"]: - return info - - backup_local, spider_mnt_address = info["backup_local"].split(divider) - info["backup_local"] = backup_local - info["spider_mnt_address"] = spider_mnt_address - - return info - - def validate(self, attrs): - for cluster_info in attrs["infos"]["clusters"]: - self.get_backup_local_params(cluster_info) - - for cluster in attrs["infos"]["clusters"]: - if cluster["backup_local"] == TenDBBackUpLocation.SPIDER_MNT and "spider_mnt_address" not in cluster: - raise serializers.ValidationError(_("备份位置选择spider_mnt时，请提供运维节点的地址")) - - return attrs - - -class TendbFullBackUpFlowParamBuilder(builders.FlowParamBuilder): - controller = SpiderController.full_backup - - -@builders.BuilderFactory.register(TicketType.TENDBCLUSTER_FULL_BACKUP) -class TendbFullBackUpFlowBuilder(BaseTendbTicketFlowBuilder): - serializer = TendbFullBackUpDetailSerializer - inner_flow_builder = TendbFullBackUpFlowParamBuilder - inner_flow_name = _("TenDB Cluster 全库备份") diff --git a/dbm-ui/backend/ticket/builders/tendbcluster/tendb_import_sqlfile.py b/dbm-ui/backend/ticket/builders/tendbcluster/tendb_import_sqlfile.py index fd174f8f46..b1ad160b04 100644 --- a/dbm-ui/backend/ticket/builders/tendbcluster/tendb_import_sqlfile.py +++ b/dbm-ui/backend/ticket/builders/tendbcluster/tendb_import_sqlfile.py @@ -22,7 +22,7 @@ MysqlSqlImportItsmParamBuilder, ) from backend.ticket.builders.tendbcluster.base import BaseTendbTicketFlowBuilder -from backend.ticket.builders.tendbcluster.tendb_full_backup import TendbFullBackUpDetailSerializer +from backend.ticket.builders.tendbcluster.full_backup import TenDBClusterFullBackUpDetailSerializer from backend.ticket.constants import TicketType logger = logging.getLogger("root") @@ -43,7 +43,7 @@ def format_ticket_data(self): super().format_ticket_data() for info in self.ticket_data["infos"]: info["backup_local"] = info["backup_on"] - TendbFullBackUpDetailSerializer.get_backup_local_params(info) + TenDBClusterFullBackUpDetailSerializer.get_backup_local_params(info) class TenDBClusterSqlImportFlowParamBuilder(MysqlSqlImportFlowParamBuilder): diff --git a/dbm-ui/backend/ticket/builders/tendbsingle/db_table_backup.py b/dbm-ui/backend/ticket/builders/tendbsingle/db_table_backup.py new file mode 100644 index 0000000000..982f6e8632 --- /dev/null +++ b/dbm-ui/backend/ticket/builders/tendbsingle/db_table_backup.py @@ -0,0 +1,28 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. +Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + +from django.utils.translation import ugettext_lazy as _ + +from backend.ticket import builders +from backend.ticket.builders.mysql.base import BaseMySQLSingleTicketFlowBuilder +from backend.ticket.builders.mysql.mysql_db_table_backup import ( + MySQLDBTableBackupDetailSerializer, + MySQLDBTableBackupFlowParamBuilder, +) +from backend.ticket.constants import FlowRetryType, TicketType + + +@builders.BuilderFactory.register(TicketType.MYSQL_SINGLE_DB_TABLE_BACKUP) +class TenDBSingleDBTableBackupFlowBuilder(BaseMySQLSingleTicketFlowBuilder): + serializer = MySQLDBTableBackupDetailSerializer + inner_flow_builder = MySQLDBTableBackupFlowParamBuilder + inner_flow_name = _("TenDBSingle 库表备份执行") + retry_type = FlowRetryType.MANUAL_RETRY diff --git a/dbm-ui/backend/ticket/builders/tendbsingle/fullbackup.py b/dbm-ui/backend/ticket/builders/tendbsingle/fullbackup.py new file mode 100644 index 0000000000..3aad765cd9 --- /dev/null +++ b/dbm-ui/backend/ticket/builders/tendbsingle/fullbackup.py @@ -0,0 +1,27 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. +Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +from django.utils.translation import ugettext_lazy as _ + +from backend.ticket import builders +from backend.ticket.builders.mysql.base import BaseMySQLSingleTicketFlowBuilder +from backend.ticket.builders.mysql.mysql_full_backup import ( + MySQLFullBackupDetailSerializer, + MySQLFullBackupFlowParamBuilder, +) +from backend.ticket.constants import FlowRetryType, TicketType + + +@builders.BuilderFactory.register(TicketType.MYSQL_SINGLE_FULL_BACKUP) +class TenDBSingleFullBackupFlowBuilder(BaseMySQLSingleTicketFlowBuilder): + serializer = MySQLFullBackupDetailSerializer + inner_flow_builder = MySQLFullBackupFlowParamBuilder + inner_flow_name = _("TenDBHA 全库备份执行") + retry_type = FlowRetryType.MANUAL_RETRY diff --git a/dbm-ui/backend/ticket/constants.py b/dbm-ui/backend/ticket/constants.py index 040753f7ab..9af6739502 100644 --- a/dbm-ui/backend/ticket/constants.py +++ b/dbm-ui/backend/ticket/constants.py @@ -194,14 +194,16 @@ def get_cluster_type_by_ticket(cls, ticket_type): MYSQL_INSTANCE_CLONE_RULES = TicketEnumField("MYSQL_INSTANCE_CLONE_RULES", _("MySQL DB实例权限克隆"), _("权限管理")) MYSQL_HA_RENAME_DATABASE = TicketEnumField("MYSQL_HA_RENAME_DATABASE", _("MySQL 高可用DB重命名"), _("集群维护")) MYSQL_HA_TRUNCATE_DATA = TicketEnumField("MYSQL_HA_TRUNCATE_DATA", _("MySQL 高可用清档"), _("数据处理")) - MYSQL_HA_DB_TABLE_BACKUP = TicketEnumField("MYSQL_HA_DB_TABLE_BACKUP", _("MySQL 高可用库表备份"), _("备份")) + MYSQL_HA_DB_TABLE_BACKUP = TicketEnumField("MYSQL_HA_DB_TABLE_BACKUP", _("TenDBHA 库表备份"), _("备份")) + MYSQL_SINGLE_DB_TABLE_BACKUP = TicketEnumField("MYSQL_SINGLE_DB_TABLE_BACKUP", _("TenDBSingle 库表备份"), _("备份")) MYSQL_CHECKSUM = TicketEnumField("MYSQL_CHECKSUM", _("MySQL 数据校验修复"), _("数据处理")) MYSQL_PARTITION = TicketEnumField("MYSQL_PARTITION", _("MySQL 分区"), _("分区管理")) MYSQL_PARTITION_CRON = TicketEnumField("MYSQL_PARTITION_CRON", _("MySQL 分区定时任务"), register_iam=False) # noqa MYSQL_DATA_REPAIR = TicketEnumField("MYSQL_DATA_REPAIR", _("MySQL 数据修复"), register_iam=False) MYSQL_FLASHBACK = TicketEnumField("MYSQL_FLASHBACK", _("MySQL 闪回"), _("回档")) MYSQL_ROLLBACK_CLUSTER = TicketEnumField("MYSQL_ROLLBACK_CLUSTER", _("MySQL 定点构造"), _("回档")) - MYSQL_HA_FULL_BACKUP = TicketEnumField("MYSQL_HA_FULL_BACKUP", _("MySQL 高可用全库备份"), _("备份")) + MYSQL_HA_FULL_BACKUP = TicketEnumField("MYSQL_HA_FULL_BACKUP", _("TenDB HA全库备份"), _("备份")) + MYSQL_SINGLE_FULL_BACKUP = TicketEnumField("MYSQL_SINGLE_FULL_BACKUP", _("TenDB Single全库备份"), _("备份")) MYSQL_SINGLE_TRUNCATE_DATA = TicketEnumField("MYSQL_SINGLE_TRUNCATE_DATA", _("MySQL 单节点清档"), _("数据处理")) MYSQL_SINGLE_RENAME_DATABASE = TicketEnumField("MYSQL_SINGLE_RENAME_DATABASE", _("MySQL 单节点DB重命名"), _("集群维护")) # noqa MYSQL_HA_STANDARDIZE = TicketEnumField("MYSQL_HA_STANDARDIZE", _("TendbHA 标准化"), register_iam=False) @@ -212,7 +214,7 @@ def get_cluster_type_by_ticket(cls, ticket_type): MYSQL_LOCAL_UPGRADE = TicketEnumField("MYSQL_LOCAL_UPGRADE", _("MySQL 原地升级"), _("版本升级")) MYSQL_MIGRATE_UPGRADE = TicketEnumField("MYSQL_MIGRATE_UPGRADE", _("MySQL 迁移升级"), _("版本升级")) MYSQL_SLAVE_MIGRATE_UPGRADE = TicketEnumField("MYSQL_SLAVE_MIGRATE_UPGRADE", _("MySQL Slave 迁移升级"), _("版本升级")) - MYSQL_RO_SLAVE_UNINSTALL = TicketEnumField("MYSQL_RO_SLAVE_UNINSTALL", _("MySQL非stanby slave下架"), _("集群维护")) + MYSQL_RO_SLAVE_UNINSTALL = TicketEnumField("MYSQL_RO_SLAVE_UNINSTALL", _("MySQL非standby slave下架"), _("集群维护")) MYSQL_PROXY_UPGRADE = TicketEnumField("MYSQL_PROXY_UPGRADE", _("MySQL Proxy升级"), _("版本升级")) MYSQL_HA_TRANSFER_TO_OTHER_BIZ = TicketEnumField("MYSQL_HA_TRANSFER_TO_OTHER_BIZ", _("TendbHA集群迁移至其他业务"), register_iam=False) # noqa MYSQL_PUSH_PERIPHERAL_CONFIG = TicketEnumField("MYSQL_PUSH_PERIPHERAL_CONFIG", _("推送周边配置"), register_iam=False)