From 86f745ff6f5c285966f7f14bf717404216916778 Mon Sep 17 00:00:00 2001 From: seanlook Date: Wed, 13 Nov 2024 10:20:47 +0800 Subject: [PATCH] =?UTF-8?q?fix(mysql):=20=E4=BF=AE=E5=A4=8Ddbbackup?= =?UTF-8?q?=E5=AF=BC=E5=85=A5gztab=E6=A0=BC=E5=BC=8F=E9=80=BB=E8=BE=91?= =?UTF-8?q?=E5=A4=87=E4=BB=BD=20#7920?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../components/mysql/dbbackup/backup_info.go | 5 +---- .../pkg/components/mysql/restore/backup.go | 13 ++++++++--- .../components/mysql/restore/dbbackup_load.go | 2 +- .../components/mysql/restore/restore_comp.go | 4 ++-- .../pkg/components/rename_dbs/on_mysql.go | 9 ++++---- .../components/rename_dbs/pkg/dump_schema.go | 10 +++++---- .../pkg/util/mysqlutil/mysql_dumper.go | 6 +++-- .../pkg/src/backupexe/execute_load.go | 3 ++- .../src/backupexe/loader_logical_mysqldump.go | 22 ++++++------------- .../pkg/src/backupexe/migrate_old.go | 17 +++++++++----- 10 files changed, 49 insertions(+), 42 deletions(-) diff --git a/dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/dbbackup/backup_info.go b/dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/dbbackup/backup_info.go index 20258ab232..8530510d10 100644 --- a/dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/dbbackup/backup_info.go +++ b/dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/dbbackup/backup_info.go @@ -7,7 +7,6 @@ import ( "os" "path/filepath" "regexp" - "sort" "strconv" "strings" "time" @@ -217,8 +216,7 @@ func (i *InfoFileDetail) getFullFileListFromInfo(checkMD5 bool) error { if i.backupFiles == nil { i.backupFiles = make(map[string][]string) } - i.backupFiles[MYSQL_FULL_BACKUP] = fullFileNames - // sort.Strings(fullFileList) + i.backupFiles[MYSQL_FULL_BACKUP] = util.SortSplitPartFiles(fullFileNames, ".") return nil } @@ -239,7 +237,6 @@ func (i *InfoFileDetail) UntarFiles(untarDir string) error { return errors.Errorf("target untar path already exists %s", i.targetDir) } fullFileList := i.backupFiles[MYSQL_FULL_BACKUP] - sort.Strings(fullFileList) if len(fullFileList) >= 2 { cmd = fmt.Sprintf( diff --git a/dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/restore/backup.go b/dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/restore/backup.go index e55848c780..e94dc59c0e 100644 --- a/dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/restore/backup.go +++ b/dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/restore/backup.go @@ -6,11 +6,11 @@ import ( "strings" "time" + "github.com/pkg/errors" + "dbm-services/common/go-pubpkg/cmutil" "dbm-services/common/go-pubpkg/logger" "dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/dbbackup" - - "github.com/pkg/errors" ) // BackupInfo backup file info @@ -82,7 +82,7 @@ func (b *BackupInfo) GetBackupMetaFile(fileType string) error { } } - if cmutil.StringsHas([]string{"gztab", "xtra"}, b.backupType) { + if b.indexObj != nil && cmutil.StringsHas([]string{"gztab", "xtra"}, b.indexObj.BackupTool) { indexInfoFile := "" for _, fileItem := range b.indexObj.FileList { if fileItem.FileType == "index" { @@ -106,6 +106,13 @@ func (b *BackupInfo) GetBackupMetaFile(fileType string) error { b.backupHost = b.infoObj.BackupHost b.backupPort = b.infoObj.BackupPort logger.Info("GetBackupMetaFile infoObj:%+v", b.infoObj) + b.indexObj = &dbbackup.BackupIndexFile{} + b.indexObj.BackupTool = b.infoObj.BackupType + if infoObj.BackupType == "gztab" { + b.indexObj.BackupType = "logical" + } else if infoObj.BackupType == "xtra" { + b.indexObj.BackupType = "physical" + } } } logger.Info("backupType=%s, backupHost=%s, backupPort=%d", b.backupType, b.backupHost, b.backupPort) diff --git a/dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/restore/dbbackup_load.go b/dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/restore/dbbackup_load.go index fe2879eb36..15c36571ee 100644 --- a/dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/restore/dbbackup_load.go +++ b/dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/restore/dbbackup_load.go @@ -189,7 +189,7 @@ func (m *DBLoader) PostCheck() error { sqlStr := fmt.Sprintf(`update infodba_schema.global_backup SET BackupStatus ='%s' where Host ='%s' and Port =%d`, spider.StatusQuit, m.TgtInstance.Host, m.TgtInstance.Port) if _, err = dbWorker.ExecMore([]string{"set session sql_log_bin=off", sqlStr}); err != nil { - logger.Warn("fail to repair data for table global_backup. ignore", err.Error()) + logger.Warn("fail to repair data for table global_backup. ignore %s", err.Error()) } return nil } diff --git a/dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/restore/restore_comp.go b/dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/restore/restore_comp.go index 9a99be6648..4fda6c6f7b 100644 --- a/dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/restore/restore_comp.go +++ b/dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/restore/restore_comp.go @@ -180,13 +180,13 @@ func (r *RestoreDRComp) ChooseType() error { return err } b.backupType = strings.ToLower(b.backupType) // 一律转成小写来判断 - if b.backupType == cst.TypeXTRA { + if b.indexObj.BackupTool == cst.TypeXTRA || b.backupType == cst.TypeXTRA { xload := XLoad{ RestoreParam: &r.Params, } r.restore = &xload logger.Info("choose recover type [%s], infoObj.BackupType=%s", b.backupType, b.infoObj.BackupType) - } else if b.backupType == cst.TypeGZTAB { + } else if b.indexObj.BackupTool == cst.TypeGZTAB || b.backupType == cst.TypeGZTAB { mload := MLoad{ RestoreParam: &r.Params, } diff --git a/dbm-services/mysql/db-tools/dbactuator/pkg/components/rename_dbs/on_mysql.go b/dbm-services/mysql/db-tools/dbactuator/pkg/components/rename_dbs/on_mysql.go index 57885c0be7..a8cfe666f7 100644 --- a/dbm-services/mysql/db-tools/dbactuator/pkg/components/rename_dbs/on_mysql.go +++ b/dbm-services/mysql/db-tools/dbactuator/pkg/components/rename_dbs/on_mysql.go @@ -4,15 +4,16 @@ import ( "bufio" "bytes" "context" - "dbm-services/common/go-pubpkg/logger" - "dbm-services/mysql/db-tools/dbactuator/pkg/components" - "dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/common" - "dbm-services/mysql/db-tools/dbactuator/pkg/components/rename_dbs/pkg" "fmt" "os" "os/exec" "strings" + "dbm-services/common/go-pubpkg/logger" + "dbm-services/mysql/db-tools/dbactuator/pkg/components" + "dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/common" + "dbm-services/mysql/db-tools/dbactuator/pkg/components/rename_dbs/pkg" + _ "github.com/go-sql-driver/mysql" "github.com/jmoiron/sqlx" ) diff --git a/dbm-services/mysql/db-tools/dbactuator/pkg/components/rename_dbs/pkg/dump_schema.go b/dbm-services/mysql/db-tools/dbactuator/pkg/components/rename_dbs/pkg/dump_schema.go index a44ffc80ec..b65b028018 100644 --- a/dbm-services/mysql/db-tools/dbactuator/pkg/components/rename_dbs/pkg/dump_schema.go +++ b/dbm-services/mysql/db-tools/dbactuator/pkg/components/rename_dbs/pkg/dump_schema.go @@ -1,13 +1,14 @@ package pkg import ( - "dbm-services/mysql/db-tools/dbactuator/pkg/core/cst" - "dbm-services/mysql/db-tools/dbactuator/pkg/util/mysqlutil" - "dbm-services/mysql/db-tools/mysql-dbbackup/pkg/config" "fmt" "os" "path/filepath" + "dbm-services/mysql/db-tools/dbactuator/pkg/core/cst" + "dbm-services/mysql/db-tools/dbactuator/pkg/util/mysqlutil" + "dbm-services/mysql/db-tools/mysql-dbbackup/pkg/config" + "gopkg.in/ini.v1" ) @@ -17,7 +18,8 @@ func DumpDBSchema(ip string, port int, user, password, dbName string, uid string mysqldumpBasePath = cst.TdbctlInstallPath } - backupCharset, backupDir, err := readBackupConfig(port) + backupCharset := "utf8" // mysql 表结构编码都是用 utf8，可能跟表数据编码不一样 + _, backupDir, err := readBackupConfig(port) if err != nil { return "", err } diff --git a/dbm-services/mysql/db-tools/dbactuator/pkg/util/mysqlutil/mysql_dumper.go b/dbm-services/mysql/db-tools/dbactuator/pkg/util/mysqlutil/mysql_dumper.go index 7ee78823fc..3b62343032 100644 --- a/dbm-services/mysql/db-tools/dbactuator/pkg/util/mysqlutil/mysql_dumper.go +++ b/dbm-services/mysql/db-tools/dbactuator/pkg/util/mysqlutil/mysql_dumper.go @@ -286,14 +286,16 @@ func (m *MySQLDumper) getDumpCmd(outputFile, errFile, dumpOption string) (dumpCm if m.Quick { dumpOption += " --quick " } + if m.Charset != "" { // charset 可能为空 + dumpOption += " --default-character-set=" + m.Charset + } dumpCmd = fmt.Sprintf( - `%s -h%s -P%d -u%s -p%s --skip-opt --create-options --single-transaction --max-allowed-packet=1G -q --no-autocommit --default-character-set=%s %s`, + `%s -h%s -P%d -u%s -p%s --skip-opt --create-options --single-transaction --max-allowed-packet=1G -q --no-autocommit %s`, m.DumpCmdFile, m.Ip, m.Port, m.DbBackupUser, m.DbBackupPwd, - m.Charset, dumpOption, ) if len(m.Tables) > 0 { diff --git a/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/execute_load.go b/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/execute_load.go index 5c065aba35..87ed01f082 100644 --- a/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/execute_load.go +++ b/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/execute_load.go @@ -1,9 +1,10 @@ package backupexe import ( + "strings" + "dbm-services/mysql/db-tools/mysql-dbbackup/pkg/config" "dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/dbareport" - "strings" ) // ExecuteLoad execute load backup command diff --git a/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/loader_logical_mysqldump.go b/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/loader_logical_mysqldump.go index 4329c79eff..d7ada3a79c 100644 --- a/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/loader_logical_mysqldump.go +++ b/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/loader_logical_mysqldump.go @@ -84,10 +84,8 @@ func (l *LogicalLoaderMysqldump) preExecute() error { // handle DBListDropIfExists // 如果有设置这个选项，会在运行前执行 drop database if exists 命令，来清理脏库 if strings.TrimSpace(dbListDrop) != "" { - if strings.TrimSpace(dbListDrop) != "" { - if err := dropDatabasesBeforeLoad(dbListDrop, &l.cnf.LogicalLoad, l.dbConn); err != nil { - return err - } + if err := dropDatabasesBeforeLoad(dbListDrop, &l.cnf.LogicalLoad, l.dbConn); err != nil { + return err } } return nil @@ -113,7 +111,6 @@ func (l *LogicalLoaderMysqldump) Execute() (err error) { if err = l.preExecute(); err != nil { return err } - defer func() { if l.initConnect != "" { logger.Log.Info("set global init_connect back:", l.initConnect) @@ -136,7 +133,6 @@ func (l *LogicalLoaderMysqldump) Execute() (err error) { } } } - args := []string{ binPath, "-h" + l.cnf.LogicalLoad.MysqlHost, @@ -158,22 +154,18 @@ func (l *LogicalLoaderMysqldump) Execute() (err error) { if len(initCommand) > 0 { args = append(args, fmt.Sprintf(`--init-command='%s'`, strings.Join(initCommand, ";"))) } - // ExtraOpt is to freely add command line arguments if l.cnf.LogicalLoadMysqldump.ExtraOpt != "" { - args = append(args, []string{ - fmt.Sprintf(`%s`, l.cnf.LogicalLoadMysqldump.ExtraOpt), - }...) + args = append(args, l.cnf.LogicalLoadMysqldump.ExtraOpt) } - sqlFiles, err := filepath.Glob(filepath.Join(l.cnf.LogicalLoad.MysqlLoadDir, "*_logical.sql*")) - if err != nil { - return errors.WithMessagef(err, "get sql file from %s", l.cnf.LogicalLoad.MysqlLoadDir) - } else if len(sqlFiles) == 0 { - return errors.WithMessagef(err, "no sql file found from %s", l.cnf.LogicalLoad.MysqlLoadDir) + sqlFiles, err := filepath.Glob(filepath.Join(l.cnf.LogicalLoad.MysqlLoadDir, "*.sql*")) + if len(sqlFiles) == 0 { + return errors.Errorf("no sql file found from %s", l.cnf.LogicalLoad.MysqlLoadDir) } else { logger.Log.Info("found sql files:", sqlFiles) } + // 取第一个 dumpedSqlFile := sqlFiles[0] if strings.HasSuffix(dumpedSqlFile, cst.ZstdSuffix) { diff --git a/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/migrate_old.go b/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/migrate_old.go index f7ea181e6c..3cf1323b90 100644 --- a/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/migrate_old.go +++ b/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/migrate_old.go @@ -39,18 +39,22 @@ func MigrateInstanceBackupInfo(infoFilePath string, cnf *config.BackupConfig) (s if err != nil { return "", nil, err } - backupInfoDir := filepath.Dir(infoFilePath) targetName := strings.TrimSuffix(filepath.Base(infoFilePath), ".info") gztabBeginFile := filepath.Join(backupInfoDir, targetName+".DUMP.BEGIN.sql.gz") xtraInfoFile := filepath.Join(backupInfoDir, targetName+".xtrabackup_info") var backupTime time.Time + var backupType, backupTool string if infoObj.BackupType == "gztab" { + backupType = "logical" backupTime = beginTime - } else { + } else if infoObj.BackupType == "xtra" { + backupType = "physical" backupTime = endTime } + backupTool = infoObj.BackupType + var isFullBackup bool if infoObj.DataOrGrant == "ALL" { isFullBackup = true @@ -117,7 +121,7 @@ func MigrateInstanceBackupInfo(infoFilePath string, cnf *config.BackupConfig) (s ClusterId: cnf.Public.ClusterId, BkBizId: cnf.Public.BkBizId, ClusterAddress: cnf.Public.ClusterAddress, - BackupType: infoObj.BackupType, + BackupType: backupType, BackupHost: infoObj.BackupHost, BackupPort: infoObj.BackupPort, MysqlRole: infoObj.BackupRole, @@ -133,6 +137,7 @@ func MigrateInstanceBackupInfo(infoFilePath string, cnf *config.BackupConfig) (s BkCloudId: 0, BackupCharset: infoObj.Charset, StorageEngine: storageEngine, + BackupTool: backupTool, }, BinlogInfo: dbareport.BinlogStatusInfo{ ShowMasterStatus: nil, @@ -143,12 +148,12 @@ func MigrateInstanceBackupInfo(infoFilePath string, cnf *config.BackupConfig) (s // 补齐 show master status, show slave status 信息 var masterStatus, slaveStatus *dbareport.StatusInfo - if infoObj.BackupType == "gztab" { + if backupTool == "gztab" { if masterStatus, slaveStatus, err = MLoadGetBackupSlaveStatus(gztabBeginFile); err != nil { return "", nil, err } } - if infoObj.BackupType == "xtra" { + if backupTool == "xtra" { if masterStatus, slaveStatus, err = XLoadGetBackupSlaveStatus(xtraInfoFile); err != nil { return "", nil, err } @@ -174,7 +179,7 @@ func MigrateInstanceBackupInfo(infoFilePath string, cnf *config.BackupConfig) (s // 使用新的 index 名 newTargetName := fmt.Sprintf("%d_%d_%s_%d_%s_%s", cnf.Public.BkBizId, cnf.Public.ClusterId, cnf.Public.MysqlHost, cnf.Public.MysqlPort, - beginTime.Format("20060102150405"), infoObj.BackupType) + beginTime.Format("20060102150405"), backupTool) cnf.Public.SetTargetName(newTargetName) indexFilePath, err := indexObj.SaveIndexContent(&cnf.Public) if err != nil {