fix(mysql): 修复dbbackup导入gztab格式逻辑备份 #7920

dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/dbbackup/backup_info.go

@@ -7,7 +7,6 @@ import (
 	"os"
 	"path/filepath"
 	"regexp"
-	"sort"
 	"strconv"
 	"strings"
 	"time"
@@ -217,8 +216,7 @@ func (i *InfoFileDetail) getFullFileListFromInfo(checkMD5 bool) error {
 	if i.backupFiles == nil {
 		i.backupFiles = make(map[string][]string)
 	}
-	i.backupFiles[MYSQL_FULL_BACKUP] = fullFileNames
-	// sort.Strings(fullFileList)
+	i.backupFiles[MYSQL_FULL_BACKUP] = util.SortSplitPartFiles(fullFileNames, ".")
 	return nil
 }
 
@@ -239,7 +237,6 @@ func (i *InfoFileDetail) UntarFiles(untarDir string) error {
 		return errors.Errorf("target untar path already exists %s", i.targetDir)
 	}
 	fullFileList := i.backupFiles[MYSQL_FULL_BACKUP]
-	sort.Strings(fullFileList)
 
 	if len(fullFileList) >= 2 {
 		cmd = fmt.Sprintf(

dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/restore/backup.go

@@ -6,11 +6,11 @@ import (
 	"strings"
 	"time"
 
+	"github.com/pkg/errors"
+
 	"dbm-services/common/go-pubpkg/cmutil"
 	"dbm-services/common/go-pubpkg/logger"
 	"dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/dbbackup"
-
-	"github.com/pkg/errors"
 )
 
 // BackupInfo backup file info
@@ -82,7 +82,7 @@ func (b *BackupInfo) GetBackupMetaFile(fileType string) error {
 		}
 	}
 
-	if cmutil.StringsHas([]string{"gztab", "xtra"}, b.backupType) {
+	if b.indexObj != nil && cmutil.StringsHas([]string{"gztab", "xtra"}, b.indexObj.BackupTool) {
 		indexInfoFile := ""
 		for _, fileItem := range b.indexObj.FileList {
 			if fileItem.FileType == "index" {
@@ -106,6 +106,13 @@ func (b *BackupInfo) GetBackupMetaFile(fileType string) error {
 			b.backupHost = b.infoObj.BackupHost
 			b.backupPort = b.infoObj.BackupPort
 			logger.Info("GetBackupMetaFile infoObj:%+v", b.infoObj)
+			b.indexObj = &dbbackup.BackupIndexFile{}
+			b.indexObj.BackupTool = b.infoObj.BackupType
+			if infoObj.BackupType == "gztab" {
+				b.indexObj.BackupType = "logical"
+			} else if infoObj.BackupType == "xtra" {
+				b.indexObj.BackupType = "physical"
+			}
 		}
 	}
 	logger.Info("backupType=%s, backupHost=%s, backupPort=%d", b.backupType, b.backupHost, b.backupPort)

dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/restore/dbbackup_load.go

@@ -189,7 +189,7 @@ func (m *DBLoader) PostCheck() error {
 	sqlStr := fmt.Sprintf(`update infodba_schema.global_backup SET BackupStatus ='%s' where Host ='%s' and Port =%d`,
 		spider.StatusQuit, m.TgtInstance.Host, m.TgtInstance.Port)
 	if _, err = dbWorker.ExecMore([]string{"set session sql_log_bin=off", sqlStr}); err != nil {
-		logger.Warn("fail to repair data for table global_backup. ignore", err.Error())
+		logger.Warn("fail to repair data for table global_backup. ignore %s", err.Error())
 	}
 	return nil
 }

dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/restore/restore_comp.go

@@ -180,13 +180,13 @@ func (r *RestoreDRComp) ChooseType() error {
 		return err
 	}
 	b.backupType = strings.ToLower(b.backupType) // 一律转成小写来判断
-	if b.backupType == cst.TypeXTRA {
+	if b.indexObj.BackupTool == cst.TypeXTRA || b.backupType == cst.TypeXTRA {
 		xload := XLoad{
 			RestoreParam: &r.Params,
 		}
 		r.restore = &xload
 		logger.Info("choose recover type [%s], infoObj.BackupType=%s", b.backupType, b.infoObj.BackupType)
-	} else if b.backupType == cst.TypeGZTAB {
+	} else if b.indexObj.BackupTool == cst.TypeGZTAB || b.backupType == cst.TypeGZTAB {
 		mload := MLoad{
 			RestoreParam: &r.Params,
 		}

dbm-services/mysql/db-tools/dbactuator/pkg/components/rename_dbs/on_mysql.go

@@ -4,15 +4,16 @@ import (
 	"bufio"
 	"bytes"
 	"context"
-	"dbm-services/common/go-pubpkg/logger"
-	"dbm-services/mysql/db-tools/dbactuator/pkg/components"
-	"dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/common"
-	"dbm-services/mysql/db-tools/dbactuator/pkg/components/rename_dbs/pkg"
 	"fmt"
 	"os"
 	"os/exec"
 	"strings"
 
+	"dbm-services/common/go-pubpkg/logger"
+	"dbm-services/mysql/db-tools/dbactuator/pkg/components"
+	"dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/common"
+	"dbm-services/mysql/db-tools/dbactuator/pkg/components/rename_dbs/pkg"
+
 	_ "github.com/go-sql-driver/mysql"
 	"github.com/jmoiron/sqlx"
 )

dbm-services/mysql/db-tools/dbactuator/pkg/components/rename_dbs/pkg/dump_schema.go

@@ -1,13 +1,14 @@
 package pkg
 
 import (
-	"dbm-services/mysql/db-tools/dbactuator/pkg/core/cst"
-	"dbm-services/mysql/db-tools/dbactuator/pkg/util/mysqlutil"
-	"dbm-services/mysql/db-tools/mysql-dbbackup/pkg/config"
 	"fmt"
 	"os"
 	"path/filepath"
 
+	"dbm-services/mysql/db-tools/dbactuator/pkg/core/cst"
+	"dbm-services/mysql/db-tools/dbactuator/pkg/util/mysqlutil"
+	"dbm-services/mysql/db-tools/mysql-dbbackup/pkg/config"
+
 	"gopkg.in/ini.v1"
 )
 
@@ -17,7 +18,8 @@ func DumpDBSchema(ip string, port int, user, password, dbName string, uid string
 		mysqldumpBasePath = cst.TdbctlInstallPath
 	}
 
-	backupCharset, backupDir, err := readBackupConfig(port)
+	backupCharset := "utf8" // mysql 表结构编码都是用 utf8，可能跟表数据编码不一样
+	_, backupDir, err := readBackupConfig(port)
 	if err != nil {
 		return "", err
 	}

dbm-services/mysql/db-tools/dbactuator/pkg/util/mysqlutil/mysql_dumper.go

@@ -286,14 +286,16 @@ func (m *MySQLDumper) getDumpCmd(outputFile, errFile, dumpOption string) (dumpCm
 	if m.Quick {
 		dumpOption += " --quick "
 	}
+	if m.Charset != "" { // charset 可能为空
+		dumpOption += " --default-character-set=" + m.Charset
+	}
 	dumpCmd = fmt.Sprintf(
-		`%s -h%s -P%d  -u%s  -p%s --skip-opt --create-options --single-transaction --max-allowed-packet=1G -q --no-autocommit --default-character-set=%s %s`,
+		`%s -h%s -P%d  -u%s  -p%s --skip-opt --create-options --single-transaction --max-allowed-packet=1G -q --no-autocommit %s`,
 		m.DumpCmdFile,
 		m.Ip,
 		m.Port,
 		m.DbBackupUser,
 		m.DbBackupPwd,
-		m.Charset,
 		dumpOption,
 	)
 	if len(m.Tables) > 0 {

dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/execute_load.go

@@ -1,9 +1,10 @@
 package backupexe
 
 import (
+	"strings"
+
 	"dbm-services/mysql/db-tools/mysql-dbbackup/pkg/config"
 	"dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/dbareport"
-	"strings"
 )
 
 // ExecuteLoad execute load backup command

dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/loader_logical_mysqldump.go

@@ -84,10 +84,8 @@ func (l *LogicalLoaderMysqldump) preExecute() error {
 	// handle DBListDropIfExists
 	// 如果有设置这个选项，会在运行前执行 drop database if exists 命令，来清理脏库
 	if strings.TrimSpace(dbListDrop) != "" {
-		if strings.TrimSpace(dbListDrop) != "" {
-			if err := dropDatabasesBeforeLoad(dbListDrop, &l.cnf.LogicalLoad, l.dbConn); err != nil {
-				return err
-			}
+		if err := dropDatabasesBeforeLoad(dbListDrop, &l.cnf.LogicalLoad, l.dbConn); err != nil {
+			return err
 		}
 	}
 	return nil
@@ -113,7 +111,6 @@ func (l *LogicalLoaderMysqldump) Execute() (err error) {
 	if err = l.preExecute(); err != nil {
 		return err
 	}
-
 	defer func() {
 		if l.initConnect != "" {
 			logger.Log.Info("set global init_connect back:", l.initConnect)
@@ -136,7 +133,6 @@ func (l *LogicalLoaderMysqldump) Execute() (err error) {
 			}
 		}
 	}
-
 	args := []string{
 		binPath,
 		"-h" + l.cnf.LogicalLoad.MysqlHost,
@@ -158,22 +154,18 @@ func (l *LogicalLoaderMysqldump) Execute() (err error) {
 	if len(initCommand) > 0 {
 		args = append(args, fmt.Sprintf(`--init-command='%s'`, strings.Join(initCommand, ";")))
 	}
-
 	// ExtraOpt is to freely add command line arguments
 	if l.cnf.LogicalLoadMysqldump.ExtraOpt != "" {
-		args = append(args, []string{
-			fmt.Sprintf(`%s`, l.cnf.LogicalLoadMysqldump.ExtraOpt),
-		}...)
+		args = append(args, l.cnf.LogicalLoadMysqldump.ExtraOpt)
 	}
 
-	sqlFiles, err := filepath.Glob(filepath.Join(l.cnf.LogicalLoad.MysqlLoadDir, "*_logical.sql*"))
-	if err != nil {
-		return errors.WithMessagef(err, "get sql file from %s", l.cnf.LogicalLoad.MysqlLoadDir)
-	} else if len(sqlFiles) == 0 {
-		return errors.WithMessagef(err, "no sql file found from %s", l.cnf.LogicalLoad.MysqlLoadDir)
+	sqlFiles, err := filepath.Glob(filepath.Join(l.cnf.LogicalLoad.MysqlLoadDir, "*.sql*"))
+	if len(sqlFiles) == 0 {
+		return errors.Errorf("no sql file found from %s", l.cnf.LogicalLoad.MysqlLoadDir)
 	} else {
 		logger.Log.Info("found sql files:", sqlFiles)
 	}
+
 	// 取第一个
 	dumpedSqlFile := sqlFiles[0]
 	if strings.HasSuffix(dumpedSqlFile, cst.ZstdSuffix) {

dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/migrate_old.go

@@ -39,18 +39,22 @@ func MigrateInstanceBackupInfo(infoFilePath string, cnf *config.BackupConfig) (s
 	if err != nil {
 		return "", nil, err
 	}
-
 	backupInfoDir := filepath.Dir(infoFilePath)
 	targetName := strings.TrimSuffix(filepath.Base(infoFilePath), ".info")
 	gztabBeginFile := filepath.Join(backupInfoDir, targetName+".DUMP.BEGIN.sql.gz")
 	xtraInfoFile := filepath.Join(backupInfoDir, targetName+".xtrabackup_info")
 
 	var backupTime time.Time
+	var backupType, backupTool string
 	if infoObj.BackupType == "gztab" {
+		backupType = "logical"
 		backupTime = beginTime
-	} else {
+	} else if infoObj.BackupType == "xtra" {
+		backupType = "physical"
 		backupTime = endTime
 	}
+	backupTool = infoObj.BackupType
+
 	var isFullBackup bool
 	if infoObj.DataOrGrant == "ALL" {
 		isFullBackup = true
@@ -117,7 +121,7 @@ func MigrateInstanceBackupInfo(infoFilePath string, cnf *config.BackupConfig) (s
 			ClusterId:            cnf.Public.ClusterId,
 			BkBizId:              cnf.Public.BkBizId,
 			ClusterAddress:       cnf.Public.ClusterAddress,
-			BackupType:           infoObj.BackupType,
+			BackupType:           backupType,
 			BackupHost:           infoObj.BackupHost,
 			BackupPort:           infoObj.BackupPort,
 			MysqlRole:            infoObj.BackupRole,
@@ -133,6 +137,7 @@ func MigrateInstanceBackupInfo(infoFilePath string, cnf *config.BackupConfig) (s
 			BkCloudId:     0,
 			BackupCharset: infoObj.Charset,
 			StorageEngine: storageEngine,
+			BackupTool:    backupTool,
 		},
 		BinlogInfo: dbareport.BinlogStatusInfo{
 			ShowMasterStatus: nil,
@@ -143,12 +148,12 @@ func MigrateInstanceBackupInfo(infoFilePath string, cnf *config.BackupConfig) (s
 
 	// 补齐  show master status, show slave status 信息
 	var masterStatus, slaveStatus *dbareport.StatusInfo
-	if infoObj.BackupType == "gztab" {
+	if backupTool == "gztab" {
 		if masterStatus, slaveStatus, err = MLoadGetBackupSlaveStatus(gztabBeginFile); err != nil {
 			return "", nil, err
 		}
 	}
-	if infoObj.BackupType == "xtra" {
+	if backupTool == "xtra" {
 		if masterStatus, slaveStatus, err = XLoadGetBackupSlaveStatus(xtraInfoFile); err != nil {
 			return "", nil, err
 		}
@@ -174,7 +179,7 @@ func MigrateInstanceBackupInfo(infoFilePath string, cnf *config.BackupConfig) (s
 	// 使用新的 index 名
 	newTargetName := fmt.Sprintf("%d_%d_%s_%d_%s_%s",
 		cnf.Public.BkBizId, cnf.Public.ClusterId, cnf.Public.MysqlHost, cnf.Public.MysqlPort,
-		beginTime.Format("20060102150405"), infoObj.BackupType)
+		beginTime.Format("20060102150405"), backupTool)
 	cnf.Public.SetTargetName(newTargetName)
 	indexFilePath, err := indexObj.SaveIndexContent(&cnf.Public)
 	if err != nil {