feat(mysql): dbbackup rocksdb恢复 TencentBlueKing#8034

dbm-services/common/db-config/assets/migrations/000021_tendbsingle_data.up.sql

@@ -125,7 +125,7 @@ INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `
 INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (13074,'tendbsingle','backup','dbbackup.ini','Public.DataSchemaGrant','STRING','{{.Public.DataSchemaGrant}}','all | schema | grant | data | none','ENUM',2,0,0,0,0,NULL,'','',-1,NULL,'','2022-05-26 20:11:23','2023-12-07 10:03:48',0);
 INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (23694,'tendbsingle','backup','dbbackup.ini','Public.IOLimitMasterFactor','FLOAT','0.5','(0, 1]','RANGE',1,0,0,0,0,NULL,'','',-1,NULL,'','2022-05-26 20:11:23','2024-06-12 10:35:19',0);
 INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (16404,'tendbsingle','backup','dbbackup.ini','Public.IOLimitMBPerSec','INT','300','[0, 4096]','RANGE',1,0,0,0,0,NULL,'','',-1,NULL,'','2022-05-26 20:11:23','2024-05-31 16:46:26',0);
-INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (13075,'tendbsingle','backup','dbbackup.ini','Public.MysqlCharset','STRING','auto','','STRING',1,0,0,0,0,NULL,'','',-1,NULL,'','2022-05-26 20:11:23','2024-11-01 17:43:12',0);
+INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (13075,'tendbsingle','backup','dbbackup.ini','Public.MysqlCharset','STRING','','| binary | auto | utf8mb4 | utf8 | latin1 | gbk','ENUM',1,0,0,0,0,NULL,'','',-1,NULL,'','2022-05-26 20:11:23','2024-11-14 10:56:52',0);
 INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (13076,'tendbsingle','backup','dbbackup.ini','Public.MysqlHost','STRING','{{.Public.MysqlHost}}','','STRING',2,0,0,0,0,NULL,'','',-1,NULL,'','2022-05-26 20:11:23','2023-04-17 17:10:41',0);
 INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (13077,'tendbsingle','backup','dbbackup.ini','Public.MysqlPasswd','STRING','{{.Public.MysqlPasswd}}','','STRING',2,0,0,1,0,NULL,'','',-1,NULL,'','2022-05-26 20:11:23','2023-04-17 17:10:41',0);
 INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (13078,'tendbsingle','backup','dbbackup.ini','Public.MysqlPort','INT','{{.Public.MysqlPort}}','','STRING',2,0,0,0,0,NULL,'','',-1,NULL,'','2022-05-26 20:11:23','2023-04-17 17:10:41',0);

dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/mycnf_clone.go

@@ -56,6 +56,7 @@ var MycnfCloneItemsDefault = []string{
 	"table_definition_cache",
 	"long_query_time",
 	"max_connections",
+	"group_concat_max_len",
 
 	"slave_parallel_workers",
 	"slave_parallel_type",

dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/dumper_physical_rocksdb.go

@@ -202,32 +202,37 @@ func (p *PhysicalRocksdbDumper) Execute(enableTimeOut bool) error {
 
 // PrepareBackupMetaInfo generate the metadata of database backup
 func (p *PhysicalRocksdbDumper) PrepareBackupMetaInfo(cnf *config.BackupConfig) (*dbareport.IndexContent, error) {
-
-	// parse the binglog position
-	xtrabackupBinlogInfoFileName := filepath.Join(cnf.Public.BackupDir, cnf.Public.TargetName(), "xtrabackup_binlog_info")
-	xtrabackupSlaveInfoFileName := filepath.Join(cnf.Public.BackupDir, cnf.Public.TargetName(), "xtrabackup_slave_info")
-
-	tmpFileName := filepath.Join(cnf.Public.BackupDir, cnf.Public.TargetName(), "tmp_dbbackup_go.txt")
+	backupTargetDir := filepath.Join(cnf.Public.BackupDir, cnf.Public.TargetName())
+	xtrabackupBinlogInfoFileName := filepath.Join(backupTargetDir, "xtrabackup_binlog_info")
+	xtrabackupSlaveInfoFileName := filepath.Join(backupTargetDir, "xtrabackup_slave_info")
+	xtrabackupTimestampFileName := filepath.Join(backupTargetDir, "xtrabackup_timestamp_info")
+	tmpFileName := filepath.Join(backupTargetDir, "tmp_dbbackup_go.txt")
 
 	// obtain the qpress command path
 	exepath, err := os.Executable()
 	if err != nil {
 		return nil, err
 	}
-
 	exepath = filepath.Dir(exepath)
 	qpressPath := filepath.Join(exepath, "bin", "qpress")
-
 	var metaInfo = dbareport.IndexContent{
 		BinlogInfo: dbareport.BinlogStatusInfo{},
 	}
 
+	// parse xtrabackup_timestamp_info
+	if err := parseXtraTimestamp(qpressPath, xtrabackupTimestampFileName, tmpFileName, &metaInfo); err != nil {
+		// 此时刚备份完成，还没有开始打包，这里把当前时间认为是 consistent_time，不完善！
+		logger.Log.Warnf("xtrabackup_timestamp_info file not found, "+
+			"use current time as Consistent Time, err: %s", err.Error())
+		metaInfo.BackupConsistentTime = metaInfo.BackupEndTime
+	}
 	// parse the binlog
 	masterStatus, err := parseXtraBinlogInfo(qpressPath, xtrabackupBinlogInfoFileName, tmpFileName)
 	if err != nil {
-		logger.Log.Errorf("do not parse xtrabackup binlog file, file name:%s, errmsg:%s",
-			xtrabackupBinlogInfoFileName, err)
-		return nil, err
+		logger.Log.Errorf("can not parse xtrabackup_binlog_info, errmsg:%s", err)
+	} else {
+		logger.Log.Infof("xtrabackup_binlog_info binlog file:%s pos:%s",
+			masterStatus.BinlogFile, masterStatus.BinlogPos)
 	}
 
 	// save the master node status

dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/loader_logical.go

@@ -211,7 +211,7 @@ func (l *LogicalLoader) Execute() (err error) {
 	if err != nil {
 		logger.Log.Error("myloader load backup failed: ", err, errStr)
 		// 尝试读取 myloader.log 里 CRITICAL 关键字
-		grepError := []string{"grep", "-E", "CRITICAL", logfile, "|", "tail", "-5"}
+		grepError := []string{"cat", logfile, "|", "grep -Ei 'CRITICAL|not found|error|fatal'", "|", "tail", "-5"}
 		errStrPrefix := fmt.Sprintf("tail 5 error from %s", logfile)
 		errStrDetail, _, _ := cmutil.ExecCommand(true, "", grepError[0], grepError[1:]...)
 		if len(strings.TrimSpace(errStr)) > 0 {

dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/prepareinfo.go

@@ -181,8 +181,29 @@ func openXtrabackupFile(binpath string, fileName string, tmpFileName string) (*b
 	return bytes.NewBuffer(content), nil
 }
 
-// parseXtraInfo get start_time / end_time from xtrabackup_info
+// parseXtraInfo get start_time / end_time / binlog pos from xtrabackup_info
 // return startTime,endTime,error
+/*
+uuid = xx-4347-11ef-8de0-xxxxxxxxx
+name =
+tool_name = xtrabackup_57
+tool_command = --defaults-file=/etc/my.cnf.3306 --host=x.x.x.x --port=3306 --user=xx --password=...
+tool_version = 2.4.11
+ibbackup_version = 2.4.11
+server_version = 5.7.20-tmysql-3.3-log
+start_time = 2024-07-16 15:44:13
+end_time = 2024-07-16 15:44:20
+lock_time = 0
+binlog_pos = filename 'binlog20000.000353', position '181942'
+innodb_from_lsn = 0
+innodb_to_lsn = 980247078
+partial = N
+incremental = N
+format = file
+compact = N
+compressed = compressed
+encrypted = N
+*/
 func parseXtraInfo(qpress string, fileName string, tmpFileName string, metaInfo *dbareport.IndexContent) error {
 	fileBytes, err := openXtrabackupFile(qpress, fileName, tmpFileName)
 	if err != nil {
@@ -193,25 +214,37 @@ func parseXtraInfo(qpress string, fileName string, tmpFileName string, metaInfo
 	var startTimeStr, endTimeStr string
 	for scanner.Scan() {
 		line := scanner.Text()
-		if strings.HasPrefix(line, "start_time = ") {
+		if strings.HasPrefix(line, "start_time = ") { // start_time = 2024-07-16 15:44:13
 			startTimeStr = strings.TrimPrefix(line, "start_time = ")
 			metaInfo.BackupBeginTime, err = time.ParseInLocation(cst.XtrabackupTimeLayout, startTimeStr, time.Local)
 			if err != nil {
 				return errors.Wrapf(err, "parse BackupBeginTime %s", startTimeStr)
 			}
 		}
-		if strings.HasPrefix(line, "end_time = ") {
+		if strings.HasPrefix(line, "end_time = ") { // end_time = 2024-07-16 15:44:20
 			endTimeStr = strings.TrimPrefix(line, "end_time = ")
 			metaInfo.BackupEndTime, err = time.ParseInLocation(cst.XtrabackupTimeLayout, endTimeStr, time.Local)
 			if err != nil {
 				return errors.Wrapf(err, "parse BackupEndTime %s", endTimeStr)
 			}
 		}
+		if strings.HasPrefix(line, "binlog_pos =") { // binlog_pos = filename 'binlog20000.000353', position '181942'
+			regBinlogPos := regexp.MustCompile(`.* filename '(.+\.\d+)', position '(\d+)'`)
+			if matches := regBinlogPos.FindStringSubmatch(line); len(matches) == 3 {
+				metaInfo.BinlogInfo.ShowMasterStatus = &dbareport.StatusInfo{
+					BinlogFile: matches[1],
+					BinlogPos:  matches[2],
+				}
+			}
+		}
 	}
 	return nil
 }
 
 // parseXtraTimestamp get consistentTime from xtrabackup_timestamp_info(if exists)
+/*
+20240716_154420
+*/
 func parseXtraTimestamp(qpress string, fileName string, tmpFileName string, metaInfo *dbareport.IndexContent) error {
 	fileBytes, err := openXtrabackupFile(qpress, fileName, tmpFileName)
 
@@ -230,7 +263,10 @@ func parseXtraTimestamp(qpress string, fileName string, tmpFileName string, meta
 	return nil
 }
 
-// parseXtraBinlogInfo parse xtrabackup_binlog_info to get master info
+// parseXtraBinlogInfo parse xtrabackup_binlog_info / xtrabackup_binlog_pos_innodb to get master info
+/*
+binlog20000.000353      181942
+*/
 func parseXtraBinlogInfo(qpress string, fileName string, tmpFileName string) (*dbareport.StatusInfo, error) {
 	fileBytes, err := openXtrabackupFile(qpress, fileName, tmpFileName)
 	if err != nil {
@@ -255,6 +291,9 @@ func parseXtraBinlogInfo(qpress string, fileName string, tmpFileName string) (*d
 }
 
 // parseXtraSlaveInfo parse xtrabackup_slave_info to get slave info
+/*
+CHANGE MASTER TO MASTER_LOG_FILE='binlog20000.009159', MASTER_LOG_POS=6488;
+*/
 func parseXtraSlaveInfo(qpress string, fileName string, tmpFileName string) (*dbareport.StatusInfo, error) {
 	fileBytes, err := openXtrabackupFile(qpress, fileName, tmpFileName)
 	if err != nil {

dbm-services/mysql/db-tools/mysql-dbbackup/pkg/util/misc.go

@@ -27,6 +27,7 @@ type DiskStatus struct {
 	Total uint64
 	Used  uint64
 	Free  uint64
+	Avail uint64
 }
 
 // FileExist check whether the file exist
@@ -144,6 +145,7 @@ func DiskUsage(path string) (disk DiskStatus, err error) {
 		}
 		disk.Total = fs.Blocks * uint64(fs.Bsize)
 		disk.Free = fs.Bfree * uint64(fs.Bsize)
+		disk.Avail = fs.Bavail * uint64(fs.Bsize)
 		disk.Used = disk.Total - disk.Free
 	default:
 		return disk, fmt.Errorf("unsupported operating system: %s", runtime.GOOS)
@@ -165,7 +167,7 @@ func CheckDiskSpace(backupDir string, mysqlPort int, backupSize uint64) (sizeLef
 	}
 	logger.Log.Infof("backupDir %s disk info: %+v", backupDir, diskSpaceInfo)
 	expectSize := backupSize*1 + 5*1024*1024*1024 // 预计备份需要多少实际空间
-	expectSizeLeft := float64(diskSpaceInfo.Free) - float64(expectSize)
+	expectSizeLeft := float64(diskSpaceInfo.Avail) - float64(expectSize)
 	if expectSizeLeft < 0 {
 		err = errors.New("free space is not enough")
 		return int64(expectSizeLeft), err