feat(backend): 权限修改支持会签模式 #6751

x

dbm-services/mysql/db-tools/mysql-dbbackup/cmd/subcmd_dump.go

@@ -195,13 +195,12 @@ func backupData(cnf *config.BackupConfig) (err error) {
 	}
 	logger.Log.Info("parse config file: end")
 	if cnf.Public.DataSchemaGrant == cst.BackupNone {
-		logger.Log.Info("backup nothing, exit")
+		logger.Log.Infof("backup nothing for %d, exit", cnf.Public.MysqlPort)
 		return nil
 	}
 	if err := precheck.BeforeDump(cnf); err != nil {
 		return err
 	}
-
 	// 备份权限 backup priv info
 	// 注意：如果只备份权限，则走 backupexe.ExecuteBackup(cnf) 逻辑
 	//  如果还备份 schema/data，则走下面这个逻辑

dbm-services/mysql/db-tools/mysql-dbbackup/docs/loadbackup.md

@@ -45,6 +45,8 @@ Global Flags:
 
 如果是 mydumper 备份出的文件，还可以通过 `--databases` 等选项控制导入的数据。mysqldump 导出的数据 使用 `--databases` 则会提示错误。
 
+逻辑备份导入 `loadbackup logical` 是不需要指定字符集，dbbackup 会自动使用导出后备份文件里面的 charset (即 .index 里面的 backup_charset ) 
+
 ### 物理备份 恢复:
 ```
 ./dbbackup loadbackup physical --help

dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/dumper.go

@@ -2,6 +2,7 @@
 package backupexe
 
 import (
+	"database/sql"
 	"fmt"
 	"strings"
 
@@ -12,6 +13,7 @@ import (
 	"dbm-services/mysql/db-tools/mysql-dbbackup/pkg/cst"
 	"dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/dbareport"
 	"dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/logger"
+	"dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/mysqlconn"
 	"dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/precheck"
 	"dbm-services/mysql/db-tools/mysql-dbbackup/pkg/util"
 )
@@ -24,7 +26,7 @@ type Dumper interface {
 }
 
 // BuildDumper return logical or physical dumper
-func BuildDumper(cnf *config.BackupConfig, storageEngine string) (dumper Dumper, err error) {
+func BuildDumper(cnf *config.BackupConfig, db *sql.DB) (dumper Dumper, err error) {
 	if cnf.Public.IfBackupGrantOnly() {
 		logger.Log.Infof("only backup grants for %d", cnf.Public.MysqlPort)
 		cnf.Public.BackupType = cst.BackupLogical
@@ -33,6 +35,10 @@ func BuildDumper(cnf *config.BackupConfig, storageEngine string) (dumper Dumper,
 		}
 		return dumper, nil
 	}
+	storageEngine, err := mysqlconn.GetStorageEngine(db)
+	if err != nil {
+		return nil, err
+	}
 	if err = precheck.CheckBackupType(cnf, storageEngine); err != nil {
 		return nil, err
 	}

dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/dumper_physical.go

@@ -209,7 +209,7 @@ func (p *PhysicalDumper) Execute(enableTimeOut bool) error {
 
 	err = cmd.Run()
 	if err != nil {
-		logger.Log.Error("run physical backup failed: ", err, stderr.Bytes())
+		logger.Log.Error("run physical backup failed: ", err, stderr.String())
 		return errors.WithMessage(err, stderr.String())
 	}
 	return nil
@@ -259,15 +259,15 @@ func (p *PhysicalDumper) PrepareBackupMetaInfo(cnf *config.BackupConfig) (*dbare
 	// parse xtrabackup_info
 	if err = parseXtraInfo(qpressPath, xtrabackupInfoFileName, tmpFileName, &metaInfo); err != nil {
 		logger.Log.Warnf("xtrabackup_info file not found, use current time as BackupEndTime, err: %s", err.Error())
-		metaInfo.BackupBeginTime, _ = time.Parse(time.DateTime, p.backupStartTime.Format(time.DateTime))
-		metaInfo.BackupEndTime, _ = time.Parse(time.DateTime, p.backupEndTime.Format(time.DateTime))
+		metaInfo.BackupBeginTime = cmutil.TimeToSecondPrecision(p.backupStartTime)
+		metaInfo.BackupEndTime = cmutil.TimeToSecondPrecision(p.backupEndTime)
 	}
 	// parse xtrabackup_timestamp_info
 	if err := parseXtraTimestamp(qpressPath, xtrabackupTimestampFileName, tmpFileName, &metaInfo); err != nil {
 		// 此时刚备份完成，还没有开始打包，这里把当前时间认为是 consistent_time，不完善！
 		logger.Log.Warnf("xtrabackup_timestamp_info file not found, "+
 			"use current time as Consistent Time, err: %s", err.Error())
-		metaInfo.BackupConsistentTime, _ = time.Parse(time.DateTime, p.backupEndTime.Format(time.DateTime))
+		metaInfo.BackupConsistentTime = cmutil.TimeToSecondPrecision(p.backupEndTime)
 	}
 	// parse xtrabackup_binlog_info 本机的 binlog file,pos
 	if masterStatus, err := parseXtraBinlogInfo(qpressPath, xtrabackupBinlogInfoFileName, tmpFileName); err != nil {

dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/execute_dump.go

@@ -34,16 +34,10 @@ func ExecuteBackup(cnf *config.BackupConfig) (*dbareport.IndexContent, error) {
 		return nil, err
 	}
 
-	storageEngine, err := mysqlconn.GetStorageEngine(db)
-	if err != nil {
-		return nil, err
-	}
-	storageEngine = strings.ToLower(storageEngine)
-
 	mysqlVersion, isOfficial := util.VersionParser(versionStr)
 	XbcryptBin = GetXbcryptBin(mysqlVersion, isOfficial)
 
-	dumper, err := BuildDumper(cnf, storageEngine) // 会在里面确定备份方式
+	dumper, err := BuildDumper(cnf, db) // 会在里面确定备份方式
 	if err != nil {
 		return nil, err
 	}

dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/mysqlconn/conn.go

@@ -122,14 +122,15 @@ func GetBinlogFormat(dbh *sql.DB) (string, string) {
 	return binlogFormat, binlogRowImage
 }
 
-// GetStorageEngine Get the storage engine from mysql server
+// GetStorageEngine Get the storage engine from mysql server, to lower
 func GetStorageEngine(dbh *sql.DB) (string, error) {
 	version, err := MysqlSingleColumnQuery("select @@default_storage_engine", dbh)
 	if err != nil {
 		logger.Log.Error("can't get default storage_engine, error :", err)
 		return "", err
 	}
-	return version[0], nil
+
+	return strings.ToLower(version[0]), nil
 }
 
 // GetDataDir get datadir from mysql server

dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/precheck/check_backup_type.go

@@ -0,0 +1,39 @@
+package precheck
+
+import (
+	"dbm-services/mysql/db-tools/mysql-dbbackup/pkg/config"
+	"dbm-services/mysql/db-tools/mysql-dbbackup/pkg/cst"
+	"dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/logger"
+	"dbm-services/mysql/db-tools/mysql-dbbackup/pkg/util"
+)
+
+// CheckBackupType check and fix backup type
+func CheckBackupType(cnf *config.BackupConfig, storageEngine string) error {
+	backupSize, err := util.CalServerDataSize(cnf.Public.MysqlPort)
+	if err != nil {
+		return err
+	}
+	if cnf.Public.BackupType == cst.BackupTypeAuto {
+		if storageEngine == cst.StorageEngineTokudb || storageEngine == cst.StorageEngineRocksdb {
+			logger.Log.Infof("BackupType auto with engine=%s, use physical", storageEngine)
+			cnf.Public.BackupType = cst.BackupPhysical
+			return nil
+		}
+		// report 时需要用真实的 backup type
+		if backupSize > cst.BackupTypeAutoDataSizeGB*1024*1024*1024 {
+			logger.Log.Infof("data size %d for port %d is larger than %d GB, use physical",
+				backupSize, cnf.Public.MysqlPort, cst.BackupTypeAutoDataSizeGB)
+			cnf.Public.BackupType = cst.BackupPhysical
+		} else {
+			cnf.Public.BackupType = cst.BackupLogical
+		}
+		if glibcVer, err := util.GetGlibcVersion(); err != nil {
+			logger.Log.Warn("failed to glibc version, err:", err)
+		} else if glibcVer < "2.14" {
+			// mydumper need glibc version >= 2.14
+			logger.Log.Infof("BackupType auto with glibc version %s < 2.14, use physical", glibcVer)
+			cnf.Public.BackupType = cst.BackupPhysical
+		}
+	}
+	return nil
+}

dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/precheck/check_charset.go

@@ -7,19 +7,18 @@ import (
 	"github.com/pkg/errors"
 
 	"dbm-services/mysql/db-tools/mysql-dbbackup/pkg/config"
-	"dbm-services/mysql/db-tools/mysql-dbbackup/pkg/cst"
 	"dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/common"
 	"dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/logger"
 	"dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/mysqlconn"
 	"dbm-services/mysql/db-tools/mysql-dbbackup/pkg/util"
 )
 
-// CheckCharset Check Mysql server charset
-func CheckCharset(cnf *config.Public, dbh *sql.DB) error {
-	if strings.ToLower(cnf.BackupType) != cst.BackupLogical {
-		return nil
-	}
-	confCharset := cnf.MysqlCharset
+// CheckCharset Check and fix mysql server charset
+func CheckCharset(cnf *config.BackupConfig, dbh *sql.DB) error {
+	//if strings.ToLower(cnf.Public.BackupType) != cst.BackupLogical {
+	//	return nil
+	//}
+	confCharset := cnf.Public.MysqlCharset
 	var superCharset string
 
 	version, verErr := mysqlconn.GetMysqlVersion(dbh)
@@ -32,17 +31,18 @@ func CheckCharset(cnf *config.Public, dbh *sql.DB) error {
 	} else {
 		superCharset = "utf8mb4"
 	}
+
 	if confCharset == "auto" || confCharset == "" {
 		// 如果 cnf.MysqlCharset 为空，则自动读取 character_set_server
 		serverCharset, err := mysqlconn.MysqlSingleColumnQuery("select @@character_set_server", dbh)
 		if err != nil {
 			logger.Log.Error("can't select mysql server charset , error :", err)
-			return errors.WithMessagef(err, "failed to get character_set_server from %d", cnf.MysqlPort)
+			return errors.WithMessagef(err, "failed to get character_set_server from %d", cnf.Public.MysqlPort)
 		}
-		cnf.MysqlCharset = serverCharset[0]
+		cnf.Public.MysqlCharset = serverCharset[0]
 		return nil
 	}
-	if confCharset != "binary" && confCharset != superCharset && strings.ToUpper(cnf.DataSchemaGrant) == "ALL" {
+	if confCharset != "binary" && confCharset != superCharset && strings.ToUpper(cnf.Public.DataSchemaGrant) == "ALL" {
 		var goodCharset = []string{"latin1", "utf8", "utf8mb4"}
 
 		serverCharset, err := mysqlconn.GetMysqlCharset(dbh)
@@ -60,26 +60,26 @@ func CheckCharset(cnf *config.Public, dbh *sql.DB) error {
 
 		if err != nil {
 			logger.Log.Warn("get_server_data_charsets query failed,use super charset")
-			cnf.MysqlCharset = superCharset
+			cnf.Public.MysqlCharset = superCharset
 		} else if len(serverCharset) > 1 {
 			logger.Log.Warn("found multi character sets on server ")
-			cnf.MysqlCharset = superCharset
+			cnf.Public.MysqlCharset = superCharset
 		} else if len(serverCharset) == 1 {
-			cnf.MysqlCharset = serverCharset[0]
+			cnf.Public.MysqlCharset = serverCharset[0]
 			if serverCharset[0] != confCharset {
 				logger.Log.Warn("backup config charset:'%s' and server charset '%s' are not the same."+
 					" You should use %s to backup,please modify config charset to remove this warning",
 					confCharset, serverCharset[0], serverCharset[0])
 			}
 		} else {
-			tableNum := common.GetTableNum(cnf.MysqlPort) // todo
+			tableNum := common.GetTableNum(cnf.Public.MysqlPort) // todo
 			if tableNum > 1000 {
-				cnf.MysqlCharset = superCharset
+				cnf.Public.MysqlCharset = superCharset
 				logger.Log.Warn("too much table, tableNum is %d,check server charset failed,"+
 					"use super charset:%s to backup.", tableNum, superCharset)
 			}
 		}
 	}
-	logger.Log.Info("use character set:", cnf.MysqlCharset, "  to backup")
+	logger.Log.Info("use character set:", cnf.Public.MysqlCharset, "  to backup")
 	return nil
 }

dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/precheck/precheck.go

@@ -10,7 +10,6 @@ import (
 	"dbm-services/mysql/db-tools/mysql-dbbackup/pkg/cst"
 	"dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/logger"
 	"dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/mysqlconn"
-	"dbm-services/mysql/db-tools/mysql-dbbackup/pkg/util"
 )
 
 // BeforeDump precheck before dumping backup
@@ -27,11 +26,13 @@ func BeforeDump(cnf *config.BackupConfig) error {
 	defer func() {
 		_ = dbh.Close()
 	}()
-	/*
-		if err := CheckBackupType(cnf); err != nil {
-			return err
-		}
-	*/
+	storageEngine, err := mysqlconn.GetStorageEngine(dbh)
+	if err != nil {
+		return err
+	}
+	if err := CheckBackupType(cnf, storageEngine); err != nil {
+		return err
+	}
 	cnfPublic := &cnf.Public
 
 	/*
@@ -40,8 +41,8 @@ func BeforeDump(cnf *config.BackupConfig) error {
 			return err
 		}
 	*/
-	// check server charset
-	if err := CheckCharset(cnfPublic, dbh); err != nil {
+	// check server charset, need correct charset
+	if err := CheckCharset(cnf, dbh); err != nil {
 		logger.Log.Errorf("failed to get Mysqlcharset for %d", cnfPublic.MysqlPort)
 		return err
 	}
@@ -62,37 +63,6 @@ func BeforeDump(cnf *config.BackupConfig) error {
 	return nil
 }
 
-// CheckBackupType check and fix backup type
-func CheckBackupType(cnf *config.BackupConfig, engine string) error {
-	backupSize, err := util.CalServerDataSize(cnf.Public.MysqlPort)
-	if err != nil {
-		return err
-	}
-	if cnf.Public.BackupType == cst.BackupTypeAuto {
-		if engine == cst.StorageEngineTokudb || engine == cst.StorageEngineRocksdb {
-			logger.Log.Infof("BackupType auto with engine=%s, use physical", engine)
-			cnf.Public.BackupType = cst.BackupPhysical
-			return nil
-		}
-		// report 时需要用真实的 backup type
-		if backupSize > cst.BackupTypeAutoDataSizeGB*1024*1024*1024 {
-			logger.Log.Infof("data size %d for port %d is larger than %d GB, use physical",
-				backupSize, cnf.Public.MysqlPort, cst.BackupTypeAutoDataSizeGB)
-			cnf.Public.BackupType = cst.BackupPhysical
-		} else {
-			cnf.Public.BackupType = cst.BackupLogical
-		}
-		if glibcVer, err := util.GetGlibcVersion(); err != nil {
-			logger.Log.Warn("failed to glibc version, err:", err)
-		} else if glibcVer < "2.14" {
-			// mydumper need glibc version >= 2.14
-			logger.Log.Infof("BackupType auto with glibc version %s < 2.14, use physical", glibcVer)
-			cnf.Public.BackupType = cst.BackupPhysical
-		}
-	}
-	return nil
-}
-
 // CheckEngineTables 只有在 master 上进行物理备份数据时，才执行检查
 func CheckEngineTables(cnf *config.BackupConfig, db *sql.DB) error {
 	if !(cnf.Public.BackupType == cst.BackupPhysical &&

dbm-ui/backend/configuration/constants.py

@@ -99,6 +99,7 @@ class SystemSettingsEnum(str, StructuredEnum):
     BKM_DBM_TOKEN = EnumField("BKM_DBM_TOKEN", _("监控数据源token"))
     BKM_DBM_REPORT = EnumField("BKM_DBM_REPORT", _("mysql/redis-监控自定义上报: dataid/token"))
     FREE_BK_MODULE_ID = EnumField("FREE_BK_MODULE_ID", _("业务空闲模块ID"))
+    VIRTUAL_USERS = EnumField("VIRTUAL_USERS", _("平台调用的虚拟账号列表"))
     # 主机默认统一转移到 DBM 业务下托管，若业务 ID 属于这个列表，则转移到对应的业务下
     INDEPENDENT_HOSTING_BIZS = EnumField("INDEPENDENT_HOSTING_BIZS", _("独立托管机器的业务列表"))
     BF_WHITELIST_BIZS = EnumField("BF_WHITELIST_BIZS", _("BF业务白名单"))
@@ -213,6 +214,7 @@ class BizSettingsEnum(str, StructuredEnum):
     [SystemSettingsEnum.AFFINITY, "list", [], _("环境的容灾要求")],
     [SystemSettingsEnum.SYSTEM_MSG_TYPE, "list", ["weixin", "mail"], _("系统消息通知方式")],
     [SystemSettingsEnum.PADDING_PROXY_CLUSTER_LIST, "list", [], _("补全proxy的集群域名列表")],
+    [SystemSettingsEnum.VIRTUAL_USERS, "list", [], _("平台调用的虚拟账户列表")],
 ]
 
 # 环境配置项 是否支持DNS解析 pulsar flow used

dbm-ui/backend/db_services/dbpermission/constants.py

@@ -107,6 +107,15 @@ class FormatType(str, StructuredEnum):
     CLUSTER = EnumField("cluster", _("cluster"))
 
 
+class RuleActionType(str, StructuredEnum):
+    """权限操作类型"""
+
+    AUTH = EnumField("auth", _("授权"))
+    CREATE = EnumField("create", _("创建"))
+    CHANGE = EnumField("change", _("修改"))
+    DELETE = EnumField("delete", _("删除"))
+
+
 class AuthorizeExcelHeader(str, StructuredEnum):
     """授权excel的头部信息"""
 
@@ -117,6 +126,14 @@ class AuthorizeExcelHeader(str, StructuredEnum):
     ERROR = EnumField("错误信息/提示信息", _("错误信息/提示信息"))
 
 
+# 授权字段和授权excel头的映射
+AUTHORIZE_KEY__EXCEL_FIELD_MAP = {
+    "user": AuthorizeExcelHeader.USER,
+    "access_dbs": AuthorizeExcelHeader.ACCESS_DBS,
+    "source_ips": AuthorizeExcelHeader.SOURCE_IPS,
+    "target_instances": AuthorizeExcelHeader.TARGET_INSTANCES,
+}
+
 # 授权数据过期时间
 AUTHORIZE_DATA_EXPIRE_TIME = 60 * 60 * 6
 
@@ -126,5 +143,4 @@ class AuthorizeExcelHeader(str, StructuredEnum):
 # 账号名称最大长度
 MAX_ACCOUNT_LENGTH = 31
 
-
 DPRIV_PARAMETER_MAP = {"account_type": "cluster_type", "rule_ids": "ids", "privilege": "privs", "access_db": "dbname"}