feat(dbm-services): 区分数据库存储的语法规则的db类型 #8094

dbm-services/common/db-resource/assets/assets.go

@@ -8,6 +8,7 @@
  * specific language governing permissions and limitations under the License.
  */
 
+// Package assets TODO
 package assets
 
 import (
@@ -29,7 +30,7 @@ import (
 var fs embed.FS
 
 // DoMigrateFromEmbed  do migrate from embed
-func DoMigrateFromEmbed(user, addr, password, dbname string, port int) (err error) {
+func DoMigrateFromEmbed(user, addr, password, dbname string) (err error) {
 	var mig *migrate.Migrate
 	var d source.Driver
 	if d, err = iofs.New(fs, "migrations"); err != nil {

dbm-services/common/db-resource/internal/model/model.go

@@ -87,7 +87,7 @@ func createSysDb() {
 	if err != nil {
 		log.Fatalf("init create db failed:%s", err.Error())
 	}
-	err = assets.DoMigrateFromEmbed(user, addr, pwd, dbname, 3306)
+	err = assets.DoMigrateFromEmbed(user, addr, pwd, dbname)
 	if err != nil {
 		log.Fatalf("init migrate from embed failed:%s", err.Error())
 	}

dbm-services/mysql/db-simulation/app/syntax/rule.go

@@ -22,6 +22,7 @@ import (
 
 	"dbm-services/common/go-pubpkg/cmutil"
 	"dbm-services/common/go-pubpkg/logger"
+	"dbm-services/mysql/db-simulation/app"
 	"dbm-services/mysql/db-simulation/app/config"
 	"dbm-services/mysql/db-simulation/model"
 )
@@ -65,7 +66,7 @@ func init() {
 	}
 	// 是否从db中加载配置覆盖配置文件
 	if config.GAppConfig.LoadRuleFromdb {
-		if err = traverseLoadRule(*R); err != nil {
+		if err = traverseLoadRule(app.MySQL, *R); err != nil {
 			logger.Error("load rule from database failed %s", err.Error())
 		}
 	}
@@ -201,7 +202,7 @@ type DmlRule struct {
 	DmlNotHasWhere *RuleItem `yaml:"DmlNotHasWhere"`
 }
 
-func traverseLoadRule(rulepointer interface{}) error {
+func traverseLoadRule(dbType string, rulepointer interface{}) error {
 	tv := reflect.TypeOf(rulepointer)
 	v := reflect.ValueOf(rulepointer)
 	var groupname, rulename string
@@ -211,7 +212,7 @@ func traverseLoadRule(rulepointer interface{}) error {
 			structField := v.Field(i).Type()
 			for j := 0; j < structField.NumField(); j++ {
 				rulename = structField.Field(j).Name
-				drule, err := model.GetRuleByName(groupname, rulename)
+				drule, err := model.GetRuleByName(groupname, dbType, rulename)
 				if err != nil {
 					if err == gorm.ErrRecordNotFound {
 						logger.Warn("not found group:%s,rule:%s rules in databases", groupname, rulename)

dbm-services/mysql/db-simulation/app/syntax/spider_rule.go

@@ -17,6 +17,7 @@ import (
 
 	"dbm-services/common/go-pubpkg/cmutil"
 	"dbm-services/common/go-pubpkg/logger"
+	"dbm-services/mysql/db-simulation/app"
 	"dbm-services/mysql/db-simulation/app/config"
 )
 
@@ -64,10 +65,8 @@ func init() {
 		logger.Fatal("yaml Unmarshal failed %s", err.Error())
 		return
 	}
-	if config.GAppConfig.LoadRuleFromdb {
-		if err = traverseLoadRule(*SR); err != nil {
-			logger.Error("load rule from database failed %s", err.Error())
-		}
+	if err = traverseLoadRule(app.Spider, *SR); err != nil {
+		logger.Error("load rule from database failed %s", err.Error())
 	}
 	var initCompiles = []*RuleItem{}
 	initCompiles = append(initCompiles, traverseRule(SR.CommandRule)...)
@@ -79,3 +78,38 @@ func init() {
 		}
 	}
 }
+
+// ReloadRuleFromDb reload rule from db
+func ReloadRuleFromDb() (err error) {
+	logger.Info("reload mysql rule from db")
+	if err = traverseLoadRule(app.Spider, *R); err != nil {
+		logger.Error("load rule from database failed %s", err.Error())
+		return err
+	}
+	var initCompiles = []*RuleItem{}
+	initCompiles = append(initCompiles, traverseRule(R.CommandRule)...)
+	initCompiles = append(initCompiles, traverseRule(R.CreateTableRule)...)
+	initCompiles = append(initCompiles, traverseRule(R.AlterTableRule)...)
+	initCompiles = append(initCompiles, traverseRule(R.DmlRule)...)
+	for _, c := range initCompiles {
+		if err = c.compile(); err != nil {
+			logger.Error("compile rule failed %s", err.Error())
+			return err
+		}
+	}
+	logger.Info("reload spider rule from db success")
+	if err = traverseLoadRule(app.Spider, *SR); err != nil {
+		logger.Error("load rule from database failed %s", err.Error())
+		return err
+	}
+	initCompiles = []*RuleItem{}
+	initCompiles = append(initCompiles, traverseRule(SR.CommandRule)...)
+	initCompiles = append(initCompiles, traverseRule(SR.SpiderCreateTableRule)...)
+	for _, c := range initCompiles {
+		if err = c.compile(); err != nil {
+			logger.Error("compile rule failed %s", err.Error())
+			return err
+		}
+	}
+	return nil
+}

dbm-services/mysql/db-simulation/assets/assets.go

@@ -0,0 +1,72 @@
+/*
+ * TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available.
+ * Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved.
+ * Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at https://opensource.org/licenses/MIT
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+
+/*
+ * TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available.
+ * Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved.
+ * Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at https://opensource.org/licenses/MIT
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+
+// Package assets TODO
+package assets
+
+import (
+	"embed"
+	"fmt"
+
+	"github.com/golang-migrate/migrate/v4"
+	_ "github.com/golang-migrate/migrate/v4/database/mysql" // mysql TODO
+	"github.com/golang-migrate/migrate/v4/source"
+	"github.com/golang-migrate/migrate/v4/source/iofs"
+	"github.com/pkg/errors"
+
+	"dbm-services/common/go-pubpkg/logger"
+)
+
+// Migrations embed migrations sqlfile
+
+//go:embed migrations/*.sql
+var fs embed.FS
+
+// DoMigrateFromEmbed  do migrate from embed
+func DoMigrateFromEmbed(user, addr, password, dbname string) (err error) {
+	var mig *migrate.Migrate
+	var d source.Driver
+	if d, err = iofs.New(fs, "migrations"); err != nil {
+		return err
+	}
+	dbURL := fmt.Sprintf(
+		"mysql://%s:%s@tcp(%s)/%s?charset=%s&parseTime=true&loc=Local&multiStatements=true&interpolateParams=true",
+		user,
+		password,
+		addr,
+		dbname,
+		"utf8",
+	)
+	mig, err = migrate.NewWithSourceInstance("iofs", d, dbURL)
+	if err != nil {
+		return errors.WithMessage(err, "migrate from embed")
+	}
+	defer mig.Close()
+	if err = mig.Up(); err != nil {
+		if err == migrate.ErrNoChange {
+			logger.Info("migrate source from embed success with", "msg", err.Error())
+			return nil
+		}
+		logger.Error("migrate source from embed failed", err)
+		return err
+	}
+	logger.Info("migrate source from embed success")
+	return nil
+}

dbm-services/mysql/db-simulation/assets/migrations/000001_init_sql.up.sql

@@ -0,0 +1,82 @@
+CREATE TABLE IF NOT EXISTS `tb_container_records` (
+    `id` int(11) NOT NULL AUTO_INCREMENT,
+    `uid` varchar(255) NOT NULL,
+    `container` varchar(255) NOT NULL,
+    `create_pod_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    `pod_ready_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    `update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    PRIMARY KEY (`id`)
+) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4;
+CREATE TABLE IF NOT EXISTS `tb_request_records` (
+    `id` int(11) NOT NULL AUTO_INCREMENT,
+    `request_id` varchar(64) NOT NULL,
+    `request_body` json DEFAULT NULL,
+    `update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    `method` varchar(16) NOT NULL,
+    `user` varchar(32) NOT NULL,
+    `path` varchar(32) NOT NULL,
+    `source_ip` varchar(32) NOT NULL,
+    `response_code` int(11) NOT NULL,
+    PRIMARY KEY (`id`),
+    UNIQUE KEY `request_id` (`request_id`)
+) ENGINE = InnoDB AUTO_INCREMENT = 77006 DEFAULT CHARSET = utf8mb4;
+CREATE TABLE IF NOT EXISTS `tb_simulation_img_cfgs` (
+    `id` int(11) NOT NULL AUTO_INCREMENT,
+    `component_type` varchar(64) NOT NULL,
+    `version` varchar(64) NOT NULL,
+    `image` varchar(128) NOT NULL,
+    `update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    PRIMARY KEY (`id`),
+    UNIQUE KEY `component_type` (`component_type`),
+    UNIQUE KEY `version` (`version`)
+) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4;
+CREATE TABLE IF NOT EXISTS `tb_simulation_tasks` (
+    `id` int(11) NOT NULL AUTO_INCREMENT,
+    `task_id` varchar(256) NOT NULL,
+    `request_id` varchar(64) NOT NULL,
+    `phase` varchar(16) NOT NULL,
+    `status` varchar(16) NOT NULL,
+    `stdout` mediumtext,
+    `stderr` mediumtext,
+    `sys_err_msg` text,
+    `extra` varchar(512) NOT NULL,
+    `heartbeat_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    `update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    `bill_task_id` varchar(128) NOT NULL,
+    `mysql_version` varchar(64) NOT NULL,
+    PRIMARY KEY (`id`),
+    UNIQUE KEY `task_id` (`task_id`),
+    UNIQUE KEY `request_id` (`request_id`)
+) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4;
+CREATE TABLE IF NOT EXISTS `tb_sql_file_simulation_infos` (
+    `id` int(11) NOT NULL AUTO_INCREMENT,
+    `task_id` varchar(128) NOT NULL,
+    `file_name_hash` varchar(65) DEFAULT NULL,
+    `file_name` text NOT NULL,
+    `status` varchar(16) NOT NULL,
+    `err_msg` varchar(512) NOT NULL,
+    `update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    `bill_task_id` varchar(128) NOT NULL,
+    `line_id` int(11) NOT NULL,
+    `mysql_version` varchar(64) NOT NULL,
+    PRIMARY KEY (`id`),
+    UNIQUE KEY `uk_tk_file` (`task_id`, `file_name_hash`)
+) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4;
+CREATE TABLE IF NOT EXISTS `tb_syntax_rules` (
+    `id` int(11) NOT NULL AUTO_INCREMENT,
+    `group_name` varchar(64) NOT NULL,
+    `rule_name` varchar(64) NOT NULL,
+    `item` varchar(1024) NOT NULL,
+    `item_type` varchar(128) NOT NULL,
+    `expr` varchar(128) NOT NULL,
+    `desc` varchar(512) NOT NULL,
+    `warn_level` smallint(2) NOT NULL,
+    `status` tinyint(1) NOT NULL,
+    PRIMARY KEY (`id`),
+    UNIQUE KEY `group` (`group_name`, `rule_name`)
+) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4;

dbm-services/mysql/db-simulation/assets/migrations/000002_add_db_type_for_rule_table.up.sql

@@ -0,0 +1,6 @@
+ALTER TABLE tb_syntax_rules
+ADD COLUMN `db_type` varchar(64) NOT NULL DEFAULT ''
+AFTER `id`;
+ALTER TABLE tb_syntax_rules DROP INDEX `group`;
+ALTER TABLE tb_syntax_rules
+ADD UNIQUE INDEX `group`(`group_name`, `db_type`, `rule_name`);