From cb635c8214933575d88f3525d6b4b3ea34d991d9 Mon Sep 17 00:00:00 2001 From: yuanruji Date: Thu, 21 Nov 2024 16:05:29 +0800 Subject: [PATCH] =?UTF-8?q?feat(dbm-services):=20=E5=8C=BA=E5=88=86?= =?UTF-8?q?=E6=95=B0=E6=8D=AE=E5=BA=93=E5=AD=98=E5=82=A8=E7=9A=84=E8=AF=AD?= =?UTF-8?q?=E6=B3=95=E8=A7=84=E5=88=99=E7=9A=84db=E7=B1=BB=E5=9E=8B=20#809?= =?UTF-8?q?4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../common/db-resource/assets/assets.go | 3 +- .../db-resource/internal/model/model.go | 2 +- .../mysql/db-simulation/app/syntax/rule.go | 7 +- .../db-simulation/app/syntax/spider_rule.go | 42 ++- .../mysql/db-simulation/assets/assets.go | 72 +++++ .../migrations/000001_init_sql.down.sql | 0 .../assets/migrations/000001_init_sql.up.sql | 82 ++++++ ...000002_add_db_type_for_rule_table.down.sql | 0 .../000002_add_db_type_for_rule_table.up.sql | 6 + .../migrations/000003_init_rule.down.sql | 0 .../assets/migrations/000003_init_rule.up.sql | 183 ++++++++++++ .../db-simulation/handler/syntax_check.go | 6 + .../db-simulation/handler/syntax_rule.go | 13 + .../mysql/db-simulation/model/model.go | 20 +- .../db-simulation/model/tb_syntax_rule.go | 277 +++++++++--------- .../mysql/db-simulation/router/router.go | 1 + 16 files changed, 557 insertions(+), 157 deletions(-) create mode 100644 dbm-services/mysql/db-simulation/assets/assets.go create mode 100644 dbm-services/mysql/db-simulation/assets/migrations/000001_init_sql.down.sql create mode 100644 dbm-services/mysql/db-simulation/assets/migrations/000001_init_sql.up.sql create mode 100644 dbm-services/mysql/db-simulation/assets/migrations/000002_add_db_type_for_rule_table.down.sql create mode 100644 dbm-services/mysql/db-simulation/assets/migrations/000002_add_db_type_for_rule_table.up.sql create mode 100644 dbm-services/mysql/db-simulation/assets/migrations/000003_init_rule.down.sql create mode 100644 dbm-services/mysql/db-simulation/assets/migrations/000003_init_rule.up.sql diff --git a/dbm-services/common/db-resource/assets/assets.go b/dbm-services/common/db-resource/assets/assets.go index 089862d1db..56a6c34f46 100644 --- a/dbm-services/common/db-resource/assets/assets.go +++ b/dbm-services/common/db-resource/assets/assets.go @@ -8,6 +8,7 @@ * specific language governing permissions and limitations under the License. */ +// Package assets TODO package assets import ( @@ -29,7 +30,7 @@ import ( var fs embed.FS // DoMigrateFromEmbed do migrate from embed -func DoMigrateFromEmbed(user, addr, password, dbname string, port int) (err error) { +func DoMigrateFromEmbed(user, addr, password, dbname string) (err error) { var mig *migrate.Migrate var d source.Driver if d, err = iofs.New(fs, "migrations"); err != nil { diff --git a/dbm-services/common/db-resource/internal/model/model.go b/dbm-services/common/db-resource/internal/model/model.go index 3644c40ecd..1f5ffe0e57 100644 --- a/dbm-services/common/db-resource/internal/model/model.go +++ b/dbm-services/common/db-resource/internal/model/model.go @@ -87,7 +87,7 @@ func createSysDb() { if err != nil { log.Fatalf("init create db failed:%s", err.Error()) } - err = assets.DoMigrateFromEmbed(user, addr, pwd, dbname, 3306) + err = assets.DoMigrateFromEmbed(user, addr, pwd, dbname) if err != nil { log.Fatalf("init migrate from embed failed:%s", err.Error()) } diff --git a/dbm-services/mysql/db-simulation/app/syntax/rule.go b/dbm-services/mysql/db-simulation/app/syntax/rule.go index 98ac363a1b..c4fdcf309f 100644 --- a/dbm-services/mysql/db-simulation/app/syntax/rule.go +++ b/dbm-services/mysql/db-simulation/app/syntax/rule.go @@ -22,6 +22,7 @@ import ( "dbm-services/common/go-pubpkg/cmutil" "dbm-services/common/go-pubpkg/logger" + "dbm-services/mysql/db-simulation/app" "dbm-services/mysql/db-simulation/app/config" "dbm-services/mysql/db-simulation/model" ) @@ -65,7 +66,7 @@ func init() { } // 是否从db中加载配置覆盖配置文件 if config.GAppConfig.LoadRuleFromdb { - if err = traverseLoadRule(*R); err != nil { + if err = traverseLoadRule(app.MySQL, *R); err != nil { logger.Error("load rule from database failed %s", err.Error()) } } @@ -201,7 +202,7 @@ type DmlRule struct { DmlNotHasWhere *RuleItem `yaml:"DmlNotHasWhere"` } -func traverseLoadRule(rulepointer interface{}) error { +func traverseLoadRule(dbType string, rulepointer interface{}) error { tv := reflect.TypeOf(rulepointer) v := reflect.ValueOf(rulepointer) var groupname, rulename string @@ -211,7 +212,7 @@ func traverseLoadRule(rulepointer interface{}) error { structField := v.Field(i).Type() for j := 0; j < structField.NumField(); j++ { rulename = structField.Field(j).Name - drule, err := model.GetRuleByName(groupname, rulename) + drule, err := model.GetRuleByName(groupname, dbType, rulename) if err != nil { if err == gorm.ErrRecordNotFound { logger.Warn("not found group:%s,rule:%s rules in databases", groupname, rulename) diff --git a/dbm-services/mysql/db-simulation/app/syntax/spider_rule.go b/dbm-services/mysql/db-simulation/app/syntax/spider_rule.go index 6671202e68..12b51caa00 100644 --- a/dbm-services/mysql/db-simulation/app/syntax/spider_rule.go +++ b/dbm-services/mysql/db-simulation/app/syntax/spider_rule.go @@ -17,6 +17,7 @@ import ( "dbm-services/common/go-pubpkg/cmutil" "dbm-services/common/go-pubpkg/logger" + "dbm-services/mysql/db-simulation/app" "dbm-services/mysql/db-simulation/app/config" ) @@ -64,10 +65,8 @@ func init() { logger.Fatal("yaml Unmarshal failed %s", err.Error()) return } - if config.GAppConfig.LoadRuleFromdb { - if err = traverseLoadRule(*SR); err != nil { - logger.Error("load rule from database failed %s", err.Error()) - } + if err = traverseLoadRule(app.Spider, *SR); err != nil { + logger.Error("load rule from database failed %s", err.Error()) } var initCompiles = []*RuleItem{} initCompiles = append(initCompiles, traverseRule(SR.CommandRule)...) @@ -79,3 +78,38 @@ func init() { } } } + +// ReloadRuleFromDb reload rule from db +func ReloadRuleFromDb() (err error) { + logger.Info("reload mysql rule from db") + if err = traverseLoadRule(app.Spider, *R); err != nil { + logger.Error("load rule from database failed %s", err.Error()) + return err + } + var initCompiles = []*RuleItem{} + initCompiles = append(initCompiles, traverseRule(R.CommandRule)...) + initCompiles = append(initCompiles, traverseRule(R.CreateTableRule)...) + initCompiles = append(initCompiles, traverseRule(R.AlterTableRule)...) + initCompiles = append(initCompiles, traverseRule(R.DmlRule)...) + for _, c := range initCompiles { + if err = c.compile(); err != nil { + logger.Error("compile rule failed %s", err.Error()) + return err + } + } + logger.Info("reload spider rule from db success") + if err = traverseLoadRule(app.Spider, *SR); err != nil { + logger.Error("load rule from database failed %s", err.Error()) + return err + } + initCompiles = []*RuleItem{} + initCompiles = append(initCompiles, traverseRule(SR.CommandRule)...) + initCompiles = append(initCompiles, traverseRule(SR.SpiderCreateTableRule)...) + for _, c := range initCompiles { + if err = c.compile(); err != nil { + logger.Error("compile rule failed %s", err.Error()) + return err + } + } + return nil +} diff --git a/dbm-services/mysql/db-simulation/assets/assets.go b/dbm-services/mysql/db-simulation/assets/assets.go new file mode 100644 index 0000000000..51eea777c7 --- /dev/null +++ b/dbm-services/mysql/db-simulation/assets/assets.go @@ -0,0 +1,72 @@ +/* + * TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. + * Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. + * Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at https://opensource.org/licenses/MIT + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +/* + * TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. + * Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. + * Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at https://opensource.org/licenses/MIT + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +// Package assets TODO +package assets + +import ( + "embed" + "fmt" + + "github.com/golang-migrate/migrate/v4" + _ "github.com/golang-migrate/migrate/v4/database/mysql" // mysql TODO + "github.com/golang-migrate/migrate/v4/source" + "github.com/golang-migrate/migrate/v4/source/iofs" + "github.com/pkg/errors" + + "dbm-services/common/go-pubpkg/logger" +) + +// Migrations embed migrations sqlfile + +//go:embed migrations/*.sql +var fs embed.FS + +// DoMigrateFromEmbed do migrate from embed +func DoMigrateFromEmbed(user, addr, password, dbname string) (err error) { + var mig *migrate.Migrate + var d source.Driver + if d, err = iofs.New(fs, "migrations"); err != nil { + return err + } + dbURL := fmt.Sprintf( + "mysql://%s:%s@tcp(%s)/%s?charset=%s&parseTime=true&loc=Local&multiStatements=true&interpolateParams=true", + user, + password, + addr, + dbname, + "utf8", + ) + mig, err = migrate.NewWithSourceInstance("iofs", d, dbURL) + if err != nil { + return errors.WithMessage(err, "migrate from embed") + } + defer mig.Close() + if err = mig.Up(); err != nil { + if err == migrate.ErrNoChange { + logger.Info("migrate source from embed success with", "msg", err.Error()) + return nil + } + logger.Error("migrate source from embed failed", err) + return err + } + logger.Info("migrate source from embed success") + return nil +} diff --git a/dbm-services/mysql/db-simulation/assets/migrations/000001_init_sql.down.sql b/dbm-services/mysql/db-simulation/assets/migrations/000001_init_sql.down.sql new file mode 100644 index 0000000000..e69de29bb2 diff --git a/dbm-services/mysql/db-simulation/assets/migrations/000001_init_sql.up.sql b/dbm-services/mysql/db-simulation/assets/migrations/000001_init_sql.up.sql new file mode 100644 index 0000000000..8d65394185 --- /dev/null +++ b/dbm-services/mysql/db-simulation/assets/migrations/000001_init_sql.up.sql @@ -0,0 +1,82 @@ +CREATE TABLE IF NOT EXISTS `tb_container_records` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `uid` varchar(255) NOT NULL, + `container` varchar(255) NOT NULL, + `create_pod_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + `pod_ready_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + `update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (`id`) +) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4; +CREATE TABLE IF NOT EXISTS `tb_request_records` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `request_id` varchar(64) NOT NULL, + `request_body` json DEFAULT NULL, + `update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + `method` varchar(16) NOT NULL, + `user` varchar(32) NOT NULL, + `path` varchar(32) NOT NULL, + `source_ip` varchar(32) NOT NULL, + `response_code` int(11) NOT NULL, + PRIMARY KEY (`id`), + UNIQUE KEY `request_id` (`request_id`) +) ENGINE = InnoDB AUTO_INCREMENT = 77006 DEFAULT CHARSET = utf8mb4; +CREATE TABLE IF NOT EXISTS `tb_simulation_img_cfgs` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `component_type` varchar(64) NOT NULL, + `version` varchar(64) NOT NULL, + `image` varchar(128) NOT NULL, + `update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (`id`), + UNIQUE KEY `component_type` (`component_type`), + UNIQUE KEY `version` (`version`) +) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4; +CREATE TABLE IF NOT EXISTS `tb_simulation_tasks` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `task_id` varchar(256) NOT NULL, + `request_id` varchar(64) NOT NULL, + `phase` varchar(16) NOT NULL, + `status` varchar(16) NOT NULL, + `stdout` mediumtext, + `stderr` mediumtext, + `sys_err_msg` text, + `extra` varchar(512) NOT NULL, + `heartbeat_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + `update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + `bill_task_id` varchar(128) NOT NULL, + `mysql_version` varchar(64) NOT NULL, + PRIMARY KEY (`id`), + UNIQUE KEY `task_id` (`task_id`), + UNIQUE KEY `request_id` (`request_id`) +) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4; +CREATE TABLE IF NOT EXISTS `tb_sql_file_simulation_infos` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `task_id` varchar(128) NOT NULL, + `file_name_hash` varchar(65) DEFAULT NULL, + `file_name` text NOT NULL, + `status` varchar(16) NOT NULL, + `err_msg` varchar(512) NOT NULL, + `update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + `bill_task_id` varchar(128) NOT NULL, + `line_id` int(11) NOT NULL, + `mysql_version` varchar(64) NOT NULL, + PRIMARY KEY (`id`), + UNIQUE KEY `uk_tk_file` (`task_id`, `file_name_hash`) +) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4; +CREATE TABLE IF NOT EXISTS `tb_syntax_rules` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `group_name` varchar(64) NOT NULL, + `rule_name` varchar(64) NOT NULL, + `item` varchar(1024) NOT NULL, + `item_type` varchar(128) NOT NULL, + `expr` varchar(128) NOT NULL, + `desc` varchar(512) NOT NULL, + `warn_level` smallint(2) NOT NULL, + `status` tinyint(1) NOT NULL, + PRIMARY KEY (`id`), + UNIQUE KEY `group` (`group_name`, `rule_name`) +) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4; \ No newline at end of file diff --git a/dbm-services/mysql/db-simulation/assets/migrations/000002_add_db_type_for_rule_table.down.sql b/dbm-services/mysql/db-simulation/assets/migrations/000002_add_db_type_for_rule_table.down.sql new file mode 100644 index 0000000000..e69de29bb2 diff --git a/dbm-services/mysql/db-simulation/assets/migrations/000002_add_db_type_for_rule_table.up.sql b/dbm-services/mysql/db-simulation/assets/migrations/000002_add_db_type_for_rule_table.up.sql new file mode 100644 index 0000000000..adacd61dae --- /dev/null +++ b/dbm-services/mysql/db-simulation/assets/migrations/000002_add_db_type_for_rule_table.up.sql @@ -0,0 +1,6 @@ +ALTER TABLE tb_syntax_rules +ADD COLUMN `db_type` varchar(64) NOT NULL DEFAULT '' +AFTER `id`; +ALTER TABLE tb_syntax_rules DROP INDEX `group`; +ALTER TABLE tb_syntax_rules +ADD UNIQUE INDEX `group`(`group_name`, `db_type`, `rule_name`); \ No newline at end of file diff --git a/dbm-services/mysql/db-simulation/assets/migrations/000003_init_rule.down.sql b/dbm-services/mysql/db-simulation/assets/migrations/000003_init_rule.down.sql new file mode 100644 index 0000000000..e69de29bb2 diff --git a/dbm-services/mysql/db-simulation/assets/migrations/000003_init_rule.up.sql b/dbm-services/mysql/db-simulation/assets/migrations/000003_init_rule.up.sql new file mode 100644 index 0000000000..29c55c0f92 --- /dev/null +++ b/dbm-services/mysql/db-simulation/assets/migrations/000003_init_rule.up.sql @@ -0,0 +1,183 @@ +delete from tb_syntax_rules; +INSERT INTO `tb_syntax_rules` +VALUES ( + 1, + 'mysql', + 'CommandRule', + 'HighRiskCommandRule', + '[\"drop_table\", \"drop_index\", \"lock_tables\", \"drop_db\", \"analyze\",\"rename_table\",\"drop_procedure\", \"drop_view\",\"drop_trigger\",\"drop_function\", \"drop_server\",\"drop_event\", \"drop_compression_dictionary\",\"optimize\", \"alter_tablespace\"]', + 'arry', + 'Val in Item', + '高危命令', + 0, + 0 + ); +INSERT INTO `tb_syntax_rules` +VALUES ( + 2, + 'mysql', + 'CommandRule', + 'BanCommandRule', + '[\"truncate\", \"revoke\", \"kill\", \"reset\", \"drop_user\", \"grant\",\"create_user\", \"revoke_all\", \"shutdown\", \"lock_tables_for_backup\",\"reset\", \"purge\", \"lock_binlog_for_backup\",\"lock_tables_for_backup\",\"install_plugin\", \"uninstall_plugin\",\"alter_user\"]', + 'arry', + 'Val in Item', + '高危变更类型', + 1, + 0 + ); +INSERT INTO `tb_syntax_rules` +VALUES ( + 3, + 'mysql', + 'CreateTableRule', + 'SuggestBlobColumCount', + '10', + 'int', + 'Val >= Item ', + '建议单表Blob字段不要过多', + 0, + 0 + ); +INSERT INTO `tb_syntax_rules` +VALUES ( + 4, + 'mysql', + 'CreateTableRule', + 'SuggestEngine', + '\"innodb\"', + 'string', + 'not (Val contains Item) and ( len(Val) != 0 )', + '建议使用Innodb表', + 0, + 0 + ); +INSERT INTO `tb_syntax_rules` +VALUES ( + 5, + 'mysql', + 'CreateTableRule', + 'NeedPrimaryKey', + '1', + 'int', + 'Val == Item', + '建议包含主键', + 0, + 0 + ); +INSERT INTO `tb_syntax_rules` +VALUES ( + 6, + 'mysql', + 'CreateTableRule', + 'DefinerRule', + '[\"ADMIN@localhost\"]', + 'arry', + 'Val not in Item ', + '必须指定definer', + 0, + 0 + ); +INSERT INTO `tb_syntax_rules` +VALUES ( + 7, + 'mysql', + 'CreateTableRule', + 'NormalizedName', + '[\"first_char_exception\", \"special_char\", \"Keyword_exception\"]', + 'arry', + 'Val in Item ', + '规范化命名', + 0, + 0 + ); +INSERT INTO `tb_syntax_rules` +VALUES ( + 8, + 'mysql', + 'AlterTableRule', + 'HighRiskType', + '[\"drop_column\"]', + 'arry', + 'Val in Item', + '高危变更类型', + 0, + 0 + ); +INSERT INTO `tb_syntax_rules` +VALUES ( + 9, + 'mysql', + 'AlterTableRule', + 'HighRiskPkAlterType', + '[\"add_column\", \"add_key\", \"change_column\"]', + 'arry', + 'Val in Item', + '主键高危变更类型', + 0, + 0 + ); +INSERT INTO `tb_syntax_rules` +VALUES ( + 10, + 'mysql', + 'AlterTableRule', + 'AlterUseAfter', + '\"\"', + 'string', + 'Val != Item', + '变更表时使用了after', + 0, + 0 + ); +INSERT INTO `tb_syntax_rules` +VALUES ( + 11, + 'mysql', + 'AlterTableRule', + 'AddColumnMixed', + '\"add_column\"', + 'string', + '( Item in Val ) && ( len(Val) > 1 )', + '加字段和其它alter table 类型混用，可能导致非在线加字段', + 0, + 0 + ); +INSERT INTO `tb_syntax_rules` +VALUES ( + 12, + 'mysql', + 'DmlRule', + 'DmlNotHasWhere', + 'true', + 'bool', + ' Val != Item ', + '没有使用WHERE或者LIMIT,可能会导致全表数据更改', + 0, + 0 + ); +INSERT INTO `tb_syntax_rules` +VALUES ( + 13, + 'spider', + 'CommandRule', + 'HighRiskCommandRule', + '[\"truncate\",\"drop_table\", \"drop_index\", \"lock_tables\", \"drop_db\", \"analyze\",\"rename_table\",\"drop_procedure\", \"drop_view\",\"drop_trigger\",\"drop_function\", \"drop_server\",\"drop_event\", \"drop_compression_dictionary\",\"optimize\", \"alter_tablespace\"]', + 'arry', + 'Val in Item', + '高危命令', + 0, + 1 + ); +INSERT INTO `tb_syntax_rules` +VALUES ( + 14, + 'spider', + 'CommandRule', + 'BanCommandRule', + '[\"revoke\", \"kill\", \"reset\", \"drop_user\", \"grant\",\"create_user\", \"revoke_all\", \"shutdown\", \"lock_tables_for_backup\",\"reset\", \"purge\", \"lock_binlog_for_backup\",\"lock_tables_for_backup\",\"install_plugin\", \"uninstall_plugin\",\"alter_user\",\"slave_start\",\"slave_stop\",\"change_master\",\"start_group_replication\",\"stop_group_replication\",\"change_replication_filter\"]', + 'arry', + 'Val in Item', + '禁止的变更类型', + 1, + 1 + ); \ No newline at end of file diff --git a/dbm-services/mysql/db-simulation/handler/syntax_check.go b/dbm-services/mysql/db-simulation/handler/syntax_check.go index 2060aadb46..ebe129aafa 100644 --- a/dbm-services/mysql/db-simulation/handler/syntax_check.go +++ b/dbm-services/mysql/db-simulation/handler/syntax_check.go @@ -14,6 +14,7 @@ import ( "os" "path" "strings" + "time" "github.com/gin-gonic/gin" "github.com/spf13/viper" @@ -77,6 +78,11 @@ func SyntaxCheckSQL(r *gin.Context) { sqlContext := strings.Join(param.Sqls, "\n") fileName := "ce_" + cmutil.RandStr(10) + ".sql" + workdir = path.Join(workdir, time.Now().Format("20060102150405")) + if err := os.MkdirAll(workdir, 0755); err != nil { + SendResponse(r, err, err.Error(), requestID) + return + } f := path.Join(workdir, fileName) err := os.WriteFile(f, []byte(sqlContext), 0600) if err != nil { diff --git a/dbm-services/mysql/db-simulation/handler/syntax_rule.go b/dbm-services/mysql/db-simulation/handler/syntax_rule.go index 10b963c64d..d505af342d 100644 --- a/dbm-services/mysql/db-simulation/handler/syntax_rule.go +++ b/dbm-services/mysql/db-simulation/handler/syntax_rule.go @@ -14,6 +14,8 @@ import ( "errors" "fmt" + "dbm-services/mysql/db-simulation/app/syntax" + "github.com/gin-gonic/gin" "dbm-services/common/go-pubpkg/logger" @@ -129,3 +131,14 @@ func errReturn(r *gin.Context, tsr *model.TbSyntaxRule) { logger.Error("Item type error: %s", err) SendResponse(r, err, nil, "") } + +// ReloadRule trigger reload rule +func ReloadRule(c *gin.Context) { + err := syntax.ReloadRuleFromDb() + if err != nil { + logger.Error("reload rule from db failed %s", err.Error()) + SendResponse(c, err, nil, "") + return + } + SendResponse(c, nil, "ok", "") +} diff --git a/dbm-services/mysql/db-simulation/model/model.go b/dbm-services/mysql/db-simulation/model/model.go index f85cf5195e..42e66597bb 100644 --- a/dbm-services/mysql/db-simulation/model/model.go +++ b/dbm-services/mysql/db-simulation/model/model.go @@ -23,6 +23,7 @@ import ( "gorm.io/gorm/logger" "dbm-services/mysql/db-simulation/app/config" + "dbm-services/mysql/db-simulation/assets" ) // DB TODO @@ -32,10 +33,10 @@ func init() { user := config.GAppConfig.DbConf.User pwd := config.GAppConfig.DbConf.Pwd addr := fmt.Sprintf("%s:%d", config.GAppConfig.DbConf.Host, config.GAppConfig.DbConf.Port) - db := config.GAppConfig.DbConf.Name + dbName := config.GAppConfig.DbConf.Name log.Printf("connect to %s", addr) testConn := openDB(user, pwd, addr, "") - err := testConn.Exec(fmt.Sprintf("create database IF NOT EXISTS `%s`;", db)).Error + err := testConn.Exec(fmt.Sprintf("create database IF NOT EXISTS `%s`;", dbName)).Error if err != nil { log.Fatalf("init create db failed:%s", err.Error()) } @@ -43,16 +44,13 @@ func init() { if err != nil { log.Fatalf("init create db failed:%s", err.Error()) } + log.Println("make dbeug dbname", dbName) + err = assets.DoMigrateFromEmbed(user, addr, pwd, dbName) + if err != nil { + log.Fatalf("init migrate from embed failed:%s", err.Error()) + } sqldb.Close() - DB = openDB(user, pwd, addr, db) - Migration() -} - -// Migration TODO -func Migration() { - //nolint - DB.AutoMigrate(&TbSimulationTask{}, &TbRequestRecord{}, &TbSyntaxRule{}, &TbContainerRecord{}, - &TbSqlFileSimulationInfo{}, &TbSimulationImgCfg{}) + DB = openDB(user, pwd, addr, dbName) } func openDB(username, password, addr, name string) *gorm.DB { diff --git a/dbm-services/mysql/db-simulation/model/tb_syntax_rule.go b/dbm-services/mysql/db-simulation/model/tb_syntax_rule.go index ab067ead95..7f4c3f0ae0 100644 --- a/dbm-services/mysql/db-simulation/model/tb_syntax_rule.go +++ b/dbm-services/mysql/db-simulation/model/tb_syntax_rule.go @@ -33,6 +33,8 @@ const ( // TbSyntaxRule [...] type TbSyntaxRule struct { ID int `gorm:"primaryKey;column:id;type:int(11);not null" json:"-"` + // 数据库类型 + DbType string `gorm:"uniqueIndex:group;column:db_type;type:varchar(32);not null" json:"db_type"` // 规则组名称 GroupName string `gorm:"uniqueIndex:group;column:group_name;type:varchar(64);not null" json:"group_name"` // 子规则项，一个规则可能包括过个子规则 @@ -54,143 +56,143 @@ func (obj *TbSyntaxRule) GetTableName() string { return "tb_syntax_rules" } -func init() { - if err := InitRule(); err != nil { - logger.Fatal("init syntax rule failed %s", err.Error()) - return - } -} +// func init() { +// if err := InitRule(); err != nil { +// logger.Fatal("init syntax rule failed %s", err.Error()) +// return +// } +// } // InitRule init rules -func InitRule() (err error) { - initRules := []TbSyntaxRule{} - initRules = append(initRules, TbSyntaxRule{ - GroupName: "CommandRule", - RuleName: "HighRiskCommandRule", - Expr: "Val in Item", - ItemType: ArryItem, - Item: []byte( - `["drop_table", "drop_index", "lock_tables", "drop_db", "analyze","rename_table", - "drop_procedure", "drop_view","drop_trigger","drop_function", "drop_server", - "drop_event", "drop_compression_dictionary","optimize", "alter_tablespace"]`), - Desc: "高危命令", - WarnLevel: 0, - Status: true, - }) - initRules = append(initRules, TbSyntaxRule{ - GroupName: "CommandRule", - RuleName: "BanCommandRule", - Expr: "Val in Item", - ItemType: ArryItem, - Item: []byte(`["truncate", "revoke", "kill", "reset", "drop_user", "grant", - "create_user", "revoke_all", "shutdown", "lock_tables_for_backup", - "reset", "purge", "lock_binlog_for_backup","lock_tables_for_backup", - "install_plugin", "uninstall_plugin","alter_user"]`), - Desc: "高危变更类型", - WarnLevel: 1, - Status: true, - }) - initRules = append(initRules, TbSyntaxRule{ - GroupName: "CreateTableRule", - RuleName: "SuggestBlobColumCount", - Expr: "Val >= Item ", - ItemType: IntItem, - Item: []byte(`10`), - Desc: "建议单表Blob字段不要过多", - WarnLevel: 0, - Status: true, - }) - initRules = append(initRules, TbSyntaxRule{ - GroupName: "CreateTableRule", - RuleName: "SuggestEngine", - Expr: "not (Val contains Item) and ( len(Val) != 0 )", - ItemType: StringItem, - Item: []byte(`"innodb"`), - Desc: "建议使用Innodb表", - WarnLevel: 0, - Status: true, - }) - initRules = append(initRules, TbSyntaxRule{ - GroupName: "CreateTableRule", - RuleName: "NeedPrimaryKey", - Expr: "Val == Item", - ItemType: IntItem, - Item: []byte(`1`), - Desc: "建议包含主键", - WarnLevel: 0, - Status: true, - }) - initRules = append(initRules, TbSyntaxRule{ - GroupName: "CreateTableRule", - RuleName: "DefinerRule", - Expr: "Val not in Item ", - ItemType: ArryItem, - Item: []byte(`["ADMIN@localhost"]`), - Desc: "必须指定definer", - WarnLevel: 0, - Status: true, - }) - - initRules = append(initRules, TbSyntaxRule{ - GroupName: "AlterTableRule", - RuleName: "HighRiskType", - Expr: "Val in Item", - ItemType: ArryItem, - Item: []byte(`["drop_column"]`), - Desc: "高危变更类型", - WarnLevel: 0, - Status: true, - }) - initRules = append(initRules, TbSyntaxRule{ - GroupName: "AlterTableRule", - RuleName: "HighRiskPkAlterType", - Expr: "Val in Item", - ItemType: ArryItem, - Item: []byte(`["add_column", "add_key", "change_column"]`), - Desc: "主键高危变更类型", - WarnLevel: 0, - Status: true, - }) - initRules = append(initRules, TbSyntaxRule{ - GroupName: "AlterTableRule", - RuleName: "AlterUseAfter", - Expr: "Val != Item", - ItemType: StringItem, - Item: []byte(`""`), - Desc: "变更表时使用了after", - WarnLevel: 0, - Status: true, - }) - initRules = append(initRules, TbSyntaxRule{ - GroupName: "AlterTableRule", - RuleName: "AddColumnMixed", - Expr: "( Item in Val ) && ( len(Val) > 1 )", - ItemType: StringItem, - Item: []byte(`"add_column"`), - Desc: "加字段和其它alter table 类型混用，可能导致非在线加字段", - WarnLevel: 0, - Status: true, - }) - - initRules = append(initRules, TbSyntaxRule{ - GroupName: "DmlRule", - RuleName: "DmlNotHasWhere", - Expr: " Val != Item ", - ItemType: BoolItem, - Item: []byte(`true`), - Desc: "没有使用WHERE或者LIMIT,可能会导致全表数据更改", - WarnLevel: 0, - Status: true, - }) - - for i := range initRules { - if err = CreateRule(&initRules[i]); err != nil { - logger.Error("初始化规则失败%s", err.Error()) - return err - } - } - return err -} +// func InitRule() (err error) { +// initRules := []TbSyntaxRule{} +// initRules = append(initRules, TbSyntaxRule{ +// GroupName: "CommandRule", +// RuleName: "HighRiskCommandRule", +// Expr: "Val in Item", +// ItemType: ArryItem, +// Item: []byte( +// `["drop_table", "drop_index", "lock_tables", "drop_db", "analyze","rename_table", +// "drop_procedure", "drop_view","drop_trigger","drop_function", "drop_server", +// "drop_event", "drop_compression_dictionary","optimize", "alter_tablespace"]`), +// Desc: "高危命令", +// WarnLevel: 0, +// Status: true, +// }) +// initRules = append(initRules, TbSyntaxRule{ +// GroupName: "CommandRule", +// RuleName: "BanCommandRule", +// Expr: "Val in Item", +// ItemType: ArryItem, +// Item: []byte(`["truncate", "revoke", "kill", "reset", "drop_user", "grant", +// "create_user", "revoke_all", "shutdown", "lock_tables_for_backup", +// "reset", "purge", "lock_binlog_for_backup","lock_tables_for_backup", +// "install_plugin", "uninstall_plugin","alter_user"]`), +// Desc: "高危变更类型", +// WarnLevel: 1, +// Status: true, +// }) +// initRules = append(initRules, TbSyntaxRule{ +// GroupName: "CreateTableRule", +// RuleName: "SuggestBlobColumCount", +// Expr: "Val >= Item ", +// ItemType: IntItem, +// Item: []byte(`10`), +// Desc: "建议单表Blob字段不要过多", +// WarnLevel: 0, +// Status: true, +// }) +// initRules = append(initRules, TbSyntaxRule{ +// GroupName: "CreateTableRule", +// RuleName: "SuggestEngine", +// Expr: "not (Val contains Item) and ( len(Val) != 0 )", +// ItemType: StringItem, +// Item: []byte(`"innodb"`), +// Desc: "建议使用Innodb表", +// WarnLevel: 0, +// Status: true, +// }) +// initRules = append(initRules, TbSyntaxRule{ +// GroupName: "CreateTableRule", +// RuleName: "NeedPrimaryKey", +// Expr: "Val == Item", +// ItemType: IntItem, +// Item: []byte(`1`), +// Desc: "建议包含主键", +// WarnLevel: 0, +// Status: true, +// }) +// initRules = append(initRules, TbSyntaxRule{ +// GroupName: "CreateTableRule", +// RuleName: "DefinerRule", +// Expr: "Val not in Item ", +// ItemType: ArryItem, +// Item: []byte(`["ADMIN@localhost"]`), +// Desc: "必须指定definer", +// WarnLevel: 0, +// Status: true, +// }) + +// initRules = append(initRules, TbSyntaxRule{ +// GroupName: "AlterTableRule", +// RuleName: "HighRiskType", +// Expr: "Val in Item", +// ItemType: ArryItem, +// Item: []byte(`["drop_column"]`), +// Desc: "高危变更类型", +// WarnLevel: 0, +// Status: true, +// }) +// initRules = append(initRules, TbSyntaxRule{ +// GroupName: "AlterTableRule", +// RuleName: "HighRiskPkAlterType", +// Expr: "Val in Item", +// ItemType: ArryItem, +// Item: []byte(`["add_column", "add_key", "change_column"]`), +// Desc: "主键高危变更类型", +// WarnLevel: 0, +// Status: true, +// }) +// initRules = append(initRules, TbSyntaxRule{ +// GroupName: "AlterTableRule", +// RuleName: "AlterUseAfter", +// Expr: "Val != Item", +// ItemType: StringItem, +// Item: []byte(`""`), +// Desc: "变更表时使用了after", +// WarnLevel: 0, +// Status: true, +// }) +// initRules = append(initRules, TbSyntaxRule{ +// GroupName: "AlterTableRule", +// RuleName: "AddColumnMixed", +// Expr: "( Item in Val ) && ( len(Val) > 1 )", +// ItemType: StringItem, +// Item: []byte(`"add_column"`), +// Desc: "加字段和其它alter table 类型混用，可能导致非在线加字段", +// WarnLevel: 0, +// Status: true, +// }) + +// initRules = append(initRules, TbSyntaxRule{ +// GroupName: "DmlRule", +// RuleName: "DmlNotHasWhere", +// Expr: " Val != Item ", +// ItemType: BoolItem, +// Item: []byte(`true`), +// Desc: "没有使用WHERE或者LIMIT,可能会导致全表数据更改", +// WarnLevel: 0, +// Status: true, +// }) + +// for i := range initRules { +// if err = CreateRule(&initRules[i]); err != nil { +// logger.Error("初始化规则失败%s", err.Error()) +// return err +// } +// } +// return err +// } // CreateRule create rule func CreateRule(m *TbSyntaxRule) (err error) { @@ -206,8 +208,9 @@ func GetAllRule() (rs []TbSyntaxRule, err error) { } // GetRuleByName get rules group by group name -func GetRuleByName(group, rulename string) (rs TbSyntaxRule, err error) { - err = DB.Where("group_name = ? and rule_name = ? ", group, rulename).First(&rs).Error +func GetRuleByName(group, dbtype, rulename string) (rs TbSyntaxRule, err error) { + err = DB.Where("group_name = ? and db_type = ? and rule_name = ? and status = 1", group, dbtype, rulename). + First(&rs).Error return } diff --git a/dbm-services/mysql/db-simulation/router/router.go b/dbm-services/mysql/db-simulation/router/router.go index 4adbaccf28..51d5f3fc6d 100644 --- a/dbm-services/mysql/db-simulation/router/router.go +++ b/dbm-services/mysql/db-simulation/router/router.go @@ -46,6 +46,7 @@ func RegisterRouter(engine *gin.Engine) { r.POST("/manage", handler.ManageRule) r.GET("/getall", handler.GetAllRule) r.POST("/update", handler.UpdateRule) + r.POST("/reload", handler.ReloadRule) // spider sp := engine.Group("/spider") sp.POST("/simulation", handler.TendbClusterSimulation)