From 481f440a0641a62a58fb4e79ba490ba2fedda698 Mon Sep 17 00:00:00 2001 From: xfwduke Date: Thu, 24 Oct 2024 10:20:44 +0800 Subject: [PATCH] =?UTF-8?q?fix(mysql):=20=E7=AB=AF=E5=8F=A33306=E7=9A=84my?= =?UTF-8?q?sql=E9=85=8D=E7=BD=AE=E6=96=87=E4=BB=B6=E8=B7=AF=E5=BE=84?= =?UTF-8?q?=E9=94=99=E8=AF=AF=20#7541?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../pkg/components/truncate/on_mysql.go | 38 +++-- .../db-tools/mysql-monitor/items-config.sql | 65 ++++---- .../db-tools/mysql-monitor/items-config.yaml | 26 +++- .../ibdstatistic/ibd_statistic.go | 8 +- .../itemscollect/ibdstatistic/report_log.go | 116 ++++++++++++++ .../ibdstatistic/report_metrics.go | 6 +- .../pkg/itemscollect/items_collect.go | 5 +- .../master_slave_heartbeat.go | 146 +++++++++++++++++- .../mysqlconfigdiff/mysql_config_diff.go | 2 +- .../pkg/itemscollect/proxyuserlist/backup.go | 1 - .../itemscollect/slavestatus/ctl_replicate.go | 44 ++++-- .../itemscollect/slavestatus/slave_status.go | 12 +- .../spiderctlchecker/getprimarychecker.go | 52 +++++++ .../spiderctlchecker/uniquechecker.go | 57 +++++++ .../itemscollect/uniquectlmaster/checker.go | 64 -------- .../connection_collect.go | 2 +- items-config.sql | 0 17 files changed, 495 insertions(+), 149 deletions(-) create mode 100644 dbm-services/mysql/db-tools/mysql-monitor/pkg/itemscollect/ibdstatistic/report_log.go create mode 100644 dbm-services/mysql/db-tools/mysql-monitor/pkg/itemscollect/spiderctlchecker/getprimarychecker.go create mode 100644 dbm-services/mysql/db-tools/mysql-monitor/pkg/itemscollect/spiderctlchecker/uniquechecker.go delete mode 100644 dbm-services/mysql/db-tools/mysql-monitor/pkg/itemscollect/uniquectlmaster/checker.go create mode 100644 items-config.sql diff --git a/dbm-services/mysql/db-tools/dbactuator/pkg/components/truncate/on_mysql.go b/dbm-services/mysql/db-tools/dbactuator/pkg/components/truncate/on_mysql.go index 55f64f2a72..136572e792 100644 --- a/dbm-services/mysql/db-tools/dbactuator/pkg/components/truncate/on_mysql.go +++ b/dbm-services/mysql/db-tools/dbactuator/pkg/components/truncate/on_mysql.go @@ -309,7 +309,15 @@ func (c *OnMySQLComponent) instanceDropSourceDBs(port int) error { func (c *OnMySQLComponent) instanceRecreateSourceTables(port int) error { for db := range c.dbTablesMap { stageDBName := generateStageDBName(c.Param.StageDBHeader, c.Param.FlowTimeStr, db) - for _, table := range c.dbTablesMap[db] { + stageTables, err := rpkg.ListDBTables(c.dbConn, stageDBName) + if err != nil { + logger.Error("list stage db tables on instance %d from %s failed: %s", port, stageDBName, err.Error()) + return err + } + + // 这里改成从 stage 库拿表列表, 在重试的时候才能幂等 + //for _, table := range c.dbTablesMap[db] { + for _, table := range stageTables { err := c.instanceRecreateSourceTable(port, db, stageDBName, table) if err != nil { logger.Error( @@ -328,25 +336,25 @@ func (c *OnMySQLComponent) instanceRecreateSourceTables(port int) error { } func (c *OnMySQLComponent) instanceRecreateSourceTable(port int, dbName, stageDBName, tableName string) error { - yes, err := rpkg.IsTableExistsIn(c.dbConn, tableName, stageDBName) - if err != nil { - logger.Error("check table %s exists in %s failed: %s", tableName, stageDBName, err.Error()) - return err - } - - if !yes { - err := fmt.Errorf("%s.%s not found", stageDBName, tableName) - logger.Error("re create source table on instance %d failed: ", port, err.Error()) - return err - } - logger.Info("source table found in stage db on instance %d, try to truncate", port) + //yes, err := rpkg.IsTableExistsIn(c.dbConn, tableName, stageDBName) + //if err != nil { + // logger.Error("check table %s exists in %s failed: %s", tableName, stageDBName, err.Error()) + // return err + //} + // + //if !yes { + // err := fmt.Errorf("%s.%s not found", stageDBName, tableName) + // logger.Error("re create source table on instance %d failed: ", port, err.Error()) + // return err + //} + //logger.Info("source table found in stage db on instance %d, try to truncate", port) ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() - _, err = c.dbConn.ExecContext( + _, err := c.dbConn.ExecContext( ctx, fmt.Sprintf( - "CREATE TABLE `%s`.`%s` LIKE `%s`.`%s`", + "CREATE TABLE IF NOT EXISTS `%s`.`%s` LIKE `%s`.`%s`", dbName, tableName, stageDBName, tableName, ), ) diff --git a/dbm-services/mysql/db-tools/mysql-monitor/items-config.sql b/dbm-services/mysql/db-tools/mysql-monitor/items-config.sql index 4fb3c2102e..37ac76fc58 100644 --- a/dbm-services/mysql/db-tools/mysql-monitor/items-config.sql +++ b/dbm-services/mysql/db-tools/mysql-monitor/items-config.sql @@ -1,33 +1,34 @@ DELETE FROM tb_config_name_def WHERE namespace = 'tendb' AND conf_type = 'mysql_monitor' AND conf_file = 'items-config.yaml'; -REPLACE INTO tb_config_name_def( namespace, conf_type, conf_file, conf_name, value_type, value_default, value_allowed, value_type_sub, flag_status, flag_disable, flag_locked, flag_encrypt, need_restart) VALUES( 'tendb', 'mysql_monitor', 'items-config.yaml', 'character-consistency', 'STRING', '{"schedule":"0 0 14 * * 1","name":"character-consistency","machine_type":["single","backend","remote","spider"],"role":[],"enable":true}', '', 'MAP', 1, 0, 0, 0, 1); -REPLACE INTO tb_config_name_def( namespace, conf_type, conf_file, conf_name, value_type, value_default, value_allowed, value_type_sub, flag_status, flag_disable, flag_locked, flag_encrypt, need_restart) VALUES( 'tendb', 'mysql_monitor', 'items-config.yaml', 'routine-definer', 'STRING', '{"schedule":"0 0 15 * * 1","machine_type":["single","backend","remote"],"name":"routine-definer","role":[],"enable":true}', '', 'MAP', 1, 0, 0, 0, 1); -REPLACE INTO tb_config_name_def( namespace, conf_type, conf_file, conf_name, value_type, value_default, value_allowed, value_type_sub, flag_status, flag_disable, flag_locked, flag_encrypt, need_restart) VALUES( 'tendb', 'mysql_monitor', 'items-config.yaml', 'view-definer', 'STRING', '{"enable":true,"role":[],"name":"view-definer","machine_type":["single","backend","remote"],"schedule":"0 0 15 * * 1"}', '', 'MAP', 1, 0, 0, 0, 1); -REPLACE INTO tb_config_name_def( namespace, conf_type, conf_file, conf_name, value_type, value_default, value_allowed, value_type_sub, flag_status, flag_disable, flag_locked, flag_encrypt, need_restart) VALUES( 'tendb', 'mysql_monitor', 'items-config.yaml', 'trigger-definer', 'STRING', '{"schedule":"0 0 15 * * 1","name":"trigger-definer","machine_type":["single","backend","remote"],"role":[],"enable":true}', '', 'MAP', 1, 0, 0, 0, 1); -REPLACE INTO tb_config_name_def( namespace, conf_type, conf_file, conf_name, value_type, value_default, value_allowed, value_type_sub, flag_status, flag_disable, flag_locked, flag_encrypt, need_restart) VALUES( 'tendb', 'mysql_monitor', 'items-config.yaml', 'engine', 'STRING', '{"enable":true,"role":["slave","orphan"],"machine_type":["single","backend","remote"],"name":"engine","schedule":"0 10 1 * * *"}', '', 'MAP', 1, 0, 0, 0, 1); -REPLACE INTO tb_config_name_def( namespace, conf_type, conf_file, conf_name, value_type, value_default, value_allowed, value_type_sub, flag_status, flag_disable, flag_locked, flag_encrypt, need_restart) VALUES( 'tendb', 'mysql_monitor', 'items-config.yaml', 'ext3-check', 'STRING', '{"schedule":"0 0 16 * * 1","machine_type":["single","backend","remote"],"name":"ext3-check","role":[],"enable":true}', '', 'MAP', 1, 0, 0, 0, 1); -REPLACE INTO tb_config_name_def( namespace, conf_type, conf_file, conf_name, value_type, value_default, value_allowed, value_type_sub, flag_status, flag_disable, flag_locked, flag_encrypt, need_restart) VALUES( 'tendb', 'mysql_monitor', 'items-config.yaml', 'ibd-statistic', 'STRING', '{"enable":true,"role":["slave","orphan"],"name":"ibd-statistic","machine_type":["single","backend","remote"],"schedule":"0 0 14 * * 1"}', '', 'MAP', 1, 0, 0, 0, 1); -REPLACE INTO tb_config_name_def( namespace, conf_type, conf_file, conf_name, value_type, value_default, value_allowed, value_type_sub, flag_status, flag_disable, flag_locked, flag_encrypt, need_restart) VALUES( 'tendb', 'mysql_monitor', 'items-config.yaml', 'master-slave-heartbeat', 'STRING', '{"machine_type":["backend","remote"],"name":"master-slave-heartbeat","schedule":"@every 1m","enable":true,"role":["master","repeater"]}', '', 'MAP', 1, 0, 0, 0, 1); -REPLACE INTO tb_config_name_def( namespace, conf_type, conf_file, conf_name, value_type, value_default, value_allowed, value_type_sub, flag_status, flag_disable, flag_locked, flag_encrypt, need_restart) VALUES( 'tendb', 'mysql_monitor', 'items-config.yaml', 'mysql-config-diff', 'STRING', '{"name":"mysql-config-diff","machine_type":["single","backend","remote","spider"],"schedule":"0 5 10 * * *","enable":true,"role":[]}', '', 'MAP', 1, 0, 0, 0, 1); -REPLACE INTO tb_config_name_def( namespace, conf_type, conf_file, conf_name, value_type, value_default, value_allowed, value_type_sub, flag_status, flag_disable, flag_locked, flag_encrypt, need_restart) VALUES( 'tendb', 'mysql_monitor', 'items-config.yaml', 'mysql-connlog-size', 'STRING', '{"machine_type":["spider"],"name":"mysql-connlog-size","schedule":"0 0 12 * * *","enable":true,"role":[]}', '', 'MAP', 1, 0, 0, 0, 1); -REPLACE INTO tb_config_name_def( namespace, conf_type, conf_file, conf_name, value_type, value_default, value_allowed, value_type_sub, flag_status, flag_disable, flag_locked, flag_encrypt, need_restart) VALUES( 'tendb', 'mysql_monitor', 'items-config.yaml', 'mysql-connlog-rotate', 'STRING', '{"schedule":"0 30 23 * * *","name":"mysql-connlog-rotate","machine_type":["spider"],"role":[],"enable":true}', '', 'MAP', 1, 0, 0, 0, 1); -REPLACE INTO tb_config_name_def( namespace, conf_type, conf_file, conf_name, value_type, value_default, value_allowed, value_type_sub, flag_status, flag_disable, flag_locked, flag_encrypt, need_restart) VALUES( 'tendb', 'mysql_monitor', 'items-config.yaml', 'mysql-err-notice', 'STRING', '{"role":[],"enable":false,"schedule":"@every 1m","machine_type":["single","backend","remote"],"name":"mysql-err-notice"}', '', 'MAP', 1, 0, 0, 0, 1); -REPLACE INTO tb_config_name_def( namespace, conf_type, conf_file, conf_name, value_type, value_default, value_allowed, value_type_sub, flag_status, flag_disable, flag_locked, flag_encrypt, need_restart) VALUES( 'tendb', 'mysql_monitor', 'items-config.yaml', 'mysql-err-critical', 'STRING', '{"machine_type":["single","backend","remote"],"name":"mysql-err-critical","schedule":"@every 1m","enable":false,"role":[]}', '', 'MAP', 1, 0, 0, 0, 1); -REPLACE INTO tb_config_name_def( namespace, conf_type, conf_file, conf_name, value_type, value_default, value_allowed, value_type_sub, flag_status, flag_disable, flag_locked, flag_encrypt, need_restart) VALUES( 'tendb', 'mysql_monitor', 'items-config.yaml', 'spider-err-notice', 'STRING', '{"name":"spider-err-notice","machine_type":["spider"],"schedule":"@every 1m","enable":false,"role":[]}', '', 'MAP', 1, 0, 0, 0, 1); -REPLACE INTO tb_config_name_def( namespace, conf_type, conf_file, conf_name, value_type, value_default, value_allowed, value_type_sub, flag_status, flag_disable, flag_locked, flag_encrypt, need_restart) VALUES( 'tendb', 'mysql_monitor', 'items-config.yaml', 'spider-err-warn', 'STRING', '{"enable":false,"role":[],"name":"spider-err-warn","machine_type":["spider"],"schedule":"@every 1m"}', '', 'MAP', 1, 0, 0, 0, 1); -REPLACE INTO tb_config_name_def( namespace, conf_type, conf_file, conf_name, value_type, value_default, value_allowed, value_type_sub, flag_status, flag_disable, flag_locked, flag_encrypt, need_restart) VALUES( 'tendb', 'mysql_monitor', 'items-config.yaml', 'spider-err-critical', 'STRING', '{"schedule":"@every 1m","machine_type":["spider"],"name":"spider-err-critical","role":[],"enable":false}', '', 'MAP', 1, 0, 0, 0, 1); -REPLACE INTO tb_config_name_def( namespace, conf_type, conf_file, conf_name, value_type, value_default, value_allowed, value_type_sub, flag_status, flag_disable, flag_locked, flag_encrypt, need_restart) VALUES( 'tendb', 'mysql_monitor', 'items-config.yaml', 'mysqld-restarted', 'STRING', '{"role":[],"enable":true,"schedule":"@every 1m","machine_type":["single","backend","remote","spider"],"name":"mysqld-restarted"}', '', 'MAP', 1, 0, 0, 0, 1); -REPLACE INTO tb_config_name_def( namespace, conf_type, conf_file, conf_name, value_type, value_default, value_allowed, value_type_sub, flag_status, flag_disable, flag_locked, flag_encrypt, need_restart) VALUES( 'tendb', 'mysql_monitor', 'items-config.yaml', 'mysql-lock', 'STRING', '{"schedule":"@every 1m","name":"mysql-lock","machine_type":["single","backend","remote","spider"],"role":["master","spider_master","orphan"],"enable":true}', '', 'MAP', 1, 0, 0, 0, 1); -REPLACE INTO tb_config_name_def( namespace, conf_type, conf_file, conf_name, value_type, value_default, value_allowed, value_type_sub, flag_status, flag_disable, flag_locked, flag_encrypt, need_restart) VALUES( 'tendb', 'mysql_monitor', 'items-config.yaml', 'mysql-inject', 'STRING', '{"schedule":"@every 1m","machine_type":["single","backend","spider"],"name":"mysql-inject","role":[],"enable":true}', '', 'MAP', 1, 0, 0, 0, 1); -REPLACE INTO tb_config_name_def( namespace, conf_type, conf_file, conf_name, value_type, value_default, value_allowed, value_type_sub, flag_status, flag_disable, flag_locked, flag_encrypt, need_restart) VALUES( 'tendb', 'mysql_monitor', 'items-config.yaml', 'proxy-backend', 'STRING', '{"schedule":"@every 1m","machine_type":["proxy"],"name":"proxy-backend","role":[],"enable":true}', '', 'MAP', 1, 0, 0, 0, 1); -REPLACE INTO tb_config_name_def( namespace, conf_type, conf_file, conf_name, value_type, value_default, value_allowed, value_type_sub, flag_status, flag_disable, flag_locked, flag_encrypt, need_restart) VALUES( 'tendb', 'mysql_monitor', 'items-config.yaml', 'proxy-user-list', 'STRING', '{"schedule":"0 55 23 * * *","machine_type":["proxy"],"name":"proxy-user-list","role":[],"enable":true}', '', 'MAP', 1, 0, 0, 0, 1); -REPLACE INTO tb_config_name_def( namespace, conf_type, conf_file, conf_name, value_type, value_default, value_allowed, value_type_sub, flag_status, flag_disable, flag_locked, flag_encrypt, need_restart) VALUES( 'tendb', 'mysql_monitor', 'items-config.yaml', 'rotate-slowlog', 'STRING', '{"role":[],"enable":true,"schedule":"0 55 23 * * *","name":"rotate-slowlog","machine_type":["single","backend","remote","spider"]}', '', 'MAP', 1, 0, 0, 0, 1); -REPLACE INTO tb_config_name_def( namespace, conf_type, conf_file, conf_name, value_type, value_default, value_allowed, value_type_sub, flag_status, flag_disable, flag_locked, flag_encrypt, need_restart) VALUES( 'tendb', 'mysql_monitor', 'items-config.yaml', 'slave-status', 'STRING', '{"role":["slave","repeater"],"enable":true,"schedule":"@every 1m","machine_type":["backend","remote"],"name":"slave-status"}', '', 'MAP', 1, 0, 0, 0, 1); -REPLACE INTO tb_config_name_def( namespace, conf_type, conf_file, conf_name, value_type, value_default, value_allowed, value_type_sub, flag_status, flag_disable, flag_locked, flag_encrypt, need_restart) VALUES( 'tendb', 'mysql_monitor', 'items-config.yaml', 'ctl-replicate', 'STRING', '{"role":["spider_master"],"enable":true,"schedule":"@every 1m","name":"ctl-replicate","machine_type":["spider"]}', '', 'MAP', 1, 0, 0, 0, 1); -REPLACE INTO tb_config_name_def( namespace, conf_type, conf_file, conf_name, value_type, value_default, value_allowed, value_type_sub, flag_status, flag_disable, flag_locked, flag_encrypt, need_restart) VALUES( 'tendb', 'mysql_monitor', 'items-config.yaml', 'spider-remote', 'STRING', '{"enable":true,"role":[],"machine_type":["spider"],"name":"spider-remote","schedule":"@every 1m"}', '', 'MAP', 1, 0, 0, 0, 1); -REPLACE INTO tb_config_name_def( namespace, conf_type, conf_file, conf_name, value_type, value_default, value_allowed, value_type_sub, flag_status, flag_disable, flag_locked, flag_encrypt, need_restart) VALUES( 'tendb', 'mysql_monitor', 'items-config.yaml', 'spider-table-schema-consistency', 'STRING', '{"enable":false,"role":["spider_master"],"name":"spider-table-schema-consistency","machine_type":["spider"],"schedule":"0 10 1 * * *"}', '', 'MAP', 1, 0, 0, 0, 1); -REPLACE INTO tb_config_name_def( namespace, conf_type, conf_file, conf_name, value_type, value_default, value_allowed, value_type_sub, flag_status, flag_disable, flag_locked, flag_encrypt, need_restart) VALUES( 'tendb', 'mysql_monitor', 'items-config.yaml', 'dbha-heartbeat', 'STRING', '{"role":[],"enable":true,"schedule":"@every 2h","machine_type":["spider","remote","backend"],"name":"dbha-heartbeat"}', '', 'MAP', 1, 0, 0, 0, 1); -REPLACE INTO tb_config_name_def( namespace, conf_type, conf_file, conf_name, value_type, value_default, value_allowed, value_type_sub, flag_status, flag_disable, flag_locked, flag_encrypt, need_restart) VALUES( 'tendb', 'mysql_monitor', 'items-config.yaml', 'unique-ctl-master', 'STRING', '{"schedule":"@every 1m","name":"unique-ctl-master","machine_type":["spider"],"role":["spider_master"],"enable":true}', '', 'MAP', 1, 0, 0, 0, 1); -REPLACE INTO tb_config_name_def( namespace, conf_type, conf_file, conf_name, value_type, value_default, value_allowed, value_type_sub, flag_status, flag_disable, flag_locked, flag_encrypt, need_restart) VALUES( 'tendb', 'mysql_monitor', 'items-config.yaml', 'scene-snapshot', 'STRING', '{"name":"scene-snapshot","machine_type":["spider","remote","backend","single"],"schedule":"@every 1m","enable":false,"role":[]}', '', 'MAP', 1, 0, 0, 0, 1); -REPLACE INTO tb_config_name_def( namespace, conf_type, conf_file, conf_name, value_type, value_default, value_allowed, value_type_sub, flag_status, flag_disable, flag_locked, flag_encrypt, need_restart) VALUES( 'tendb', 'mysql_monitor', 'items-config.yaml', 'mysql-timezone-change', 'STRING', '{"name":"mysql-timezone-change","machine_type":["spider","remote","backend","single"],"schedule":"@every 1m","enable":true,"role":[]}', '', 'MAP', 1, 0, 0, 0, 1); -REPLACE INTO tb_config_name_def( namespace, conf_type, conf_file, conf_name, value_type, value_default, value_allowed, value_type_sub, flag_status, flag_disable, flag_locked, flag_encrypt, need_restart) VALUES( 'tendb', 'mysql_monitor', 'items-config.yaml', 'sys-timezone-change', 'STRING', '{"enable":true,"role":[],"machine_type":["spider","proxy","remote","backend","single"],"name":"sys-timezone-change","schedule":"@every 1m"}', '', 'MAP', 1, 0, 0, 0, 1); -REPLACE INTO tb_config_name_def( namespace, conf_type, conf_file, conf_name, value_type, value_default, value_allowed, value_type_sub, flag_status, flag_disable, flag_locked, flag_encrypt, need_restart) VALUES( 'tendb', 'mysql_monitor', 'items-config.yaml', 'rotate-proxy-connlog', 'STRING', '{"schedule":"0 55 23 * * *","machine_type":["proxy"],"name":"rotate-proxy-connlog","role":[],"enable":true}', '', 'MAP', 1, 0, 0, 0, 1); +REPLACE INTO tb_config_name_def( namespace, conf_type, conf_file, conf_name, value_type, value_default, value_allowed, value_type_sub, flag_status, flag_disable, flag_locked, flag_encrypt, need_restart) VALUES( 'tendb', 'mysql_monitor', 'items-config.yaml', 'character-consistency', 'STRING', '{"role":[],"schedule":"0 0 14 * * 1","machine_type":["single","backend","remote","spider"],"name":"character-consistency","enable":true}', '', 'MAP', 1, 0, 0, 0, 1); +REPLACE INTO tb_config_name_def( namespace, conf_type, conf_file, conf_name, value_type, value_default, value_allowed, value_type_sub, flag_status, flag_disable, flag_locked, flag_encrypt, need_restart) VALUES( 'tendb', 'mysql_monitor', 'items-config.yaml', 'routine-definer', 'STRING', '{"role":[],"machine_type":["single","backend","remote"],"schedule":"0 0 15 * * 1","name":"routine-definer","enable":true}', '', 'MAP', 1, 0, 0, 0, 1); +REPLACE INTO tb_config_name_def( namespace, conf_type, conf_file, conf_name, value_type, value_default, value_allowed, value_type_sub, flag_status, flag_disable, flag_locked, flag_encrypt, need_restart) VALUES( 'tendb', 'mysql_monitor', 'items-config.yaml', 'view-definer', 'STRING', '{"enable":true,"name":"view-definer","machine_type":["single","backend","remote"],"schedule":"0 0 15 * * 1","role":[]}', '', 'MAP', 1, 0, 0, 0, 1); +REPLACE INTO tb_config_name_def( namespace, conf_type, conf_file, conf_name, value_type, value_default, value_allowed, value_type_sub, flag_status, flag_disable, flag_locked, flag_encrypt, need_restart) VALUES( 'tendb', 'mysql_monitor', 'items-config.yaml', 'trigger-definer', 'STRING', '{"role":[],"schedule":"0 0 15 * * 1","machine_type":["single","backend","remote"],"name":"trigger-definer","enable":true}', '', 'MAP', 1, 0, 0, 0, 1); +REPLACE INTO tb_config_name_def( namespace, conf_type, conf_file, conf_name, value_type, value_default, value_allowed, value_type_sub, flag_status, flag_disable, flag_locked, flag_encrypt, need_restart) VALUES( 'tendb', 'mysql_monitor', 'items-config.yaml', 'engine', 'STRING', '{"enable":true,"name":"engine","schedule":"0 10 1 * * *","machine_type":["single","backend","remote"],"role":["slave","orphan"]}', '', 'MAP', 1, 0, 0, 0, 1); +REPLACE INTO tb_config_name_def( namespace, conf_type, conf_file, conf_name, value_type, value_default, value_allowed, value_type_sub, flag_status, flag_disable, flag_locked, flag_encrypt, need_restart) VALUES( 'tendb', 'mysql_monitor', 'items-config.yaml', 'ext3-check', 'STRING', '{"enable":true,"name":"ext3-check","schedule":"0 0 16 * * 1","machine_type":["single","backend","remote"],"role":[]}', '', 'MAP', 1, 0, 0, 0, 1); +REPLACE INTO tb_config_name_def( namespace, conf_type, conf_file, conf_name, value_type, value_default, value_allowed, value_type_sub, flag_status, flag_disable, flag_locked, flag_encrypt, need_restart) VALUES( 'tendb', 'mysql_monitor', 'items-config.yaml', 'ibd-statistic', 'STRING', '{"role":["slave","orphan"],"machine_type":["single","backend","remote"],"schedule":"0 45 23 * * *","name":"ibd-statistic","enable":true}', '', 'MAP', 1, 0, 0, 0, 1); +REPLACE INTO tb_config_name_def( namespace, conf_type, conf_file, conf_name, value_type, value_default, value_allowed, value_type_sub, flag_status, flag_disable, flag_locked, flag_encrypt, need_restart) VALUES( 'tendb', 'mysql_monitor', 'items-config.yaml', 'master-slave-heartbeat', 'STRING', '{"role":["master","repeater","slave","spider_master"],"schedule":"@every 1m","machine_type":["backend","remote","spider"],"name":"master-slave-heartbeat","enable":true}', '', 'MAP', 1, 0, 0, 0, 1); +REPLACE INTO tb_config_name_def( namespace, conf_type, conf_file, conf_name, value_type, value_default, value_allowed, value_type_sub, flag_status, flag_disable, flag_locked, flag_encrypt, need_restart) VALUES( 'tendb', 'mysql_monitor', 'items-config.yaml', 'mysql-config-diff', 'STRING', '{"role":[],"machine_type":["single","backend","remote","spider"],"schedule":"0 5 10 * * *","name":"mysql-config-diff","enable":true}', '', 'MAP', 1, 0, 0, 0, 1); +REPLACE INTO tb_config_name_def( namespace, conf_type, conf_file, conf_name, value_type, value_default, value_allowed, value_type_sub, flag_status, flag_disable, flag_locked, flag_encrypt, need_restart) VALUES( 'tendb', 'mysql_monitor', 'items-config.yaml', 'mysql-connlog-size', 'STRING', '{"schedule":"0 0 12 * * *","machine_type":["single","backend","remote","spider"],"role":[],"enable":false,"name":"mysql-connlog-size"}', '', 'MAP', 1, 0, 0, 0, 1); +REPLACE INTO tb_config_name_def( namespace, conf_type, conf_file, conf_name, value_type, value_default, value_allowed, value_type_sub, flag_status, flag_disable, flag_locked, flag_encrypt, need_restart) VALUES( 'tendb', 'mysql_monitor', 'items-config.yaml', 'mysql-connlog-rotate', 'STRING', '{"role":[],"machine_type":["single","backend","remote","spider"],"schedule":"0 30 23 * * *","name":"mysql-connlog-rotate","enable":false}', '', 'MAP', 1, 0, 0, 0, 1); +REPLACE INTO tb_config_name_def( namespace, conf_type, conf_file, conf_name, value_type, value_default, value_allowed, value_type_sub, flag_status, flag_disable, flag_locked, flag_encrypt, need_restart) VALUES( 'tendb', 'mysql_monitor', 'items-config.yaml', 'mysql-err-notice', 'STRING', '{"role":[],"schedule":"@every 1m","machine_type":["single","backend","remote"],"name":"mysql-err-notice","enable":false}', '', 'MAP', 1, 0, 0, 0, 1); +REPLACE INTO tb_config_name_def( namespace, conf_type, conf_file, conf_name, value_type, value_default, value_allowed, value_type_sub, flag_status, flag_disable, flag_locked, flag_encrypt, need_restart) VALUES( 'tendb', 'mysql_monitor', 'items-config.yaml', 'mysql-err-critical', 'STRING', '{"schedule":"@every 1m","machine_type":["single","backend","remote"],"role":[],"enable":false,"name":"mysql-err-critical"}', '', 'MAP', 1, 0, 0, 0, 1); +REPLACE INTO tb_config_name_def( namespace, conf_type, conf_file, conf_name, value_type, value_default, value_allowed, value_type_sub, flag_status, flag_disable, flag_locked, flag_encrypt, need_restart) VALUES( 'tendb', 'mysql_monitor', 'items-config.yaml', 'spider-err-notice', 'STRING', '{"enable":false,"name":"spider-err-notice","machine_type":["spider"],"schedule":"@every 1m","role":[]}', '', 'MAP', 1, 0, 0, 0, 1); +REPLACE INTO tb_config_name_def( namespace, conf_type, conf_file, conf_name, value_type, value_default, value_allowed, value_type_sub, flag_status, flag_disable, flag_locked, flag_encrypt, need_restart) VALUES( 'tendb', 'mysql_monitor', 'items-config.yaml', 'spider-err-warn', 'STRING', '{"schedule":"@every 1m","machine_type":["spider"],"role":[],"enable":false,"name":"spider-err-warn"}', '', 'MAP', 1, 0, 0, 0, 1); +REPLACE INTO tb_config_name_def( namespace, conf_type, conf_file, conf_name, value_type, value_default, value_allowed, value_type_sub, flag_status, flag_disable, flag_locked, flag_encrypt, need_restart) VALUES( 'tendb', 'mysql_monitor', 'items-config.yaml', 'spider-err-critical', 'STRING', '{"machine_type":["spider"],"schedule":"@every 1m","role":[],"enable":false,"name":"spider-err-critical"}', '', 'MAP', 1, 0, 0, 0, 1); +REPLACE INTO tb_config_name_def( namespace, conf_type, conf_file, conf_name, value_type, value_default, value_allowed, value_type_sub, flag_status, flag_disable, flag_locked, flag_encrypt, need_restart) VALUES( 'tendb', 'mysql_monitor', 'items-config.yaml', 'mysqld-restarted', 'STRING', '{"role":[],"machine_type":["single","backend","remote","spider"],"schedule":"@every 1m","name":"mysqld-restarted","enable":true}', '', 'MAP', 1, 0, 0, 0, 1); +REPLACE INTO tb_config_name_def( namespace, conf_type, conf_file, conf_name, value_type, value_default, value_allowed, value_type_sub, flag_status, flag_disable, flag_locked, flag_encrypt, need_restart) VALUES( 'tendb', 'mysql_monitor', 'items-config.yaml', 'mysql-lock', 'STRING', '{"enable":true,"name":"mysql-lock","machine_type":["single","backend","remote","spider"],"schedule":"@every 1m","role":["master","spider_master","orphan"]}', '', 'MAP', 1, 0, 0, 0, 1); +REPLACE INTO tb_config_name_def( namespace, conf_type, conf_file, conf_name, value_type, value_default, value_allowed, value_type_sub, flag_status, flag_disable, flag_locked, flag_encrypt, need_restart) VALUES( 'tendb', 'mysql_monitor', 'items-config.yaml', 'mysql-inject', 'STRING', '{"schedule":"@every 1m","machine_type":["single","backend","spider"],"role":[],"enable":true,"name":"mysql-inject"}', '', 'MAP', 1, 0, 0, 0, 1); +REPLACE INTO tb_config_name_def( namespace, conf_type, conf_file, conf_name, value_type, value_default, value_allowed, value_type_sub, flag_status, flag_disable, flag_locked, flag_encrypt, need_restart) VALUES( 'tendb', 'mysql_monitor', 'items-config.yaml', 'proxy-backend', 'STRING', '{"enable":true,"name":"proxy-backend","schedule":"@every 1m","machine_type":["proxy"],"role":[]}', '', 'MAP', 1, 0, 0, 0, 1); +REPLACE INTO tb_config_name_def( namespace, conf_type, conf_file, conf_name, value_type, value_default, value_allowed, value_type_sub, flag_status, flag_disable, flag_locked, flag_encrypt, need_restart) VALUES( 'tendb', 'mysql_monitor', 'items-config.yaml', 'proxy-user-list', 'STRING', '{"role":[],"schedule":"0 55 23 * * *","machine_type":["proxy"],"name":"proxy-user-list","enable":true}', '', 'MAP', 1, 0, 0, 0, 1); +REPLACE INTO tb_config_name_def( namespace, conf_type, conf_file, conf_name, value_type, value_default, value_allowed, value_type_sub, flag_status, flag_disable, flag_locked, flag_encrypt, need_restart) VALUES( 'tendb', 'mysql_monitor', 'items-config.yaml', 'rotate-slowlog', 'STRING', '{"enable":true,"name":"rotate-slowlog","schedule":"0 55 23 * * *","machine_type":["single","backend","remote","spider"],"role":[]}', '', 'MAP', 1, 0, 0, 0, 1); +REPLACE INTO tb_config_name_def( namespace, conf_type, conf_file, conf_name, value_type, value_default, value_allowed, value_type_sub, flag_status, flag_disable, flag_locked, flag_encrypt, need_restart) VALUES( 'tendb', 'mysql_monitor', 'items-config.yaml', 'slave-status', 'STRING', '{"schedule":"@every 1m","machine_type":["backend","remote"],"role":["slave","repeater"],"enable":true,"name":"slave-status"}', '', 'MAP', 1, 0, 0, 0, 1); +REPLACE INTO tb_config_name_def( namespace, conf_type, conf_file, conf_name, value_type, value_default, value_allowed, value_type_sub, flag_status, flag_disable, flag_locked, flag_encrypt, need_restart) VALUES( 'tendb', 'mysql_monitor', 'items-config.yaml', 'ctl-replicate', 'STRING', '{"name":"ctl-replicate","enable":true,"role":["spider_master"],"machine_type":["spider"],"schedule":"@every 1m"}', '', 'MAP', 1, 0, 0, 0, 1); +REPLACE INTO tb_config_name_def( namespace, conf_type, conf_file, conf_name, value_type, value_default, value_allowed, value_type_sub, flag_status, flag_disable, flag_locked, flag_encrypt, need_restart) VALUES( 'tendb', 'mysql_monitor', 'items-config.yaml', 'spider-remote', 'STRING', '{"schedule":"@every 1m","machine_type":["spider"],"role":[],"enable":true,"name":"spider-remote"}', '', 'MAP', 1, 0, 0, 0, 1); +REPLACE INTO tb_config_name_def( namespace, conf_type, conf_file, conf_name, value_type, value_default, value_allowed, value_type_sub, flag_status, flag_disable, flag_locked, flag_encrypt, need_restart) VALUES( 'tendb', 'mysql_monitor', 'items-config.yaml', 'spider-table-schema-consistency', 'STRING', '{"role":["spider_master"],"schedule":"0 10 1 * * *","machine_type":["spider"],"name":"spider-table-schema-consistency","enable":false}', '', 'MAP', 1, 0, 0, 0, 1); +REPLACE INTO tb_config_name_def( namespace, conf_type, conf_file, conf_name, value_type, value_default, value_allowed, value_type_sub, flag_status, flag_disable, flag_locked, flag_encrypt, need_restart) VALUES( 'tendb', 'mysql_monitor', 'items-config.yaml', 'dbha-heartbeat', 'STRING', '{"name":"dbha-heartbeat","enable":true,"role":[],"machine_type":["spider","remote","backend"],"schedule":"@every 2h"}', '', 'MAP', 1, 0, 0, 0, 1); +REPLACE INTO tb_config_name_def( namespace, conf_type, conf_file, conf_name, value_type, value_default, value_allowed, value_type_sub, flag_status, flag_disable, flag_locked, flag_encrypt, need_restart) VALUES( 'tendb', 'mysql_monitor', 'items-config.yaml', 'unique-ctl-master', 'STRING', '{"enable":false,"name":"unique-ctl-master","schedule":"@every 1m","machine_type":["spider"],"role":["spider_master"]}', '', 'MAP', 1, 0, 0, 0, 1); +REPLACE INTO tb_config_name_def( namespace, conf_type, conf_file, conf_name, value_type, value_default, value_allowed, value_type_sub, flag_status, flag_disable, flag_locked, flag_encrypt, need_restart) VALUES( 'tendb', 'mysql_monitor', 'items-config.yaml', 'scene-snapshot', 'STRING', '{"role":[],"schedule":"@every 1m","machine_type":["spider","remote","backend","single"],"name":"scene-snapshot","enable":false}', '', 'MAP', 1, 0, 0, 0, 1); +REPLACE INTO tb_config_name_def( namespace, conf_type, conf_file, conf_name, value_type, value_default, value_allowed, value_type_sub, flag_status, flag_disable, flag_locked, flag_encrypt, need_restart) VALUES( 'tendb', 'mysql_monitor', 'items-config.yaml', 'mysql-timezone-change', 'STRING', '{"schedule":"@every 1m","machine_type":["spider","remote","backend","single"],"role":[],"enable":true,"name":"mysql-timezone-change"}', '', 'MAP', 1, 0, 0, 0, 1); +REPLACE INTO tb_config_name_def( namespace, conf_type, conf_file, conf_name, value_type, value_default, value_allowed, value_type_sub, flag_status, flag_disable, flag_locked, flag_encrypt, need_restart) VALUES( 'tendb', 'mysql_monitor', 'items-config.yaml', 'sys-timezone-change', 'STRING', '{"enable":true,"name":"sys-timezone-change","schedule":"@every 1m","machine_type":["spider","proxy","remote","backend","single"],"role":[]}', '', 'MAP', 1, 0, 0, 0, 1); +REPLACE INTO tb_config_name_def( namespace, conf_type, conf_file, conf_name, value_type, value_default, value_allowed, value_type_sub, flag_status, flag_disable, flag_locked, flag_encrypt, need_restart) VALUES( 'tendb', 'mysql_monitor', 'items-config.yaml', 'rotate-proxy-connlog', 'STRING', '{"role":[],"schedule":"0 55 23 * * *","machine_type":["proxy"],"name":"rotate-proxy-connlog","enable":true}', '', 'MAP', 1, 0, 0, 0, 1); +REPLACE INTO tb_config_name_def( namespace, conf_type, conf_file, conf_name, value_type, value_default, value_allowed, value_type_sub, flag_status, flag_disable, flag_locked, flag_encrypt, need_restart) VALUES( 'tendb', 'mysql_monitor', 'items-config.yaml', 'get-ctl-primary', 'STRING', '{"role":["spider_master"],"machine_type":["spider"],"schedule":"@every 1m","name":"get-ctl-primary","enable":false}', '', 'MAP', 1, 0, 0, 0, 1); diff --git a/dbm-services/mysql/db-tools/mysql-monitor/items-config.yaml b/dbm-services/mysql/db-tools/mysql-monitor/items-config.yaml index 4637b33489..b2ef6d713f 100644 --- a/dbm-services/mysql/db-tools/mysql-monitor/items-config.yaml +++ b/dbm-services/mysql/db-tools/mysql-monitor/items-config.yaml @@ -51,7 +51,7 @@ role: [] - name: ibd-statistic enable: true - schedule: 0 0 14 * * 1 + schedule: 0 45 23 * * * machine_type: - single - backend @@ -65,9 +65,12 @@ machine_type: - backend - remote + - spider role: - master - repeater + - slave + - spider_master - name: mysql-config-diff enable: true schedule: 0 5 10 * * * @@ -78,15 +81,21 @@ - spider role: [] - name: mysql-connlog-size - enable: true + enable: false schedule: 0 0 12 * * * machine_type: + - single + - backend + - remote - spider role: [] - name: mysql-connlog-rotate - enable: true + enable: false schedule: 0 30 23 * * * machine_type: + - single + - backend + - remote - spider role: [] - name: mysql-err-notice @@ -211,7 +220,7 @@ - backend role: [] - name: unique-ctl-master - enable: true + enable: false schedule: '@every 1m' machine_type: - spider @@ -250,4 +259,11 @@ schedule: 0 55 23 * * * machine_type: - proxy - role: [] \ No newline at end of file + role: [] +- name: get-ctl-primary + enable: false + schedule: '@every 1m' + machine_type: + - spider + role: + - spider_master \ No newline at end of file diff --git a/dbm-services/mysql/db-tools/mysql-monitor/pkg/itemscollect/ibdstatistic/ibd_statistic.go b/dbm-services/mysql/db-tools/mysql-monitor/pkg/itemscollect/ibdstatistic/ibd_statistic.go index 5788a69218..2680f9fa25 100644 --- a/dbm-services/mysql/db-tools/mysql-monitor/pkg/itemscollect/ibdstatistic/ibd_statistic.go +++ b/dbm-services/mysql/db-tools/mysql-monitor/pkg/itemscollect/ibdstatistic/ibd_statistic.go @@ -35,6 +35,7 @@ var systemDBs = []string{ "infodba_schema", "performance_schema", "test", + "db_infobase", } func init() { @@ -70,7 +71,12 @@ func (c *ibdStatistic) Run() (msg string, err error) { return "", err } - err = reportMetrics(result) + //err = reportMetrics(result) + //if err != nil { + // return "", err + //} + + err = reportLog(result) if err != nil { return "", err } diff --git a/dbm-services/mysql/db-tools/mysql-monitor/pkg/itemscollect/ibdstatistic/report_log.go b/dbm-services/mysql/db-tools/mysql-monitor/pkg/itemscollect/ibdstatistic/report_log.go new file mode 100644 index 0000000000..580cf09081 --- /dev/null +++ b/dbm-services/mysql/db-tools/mysql-monitor/pkg/itemscollect/ibdstatistic/report_log.go @@ -0,0 +1,116 @@ +package ibdstatistic + +import ( + "dbm-services/mysql/db-tools/mysql-monitor/pkg/config" + "dbm-services/mysql/db-tools/mysql-monitor/pkg/internal/cst" + "encoding/json" + "fmt" + "log/slog" + "os" + "path/filepath" + "slices" + "time" + + "github.com/pkg/errors" +) + +type tableSizeStruct struct { + BkCloudId int `json:"bk_cloud_id"` + BkBizId int `json:"bk_biz_id"` + ImmuteDomain string `json:"cluster_domain"` + DBModule int `json:"db_module"` + MachineType string `json:"machine_type"` + Ip string `json:"instance_host"` + Port int `json:"instance_port"` + Role string `json:"instance_role"` + ServiceInstanceId int64 `json:"bk_target_service_instance_id"` + OriginalDBName string `json:"original_database_name"` + DBName string `json:"database_name"` + DBSize int64 `json:"database_size"` + TableName string `json:"table_name"` + TableSize int64 `json:"table_size"` +} + +func reportLog(result map[string]map[string]int64) error { + dbsizeReportBaseDir := filepath.Join(cst.DBAReportBase, "mysql/dbsize") + err := os.MkdirAll(dbsizeReportBaseDir, os.ModePerm) + if err != nil { + slog.Error("failed to create database size reports directory", slog.String("error", err.Error())) + return errors.Wrap(err, "failed to create database size reports directory") + } + + filePath := filepath.Join( + dbsizeReportBaseDir, + fmt.Sprintf(`report.log.%d`, time.Now().Weekday()), + ) + err = os.RemoveAll(filePath) + if err != nil { + slog.Error("failed to remove database size reports directory", slog.String("error", err.Error())) + return errors.Wrap(err, "failed to remove database size reports directory") + } + + f, err := os.OpenFile(filePath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0666) + if err != nil { + slog.Error("failed to open log file", "file", filePath) + return errors.Wrap(err, "failed to open file") + } + defer func() { + _ = f.Close() + }() + + for originalDBName, dbInfo := range result { + // 根据 dbm 枚举约定, remote 是 tendbcluster 的存储机器类型 + dbName := originalDBName + if config.MonitorConfig.MachineType == "remote" && slices.Index(systemDBs, originalDBName) < 0 { + match := tenDBClusterDbNamePattern.FindStringSubmatch(originalDBName) + if match == nil { + err := errors.Errorf( + "invalid dbname: '%s' on %s", + originalDBName, config.MonitorConfig.MachineType, + ) + slog.Error("ibd-statistic report", slog.String("error", err.Error())) + return err + } + dbName = match[1] + } + + var dbSize int64 + var tablesInfo []tableSizeStruct + for tableName, tableSize := range dbInfo { + tablesInfo = append(tablesInfo, tableSizeStruct{ + BkCloudId: *config.MonitorConfig.BkCloudID, + BkBizId: config.MonitorConfig.BkBizId, + ImmuteDomain: config.MonitorConfig.ImmuteDomain, + DBModule: *config.MonitorConfig.DBModuleID, + MachineType: config.MonitorConfig.MachineType, + Ip: config.MonitorConfig.Ip, + Port: config.MonitorConfig.Port, + Role: *config.MonitorConfig.Role, + ServiceInstanceId: config.MonitorConfig.BkInstanceId, + OriginalDBName: originalDBName, + DBName: dbName, + DBSize: 0, + TableName: tableName, + TableSize: tableSize, + }) + dbSize += tableSize + } + + for _, row := range tablesInfo { + row.DBSize = dbSize + b, err := json.Marshal(row) + if err != nil { + slog.Error("ibd-statistic report", slog.String("error", err.Error())) + return errors.Wrap(err, "failed to marshal row") + } + + b = append(b, '\n') + _, err = f.Write(b) + if err != nil { + slog.Error("ibd-statistic report", slog.String("error", err.Error())) + return errors.Wrap(err, "failed to write row") + } + } + } + return nil +} diff --git a/dbm-services/mysql/db-tools/mysql-monitor/pkg/itemscollect/ibdstatistic/report_metrics.go b/dbm-services/mysql/db-tools/mysql-monitor/pkg/itemscollect/ibdstatistic/report_metrics.go index 27813e9a68..ae6e866755 100644 --- a/dbm-services/mysql/db-tools/mysql-monitor/pkg/itemscollect/ibdstatistic/report_metrics.go +++ b/dbm-services/mysql/db-tools/mysql-monitor/pkg/itemscollect/ibdstatistic/report_metrics.go @@ -22,13 +22,13 @@ import ( var tableSizeMetricName string var dbSizeMetricName string -var tendbClusterDbNamePattern *regexp.Regexp +var tenDBClusterDbNamePattern *regexp.Regexp func init() { tableSizeMetricName = "mysql_table_size" dbSizeMetricName = "mysql_database_size" - tendbClusterDbNamePattern = regexp.MustCompile(`^(.*)_[0-9]+$`) + tenDBClusterDbNamePattern = regexp.MustCompile(`^(.*)_[0-9]+$`) } func reportMetrics(result map[string]map[string]int64) error { @@ -38,7 +38,7 @@ func reportMetrics(result map[string]map[string]int64) error { // 根据 dbm 枚举约定, remote 是 tendbcluster 的存储机器类型 if config.MonitorConfig.MachineType == "remote" && slices.Index(systemDBs, dbName) < 0 { - match := tendbClusterDbNamePattern.FindStringSubmatch(dbName) + match := tenDBClusterDbNamePattern.FindStringSubmatch(dbName) if match == nil { err := errors.Errorf( "invalid dbname: '%s' on %s", diff --git a/dbm-services/mysql/db-tools/mysql-monitor/pkg/itemscollect/items_collect.go b/dbm-services/mysql/db-tools/mysql-monitor/pkg/itemscollect/items_collect.go index bc1f1d560e..571648d521 100644 --- a/dbm-services/mysql/db-tools/mysql-monitor/pkg/itemscollect/items_collect.go +++ b/dbm-services/mysql/db-tools/mysql-monitor/pkg/itemscollect/items_collect.go @@ -30,10 +30,10 @@ import ( "dbm-services/mysql/db-tools/mysql-monitor/pkg/itemscollect/rotateslowlog" "dbm-services/mysql/db-tools/mysql-monitor/pkg/itemscollect/scenesnapshot" "dbm-services/mysql/db-tools/mysql-monitor/pkg/itemscollect/slavestatus" + "dbm-services/mysql/db-tools/mysql-monitor/pkg/itemscollect/spiderctlchecker" "dbm-services/mysql/db-tools/mysql-monitor/pkg/itemscollect/spiderremote" "dbm-services/mysql/db-tools/mysql-monitor/pkg/itemscollect/timezonechange" "dbm-services/mysql/db-tools/mysql-monitor/pkg/itemscollect/tscc" - "dbm-services/mysql/db-tools/mysql-monitor/pkg/itemscollect/uniquectlmaster" mi "dbm-services/mysql/db-tools/mysql-monitor/pkg/monitoriteminterface" ) @@ -89,9 +89,10 @@ func init() { _ = registerItemConstructor(spiderremote.Register()) _ = registerItemConstructor(tscc.Register()) _ = registerItemConstructor(dbhaheartbeat.Register()) - _ = registerItemConstructor(uniquectlmaster.Register()) + _ = registerItemConstructor(spiderctlchecker.UniqueCtlCheckerRegister()) _ = registerItemConstructor(scenesnapshot.Register()) _ = registerItemConstructor(timezonechange.RegisterSysTimezoneChange()) _ = registerItemConstructor(timezonechange.RegisterMySQLTimezoneChange()) _ = registerItemConstructor(rotateproxyconnlog.RegisterRotateProxyConnlog()) + _ = registerItemConstructor(spiderctlchecker.GetCtlPrimaryRegister()) } diff --git a/dbm-services/mysql/db-tools/mysql-monitor/pkg/itemscollect/masterslaveheartbeat/master_slave_heartbeat.go b/dbm-services/mysql/db-tools/mysql-monitor/pkg/itemscollect/masterslaveheartbeat/master_slave_heartbeat.go index 9f0fea0aff..b45e679e75 100644 --- a/dbm-services/mysql/db-tools/mysql-monitor/pkg/itemscollect/masterslaveheartbeat/master_slave_heartbeat.go +++ b/dbm-services/mysql/db-tools/mysql-monitor/pkg/itemscollect/masterslaveheartbeat/master_slave_heartbeat.go @@ -4,8 +4,12 @@ package masterslaveheartbeat import ( "context" "database/sql" + "dbm-services/mysql/db-tools/mysql-monitor/pkg/utils" + "fmt" "log/slog" + "strconv" + "strings" "dbm-services/mysql/db-tools/mysql-monitor/pkg/config" "dbm-services/mysql/db-tools/mysql-monitor/pkg/internal/cst" @@ -38,6 +42,13 @@ type Checker struct { heartBeatTable string } +type primaryDesc struct { + ServerName string `db:"SERVER_NAME"` + Host string `db:"HOST"` + Port uint32 `db:"PORT"` + IsThisServer uint32 `db:"IS_THIS_SERVER"` +} + func (c *Checker) updateHeartbeat() error { ctx, cancel := context.WithTimeout(context.Background(), config.MonitorConfig.InteractTimeout) defer cancel() @@ -48,13 +59,13 @@ func (c *Checker) updateHeartbeat() error { Scan(&masterServerId, &binlogFormatOld) if err != nil { slog.Error( - "master-slave-heartbeat query server_id, binlog_format", + name, slog.String("error", err.Error()), ) return err } - slog.Debug( - "master-slave-heartbeat", + slog.Info( + name, slog.String("server_id", masterServerId), slog.String("binlog_format", binlogFormatOld), ) @@ -62,7 +73,7 @@ func (c *Checker) updateHeartbeat() error { // will set session variables, so get a connection from pool conn, err := c.db.DB.Conn(context.Background()) if err != nil { - slog.Error("master-slave-heartbeat get conn from db", slog.String("error", err.Error())) + slog.Error(name, slog.String("error", err.Error())) return err } defer func() { @@ -85,7 +96,7 @@ VALUES('%s', @@server_id, now(), sysdate(), timestampdiff(SECOND, now(),sysdate( c.heartBeatTable, masterServerId) if _, err = conn.ExecContext(ctx, txrrSQL); err != nil { - err := errors.WithMessage(err, "update heartbeat need SET SESSION tx_isolation = 'REPEATABLE-READ'") + err := errors.Wrapf(err, "update heartbeat need SET SESSION tx_isolation = 'REPEATABLE-READ'") slog.Error("master-slave-heartbeat", slog.String("error", err.Error())) return err } @@ -102,10 +113,74 @@ VALUES('%s', @@server_id, now(), sysdate(), timestampdiff(SECOND, now(),sysdate( return err } else { if num, _ := res.RowsAffected(); num > 0 { - slog.Debug("master-slave-heartbeat insert success") + slog.Info("master-slave-heartbeat insert success") } } - slog.Debug("master-slave-heartbeat update slave success") + slog.Info("master-slave-heartbeat update slave success") + return nil +} + +func (c *Checker) reportHeartbeatDelay() error { + slaveStatus := make(map[string]interface{}) + rows, err := c.db.Queryx(`SHOW SLAVE STATUS`) + if err != nil { + slog.Error(name, slog.String("error", err.Error())) + return err + } + defer func() { + _ = rows.Close() + }() + + for rows.Next() { + err := rows.MapScan(slaveStatus) + if err != nil { + slog.Error(name, slog.String("error", err.Error())) + return err + } + break + } + + for k, v := range slaveStatus { + if value, ok := v.([]byte); ok { + slaveStatus[k] = strings.TrimSpace(string(value)) + } + } + + masterServerId, err := strconv.ParseInt(slaveStatus["Master_Server_Id"].(string), 10, 64) + if err != nil { + slog.Error(name, slog.String("error", err.Error())) + return err + } + + slog.Info(name, slog.Any("master server id", masterServerId)) + + var timeDelay int64 + err = c.db.QueryRowx( + `select convert((unix_timestamp(now())-unix_timestamp(master_time)),UNSIGNED) as time_delay + from infodba_schema.master_slave_heartbeat + where master_server_id = ? and slave_server_id != master_server_id`, + masterServerId, + ).Scan(&timeDelay) + if err != nil { + if errors.Is(err, sql.ErrNoRows) { + timeDelay = 99999999 + } else { + return err + } + } + + slog.Info("master_slave_heartbeat delay", slog.Int64("delay", timeDelay)) + + utils.SendMonitorMetrics( + strings.Replace(name, "-", "_", -1), + timeDelay, + map[string]interface{}{ + "master-server_id": masterServerId, + "master-host": slaveStatus["Master_Host"], + "master-port": slaveStatus["Master_Port"], + }, + ) + return nil } @@ -117,10 +192,65 @@ func (c *Checker) initTableHeartbeat() (sql.Result, error) { // Run TODO func (c *Checker) Run() (msg string, err error) { // check if dbbackup loadbackup running, skip this round - err = c.updateHeartbeat() + slog.Info(name, slog.String("role", *config.MonitorConfig.Role)) + slog.Info(name, slog.String("machine type", config.MonitorConfig.MachineType)) + if config.MonitorConfig.MachineType == "spider" { + return c.heartBeatOnSpider() + } else { + return c.heartBeatOnStorage() + + } +} + +func (c *Checker) heartBeatOnSpider() (msg string, err error) { + if *config.MonitorConfig.Role != "spider_master" { + return "", nil + } + + res := primaryDesc{} + err = c.db.QueryRowx(`tdbctl get primary`).StructScan(&res) + if err != nil { + slog.Error(name, slog.String("err", err.Error())) + return "", err + } + slog.Info(name, slog.Bool("is primary", res.IsThisServer == 1)) + if res.IsThisServer == 1 { + err = c.updateHeartbeat() + } else { + err = c.reportHeartbeatDelay() + } + return "", err } +func (c *Checker) heartBeatOnStorage() (msg string, err error) { + slog.Info(name, slog.String("machine type", config.MonitorConfig.MachineType)) + switch *config.MonitorConfig.Role { + case "master": + err = c.updateHeartbeat() + if err != nil { + return "", err + } + case "slave": + err = c.reportHeartbeatDelay() + if err != nil { + return "", err + } + case "repeater": + err = c.updateHeartbeat() + if err != nil { + return "", err + } + err = c.reportHeartbeatDelay() + if err != nil { + return "", err + } + default: + return "", errors.Errorf("unkown role: %s", *config.MonitorConfig.Role) + } + return "", nil +} + // Name TODO func (c *Checker) Name() string { return name diff --git a/dbm-services/mysql/db-tools/mysql-monitor/pkg/itemscollect/mysqlconfigdiff/mysql_config_diff.go b/dbm-services/mysql/db-tools/mysql-monitor/pkg/itemscollect/mysqlconfigdiff/mysql_config_diff.go index 07b2a31f12..81f6d3a5e2 100644 --- a/dbm-services/mysql/db-tools/mysql-monitor/pkg/itemscollect/mysqlconfigdiff/mysql_config_diff.go +++ b/dbm-services/mysql/db-tools/mysql-monitor/pkg/itemscollect/mysqlconfigdiff/mysql_config_diff.go @@ -80,7 +80,7 @@ type Checker struct { func (c *Checker) Run() (msg string, err error) { var cnfFile string if config.MonitorConfig.Port == 3306 { - cnfFile = "etc/my.cnf.3306" + cnfFile = "/etc/my.cnf.3306" if _, err := os.Stat(cnfFile); os.IsNotExist(err) { cnfFile = "/etc/my.cnf" } diff --git a/dbm-services/mysql/db-tools/mysql-monitor/pkg/itemscollect/proxyuserlist/backup.go b/dbm-services/mysql/db-tools/mysql-monitor/pkg/itemscollect/proxyuserlist/backup.go index 0a283a5b70..41949b0ae2 100644 --- a/dbm-services/mysql/db-tools/mysql-monitor/pkg/itemscollect/proxyuserlist/backup.go +++ b/dbm-services/mysql/db-tools/mysql-monitor/pkg/itemscollect/proxyuserlist/backup.go @@ -96,7 +96,6 @@ func (c *Checker) cleanOldBackup() error { ctx, `DELETE FROM infodba_schema.proxy_user_list WHERE create_at < DATE(NOW() - INTERVAL 7 DAY)`, - config.MonitorConfig.Ip, ) if err != nil { slog.Error("backup proxy user list delete", slog.String("err", err.Error())) diff --git a/dbm-services/mysql/db-tools/mysql-monitor/pkg/itemscollect/slavestatus/ctl_replicate.go b/dbm-services/mysql/db-tools/mysql-monitor/pkg/itemscollect/slavestatus/ctl_replicate.go index 4fae023462..443e761849 100644 --- a/dbm-services/mysql/db-tools/mysql-monitor/pkg/itemscollect/slavestatus/ctl_replicate.go +++ b/dbm-services/mysql/db-tools/mysql-monitor/pkg/itemscollect/slavestatus/ctl_replicate.go @@ -21,6 +21,14 @@ var ctlReplicateName = "ctl-replicate" type ctlReplicateChecker struct { slaveStatusChecker + primary *getPrimaryRes +} + +type getPrimaryRes struct { + ServerName string `db:"SERVER_NAME"` + Host string `db:"HOST"` + Port uint32 `db:"PORT"` + IsThisServer uint32 `db:"IS_THIS_SERVER"` } // Run 运行 @@ -51,6 +59,19 @@ func (c *ctlReplicateChecker) Run() (msg string, err error) { return fmt.Sprintf("IO/SQL thread not running: %s", slaveErr), nil } + slog.Info( + "tdbctl primary is master", + slog.String("primary", c.primary.Host), + slog.String("master", c.masterHost()), + ) + if c.masterHost() != c.primary.Host { + err = fmt.Errorf( + "tdbctl slave's master host [%s] != primary host [%s]", + c.masterHost(), c.primary.Host, + ) + slog.Error("tdbctl primary is master", slog.String("err", err.Error())) + return "", err + } return "", nil } @@ -58,20 +79,16 @@ func (c *ctlReplicateChecker) isPrimary() (bool, error) { ctx, cancel := context.WithTimeout(context.Background(), config.MonitorConfig.InteractTimeout) defer cancel() - var getPrimaryRes []struct { - ServerName string `db:"SERVER_NAME"` - Host string `db:"HOST"` - Port string `db:"PORT"` - IsThis int `db:"IS_THIS_SERVER"` - } + res := getPrimaryRes{} - err := c.db.SelectContext(ctx, &getPrimaryRes, `TDBCTL GET PRIMARY`) + err := c.db.QueryRowxContext(ctx, `TDBCTL GET PRIMARY`).StructScan(&res) if err != nil { slog.Error("TDBCTL GET PRIMARY", slog.String("error", err.Error())) return false, err } + c.primary = &res - return getPrimaryRes[0].IsThis == 1, nil + return res.IsThisServer == 1, nil } // Name 监控项名 @@ -81,10 +98,13 @@ func (c *ctlReplicateChecker) Name() string { // NewCtlReplicateChecker 新建监控项实例 func NewCtlReplicateChecker(cc *monitoriteminterface.ConnectionCollect) monitoriteminterface.MonitorItemInterface { - return &ctlReplicateChecker{slaveStatusChecker{ - db: cc.CtlDB, - slaveStatus: make(map[string]interface{}), - }} + return &ctlReplicateChecker{ + slaveStatusChecker{ + db: cc.CtlDB, + slaveStatus: make(map[string]interface{}), + }, + nil, + } } // RegisterCtlReplicateChecker 注册监控项 diff --git a/dbm-services/mysql/db-tools/mysql-monitor/pkg/itemscollect/slavestatus/slave_status.go b/dbm-services/mysql/db-tools/mysql-monitor/pkg/itemscollect/slavestatus/slave_status.go index 14de0c7b05..3f17b46999 100644 --- a/dbm-services/mysql/db-tools/mysql-monitor/pkg/itemscollect/slavestatus/slave_status.go +++ b/dbm-services/mysql/db-tools/mysql-monitor/pkg/itemscollect/slavestatus/slave_status.go @@ -48,11 +48,11 @@ func (s *slaveStatusChecker) Run() (msg string, err error) { if err != nil { slog.Warn("invalid errno", err) } else { - slog.Debug("err no found", slog.Int("io err", ioErrNo), slog.Int("sql err", sqlErrNo)) - slog.Debug("io err if is skip", slog.Int("id", slices.Index(skipErrNos, ioErrNo))) - slog.Debug("sql err if is skip", slog.Int("id", slices.Index(skipErrNos, sqlErrNo))) + slog.Info("err no found", slog.Int("io err", ioErrNo), slog.Int("sql err", sqlErrNo)) + slog.Info("io err if is skip", slog.Int("id", slices.Index(skipErrNos, ioErrNo))) + slog.Info("sql err if is skip", slog.Int("id", slices.Index(skipErrNos, sqlErrNo))) if slices.Index(skipErrNos, ioErrNo) >= 0 || slices.Index(skipErrNos, sqlErrNo) >= 0 { - slog.Debug("need skip errno found") + slog.Info("need skip errno found") err := s.skipErr() if err != nil { slog.Warn( @@ -110,6 +110,10 @@ func (s *slaveStatusChecker) isOk() bool { strings.ToUpper(s.slaveStatus["Slave_SQL_Running"].(string)) == "YES" } +func (s *slaveStatusChecker) masterHost() string { + return s.slaveStatus["Master_Host"].(string) +} + func (s *slaveStatusChecker) getErrNo() (ioErrNo int, sqlErrNo int, err error) { ioErrNo, err = strconv.Atoi(s.slaveStatus["Last_IO_Errno"].(string)) if err != nil { diff --git a/dbm-services/mysql/db-tools/mysql-monitor/pkg/itemscollect/spiderctlchecker/getprimarychecker.go b/dbm-services/mysql/db-tools/mysql-monitor/pkg/itemscollect/spiderctlchecker/getprimarychecker.go new file mode 100644 index 0000000000..221613e267 --- /dev/null +++ b/dbm-services/mysql/db-tools/mysql-monitor/pkg/itemscollect/spiderctlchecker/getprimarychecker.go @@ -0,0 +1,52 @@ +package spiderctlchecker + +import ( + "dbm-services/mysql/db-tools/mysql-monitor/pkg/monitoriteminterface" + "log/slog" + + "github.com/jmoiron/sqlx" +) + +var getCtlPrimaryCheckerName = "get-ctl-primary" + +type GetCtlPrimaryChecker struct { + db *sqlx.DB +} + +type primaryDesc struct { + ServerName string `db:"SERVER_NAME"` + Host string `db:"HOST"` + Port uint32 `db:"PORT"` + IsThisServer uint32 `db:"IS_THIS_SERVER"` +} + +func (c *GetCtlPrimaryChecker) Run() (msg string, err error) { + _, err = c.getPrimary() + if err != nil { + slog.Error("Get primary checker error", slog.String("err", err.Error())) + return err.Error(), nil + } + return "", nil +} + +func (c *GetCtlPrimaryChecker) getPrimary() (*primaryDesc, error) { + res := primaryDesc{} + err := c.db.QueryRowx(`tdbctl get primary`).StructScan(&res) + if err != nil { + return nil, err + } + + return &res, nil +} + +func (c *GetCtlPrimaryChecker) Name() string { + return getCtlPrimaryCheckerName +} + +func NewGetCtlPrimaryChecker(cc *monitoriteminterface.ConnectionCollect) monitoriteminterface.MonitorItemInterface { + return &GetCtlPrimaryChecker{db: cc.CtlDB} +} + +func GetCtlPrimaryRegister() (string, monitoriteminterface.MonitorItemConstructorFuncType) { + return getCtlPrimaryCheckerName, NewGetCtlPrimaryChecker +} diff --git a/dbm-services/mysql/db-tools/mysql-monitor/pkg/itemscollect/spiderctlchecker/uniquechecker.go b/dbm-services/mysql/db-tools/mysql-monitor/pkg/itemscollect/spiderctlchecker/uniquechecker.go new file mode 100644 index 0000000000..c5dae00c8b --- /dev/null +++ b/dbm-services/mysql/db-tools/mysql-monitor/pkg/itemscollect/spiderctlchecker/uniquechecker.go @@ -0,0 +1,57 @@ +package spiderctlchecker + +import ( + "log/slog" + "math/big" + "net" + + "dbm-services/mysql/db-tools/mysql-monitor/pkg/monitoriteminterface" + "dbm-services/mysql/db-tools/mysql-monitor/pkg/utils" +) + +/* +1. 每一个 spider master 上的中控节点都上报自己看到的 ctl master +2. 在监控平台做 count 计数, > 1 则告警 +*/ + +var uniqueCheckerName = "unique-ctl-master" + +type UniqueCtlChecker struct { + //db *sqlx.DB + GetCtlPrimaryChecker +} + +func (c *UniqueCtlChecker) Run() (msg string, err error) { + p, err := c.getPrimary() + if err != nil { + return "", err + } + + slog.Info("unique-ctl-master", + slog.String("ctl master", p.Host)) + + ret := big.NewInt(0) + ret.SetBytes(net.ParseIP(p.Host).To4()) + + utils.SendMonitorMetrics( + "unique_ctl_master", + ret.Int64(), + map[string]interface{}{ + "ctl-master": p.Host, + }, + ) + + return "", nil +} + +func (c *UniqueCtlChecker) Name() string { + return uniqueCheckerName +} + +func NewUniqueCtlChecker(cc *monitoriteminterface.ConnectionCollect) monitoriteminterface.MonitorItemInterface { + return &UniqueCtlChecker{GetCtlPrimaryChecker{db: cc.CtlDB}} +} + +func UniqueCtlCheckerRegister() (string, monitoriteminterface.MonitorItemConstructorFuncType) { + return uniqueCheckerName, NewUniqueCtlChecker +} diff --git a/dbm-services/mysql/db-tools/mysql-monitor/pkg/itemscollect/uniquectlmaster/checker.go b/dbm-services/mysql/db-tools/mysql-monitor/pkg/itemscollect/uniquectlmaster/checker.go deleted file mode 100644 index 93d252a422..0000000000 --- a/dbm-services/mysql/db-tools/mysql-monitor/pkg/itemscollect/uniquectlmaster/checker.go +++ /dev/null @@ -1,64 +0,0 @@ -package uniquectlmaster - -import ( - "log/slog" - "math/big" - "net" - - "github.com/jmoiron/sqlx" - - "dbm-services/mysql/db-tools/mysql-monitor/pkg/monitoriteminterface" - "dbm-services/mysql/db-tools/mysql-monitor/pkg/utils" -) - -/* -1. 每一个 spider master 上的中控节点都上报自己看到的 ctl master -2. 在监控平台做 count 计数, > 1 则告警 -*/ - -var name = "unique-ctl-master" - -type Checker struct { - db *sqlx.DB -} - -func (c *Checker) Run() (msg string, err error) { - var res struct { - ServerName string `db:"SERVER_NAME"` - Host string `db:"HOST"` - Port uint32 `db:"PORT"` - IsThisServer uint32 `db:"IS_THIS_SERVER"` - } - err = c.db.QueryRowx(`tdbctl get primary`).StructScan(&res) - if err != nil { - return "", err - } - - slog.Info("unique-ctl-master", - slog.String("ctl master", res.Host)) - - ret := big.NewInt(0) - ret.SetBytes(net.ParseIP(res.Host).To4()) - - utils.SendMonitorMetrics( - "unique_ctl_master", - ret.Int64(), - map[string]interface{}{ - "ctl-master": res.Host, - }, - ) - - return "", nil -} - -func (c *Checker) Name() string { - return name -} - -func NewChecker(cc *monitoriteminterface.ConnectionCollect) monitoriteminterface.MonitorItemInterface { - return &Checker{db: cc.CtlDB} -} - -func Register() (string, monitoriteminterface.MonitorItemConstructorFuncType) { - return name, NewChecker -} diff --git a/dbm-services/mysql/db-tools/mysql-monitor/pkg/monitoriteminterface/connection_collect.go b/dbm-services/mysql/db-tools/mysql-monitor/pkg/monitoriteminterface/connection_collect.go index 55e696d976..356f3b272a 100644 --- a/dbm-services/mysql/db-tools/mysql-monitor/pkg/monitoriteminterface/connection_collect.go +++ b/dbm-services/mysql/db-tools/mysql-monitor/pkg/monitoriteminterface/connection_collect.go @@ -149,7 +149,7 @@ func NewConnectionCollect() (*ConnectionCollect, error) { slog.String("ip", config.MonitorConfig.Ip), slog.Int("port", ctlPort), ) - return nil, err + return nil, errors.Wrap(err, "connect spider ctl") } } diff --git a/items-config.sql b/items-config.sql new file mode 100644 index 0000000000..e69de29bb2