feat(mongodb): dbmon for mongodb TencentBlueKing#2788

dbm-services/go.work

@@ -27,4 +27,6 @@ use (
 	riak/db-tools/dbactuator
 	riak/db-tools/riak-monitor
 	sqlserver/db-tools/dbactuator
+	mongo/db-tools/dbmon
+	mongo/db-tools/mongo-toolkit-go
 )

dbm-services/mongo/db-tools/dbmon/.ci/codecc.yml

@@ -0,0 +1,29 @@
+version: v2.0
+resources:
+  repositories:
+    - repository: ci_templates/public/codecc
+      name: codecc
+on:
+  mr:
+    target-branches:  [ "*" ]
+stages:
+  - name: "代码检查"
+    check-out:
+      gates:
+        - template: commonGate.yml@codecc
+      timeout-hours: 10
+    jobs:
+      codecc:
+        name: "CodeCC代码检查"
+        runs-on:
+          pool-name: docker  #docker-on-devcloud、docker、local、agentless
+          container:
+            image: mirrors.tencent.com/ci/tlinux3_ci:2.0.0
+        steps:
+          - checkout: self
+          - uses: CodeccCheckAtomDebug@4.*
+            name: 腾讯代码分析
+            with:
+                beAutoLang: true # 自动检测项目语言
+                checkerSetType: "openScan" # 规则集类型，normal对应自主配置规则集，openScan对应按开源治理要求配置
+                toolScanType: "2" # 扫描方式。快速全量扫描[1] | 全量扫描[0] | 差异扫描[6] | MR/PR扫描[2]，默认为1

dbm-services/mongo/db-tools/dbmon/.ci/open_source_check.yml

@@ -0,0 +1,84 @@
+version: "v2.0"
+name: "开源检查"
+label: []
+variables: {}
+stages:
+- name: "开源检查"
+  label:
+  - "Build"
+  jobs:
+    job_AfK:
+      name: "构建环境-LINUX"
+      runs-on:
+        pool-name: "docker"
+        container:
+          image: "mirrors.tencent.com/ci/tlinux3_ci:2.3.0"
+        needs: {}
+      steps:
+      - checkout: self
+      - name: "敏感信息检查-部门RTX"
+        uses: "SensitiveRtxChecker@3.*"
+      - name: "腾讯代码分析(官方-代码分析工作组)"
+        uses: "CodeccCheckAtomDebug@4.*"
+        with:
+          beAutoLang: true
+          languages:
+          - "GOLANG"
+          checkerSetType: "communityOpenScan"
+          tools:
+          - "WOODPECKER_COMMITSCAN"
+          - "SCC"
+          - "PECKER_SECURITY"
+          - "SENSITIVE"
+          - "DUPC"
+          - "IP_CHECK"
+          - "WOODPECKER_SENSITIVE"
+          - "HORUSPY"
+          - "XCHECK"
+          - "CCN"
+          asyncTask: false
+          asyncTaskId: ""
+          scriptType: "SHELL"
+          script: |-
+            # Coverity/Klocwork将通过调用编译脚本来编译您的代码，以追踪深层次的缺陷
+            # 请使用依赖的构建工具如maven/cmake等写一个编译脚本build.sh
+            # 确保build.sh能够编译代码
+            # cd path/to/build.sh
+            # sh build.sh
+          languageRuleSetMap: {}
+          checkerSetEnvType: "prod"
+          multiPipelineMark: ""
+          rtxReceiverType: "1"
+          botWebhookUrl: ""
+          botRemindRange: "2"
+          botRemindSeverity: "7"
+          botRemaindTools: []
+          emailReceiverType: "1"
+          emailCCReceiverList: []
+          instantReportStatus: "2"
+          reportDate: []
+          reportTime: ""
+          reportTools: []
+          toolScanType: "1"
+          diffBranch: ""
+          byFile: false
+          mrCommentEnable: true
+          prohibitIgnore: false
+          newDefectJudgeFromDate: ""
+          transferAuthorList: []
+          path: []
+          customPath: []
+          scanTestSource: false
+          openScanPrj: false
+          openScanFilterEnable: false
+          issueSystem: "TAPD"
+          issueSubSystem: ""
+          issueResolvers: []
+          issueReceivers: []
+          issueFindByVersion: ""
+          maxIssue: 1000
+          issueAutoCommit: false
+  check-out:
+    gates:
+      - template: open_source_gate.yml
+    timeout-hours: 10

dbm-services/mongo/db-tools/dbmon/.ci/templates/open_source_gate.yml

@@ -0,0 +1,26 @@
+parameters:
+- name: receivers
+  type: array
+  default: [ "${{ ci.actor }}" ]
+
+gates:
+- name: open-source-gate
+  rule:
+    - "CodeccCheckAtomDebug.all_risk <= 0"
+    - "CodeccCheckAtomDebug.high_med_new_issue <= 0"
+    - "CodeccCheckAtomDebug.ccn_new_max_value <= 40"
+    - "CodeccCheckAtomDebug.sensitive_defect <= 0"
+    - "CodeccCheckAtomDebug.dupc_average <= 15"
+    - "CodeccCheckAtomDebug.ccn_average <= 3"
+    - "CodeccCheckAtomDebug.ccn_new_defect <= 0"
+    - "CodeccCheckAtomDebug.ccn_funcmax <= 20"
+    - "CodeccCheckAtomDebug.woodpecker_all_defect <= 0"
+    - "CodeccCheckAtomDebug.horuspy_all_defect <= 0"
+    - "CodeccCheckAtomDebug.go_serious_defect <= 0"
+    - "CodeccCheckAtomDebug.go_all_defect <= 100"
+  notify-on-fail:
+  - type: wework-message
+    receivers: ${{ parameters.receivers }}
+  continue-on-fail:
+    gatekeepers:
+    - "${{ ci.actor }}"

dbm-services/mongo/db-tools/dbmon/.gitignore

@@ -0,0 +1,9 @@
+logs
+.vscode
+.codecc
+bin
+build.yml
+bk-dbmon/
+build
+nohup.out
+.idea

dbm-services/mongo/db-tools/dbmon/Makefile

@@ -0,0 +1,18 @@
+SRV_NAME=bk-dbmon
+SHELL := /bin/bash
+VERSION = 0.0.1
+GITHASH = ""
+MODULE="dbm-services/redis/db-tools/dbmon/cmd"
+BUILD_FLAG = " -X ${MODULE}.version=${VERSION} -X ${MODULE}.githash=${GITHASH} -X ${MODULE}.buildstamp=`date +%Y-%m-%d_%H:%M:%S`"
+
+local:
+	CGO_ENABLED=0 go build -gcflags="all=-trimpath=${PWD}" -asmflags="all=-trimpath=${PWD}"   -ldflags ${BUILD_FLAG}  -o ./build/$(SRV_NAME) -v main.go
+
+build:
+	CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -gcflags="all=-trimpath=${PWD}" -asmflags="all=-trimpath=${PWD}"   -ldflags ${BUILD_FLAG}  -o ./build/$(SRV_NAME) -v main.go
+
+clean:
+	rm -rf ./build && go clean -cache
+
+
+.PHONY: init clean build

dbm-services/mongo/db-tools/dbmon/README.md

@@ -0,0 +1,49 @@
+### bk-dbmon
+本地例行任务集合,含例行全备、binlog备份、心跳等例行任务。
+
+#### 使用示例
+- **配置示例,文件名: dbmon-config.yaml**
+```yaml
+report_save_dir: /home/mysql/dbareport/
+report_left_day: 15
+backup_client_storage_type: ""
+http_address: 127.0.0.1:6677
+bkmonitorbeat:
+  agent_address: /usr/local/gse_bkte/agent/data/ipc.state.report
+  beat_path: /usr/local/gse_bkte/plugins/bin/bkmonitorbeat
+  event_config:
+    data_id: 1572877
+    token: 91049f4d2cb74881bcef201ebb7302fe
+  metric_config:
+    data_id: 1572876
+    token: 122a95858b174a908a2ab5f7443d546a
+
+servers:
+  - bk_biz_id: "3"
+    username: root
+    password: root
+    bk_cloud_id: 0
+    app: dba
+    app_name: DBA业务
+    cluster_domain: m1.test.dba.db
+    cluster_id: "12345"
+    cluster_name: test1
+    cluster_type: ReplicaSet
+    role_type: shardsvr
+    meta_role: backup
+    server_ip: 127.0.0.1
+    server_port: 27030
+    setname: set1   
+
+```
+
+- **启动**
+```sh
+./bk-dbmon-mg --config=dbmon-config.yaml
+```
+- **调试**
+```sh
+./bk-dbmon debug sendmsg   --config ./bk-dbmon-config.yaml  --port 27001 --type event --msg "test msg"
+./bk-dbmon debug sendmsg   --config ./bk-dbmon-config.yaml  --port 27001 --type ts
+```
+#### 架构

dbm-services/mongo/db-tools/dbmon/cmd/dbmonheartbeat/dbmonhearbeat.go

@@ -0,0 +1,2 @@
+// Package dbmonheartbeat TODO
+package dbmonheartbeat

dbm-services/mongo/db-tools/dbmon/cmd/dbmonheartbeat/job.go

@@ -0,0 +1,76 @@
+// Package dbmonheartbeat 心跳
+package dbmonheartbeat
+
+import (
+	"fmt"
+	"sync"
+
+	"dbm-services/mongo/db-tools/dbmon/config"
+	"dbm-services/mongo/db-tools/dbmon/mylog"
+	"dbm-services/mongo/db-tools/dbmon/pkg/sendwarning"
+)
+
+const MongoDbmonHeartBeatMetricName = "mongo_dbmon_heart_beat"
+
+// GlobDbmonHeartbeatJob global var
+var globDbmonHeartbeatJob *Job
+var dbmonHeartOnce sync.Once
+
+// Job 心跳job
+type Job struct {
+	Conf *config.Configuration `json:"conf"`
+	Name string                `json:"name"`
+	Err  error                 `json:"-"`
+}
+
+// GetGlobDbmonHeartbeatJob 新建上报心跳任务
+func GetGlobDbmonHeartbeatJob(conf *config.Configuration) *Job {
+	dbmonHeartOnce.Do(func() {
+		globDbmonHeartbeatJob = &Job{
+			Conf: conf,
+			Name: "dbmonHeartbeat",
+		}
+	})
+	return globDbmonHeartbeatJob
+}
+
+// Run 执行例行心跳metric上报 会带第一个实例的维度信息
+func (job *Job) Run() {
+	mylog.Logger.Info("SendDbmonHeartBeat start")
+	if len(job.Conf.Servers) == 0 {
+		mylog.Logger.Warn("no server in config")
+		return
+	}
+	err := SendHeartBeat(&job.Conf.BkMonitorBeat, &job.Conf.Servers[0])
+	if err != nil {
+		mylog.Logger.Warn(fmt.Sprintf("SendHeartBeat return err %s", err.Error()))
+	} else {
+		mylog.Logger.Info("SendDbmonHeartBeat done")
+	}
+
+}
+
+// SendHeartBeat 发送心跳
+func SendHeartBeat(conf *config.BkMonitorBeatConfig, serverConf *config.ConfServerItem) error {
+	msgH, err := sendwarning.NewBkMonitorEventSender(
+		conf.BeatPath,
+		conf.AgentAddress,
+	)
+	if err != nil {
+		return err
+	}
+	return msgH.SetBkBizID(serverConf.BkBizID).
+		SetBkCloudID(serverConf.BkCloudID).
+		SetApp(serverConf.App).
+		SetAppName(serverConf.AppName).
+		SetClusterDomain(serverConf.ClusterDomain).
+		SetClusterName(serverConf.ClusterName).
+		SetClusterType(serverConf.ClusterType).
+		SetInstanceRole(serverConf.MetaRole).
+		SendTimeSeriesMsg(conf.MetricConfig.DataID,
+			conf.MetricConfig.Token,
+			serverConf.ServerIP,
+			MongoDbmonHeartBeatMetricName,
+			1)
+
+}