feat(mongodb): 增加原子任务，cluster安装flow TencentBlueKing#3018

dbm-services/go.work

@@ -29,4 +29,5 @@ use (
 	sqlserver/db-tools/dbactuator
 	mongo/db-tools/dbmon
 	mongo/db-tools/mongo-toolkit-go
+	mongo/db-tools/dbactuator
 )

dbm-services/mongo/db-tools/dbactuator/.gitignore

@@ -0,0 +1,31 @@
+.idea
+.vscode
+logs/*
+bin/mongo-dbactuator
+scripts/upload_media.sh
+.codecc
+gonote
+goimports
+.agent.properties
+agent.zip
+codecc/
+devopsAgent
+devopsDaemon
+install.sh
+jre.zip
+jre/
+latest_version.txt
+preci
+preci.log
+preci.pid
+preci_server.jar
+runtime/
+start.sh
+stop.sh
+telegraf.conf
+tmp/
+uninstall.sh
+worker-agent.jar
+preci.port
+build.yml
+tests/dbactuator-test

dbm-services/mongo/db-tools/dbactuator/Makefile

@@ -0,0 +1,9 @@
+SRV_NAME=mongo-dbactuator
+
+clean:
+	-rm ./bin/${SRV_NAME}
+
+build:clean
+	CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build  -o ./bin/$(SRV_NAME) -v main.go
+
+.PHONY: init clean build

dbm-services/mongo/db-tools/dbactuator/README.md

@@ -0,0 +1,85 @@
+## mongo-dbactuator
+mongo原子任务合集,包含mongo复制集、cluster的创建，备份，回档等等原子任务。
+
+使用方式:
+```go
+./bin/mongo-dbactuator -h
+mongo原子任务合集,包含mongo复制集、cluster的创建，备份，回档等等原子任务。
+
+Usage:
+  mongo-dbactuator [flags]
+
+
+Flags:
+  -A, --atom-job-list string   多个原子任务名用','分割,如 redis_install,redis_replicaof
+  -B, --backup_dir string      备份保存路径,亦可通过环境变量MONGO_BACKUP_DIR指定
+  -D, --data_dir string        数据保存路径,亦可通过环境变量 MONGO_DATA_DIR 指定
+  -h, --help                   help for mongo-dbactuator
+  -N, --node_id string         节点id
+  -p, --payload string         原子任务参数信息,base64包裹
+  -f, --payload_file string    原子任务参数信息,json/yaml文件
+  -R, --root_id string         流程id
+  -t, --toggle                 Help message for toggle
+  -U, --uid string             单据id
+  -V, --version_id string      运行版本id
+  -u, --user string            db进程运行的os用户
+  -g, --group string           db进程运行的os用户的属主
+
+//执行示例
+./bin/dbactuator_redis --uid=1111 --root_id=2222 --node_id=3333 --version_id=v1 --payload='eyJkaXIiOiIvZGF0YS9yZWRpcy8zMDAwMCIsInBvcnQiOjMwMDAwLCJwYXNzd29yZCI6InBhc3MwMSIsInZlcnNpb24iOiJyZWRpcy00LjExLjEyIiwiZGF0YWJhc2VzIjoyfQ==' --atom-job-list="mongod_install"
+```
+
+### 架构图
+![架构图](./imgs/bk-dbactuator-mongo_structur.png)
+
+### 开发规范
+go开发规范参考: [https://google.github.io/styleguide/go/decisions](https://google.github.io/styleguide/go/decisions)
+
+### 开发流程
+- **step1(必须):`pkg/atomJobs`目录下添加类对象,如`pkg/atomJobs/atommongodb/mongod_install.go`**;
+以`type MongoDBInstall`为例。  
+需实现`JobRunner`中的相关接口:
+```go
+//JobRunner defines a behavior of a job
+type JobRunner interface {
+	// Init doing some operation before run a job
+	// such as reading parametes
+	Init(*JobGenericRuntime) error
+
+	// Name return the name of the job
+	Name() string
+
+	// Run run a job
+	Run() error
+
+	Retry() uint
+
+	// Rollback you can define some rollback logic here when job fails
+	Rollback() error
+}
+```
+而后实现一个New函数,该函数简单返回一个`*MongoDBInstall{}`即可,如:`func NewMongoDBInstall() jobruntime.JobRunner`;
+- **step2(必须):`pkg/jobmanager/jobmanager.go`中修改`GetAtomJobInstance()`函数**
+加一行代码即可。    
+```go
+//key名必须和./mongo-dbactuator --atom-job-list 参数中的保持一致;
+//value就是step1中的New函数;
+m.atomJobMapper["NewMongoDBInstall"] =  atommongodb.NewMongoDBInstall
+```
+- **step3(非必须):更新README.md中的“当前支持的原子任务”**
+
+### 注意事项
+- 第一: **`mongo-dbactuator`中每个原子任务，强烈建议可重入，即可反复执行**  
+虽然接口`JobRunner`中有`Rollback() error`实现需求，但其实不那么重要。    
+相比可回档，实现可重入有如下优势:
+  - **可重入实现难度更低, 基本上每个动作前先判断该动作是否已做过即可，而回档操作难度大，如100个redis实例建立主从关系，其中1个失败，99个成功，可重入实现简单，回档操作则非常麻烦；**
+  - **可重入风险更低，创建的回档动作是删除，删除的回档动作是创建。回档操作代码细微bug，影响很大；**
+  - **可重入对DBA和用户更实用，用户执行某个操作失败，用户基本诉求是重跑，完全不执行该操作了恢复环境需求很少；**
+
+### 当前支持的原子任务
+```go
+os_mongo_init   // mongo安装前，os初始化
+mongod_install // mongod安装
+mongos_replicaof // mongos安装
+...
+```

dbm-services/mongo/db-tools/dbactuator/cmd/root.go

@@ -0,0 +1,126 @@
+// Package cmd 根目录
+/*
+Copyright © 2022 NAME HERE <EMAIL ADDRESS>
+
+*/
+package cmd
+
+import (
+	"encoding/base64"
+	"log"
+	"os"
+
+	"dbm-services/mongo/db-tools/dbactuator/pkg/consts"
+	"dbm-services/mongo/db-tools/dbactuator/pkg/jobmanager"
+	"dbm-services/mongo/db-tools/dbactuator/pkg/util"
+
+	"github.com/spf13/cobra"
+)
+
+var uid string
+var rootID string
+var nodeID string
+var versionID string
+var dataDir string
+var backupDir string
+var payLoad string
+var payLoadFormat string
+var payLoadFile string
+var atomJobList string
+var user string
+var group string
+
+// RootCmd represents the base command when called without any subcommands
+var RootCmd = &cobra.Command{
+	Use:   "mongo-dbactuator",
+	Short: "mongo原子任务合集",
+	Long:  `mongo原子任务合集,包含mongo复制集、cluster的创建，备份，回档等等原子任务。`,
+	// Uncomment the following line if your bare application
+	// has an action associated with it:
+	Run: func(cmd *cobra.Command, args []string) {
+		var err error
+		dir, _ := util.GetCurrentDirectory()
+
+		// 优先使用payLoad。 payLoadFile 个人测试的时候使用的.
+		if payLoad == "" && payLoadFile != "" {
+			if o, err := os.ReadFile(payLoadFile); err == nil {
+				payLoad = base64.StdEncoding.EncodeToString(o)
+				log.Printf("using payload file %s", payLoadFile)
+			} else {
+				log.Printf("using payload file %s err %v", payLoadFile, err)
+			}
+		}
+
+		// 设置mongo环境变量
+		err = consts.SetMongoDataDir(dataDir)
+		if err != nil {
+			log.Println(err.Error())
+			os.Exit(-1)
+		}
+		err = consts.SetMongoBackupDir(backupDir)
+		if err != nil {
+			log.Println(err.Error())
+			os.Exit(-1)
+		}
+
+		err = consts.SetProcessUser(user)
+		if err != nil {
+			log.Println(err.Error())
+			os.Exit(-1)
+		}
+		err = consts.SetProcessUserGroup(group)
+		if err != nil {
+			log.Println(err.Error())
+			os.Exit(-1)
+		}
+
+		manager, err := jobmanager.NewJobGenericManager(uid, rootID, nodeID, versionID,
+			payLoad, payLoadFormat, atomJobList, dir)
+		if err != nil {
+			return
+		}
+		err = manager.LoadAtomJobs()
+		if err != nil {
+			os.Exit(-1)
+		}
+		err = manager.RunAtomJobs()
+		if err != nil {
+			os.Exit(-1)
+		}
+	},
+}
+
+// Execute adds all child commands to the root command and sets flags appropriately.
+// This is called by main.main(). It only needs to happen once to the rootCmd.
+func Execute() {
+	err := RootCmd.Execute()
+	if err != nil {
+		os.Exit(1)
+	}
+}
+
+func init() {
+	// Here you will define your flags and configuration settings.
+	// Cobra supports persistent flags, which, if defined here,
+	// will be global for your application.
+
+	// Cobra also supports local flags, which will only run
+	// when this action is called directly.
+	RootCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle")
+	RootCmd.PersistentFlags().StringVarP(&uid, "uid", "U", "", "单据id")
+	RootCmd.PersistentFlags().StringVarP(&rootID, "root_id", "R", "", "流程id")
+	RootCmd.PersistentFlags().StringVarP(&nodeID, "node_id", "N", "", "节点id")
+	RootCmd.PersistentFlags().StringVarP(&versionID, "version_id", "V", "", "运行版本id")
+	RootCmd.PersistentFlags().StringVarP(&dataDir, "data_dir", "D", "",
+		"数据保存路径,亦可通过环境变量 REDIS_DATA_DIR 指定")
+	RootCmd.PersistentFlags().StringVarP(&backupDir, "backup_dir", "B", "",
+		"备份保存路径,亦可通过环境变量REDIS_BACKUP_DIR指定")
+	RootCmd.PersistentFlags().StringVarP(&payLoad, "payload", "p", "", "原子任务参数信息,base64包裹")
+	RootCmd.PersistentFlags().StringVarP(&payLoadFormat, "payload-format", "m", "",
+		"command payload format, default base64, value_allowed: base64|raw")
+	RootCmd.PersistentFlags().StringVarP(&atomJobList, "atom-job-list", "A", "",
+		"多个原子任务名用','分割,如 redis_install,redis_replicaof")
+	RootCmd.PersistentFlags().StringVarP(&payLoadFile, "payload_file", "f", "", "原子任务参数信息,json文件")
+	RootCmd.PersistentFlags().StringVarP(&user, "user", "u", "", "开启进程的os用户")
+	RootCmd.PersistentFlags().StringVarP(&group, "group", "g", "", "开启进程的os用户属主")
+}

dbm-services/mongo/db-tools/dbactuator/example/add_shard_to_cluster.example.md

@@ -0,0 +1,24 @@
+### add_shard_to_cluster
+初始化新机器:
+
+```json
+./dbactuator_redis  --uid={{uid}} --root_id={{root_id}} --node_id={{node_id}} --version_id={{version_id}} --atom-job-list="add_shard_to_cluster"  --payload='{{payload_base64}}'
+```
+
+
+原始payload
+
+```json
+{
+  "ip":"10.1.1.1",
+  "port":27021,
+  "adminUsername":"xxx",
+  "adminPassword":"xxxxxxx",
+  "shard":{
+    "test-test1-s1":"10.1.1.2:27001,10.1.1.3:27002",
+    "test-test1-s2":"10.1.1.2:27004,10.1.1.3:27005",
+    "test-test1-s3":"10.1.1.3:27001,10.1.1.4:27002",
+    "test-test1-s4":"10.1.1.3:27004,10.1.1.4:27005"
+  }
+}
+```

dbm-services/mongo/db-tools/dbactuator/example/cluster_balancer.example.md

@@ -0,0 +1,20 @@
+### mongod_replace
+初始化新机器:
+
+```json
+./dbactuator_redis  --uid={{uid}} --root_id={{root_id}} --node_id={{node_id}} --version_id={{version_id}} --atom-job-list="cluster_balancer"  --payload='{{payload_base64}}'
+```
+
+
+原始payload
+
+```json
+{
+  "ip":"10.1.1.1",
+  "port":27021,
+  "open": false,
+  "adminUsername":"xxx",
+  "adminPassword":"xxxxxxxxx"
+}
+```
+"open"字段 true：表示打开balancer  false：表示关闭balancer

dbm-services/mongo/db-tools/dbactuator/example/initiate_replicaset.example.md

@@ -0,0 +1,35 @@
+### init_replicaset
+初始化新机器:
+
+```json
+./dbactuator_redis  --uid={{uid}} --root_id={{root_id}} --node_id={{node_id}} --version_id={{version_id}} --atom-job-list="init_replicaset"  --payload='{{payload_base64}}'
+```
+--data_dir、--backup_dir 可以留空. --user启动进程用户名，--group启动进程用户名的属组，如果为空默认都为mysql。
+
+原始payload
+
+```json
+{
+  "ip":"10.1.1.1",
+  "port":27001,
+  "app":"test",
+  "areaId":"test1",
+  "setId":"s1",
+  "configSvr":false,
+  "ips":[
+    "10.1.1.1:27001",
+    "10.1.1.2:27002",
+    "10.1.1.3:27003"
+  ],
+  "priority":{
+    "10.1.1.1:27001":1,
+    "10.1.1.2:27002":1,
+    "10.1.1.3:27003":0
+  },
+  "hidden":{
+    "10.1.1.1:27001":false,
+    "10.1.1.2:27002":false,
+    "10.1.1.3:27003":true
+  }
+}
+```

dbm-services/mongo/db-tools/dbactuator/example/mongo_add_user.example.md

@@ -0,0 +1,52 @@
+### add_user
+初始化新机器:
+
+```json
+./dbactuator_redis  --uid={{uid}} --root_id={{root_id}} --node_id={{node_id}} --version_id={{version_id}} --atom-job-list="add_user"  --payload='{{payload_base64}}'
+```
+
+
+原始payload
+
+创建管理员用户
+```json
+{
+  "ip":"10.1.1.1",
+  "port":27001,
+  "instanceType":"mongod",
+  "username":"xxx",
+  "password":"xxxxxxx",
+  "adminUsername":"",
+  "adminPassword":"",
+  "authDb":"admin",
+  "dbs":[
+
+  ],
+  "privileges":[
+    "root"
+  ]
+}
+```
+
+创建业务用户
+```json
+{
+  "ip":"10.1.1.1",
+  "port":27001,
+  "instanceType":"mongod",
+  "username":"xxx",
+  "password":"xxxxxxx",
+  "adminUsername":"xxx",
+  "adminPassword":"xxxxxxxx",
+  "authDb":"admin",
+  "dbs":[
+
+  ],
+  "privileges":[
+    "xxx"
+  ]
+}
+```
+
+
+"instanceType"字段   "mongod"：表示在复制集或者复制集单点进行创建用户  "mongos"：表示cluster进行创建用户