diff --git a/dbm-services/go.work b/dbm-services/go.work index 11f6a002d0..2d782a1dc9 100644 --- a/dbm-services/go.work +++ b/dbm-services/go.work @@ -29,4 +29,5 @@ use ( sqlserver/db-tools/dbactuator mongo/db-tools/dbmon mongo/db-tools/mongo-toolkit-go + mongo/db-tools/dbactuator ) diff --git a/dbm-services/mongo/db-tools/dbactuator/.gitignore b/dbm-services/mongo/db-tools/dbactuator/.gitignore new file mode 100644 index 0000000000..f6b4016d07 --- /dev/null +++ b/dbm-services/mongo/db-tools/dbactuator/.gitignore @@ -0,0 +1,31 @@ +.idea +.vscode +logs/* +bin/mongo-dbactuator +scripts/upload_media.sh +.codecc +gonote +goimports +.agent.properties +agent.zip +codecc/ +devopsAgent +devopsDaemon +install.sh +jre.zip +jre/ +latest_version.txt +preci +preci.log +preci.pid +preci_server.jar +runtime/ +start.sh +stop.sh +telegraf.conf +tmp/ +uninstall.sh +worker-agent.jar +preci.port +build.yml +tests/dbactuator-test diff --git a/dbm-services/mongo/db-tools/dbactuator/LICENSE b/dbm-services/mongo/db-tools/dbactuator/LICENSE new file mode 100644 index 0000000000..e69de29bb2 diff --git a/dbm-services/mongo/db-tools/dbactuator/Makefile b/dbm-services/mongo/db-tools/dbactuator/Makefile new file mode 100644 index 0000000000..04a58a4c97 --- /dev/null +++ b/dbm-services/mongo/db-tools/dbactuator/Makefile @@ -0,0 +1,9 @@ +SRV_NAME=mongo-dbactuator + +clean: + -rm ./bin/${SRV_NAME} + +build:clean + CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o ./bin/$(SRV_NAME) -v main.go + +.PHONY: init clean build diff --git a/dbm-services/mongo/db-tools/dbactuator/README.md b/dbm-services/mongo/db-tools/dbactuator/README.md new file mode 100644 index 0000000000..c34a3996ae --- /dev/null +++ b/dbm-services/mongo/db-tools/dbactuator/README.md @@ -0,0 +1,85 @@ +## mongo-dbactuator +mongo原子任务合集,包含mongo复制集、cluster的创建，备份，回档等等原子任务。 + +使用方式: +```go +./bin/mongo-dbactuator -h +mongo原子任务合集,包含mongo复制集、cluster的创建，备份，回档等等原子任务。 + +Usage: + mongo-dbactuator [flags] + + +Flags: + -A, --atom-job-list string 多个原子任务名用','分割,如 redis_install,redis_replicaof + -B, --backup_dir string 备份保存路径,亦可通过环境变量MONGO_BACKUP_DIR指定 + -D, --data_dir string 数据保存路径,亦可通过环境变量 MONGO_DATA_DIR 指定 + -h, --help help for mongo-dbactuator + -N, --node_id string 节点id + -p, --payload string 原子任务参数信息,base64包裹 + -f, --payload_file string 原子任务参数信息,json/yaml文件 + -R, --root_id string 流程id + -t, --toggle Help message for toggle + -U, --uid string 单据id + -V, --version_id string 运行版本id + -u, --user string db进程运行的os用户 + -g, --group string db进程运行的os用户的属主 + +//执行示例 +./bin/dbactuator_redis --uid=1111 --root_id=2222 --node_id=3333 --version_id=v1 --payload='eyJkaXIiOiIvZGF0YS9yZWRpcy8zMDAwMCIsInBvcnQiOjMwMDAwLCJwYXNzd29yZCI6InBhc3MwMSIsInZlcnNpb24iOiJyZWRpcy00LjExLjEyIiwiZGF0YWJhc2VzIjoyfQ==' --atom-job-list="mongod_install" +``` + +### 架构图 +![架构图](./imgs/bk-dbactuator-mongo_structur.png) + +### 开发规范 +go开发规范参考: [https://google.github.io/styleguide/go/decisions](https://google.github.io/styleguide/go/decisions) + +### 开发流程 +- **step1(必须):`pkg/atomJobs`目录下添加类对象,如`pkg/atomJobs/atommongodb/mongod_install.go`**; +以`type MongoDBInstall`为例。 +需实现`JobRunner`中的相关接口: +```go +//JobRunner defines a behavior of a job +type JobRunner interface { + // Init doing some operation before run a job + // such as reading parametes + Init(*JobGenericRuntime) error + + // Name return the name of the job + Name() string + + // Run run a job + Run() error + + Retry() uint + + // Rollback you can define some rollback logic here when job fails + Rollback() error +} +``` +而后实现一个New函数,该函数简单返回一个`*MongoDBInstall{}`即可,如:`func NewMongoDBInstall() jobruntime.JobRunner`; +- **step2(必须):`pkg/jobmanager/jobmanager.go`中修改`GetAtomJobInstance()`函数** +加一行代码即可。 +```go +//key名必须和./mongo-dbactuator --atom-job-list 参数中的保持一致; +//value就是step1中的New函数; +m.atomJobMapper["NewMongoDBInstall"] = atommongodb.NewMongoDBInstall +``` +- **step3(非必须):更新README.md中的“当前支持的原子任务”** + +### 注意事项 +- 第一: **`mongo-dbactuator`中每个原子任务，强烈建议可重入，即可反复执行** +虽然接口`JobRunner`中有`Rollback() error`实现需求，但其实不那么重要。 +相比可回档，实现可重入有如下优势: + - **可重入实现难度更低, 基本上每个动作前先判断该动作是否已做过即可，而回档操作难度大，如100个redis实例建立主从关系，其中1个失败，99个成功，可重入实现简单，回档操作则非常麻烦；** + - **可重入风险更低，创建的回档动作是删除，删除的回档动作是创建。回档操作代码细微bug，影响很大；** + - **可重入对DBA和用户更实用，用户执行某个操作失败，用户基本诉求是重跑，完全不执行该操作了恢复环境需求很少；** + +### 当前支持的原子任务 +```go +os_mongo_init // mongo安装前，os初始化 +mongod_install // mongod安装 +mongos_replicaof // mongos安装 +... +``` \ No newline at end of file diff --git a/dbm-services/mongo/db-tools/dbactuator/cmd/root.go b/dbm-services/mongo/db-tools/dbactuator/cmd/root.go new file mode 100644 index 0000000000..71291f6cc4 --- /dev/null +++ b/dbm-services/mongo/db-tools/dbactuator/cmd/root.go @@ -0,0 +1,126 @@ +// Package cmd 根目录 +/* +Copyright © 2022 NAME HERE + +*/ +package cmd + +import ( + "encoding/base64" + "log" + "os" + + "dbm-services/mongo/db-tools/dbactuator/pkg/consts" + "dbm-services/mongo/db-tools/dbactuator/pkg/jobmanager" + "dbm-services/mongo/db-tools/dbactuator/pkg/util" + + "github.com/spf13/cobra" +) + +var uid string +var rootID string +var nodeID string +var versionID string +var dataDir string +var backupDir string +var payLoad string +var payLoadFormat string +var payLoadFile string +var atomJobList string +var user string +var group string + +// RootCmd represents the base command when called without any subcommands +var RootCmd = &cobra.Command{ + Use: "mongo-dbactuator", + Short: "mongo原子任务合集", + Long: `mongo原子任务合集,包含mongo复制集、cluster的创建，备份，回档等等原子任务。`, + // Uncomment the following line if your bare application + // has an action associated with it: + Run: func(cmd *cobra.Command, args []string) { + var err error + dir, _ := util.GetCurrentDirectory() + + // 优先使用payLoad。 payLoadFile 个人测试的时候使用的. + if payLoad == "" && payLoadFile != "" { + if o, err := os.ReadFile(payLoadFile); err == nil { + payLoad = base64.StdEncoding.EncodeToString(o) + log.Printf("using payload file %s", payLoadFile) + } else { + log.Printf("using payload file %s err %v", payLoadFile, err) + } + } + + // 设置mongo环境变量 + err = consts.SetMongoDataDir(dataDir) + if err != nil { + log.Println(err.Error()) + os.Exit(-1) + } + err = consts.SetMongoBackupDir(backupDir) + if err != nil { + log.Println(err.Error()) + os.Exit(-1) + } + + err = consts.SetProcessUser(user) + if err != nil { + log.Println(err.Error()) + os.Exit(-1) + } + err = consts.SetProcessUserGroup(group) + if err != nil { + log.Println(err.Error()) + os.Exit(-1) + } + + manager, err := jobmanager.NewJobGenericManager(uid, rootID, nodeID, versionID, + payLoad, payLoadFormat, atomJobList, dir) + if err != nil { + return + } + err = manager.LoadAtomJobs() + if err != nil { + os.Exit(-1) + } + err = manager.RunAtomJobs() + if err != nil { + os.Exit(-1) + } + }, +} + +// Execute adds all child commands to the root command and sets flags appropriately. +// This is called by main.main(). It only needs to happen once to the rootCmd. +func Execute() { + err := RootCmd.Execute() + if err != nil { + os.Exit(1) + } +} + +func init() { + // Here you will define your flags and configuration settings. + // Cobra supports persistent flags, which, if defined here, + // will be global for your application. + + // Cobra also supports local flags, which will only run + // when this action is called directly. + RootCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle") + RootCmd.PersistentFlags().StringVarP(&uid, "uid", "U", "", "单据id") + RootCmd.PersistentFlags().StringVarP(&rootID, "root_id", "R", "", "流程id") + RootCmd.PersistentFlags().StringVarP(&nodeID, "node_id", "N", "", "节点id") + RootCmd.PersistentFlags().StringVarP(&versionID, "version_id", "V", "", "运行版本id") + RootCmd.PersistentFlags().StringVarP(&dataDir, "data_dir", "D", "", + "数据保存路径,亦可通过环境变量 REDIS_DATA_DIR 指定") + RootCmd.PersistentFlags().StringVarP(&backupDir, "backup_dir", "B", "", + "备份保存路径,亦可通过环境变量REDIS_BACKUP_DIR指定") + RootCmd.PersistentFlags().StringVarP(&payLoad, "payload", "p", "", "原子任务参数信息,base64包裹") + RootCmd.PersistentFlags().StringVarP(&payLoadFormat, "payload-format", "m", "", + "command payload format, default base64, value_allowed: base64|raw") + RootCmd.PersistentFlags().StringVarP(&atomJobList, "atom-job-list", "A", "", + "多个原子任务名用','分割,如 redis_install,redis_replicaof") + RootCmd.PersistentFlags().StringVarP(&payLoadFile, "payload_file", "f", "", "原子任务参数信息,json文件") + RootCmd.PersistentFlags().StringVarP(&user, "user", "u", "", "开启进程的os用户") + RootCmd.PersistentFlags().StringVarP(&group, "group", "g", "", "开启进程的os用户属主") +} diff --git a/dbm-services/mongo/db-tools/dbactuator/example/add_shard_to_cluster.example.md b/dbm-services/mongo/db-tools/dbactuator/example/add_shard_to_cluster.example.md new file mode 100644 index 0000000000..b42c70cc4e --- /dev/null +++ b/dbm-services/mongo/db-tools/dbactuator/example/add_shard_to_cluster.example.md @@ -0,0 +1,24 @@ +### add_shard_to_cluster +初始化新机器: + +```json +./dbactuator_redis --uid={{uid}} --root_id={{root_id}} --node_id={{node_id}} --version_id={{version_id}} --atom-job-list="add_shard_to_cluster" --payload='{{payload_base64}}' +``` + + +原始payload + +```json +{ + "ip":"10.1.1.1", + "port":27021, + "adminUsername":"xxx", + "adminPassword":"xxxxxxx", + "shard":{ + "test-test1-s1":"10.1.1.2:27001,10.1.1.3:27002", + "test-test1-s2":"10.1.1.2:27004,10.1.1.3:27005", + "test-test1-s3":"10.1.1.3:27001,10.1.1.4:27002", + "test-test1-s4":"10.1.1.3:27004,10.1.1.4:27005" + } +} +``` \ No newline at end of file diff --git a/dbm-services/mongo/db-tools/dbactuator/example/cluster_balancer.example.md b/dbm-services/mongo/db-tools/dbactuator/example/cluster_balancer.example.md new file mode 100644 index 0000000000..579242159b --- /dev/null +++ b/dbm-services/mongo/db-tools/dbactuator/example/cluster_balancer.example.md @@ -0,0 +1,20 @@ +### mongod_replace +初始化新机器: + +```json +./dbactuator_redis --uid={{uid}} --root_id={{root_id}} --node_id={{node_id}} --version_id={{version_id}} --atom-job-list="cluster_balancer" --payload='{{payload_base64}}' +``` + + +原始payload + +```json +{ + "ip":"10.1.1.1", + "port":27021, + "open": false, + "adminUsername":"xxx", + "adminPassword":"xxxxxxxxx" +} +``` +"open"字段 true：表示打开balancer false：表示关闭balancer diff --git a/dbm-services/mongo/db-tools/dbactuator/example/initiate_replicaset.example.md b/dbm-services/mongo/db-tools/dbactuator/example/initiate_replicaset.example.md new file mode 100644 index 0000000000..1ddff6024c --- /dev/null +++ b/dbm-services/mongo/db-tools/dbactuator/example/initiate_replicaset.example.md @@ -0,0 +1,35 @@ +### init_replicaset +初始化新机器: + +```json +./dbactuator_redis --uid={{uid}} --root_id={{root_id}} --node_id={{node_id}} --version_id={{version_id}} --atom-job-list="init_replicaset" --payload='{{payload_base64}}' +``` +--data_dir、--backup_dir 可以留空. --user启动进程用户名，--group启动进程用户名的属组，如果为空默认都为mysql。 + +原始payload + +```json +{ + "ip":"10.1.1.1", + "port":27001, + "app":"test", + "areaId":"test1", + "setId":"s1", + "configSvr":false, + "ips":[ + "10.1.1.1:27001", + "10.1.1.2:27002", + "10.1.1.3:27003" + ], + "priority":{ + "10.1.1.1:27001":1, + "10.1.1.2:27002":1, + "10.1.1.3:27003":0 + }, + "hidden":{ + "10.1.1.1:27001":false, + "10.1.1.2:27002":false, + "10.1.1.3:27003":true + } +} +``` \ No newline at end of file diff --git a/dbm-services/mongo/db-tools/dbactuator/example/mongo_add_user.example.md b/dbm-services/mongo/db-tools/dbactuator/example/mongo_add_user.example.md new file mode 100644 index 0000000000..1920afbca5 --- /dev/null +++ b/dbm-services/mongo/db-tools/dbactuator/example/mongo_add_user.example.md @@ -0,0 +1,52 @@ +### add_user +初始化新机器: + +```json +./dbactuator_redis --uid={{uid}} --root_id={{root_id}} --node_id={{node_id}} --version_id={{version_id}} --atom-job-list="add_user" --payload='{{payload_base64}}' +``` + + +原始payload + +创建管理员用户 +```json +{ + "ip":"10.1.1.1", + "port":27001, + "instanceType":"mongod", + "username":"xxx", + "password":"xxxxxxx", + "adminUsername":"", + "adminPassword":"", + "authDb":"admin", + "dbs":[ + + ], + "privileges":[ + "root" + ] +} +``` + +创建业务用户 +```json +{ + "ip":"10.1.1.1", + "port":27001, + "instanceType":"mongod", + "username":"xxx", + "password":"xxxxxxx", + "adminUsername":"xxx", + "adminPassword":"xxxxxxxx", + "authDb":"admin", + "dbs":[ + + ], + "privileges":[ + "xxx" + ] +} +``` + + +"instanceType"字段 "mongod"：表示在复制集或者复制集单点进行创建用户 "mongos"：表示cluster进行创建用户 \ No newline at end of file diff --git a/dbm-services/mongo/db-tools/dbactuator/example/mongo_deinstall.example.md b/dbm-services/mongo/db-tools/dbactuator/example/mongo_deinstall.example.md new file mode 100644 index 0000000000..9374f50c74 --- /dev/null +++ b/dbm-services/mongo/db-tools/dbactuator/example/mongo_deinstall.example.md @@ -0,0 +1,22 @@ +### mongo_deinstall +初始化新机器: + +```json +./dbactuator_redis --uid={{uid}} --root_id={{root_id}} --node_id={{node_id}} --version_id={{version_id}} --atom-job-list="mongo_deinstall" --payload='{{payload_base64}}' +``` + +原始payload +```json +{ + "ip":"10.1.1.1", + "port":27002, + "app":"test", + "areaId":"test1", + "nodeInfo":[ + "10.1.1.1", + "10.1.1.2" + ], + "instanceType":"mongod" +} +``` + diff --git a/dbm-services/mongo/db-tools/dbactuator/example/mongo_del_user.example.md b/dbm-services/mongo/db-tools/dbactuator/example/mongo_del_user.example.md new file mode 100644 index 0000000000..d9f284ab45 --- /dev/null +++ b/dbm-services/mongo/db-tools/dbactuator/example/mongo_del_user.example.md @@ -0,0 +1,34 @@ +### delete_user +初始化新机器: + +```json +./dbactuator_redis --uid={{uid}} --root_id={{root_id}} --node_id={{node_id}} --version_id={{version_id}} --atom-job-list="delete_user" --payload='{{payload_base64}}' +``` + + +原始payload +mongos删除业务用户 +```json +{ + "ip":"10.1.1.1", + "port":27023, + "instanceType":"mongos", + "adminUsername":"xxx", + "adminPassword":"xxxxx", + "username":"xx", + "authDb":"admin" +} +``` + +mongod删除业务用户 +```json +{ + "ip":"10.1.1.1", + "port":27001, + "instanceType":"mongod", + "adminUsername":"xxx", + "adminPassword":"xxxx", + "username":"xx", + "authDb":"admin" +} +``` \ No newline at end of file diff --git a/dbm-services/mongo/db-tools/dbactuator/example/mongo_execute_script.example.md b/dbm-services/mongo/db-tools/dbactuator/example/mongo_execute_script.example.md new file mode 100644 index 0000000000..54356f9e28 --- /dev/null +++ b/dbm-services/mongo/db-tools/dbactuator/example/mongo_execute_script.example.md @@ -0,0 +1,30 @@ +### mongo_execute_script +初始化新机器: + +```json +./dbactuator_redis --uid={{uid}} --root_id={{root_id}} --node_id={{node_id}} --version_id={{version_id}} --atom-job-list="mongo_execute_script" --payload='{{payload_base64}}' +``` + + +原始payload + +# 原始payload +```json +{ + "ip":"10.1.1.1", + "port":27021, + "script":"xxx", + "type":"cluster", + "secondary": false, + "adminUsername":"xxx", + "adminPassword":"xxxxxx", + "repoUrl":"url", + "repoUsername":"username", + "repoToken":"token", + "repoProject":"project", + "repoRepo":"project-package", + "repoPath":"path" +} +``` + +以repo为前缀的字段为制品库信息 \ No newline at end of file diff --git a/dbm-services/mongo/db-tools/dbactuator/example/mongo_process_restart.example.md b/dbm-services/mongo/db-tools/dbactuator/example/mongo_process_restart.example.md new file mode 100644 index 0000000000..4fe5bd03f8 --- /dev/null +++ b/dbm-services/mongo/db-tools/dbactuator/example/mongo_process_restart.example.md @@ -0,0 +1,42 @@ +### mongo_restart +初始化新机器: + +```json +./dbactuator_redis --uid={{uid}} --root_id={{root_id}} --node_id={{node_id}} --version_id={{version_id}} --atom-job-list="mongo_restart" --payload='{{payload_base64}}' +``` + + +原始payload + +## mongod +```json +{ + "ip":"10.1.1.1", + "port":27001, + "instanceType":"mongod", + "singleNodeInstallRestart":false, + "auth":true, + "cacheSizeGB": null, + "mongoSConfDbOld":"", + "MongoSConfDbNew":"", + "adminUsername":"", + "adminPassword":"" +} +``` +"singleNodeInstallRestart"字段表示安装替换节点时mongod单节点重启 true：替换节点单节点重启 false：复制集节点重启 +"adminUsername"和"adminPassword"字段为空时表示安装时最后一步重启进程，不为空时表示提供服务期间重启 +## mongos +```json +{ + "ip":"10.1.1.1", + "port":27021, + "instanceType":"mongos", + "singleNodeInstallRestart":false, + "auth":true, + "cacheSizeGB": null, + "mongoSConfDbOld":"10.1.1.2:27001", + "MongoSConfDbNew":"10.1.1.2:27004", + "adminUsername":"", + "adminPassword":"" +} +``` \ No newline at end of file diff --git a/dbm-services/mongo/db-tools/dbactuator/example/mongod_install.example.md b/dbm-services/mongo/db-tools/dbactuator/example/mongod_install.example.md new file mode 100644 index 0000000000..5aa9bfb27b --- /dev/null +++ b/dbm-services/mongo/db-tools/dbactuator/example/mongod_install.example.md @@ -0,0 +1,60 @@ +### mongod_install +初始化新机器: + +```json +./dbactuator_redis --uid={{uid}} --root_id={{root_id}} --node_id={{node_id}} --version_id={{version_id}} --atom-job-list="mongod_install" --data_dir=/path/to/data --backup_dir=/path/to/backup --user="xxx" --group="xxx" --payload='{{payload_base64}}' +``` +--data_dir、--backup_dir 可以留空. --user启动进程用户名，--group启动进程用户名的属组，如果为空默认都为mysql。 + +原始payload + +## shardsvr +```json +{ + "mediapkg":{ + "pkg":"mongodb-linux-x86_64-3.4.20.tar.gz", + "pkg_md5":"e68d998d75df81b219e99795dec43ffb" + }, + "ip":"10.1.1.1", + "port":27001, + "dbVersion":"3.4.20", + "instanceType":"mongod", + "app":"test", + "areaId":"test1", + "setId":"s1", + "auth": true, + "clusterRole":"shardsvr", + "dbConfig":{ + "slowOpThresholdMs":200, + "cacheSizeGB":1, + "oplogSizeMB":500, + "destination":"file" + } +} +``` +部署复制集时"clusterRole"字段为空 + +## configsvr +```json +{ + "mediapkg":{ + "pkg":"mongodb-linux-x86_64-3.4.20.tar.gz", + "pkg_md5":"e68d998d75df81b219e99795dec43ffb" + }, + "ip":"10.1.1.1", + "port":27002, + "dbVersion":"3.4.20", + "instanceType":"mongod", + "app":"test", + "areaId":"test1", + "setId":"conf", + "auth": true, + "clusterRole":"configsvr", + "dbConfig":{ + "slowOpThresholdMs":200, + "cacheSizeGB":1, + "oplogSizeMB":500, + "destination":"file" + } +} +``` \ No newline at end of file diff --git a/dbm-services/mongo/db-tools/dbactuator/example/mongod_replace.example.md b/dbm-services/mongo/db-tools/dbactuator/example/mongod_replace.example.md new file mode 100644 index 0000000000..ca568d908c --- /dev/null +++ b/dbm-services/mongo/db-tools/dbactuator/example/mongod_replace.example.md @@ -0,0 +1,29 @@ +### mongod_replace +初始化新机器: + +```json +./dbactuator_redis --uid={{uid}} --root_id={{root_id}} --node_id={{node_id}} --version_id={{version_id}} --atom-job-list="mongod_replace" --payload='{{payload_base64}}' +``` + + +原始payload + +## mongod +```json +{ + "ip":"10.1.1.1", + "port":27002, + "sourceIP":"10.1.1.3", + "sourcePort":27007, + "sourceDown":true, + "adminUsername":"xxx", + "adminPassword":"xxxxxxxx", + "targetIP":"10.1.1.1", + "targetPort":27004, + "targetPriority":"", + "targetHidden":"" +} +``` +"sourceDown" 源端是否已down机 +"targetPriority"可以指定替换节点的优先级 +"targetHidden"可以指定替换节点是否为隐藏节点 \ No newline at end of file diff --git a/dbm-services/mongo/db-tools/dbactuator/example/mongod_step_down.example.md b/dbm-services/mongo/db-tools/dbactuator/example/mongod_step_down.example.md new file mode 100644 index 0000000000..8730c595d1 --- /dev/null +++ b/dbm-services/mongo/db-tools/dbactuator/example/mongod_step_down.example.md @@ -0,0 +1,18 @@ +### replicaset_stepdown +初始化新机器: + +```json +./dbactuator_redis --uid={{uid}} --root_id={{root_id}} --node_id={{node_id}} --version_id={{version_id}} --atom-job-list="replicaset_stepdown" --payload='{{payload_base64}}' +``` + + +原始payload + +```json +{ + "ip":"10.1.1.1", + "port":27001, + "adminUsername":"xxx", + "adminPassword":"xxx" +} +``` \ No newline at end of file diff --git a/dbm-services/mongo/db-tools/dbactuator/example/mongos_install.example.md b/dbm-services/mongo/db-tools/dbactuator/example/mongos_install.example.md new file mode 100644 index 0000000000..794ea00679 --- /dev/null +++ b/dbm-services/mongo/db-tools/dbactuator/example/mongos_install.example.md @@ -0,0 +1,30 @@ +### mongos_install +初始化新机器: + +```json +./dbactuator_redis --uid={{uid}} --root_id={{root_id}} --node_id={{node_id}} --version_id={{version_id}} --atom-job-list="mongos_install" --data_dir=/path/to/data --backup_dir=/path/to/backup --user="xxx" --group="xxx" --payload='{{payload_base64}}' +``` +--data_dir、--backup_dir 可以留空. --user启动进程用户名，--group启动进程用户名的属组，如果为空默认都为mysql。 + +原始payload + +```json +{ + "mediapkg":{ + "pkg":"mongodb-linux-x86_64-3.4.20.tar.gz", + "pkg_md5":"e68d998d75df81b219e99795dec43ffb" + }, + "ip":"10.1.1.1", + "port":27021, + "dbVersion":"3.4.20", + "instanceType":"mongos", + "app":"test", + "areaId":"test1", + "auth": true, + "configDB":["10.1.1.2:27001","10.1.1.3:27002","10.1.1.4:27003"], + "dbConfig":{ + "slowOpThresholdMs":200, + "destination":"file" + } +} +``` \ No newline at end of file diff --git a/dbm-services/mongo/db-tools/dbactuator/example/os_mongo_init.example.md b/dbm-services/mongo/db-tools/dbactuator/example/os_mongo_init.example.md new file mode 100644 index 0000000000..3aa463c70e --- /dev/null +++ b/dbm-services/mongo/db-tools/dbactuator/example/os_mongo_init.example.md @@ -0,0 +1,16 @@ +### os_mongo_init +初始化新机器: + +```json +./dbactuator_redis --uid={{uid}} --root_id={{root_id}} --node_id={{node_id}} --version_id={{version_id}} --atom-job-list="os_mongo_init" --data_dir=/path/to/data --backup_dir=/path/to/backup --user="xxx" --group="xxx" --payload='{{payload_base64}}' +``` +--data_dir、--backup_dir 可以留空. --user启动进程用户名，--group启动进程用户名的属组，如果为空默认都为mysql。 + +原始payload + +```json +{ +"user":"xxx", +"password":"xxxxxxx" +} +``` \ No newline at end of file diff --git a/dbm-services/mongo/db-tools/dbactuator/go.mod b/dbm-services/mongo/db-tools/dbactuator/go.mod new file mode 100644 index 0000000000..3d1fc24785 --- /dev/null +++ b/dbm-services/mongo/db-tools/dbactuator/go.mod @@ -0,0 +1,28 @@ +module dbm-services/mongo/db-tools/dbactuator + +go 1.18 + +require ( + github.com/dustin/go-humanize v1.0.0 + github.com/go-playground/validator/v10 v10.11.0 + github.com/shirou/gopsutil/v3 v3.23.1 + github.com/spf13/cobra v1.7.0 + golang.org/x/sys v0.11.0 + gopkg.in/yaml.v2 v2.4.0 +) + +require ( + github.com/go-ole/go-ole v1.2.6 // indirect + github.com/go-playground/locales v0.14.0 // indirect + github.com/go-playground/universal-translator v0.18.0 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/leodido/go-urn v1.2.1 // indirect + github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect + github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect + github.com/spf13/pflag v1.0.5 // indirect + github.com/tklauser/go-sysconf v0.3.11 // indirect + github.com/tklauser/numcpus v0.6.0 // indirect + github.com/yusufpapurcu/wmi v1.2.2 // indirect + golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4 // indirect + golang.org/x/text v0.3.7 // indirect +) diff --git a/dbm-services/mongo/db-tools/dbactuator/go.sum b/dbm-services/mongo/db-tools/dbactuator/go.sum new file mode 100644 index 0000000000..91ee4b3f96 --- /dev/null +++ b/dbm-services/mongo/db-tools/dbactuator/go.sum @@ -0,0 +1,95 @@ +github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= +github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= +github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= +github.com/go-playground/assert/v2 v2.0.1 h1:MsBgLAaY856+nPRTKrp3/OZK38U/wa0CcBYNjji3q3A= +github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= +github.com/go-playground/locales v0.14.0 h1:u50s323jtVGugKlcYeyzC0etD1HifMjqmJqb8WugfUU= +github.com/go-playground/locales v0.14.0/go.mod h1:sawfccIbzZTqEDETgFXqTho0QybSa7l++s0DH+LDiLs= +github.com/go-playground/universal-translator v0.18.0 h1:82dyy6p4OuJq4/CByFNOn/jYrnRPArHwAcmLoJZxyho= +github.com/go-playground/universal-translator v0.18.0/go.mod h1:UvRDBj+xPUEGrFYl+lu/H90nyDXpg0fqeB/AQUGNTVA= +github.com/go-playground/validator/v10 v10.11.0 h1:0W+xRM511GY47Yy3bZUbJVitCNg2BOGlCyvTqsp/xIw= +github.com/go-playground/validator/v10 v10.11.0/go.mod h1:i+3WkQ1FvaUjjxh1kSvIA4dMGDBiPU55YFDl0WbKdWU= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= +github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/leodido/go-urn v1.2.1 h1:BqpAaACuzVSgi/VLzGZIobT2z4v53pjosyNd9Yv6n/w= +github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= +github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= +github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw= +github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= +github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= +github.com/rogpeppe/go-internal v1.8.0 h1:FCbCCtXNOY3UtUuHUYaghJg4y7Fd14rXifAYUAtL9R8= +github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/shirou/gopsutil/v3 v3.23.1 h1:a9KKO+kGLKEvcPIs4W62v0nu3sciVDOOOPUD0Hz7z/4= +github.com/shirou/gopsutil/v3 v3.23.1/go.mod h1:NN6mnm5/0k8jw4cBfCnJtr5L7ErOTg18tMNpgFkn0hA= +github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= +github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/tklauser/go-sysconf v0.3.11 h1:89WgdJhk5SNwJfu+GKyYveZ4IaJ7xAkecBo+KdJV0CM= +github.com/tklauser/go-sysconf v0.3.11/go.mod h1:GqXfhXY3kiPa0nAXPDIQIWzJbMCB7AmcWpGR8lSZfqI= +github.com/tklauser/numcpus v0.6.0 h1:kebhY2Qt+3U6RNK7UqpYNA+tJ23IBEGKkB7JQBfDYms= +github.com/tklauser/numcpus v0.6.0/go.mod h1:FEZLMke0lhOUG6w2JadTzp0a+Nl8PF/GFkQ5UVIcaL4= +github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg= +github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4 h1:kUhD7nTDoI3fVd9G4ORWrbV5NY0liEs/Jg2pv5f+bBA= +golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= +golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/dbm-services/mongo/db-tools/dbactuator/imgs/bk-dbactuator-mongo_structur.png b/dbm-services/mongo/db-tools/dbactuator/imgs/bk-dbactuator-mongo_structur.png new file mode 100644 index 0000000000..4cc723ed15 Binary files /dev/null and b/dbm-services/mongo/db-tools/dbactuator/imgs/bk-dbactuator-mongo_structur.png differ diff --git a/dbm-services/mongo/db-tools/dbactuator/main.go b/dbm-services/mongo/db-tools/dbactuator/main.go new file mode 100644 index 0000000000..9ae21d63e2 --- /dev/null +++ b/dbm-services/mongo/db-tools/dbactuator/main.go @@ -0,0 +1,12 @@ +// Package main main func +/* +Copyright © 2022 NAME HERE + +*/ +package main + +import "dbm-services/mongo/db-tools/dbactuator/cmd" + +func main() { + cmd.Execute() +} diff --git a/dbm-services/mongo/db-tools/dbactuator/mylog/mylog.go b/dbm-services/mongo/db-tools/dbactuator/mylog/mylog.go new file mode 100644 index 0000000000..8c50d50fa0 --- /dev/null +++ b/dbm-services/mongo/db-tools/dbactuator/mylog/mylog.go @@ -0,0 +1,29 @@ +// Package mylog 便于全局日志操作 +package mylog + +import ( + "os" + + "dbm-services/common/go-pubpkg/logger" +) + +// Logger 和 jobruntime.Logger 是同一个 logger +var Logger *logger.Logger + +// SetDefaultLogger 设置默认logger +func SetDefaultLogger(log *logger.Logger) { + Logger = log +} + +// UnitTestInitLog 单元测试初始化Logger +func UnitTestInitLog() { + extMap := map[string]string{ + "uid": "1111", + "node_id": "localhost", + "root_id": "2222", + "version_id": "3333", + } + log01 := logger.New(os.Stdout, true, logger.InfoLevel, extMap) + log01.Sync() + SetDefaultLogger(log01) +} diff --git a/dbm-services/mongo/db-tools/dbactuator/pkg/atomjobs/atommongodb/add_shard_to_cluster.go b/dbm-services/mongo/db-tools/dbactuator/pkg/atomjobs/atommongodb/add_shard_to_cluster.go new file mode 100644 index 0000000000..5936df4598 --- /dev/null +++ b/dbm-services/mongo/db-tools/dbactuator/pkg/atomjobs/atommongodb/add_shard_to_cluster.go @@ -0,0 +1,248 @@ +package atommongodb + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + "time" + + "dbm-services/mongo/db-tools/dbactuator/pkg/common" + "dbm-services/mongo/db-tools/dbactuator/pkg/consts" + "dbm-services/mongo/db-tools/dbactuator/pkg/jobruntime" + "dbm-services/mongo/db-tools/dbactuator/pkg/util" + + "github.com/go-playground/validator/v10" +) + +// AddConfParams 参数 +type AddConfParams struct { + IP string `json:"ip" validate:"required"` + Port int `json:"port" validate:"required"` + AdminUsername string `json:"adminUsername" validate:"required"` + AdminPassword string `json:"adminPassword" validate:"required"` + Shards map[string]string `json:"shards" validate:"required"` // key->clusterId,value->ip:port,ip:port 不包含隐藏节点 +} + +// AddShardToCluster 添加分片到集群 +type AddShardToCluster struct { + runtime *jobruntime.JobGenericRuntime + BinDir string + Mongo string + OsUser string + ConfFilePath string + ConfFileContent string + ConfParams *AddConfParams +} + +// NewAddShardToCluster 实例化结构体 +func NewAddShardToCluster() jobruntime.JobRunner { + return &AddShardToCluster{} +} + +// Name 获取原子任务的名字 +func (a *AddShardToCluster) Name() string { + return "add_shard_to_cluster" +} + +// Run 运行原子任务 +func (a *AddShardToCluster) Run() error { + // 获取配置内容 + if err := a.makeConfContent(); err != nil { + return err + } + + // 生成js脚本 + if err := a.createAddShardToClusterScript(); err != nil { + return err + } + + // 执行js脚本 + if err := a.execScript(); err != nil { + return err + } + + return nil +} + +// Retry 重试 +func (a *AddShardToCluster) Retry() uint { + return 2 +} + +// Rollback 回滚 +func (a *AddShardToCluster) Rollback() error { + return nil +} + +// Init 初始化 +func (a *AddShardToCluster) Init(runtime *jobruntime.JobGenericRuntime) error { + // 获取安装参数 + a.runtime = runtime + a.runtime.Logger.Info("start to init") + a.BinDir = consts.UsrLocal + a.Mongo = filepath.Join(a.BinDir, "mongodb", "bin", "mongo") + a.ConfFilePath = filepath.Join("/", "tmp", "addShardToCluster.js") + a.OsUser = consts.GetProcessUser() + + // 获取MongoDB配置文件参数 + if err := json.Unmarshal([]byte(a.runtime.PayloadDecoded), &a.ConfParams); err != nil { + a.runtime.Logger.Error(fmt.Sprintf( + "get parameters of initiateReplicaset fail by json.Unmarshal, error:%s", err)) + return fmt.Errorf("get parameters of initiateReplicaset fail by json.Unmarshal, error:%s", err) + } + a.runtime.Logger.Info("init successfully") + + // 进行校验 + if err := a.checkParams(); err != nil { + return err + } + + return nil +} + +// checkParams 校验参数 +func (a *AddShardToCluster) checkParams() error { + // 校验重启配置参数 + validate := validator.New() + a.runtime.Logger.Info("start to validate parameters of addShardToCluster") + if err := validate.Struct(a.ConfParams); err != nil { + a.runtime.Logger.Error(fmt.Sprintf("validate parameters of addShardToCluster fail, error:%s", err)) + return fmt.Errorf("validate parameters of addShardToCluster fail, error:%s", err) + } + a.runtime.Logger.Info("validate parameters of addShardToCluster successfully") + return nil +} + +// makeConfContent 生成配置内容 +func (a *AddShardToCluster) makeConfContent() error { + a.runtime.Logger.Info("start to make config content of addShardToCluster") + var shards []string + for key, value := range a.ConfParams.Shards { + shards = append(shards, strings.Join([]string{key, "/", value}, "")) + } + + for _, v := range shards { + a.ConfFileContent += strings.Join([]string{"sh.addShard(\"", v, "\")\n"}, "") + } + a.runtime.Logger.Info("make config content of addShardToCluster successfully") + return nil +} + +// createAddShardToClusterScript 生成js脚本 +func (a *AddShardToCluster) createAddShardToClusterScript() error { + a.runtime.Logger.Info("start to create addShardToCluster script") + confFile, err := os.OpenFile(a.ConfFilePath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, DefaultPerm) + defer confFile.Close() + if err != nil { + a.runtime.Logger.Error( + fmt.Sprintf("create script file of addShardToCluster fail, error:%s", err)) + return fmt.Errorf("create script file of addShardToCluster fail, error:%s", err) + } + + if _, err = confFile.WriteString(a.ConfFileContent); err != nil { + a.runtime.Logger.Error( + fmt.Sprintf("create script file of addShardToCluster write content fail, error:%s", + err)) + return fmt.Errorf("create script file of addShardToCluster write content fail, error:%s", + err) + } + a.runtime.Logger.Info("create addShardToCluster script successfully") + return nil +} + +// checkShard 检查shard是否已经加入到cluster中 +func (a *AddShardToCluster) checkShard() (bool, error) { + a.runtime.Logger.Info("start to check shard") + cmd := fmt.Sprintf( + "%s -u %s -p '%s' --host %s --port %d --quiet --authenticationDatabase=admin --eval \"db.getMongo().getDB('config').shards.find()\" admin", + a.Mongo, a.ConfParams.AdminUsername, a.ConfParams.AdminPassword, a.ConfParams.IP, a.ConfParams.Port) + result, err := util.RunBashCmd( + cmd, + "", nil, + 10*time.Second) + if err != nil { + a.runtime.Logger.Error(fmt.Sprintf("get shard info fail, error:%s", err)) + return false, fmt.Errorf("get shard info fail, error:%s", err) + } + result = strings.Replace(result, "\n", "", -1) + if result == "" { + a.runtime.Logger.Info("shard is not existed") + return false, nil + } + + for k, _ := range a.ConfParams.Shards { + + if strings.Contains(result, k) { + continue + } + + return false, fmt.Errorf("add shard %s fail", k) + } + a.runtime.Logger.Info("check shard successfully") + return true, nil +} + +// execScript 执行脚本 +func (a *AddShardToCluster) execScript() error { + // 检查 + flag, err := a.checkShard() + if err != nil { + return err + } + if flag == true { + a.runtime.Logger.Info(fmt.Sprintf("shards have been added")) + // 删除脚本 + if err = a.removeScript(); err != nil { + return err + } + + return nil + } + + // 执行脚本 + a.runtime.Logger.Info("start to execute addShardToCluster script") + cmd := fmt.Sprintf("%s -u %s -p '%s' --host %s --port %d --authenticationDatabase=admin --quiet %s", + a.Mongo, a.ConfParams.AdminUsername, a.ConfParams.AdminPassword, a.ConfParams.IP, a.ConfParams.Port, + a.ConfFilePath) + if _, err = util.RunBashCmd( + cmd, + "", nil, + 10*time.Second); err != nil { + a.runtime.Logger.Error(fmt.Sprintf("execute addShardToCluster script fail, error:%s", err)) + return fmt.Errorf("execute addShardToCluster script fail, error:%s", err) + } + a.runtime.Logger.Info("execute addShardToCluster script successfully") + + time.Sleep(5 * time.Second) + + // 检查 + flag, err = a.checkShard() + if err != nil { + return err + } + if flag == false { + a.runtime.Logger.Error(fmt.Sprintf("add shard fail, error:%s", err)) + return fmt.Errorf("add shard fail, error:%s", err) + } + + // 删除脚本 + if err = a.removeScript(); err != nil { + return err + } + + return nil +} + +// removeScript 删除脚本 +func (a *AddShardToCluster) removeScript() error { + // 删除脚本 + a.runtime.Logger.Info("start to remove addShardToCluster script") + if err := common.RemoveFile(a.ConfFilePath); err != nil { + a.runtime.Logger.Error(fmt.Sprintf("remove addShardToCluster script fail, error:%s", err)) + return fmt.Errorf("remove addShardToCluster script fail, error:%s", err) + } + a.runtime.Logger.Info("remove addShardToCluster script successfully") + return nil +} diff --git a/dbm-services/mongo/db-tools/dbactuator/pkg/atomjobs/atommongodb/add_user.go b/dbm-services/mongo/db-tools/dbactuator/pkg/atomjobs/atommongodb/add_user.go new file mode 100644 index 0000000000..67ede47f53 --- /dev/null +++ b/dbm-services/mongo/db-tools/dbactuator/pkg/atomjobs/atommongodb/add_user.go @@ -0,0 +1,264 @@ +package atommongodb + +import ( + "encoding/json" + "fmt" + "path/filepath" + "strconv" + "strings" + "time" + + "dbm-services/mongo/db-tools/dbactuator/pkg/common" + "dbm-services/mongo/db-tools/dbactuator/pkg/consts" + "dbm-services/mongo/db-tools/dbactuator/pkg/jobruntime" + "dbm-services/mongo/db-tools/dbactuator/pkg/util" + + "github.com/go-playground/validator/v10" +) + +// AddUserConfParams 参数 +type AddUserConfParams struct { + IP string `json:"ip" validate:"required"` + Port int `json:"port" validate:"required"` + InstanceType string `json:"instanceType" validate:"required"` + Username string `json:"username" validate:"required"` + Password string `json:"password" validate:"required"` + AdminUsername string `json:"adminUsername"` + AdminPassword string `json:"adminPassword"` + AuthDb string `json:"authDb"` // 为方便管理用户，验证库默认为admin库 + Dbs []string `json:"dbs"` // 业务库 + Privileges []string `json:"privileges"` // 权限 + +} + +// AddUser 添加分片到集群 +type AddUser struct { + runtime *jobruntime.JobGenericRuntime + BinDir string + Mongo string + PrimaryIP string + PrimaryPort int + OsUser string + ScriptContent string + ConfParams *AddUserConfParams +} + +// NewAddUser 实例化结构体 +func NewAddUser() jobruntime.JobRunner { + return &AddUser{} +} + +// Name 获取原子任务的名字 +func (u *AddUser) Name() string { + return "add_user" +} + +// Run 运行原子任务 +func (u *AddUser) Run() error { + // 生成脚本内容 + if err := u.makeScriptContent(); err != nil { + return err + } + + // 执行js脚本 + if err := u.execScript(); err != nil { + return err + } + + return nil +} + +// Retry 重试 +func (u *AddUser) Retry() uint { + return 2 +} + +// Rollback 回滚 +func (u *AddUser) Rollback() error { + return nil +} + +// Init 初始化 +func (u *AddUser) Init(runtime *jobruntime.JobGenericRuntime) error { + // 获取安装参数 + u.runtime = runtime + u.runtime.Logger.Info("start to init") + u.BinDir = consts.UsrLocal + u.Mongo = filepath.Join(u.BinDir, "mongodb", "bin", "mongo") + u.OsUser = consts.GetProcessUser() + + // 获取MongoDB配置文件参数 + if err := json.Unmarshal([]byte(u.runtime.PayloadDecoded), &u.ConfParams); err != nil { + u.runtime.Logger.Error(fmt.Sprintf( + "get parameters of addUser fail by json.Unmarshal, error:%s", err)) + return fmt.Errorf("get parameters of addUser fail by json.Unmarshal, error:%s", err) + } + + // 获取primary信息 + if u.ConfParams.InstanceType == "mongos" { + u.PrimaryIP = u.ConfParams.IP + u.PrimaryPort = u.ConfParams.Port + } else { + var info string + var err error + // 安装时无需密码验证。安装成功后需要密码验证 + if u.ConfParams.AdminUsername != "" && u.ConfParams.AdminPassword != "" { + info, err = common.AuthGetPrimaryInfo(u.Mongo, u.ConfParams.AdminUsername, + u.ConfParams.AdminPassword, u.ConfParams.IP, u.ConfParams.Port) + if err != nil { + u.runtime.Logger.Error(fmt.Sprintf( + "get primary db info of addUser fail, error:%s", err)) + return fmt.Errorf("get primary db info of addUser fail, error:%s", err) + } + getInfo := strings.Split(info, ":") + u.PrimaryIP = getInfo[0] + u.PrimaryPort, _ = strconv.Atoi(getInfo[1]) + } + } + u.runtime.Logger.Info("init successfully") + + // 进行校验 + if err := u.checkParams(); err != nil { + return err + } + + return nil +} + +// checkParams 校验参数 +func (u *AddUser) checkParams() error { + // 校验重启配置参数 + validate := validator.New() + u.runtime.Logger.Info("start to validate parameters of addUser") + if err := validate.Struct(u.ConfParams); err != nil { + u.runtime.Logger.Error(fmt.Sprintf("validate parameters of addUser fail, error:%s", err)) + return fmt.Errorf("validate parameters of addUser fail, error:%s", err) + } + u.runtime.Logger.Info("validate parameters of addUser successfully") + return nil +} + +// makeScriptContent 生成user配置内容 +func (u *AddUser) makeScriptContent() error { + u.runtime.Logger.Info("start to make script content") + user := common.NewMongoUser() + user.User = u.ConfParams.Username + user.Pwd = u.ConfParams.Password + + // 判断验证db + if u.ConfParams.AuthDb == "" { + u.ConfParams.AuthDb = "admin" + } + + // 判断业务db是否存在 + if len(u.ConfParams.Dbs) == 0 { + u.ConfParams.Dbs = []string{"admin"} + } + + for _, db := range u.ConfParams.Dbs { + for _, privilege := range u.ConfParams.Privileges { + role := common.NewMongoRole() + role.Role = privilege + role.Db = db + user.Roles = append(user.Roles, role) + } + } + + content, err := user.GetContent() + if err != nil { + u.runtime.Logger.Error(fmt.Sprintf("make config content of addUser fail, error:%s", err)) + return fmt.Errorf("make config content of addUser fail, error:%s", err) + } + // content = strings.Replace(content, "\"", "\\\"", -1) + + // 获取mongo版本 + mongoName := "mongod" + if u.ConfParams.InstanceType == "mongos" { + mongoName = "mongos" + } + version, err := common.CheckMongoVersion(u.BinDir, mongoName) + if err != nil { + u.runtime.Logger.Error(fmt.Sprintf("check mongo version fail, error:%s", err)) + return fmt.Errorf("check mongo version fail, error:%s", err) + } + mainVersion, _ := strconv.Atoi(strings.Split(version, ".")[0]) + if mainVersion >= 3 { + u.ScriptContent = strings.Join([]string{"db", + fmt.Sprintf("createUser(%s)", content)}, ".") + u.runtime.Logger.Info("make script content successfully") + return nil + } + u.ScriptContent = strings.Join([]string{"db", + fmt.Sprintf("addUser(%s)", content)}, ".") + u.runtime.Logger.Info("make script content successfully") + + return nil +} + +// checkUser 检查用户是否存在 +func (u *AddUser) checkUser() (bool, error) { + var flag bool + var err error + time.Sleep(time.Second * 3) + // 安装时检查管理用户是否存在无需密码验证。安装后检查业务用户是否存在需密码验证 + if u.ConfParams.AdminUsername != "" && u.ConfParams.AdminPassword != "" { + flag, err = common.AuthCheckUser(u.Mongo, u.ConfParams.AdminUsername, u.ConfParams.AdminPassword, + u.PrimaryIP, u.PrimaryPort, u.ConfParams.AuthDb, u.ConfParams.Username) + } else { + flag, err = common.AuthCheckUser(u.Mongo, u.ConfParams.Username, u.ConfParams.Password, + u.ConfParams.IP, u.ConfParams.Port, u.ConfParams.AuthDb, u.ConfParams.Username) + } + return flag, err +} + +// execScript 执行脚本 +func (u *AddUser) execScript() error { + var cmd string + if u.ConfParams.AdminUsername != "" && u.ConfParams.AdminPassword != "" { + // 检查用户是否存在 + flag, err := u.checkUser() + if err != nil { + return err + } + if flag == true { + u.runtime.Logger.Info("user:%s has been existed", u.ConfParams.Username) + return nil + } + cmd = fmt.Sprintf( + "%s -u %s -p '%s' --host %s --port %d --authenticationDatabase=admin --quiet --eval '%s' %s", + u.Mongo, u.ConfParams.AdminUsername, u.ConfParams.AdminPassword, u.PrimaryIP, u.PrimaryPort, + u.ScriptContent, u.ConfParams.AuthDb) + } else if u.ConfParams.AdminUsername == "" && u.ConfParams.AdminPassword == "" { + // 复制集初始化后，马上创建db管理员用户，需要等3秒 + time.Sleep(time.Second * 3) + cmd = fmt.Sprintf( + "%s --host %s --port %d --quiet --eval '%s' %s", + u.Mongo, "127.0.0.1", u.ConfParams.Port, u.ScriptContent, u.ConfParams.AuthDb) + if u.ConfParams.AdminUsername != "" && u.ConfParams.AdminPassword != "" { + + } + } + + // 执行脚本 + u.runtime.Logger.Info("start to execute addUser script") + if _, err := util.RunBashCmd( + cmd, + "", nil, + 10*time.Second); err != nil { + u.runtime.Logger.Error("execute addUser script fail, error:%s", err) + return fmt.Errorf("execute addUser script fail, error:%s", err) + } + u.runtime.Logger.Info("execute addUser script successfully") + + // 检查用户是否存在 + flag, err := u.checkUser() + if err != nil { + return err + } + if flag == false { + u.runtime.Logger.Error("add user:%s fail, error:%s", u.ConfParams.Username, err) + return fmt.Errorf("add user:%s fail, error:%s", u.ConfParams.Username, err) + } + + return nil +} diff --git a/dbm-services/mongo/db-tools/dbactuator/pkg/atomjobs/atommongodb/atommongodb.go b/dbm-services/mongo/db-tools/dbactuator/pkg/atomjobs/atommongodb/atommongodb.go new file mode 100644 index 0000000000..f022fb23e5 --- /dev/null +++ b/dbm-services/mongo/db-tools/dbactuator/pkg/atomjobs/atommongodb/atommongodb.go @@ -0,0 +1,2 @@ +// Package atommongodb mongodb原子任务 +package atommongodb diff --git a/dbm-services/mongo/db-tools/dbactuator/pkg/atomjobs/atommongodb/cluster_balancer.go b/dbm-services/mongo/db-tools/dbactuator/pkg/atomjobs/atommongodb/cluster_balancer.go new file mode 100644 index 0000000000..95758fe350 --- /dev/null +++ b/dbm-services/mongo/db-tools/dbactuator/pkg/atomjobs/atommongodb/cluster_balancer.go @@ -0,0 +1,159 @@ +package atommongodb + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "strconv" + "time" + + "dbm-services/mongo/db-tools/dbactuator/pkg/common" + "dbm-services/mongo/db-tools/dbactuator/pkg/consts" + "dbm-services/mongo/db-tools/dbactuator/pkg/jobruntime" + "dbm-services/mongo/db-tools/dbactuator/pkg/util" + + "github.com/go-playground/validator/v10" +) + +// BalancerConfParams 参数 +type BalancerConfParams struct { + IP string `json:"ip" validate:"required"` + Port int `json:"port" validate:"required"` + Open bool `json:"open"` // true：打开 false：关闭 + AdminUsername string `json:"adminUsername" validate:"required"` + AdminPassword string `json:"adminPassword" validate:"required"` +} + +// Balancer 添加分片到集群 +type Balancer struct { + runtime *jobruntime.JobGenericRuntime + BinDir string + Mongo string + OsUser string + ConfParams *BalancerConfParams +} + +// NewBalancer 实例化结构体 +func NewBalancer() jobruntime.JobRunner { + return &Balancer{} +} + +// Name 获取原子任务的名字 +func (b *Balancer) Name() string { + return "cluster_balancer" +} + +// Run 运行原子任务 +func (b *Balancer) Run() error { + // 执行脚本 + if err := b.execScript(); err != nil { + return err + } + + return nil +} + +// Retry 重试 +func (b *Balancer) Retry() uint { + return 2 +} + +// Rollback 回滚 +func (b *Balancer) Rollback() error { + return nil +} + +// Init 初始化 +func (b *Balancer) Init(runtime *jobruntime.JobGenericRuntime) error { + // 获取安装参数 + b.runtime = runtime + b.runtime.Logger.Info("start to init") + b.BinDir = consts.UsrLocal + b.Mongo = filepath.Join(b.BinDir, "mongodb", "bin", "mongo") + b.OsUser = consts.GetProcessUser() + + // 获取MongoDB配置文件参数 + if err := json.Unmarshal([]byte(b.runtime.PayloadDecoded), &b.ConfParams); err != nil { + b.runtime.Logger.Error( + "get parameters of clusterBalancer fail by json.Unmarshal, error:%s", err) + return fmt.Errorf("get parameters of clusterBalancer fail by json.Unmarshal, error:%s", err) + } + b.runtime.Logger.Info("init successfully") + + // 进行校验 + if err := b.checkParams(); err != nil { + return err + } + + return nil +} + +// checkParams 校验参数 +func (b *Balancer) checkParams() error { + // 校验配置参数 + validate := validator.New() + b.runtime.Logger.Info("start to validate parameters of clusterBalancer") + if err := validate.Struct(b.ConfParams); err != nil { + b.runtime.Logger.Error(fmt.Sprintf("validate parameters of clusterBalancer fail, error:%s", err)) + return fmt.Errorf("validate parameters of clusterBalancer fail, error:%s", err) + } + b.runtime.Logger.Info("validate parameters of clusterBalancer successfully") + return nil +} + +// execScript 执行脚本 +func (b *Balancer) execScript() error { + // 检查状态 + b.runtime.Logger.Info("start to get balancer status") + result, err := common.CheckBalancer(b.Mongo, b.ConfParams.IP, b.ConfParams.Port, + b.ConfParams.AdminUsername, b.ConfParams.AdminPassword) + if err != nil { + b.runtime.Logger.Error("get cluster balancer status fail, error:%s", err) + return fmt.Errorf("get cluster balancer status fail, error:%s", err) + } + flag, _ := strconv.ParseBool(result) + b.runtime.Logger.Info("get balancer status successfully") + if flag == b.ConfParams.Open { + b.runtime.Logger.Info("balancer status has been %t", b.ConfParams.Open) + os.Exit(0) + } + + // 执行脚本 + var cmd string + if b.ConfParams.Open == true { + cmd = fmt.Sprintf( + "%s -u %s -p '%s' --host %s --port %d --authenticationDatabase=admin --quiet --eval \"sh.startBalancer()\"", + b.Mongo, b.ConfParams.AdminUsername, b.ConfParams.AdminPassword, b.ConfParams.IP, b.ConfParams.Port) + } else { + cmd = fmt.Sprintf( + "%s -u %s -p '%s' --host %s --port %d --authenticationDatabase=admin --quiet --eval \"sh.stopBalancer()\"", + b.Mongo, b.ConfParams.AdminUsername, b.ConfParams.AdminPassword, b.ConfParams.IP, b.ConfParams.Port) + } + b.runtime.Logger.Info("start to execute script") + _, err = util.RunBashCmd( + cmd, + "", nil, + 10*time.Second) + if err != nil { + b.runtime.Logger.Error("set cluster balancer status fail, error:%s", err) + return fmt.Errorf("set cluster balancer status fail, error:%s", err) + } + b.runtime.Logger.Info("execute script successfully") + + // 检查状态 + b.runtime.Logger.Info("start to check balancer status") + result, err = common.CheckBalancer(b.Mongo, b.ConfParams.IP, b.ConfParams.Port, + b.ConfParams.AdminUsername, b.ConfParams.AdminPassword) + if err != nil { + b.runtime.Logger.Error("get cluster balancer status fail, error:%s", err) + return fmt.Errorf("get cluster balancer status fail, error:%s", err) + } + flag, _ = strconv.ParseBool(result) + b.runtime.Logger.Info("check balancer status successfully") + if flag == b.ConfParams.Open { + b.runtime.Logger.Info("set balancer status:%t successfully", b.ConfParams.Open) + } + + return nil +} diff --git a/dbm-services/mongo/db-tools/dbactuator/pkg/atomjobs/atommongodb/del_user.go b/dbm-services/mongo/db-tools/dbactuator/pkg/atomjobs/atommongodb/del_user.go new file mode 100644 index 0000000000..393d3a3d6c --- /dev/null +++ b/dbm-services/mongo/db-tools/dbactuator/pkg/atomjobs/atommongodb/del_user.go @@ -0,0 +1,204 @@ +package atommongodb + +import ( + "encoding/json" + "fmt" + "path/filepath" + "strconv" + "strings" + "time" + + "dbm-services/mongo/db-tools/dbactuator/pkg/common" + "dbm-services/mongo/db-tools/dbactuator/pkg/consts" + "dbm-services/mongo/db-tools/dbactuator/pkg/jobruntime" + "dbm-services/mongo/db-tools/dbactuator/pkg/util" + + "github.com/go-playground/validator/v10" +) + +// DelUserConfParams 参数 +type DelUserConfParams struct { + IP string `json:"ip" validate:"required"` + Port int `json:"port" validate:"required"` + InstanceType string `json:"instanceType" validate:"required"` + AdminUsername string `json:"adminUsername" validate:"required"` + AdminPassword string `json:"adminPassword" validate:"required"` + Username string `json:"username" validate:"required"` + AuthDb string `json:"authDb"` // 为方便管理用户，验证库默认为admin库 +} + +// DelUser 添加分片到集群 +type DelUser struct { + runtime *jobruntime.JobGenericRuntime + BinDir string + Mongo string + OsUser string + PrimaryIP string + PrimaryPort int + ScriptContent string + ConfParams *DelUserConfParams +} + +// NewDelUser 实例化结构体 +func NewDelUser() jobruntime.JobRunner { + return &DelUser{} +} + +// Name 获取原子任务的名字 +func (d *DelUser) Name() string { + return "delete_user" +} + +// Run 运行原子任务 +func (d *DelUser) Run() error { + // 生成脚本内容 + if err := d.makeScriptContent(); err != nil { + return err + } + + // 执行js脚本 + if err := d.execScript(); err != nil { + return err + } + + return nil +} + +// Retry 重试 +func (d *DelUser) Retry() uint { + return 2 +} + +// Rollback 回滚 +func (d *DelUser) Rollback() error { + return nil +} + +// Init 初始化 +func (d *DelUser) Init(runtime *jobruntime.JobGenericRuntime) error { + // 获取安装参数 + d.runtime = runtime + d.runtime.Logger.Info("start to init") + d.BinDir = consts.UsrLocal + d.Mongo = filepath.Join(d.BinDir, "mongodb", "bin", "mongo") + d.OsUser = consts.GetProcessUser() + + // 获取MongoDB配置文件参数 + if err := json.Unmarshal([]byte(d.runtime.PayloadDecoded), &d.ConfParams); err != nil { + d.runtime.Logger.Error(fmt.Sprintf( + "get parameters of deleteUser fail by json.Unmarshal, error:%s", err)) + return fmt.Errorf("get parameters of deleteUser fail by json.Unmarshal, error:%s", err) + } + + // 获取primary信息 + if d.ConfParams.InstanceType == "mongos" { + d.PrimaryIP = d.ConfParams.IP + d.PrimaryPort = d.ConfParams.Port + } else { + info, err := common.AuthGetPrimaryInfo(d.Mongo, d.ConfParams.AdminUsername, d.ConfParams.AdminPassword, + d.ConfParams.IP, d.ConfParams.Port) + if err != nil { + d.runtime.Logger.Error(fmt.Sprintf( + "get primary db info of addUser fail, error:%s", err)) + return fmt.Errorf("get primary db info of addUser fail, error:%s", err) + } + getInfo := strings.Split(info, ":") + d.PrimaryIP = getInfo[0] + d.PrimaryPort, _ = strconv.Atoi(getInfo[1]) + } + d.runtime.Logger.Info("init successfully") + + // 进行校验 + if err := d.checkParams(); err != nil { + return err + } + + return nil +} + +// checkParams 校验参数 +func (d *DelUser) checkParams() error { + // 校验重启配置参数 + validate := validator.New() + d.runtime.Logger.Info("start to validate parameters of deleteUser") + if err := validate.Struct(d.ConfParams); err != nil { + d.runtime.Logger.Error(fmt.Sprintf("validate parameters of deleteUser fail, error:%s", err)) + return fmt.Errorf("validate parameters of deleteUser fail, error:%s", err) + } + d.runtime.Logger.Info("validate parameters of deleteUser successfully") + return nil +} + +// makeScriptContent 生成user配置内容 +func (d *DelUser) makeScriptContent() error { + d.runtime.Logger.Info("start to make deleteUser script content") + // 判断验证db + if d.ConfParams.AuthDb == "" { + d.ConfParams.AuthDb = "admin" + } + + // 获取mongo版本 + mongoName := "mongod" + if d.ConfParams.InstanceType == "mongos" { + mongoName = "mongos" + } + version, err := common.CheckMongoVersion(d.BinDir, mongoName) + if err != nil { + d.runtime.Logger.Error(fmt.Sprintf("check mongo version fail, error:%s", err)) + return fmt.Errorf("check mongo version fail, error:%s", err) + } + mainVersion, _ := strconv.Atoi(strings.Split(version, ".")[0]) + if mainVersion >= 3 { + d.ScriptContent = strings.Join([]string{fmt.Sprintf("db.getMongo().getDB('%s')", d.ConfParams.AuthDb), + fmt.Sprintf("dropUser('%s')", d.ConfParams.Username)}, ".") + d.runtime.Logger.Info("make deleteUser script content successfully") + return nil + } + d.ScriptContent = strings.Join([]string{fmt.Sprintf("db.getMongo().getDB('%s')", d.ConfParams.AuthDb), + fmt.Sprintf("removeUser('%s')", d.ConfParams.Username)}, ".") + d.runtime.Logger.Info("make deleteUser script content successfully") + return nil +} + +// execScript 执行脚本 +func (d *DelUser) execScript() error { + // 检查 + flag, err := common.AuthCheckUser(d.Mongo, d.ConfParams.AdminUsername, d.ConfParams.AdminPassword, + d.PrimaryIP, d.PrimaryPort, d.ConfParams.AuthDb, d.ConfParams.Username) + if err != nil { + return err + } + if flag == false { + d.runtime.Logger.Info(fmt.Sprintf("user:%s is not existed", d.ConfParams.Username)) + return nil + } + + // 执行脚本 + d.runtime.Logger.Info("start to execute deleteUser script") + cmd := fmt.Sprintf( + "%s -u %s -p '%s' --host %s --port %d --authenticationDatabase=admin --quiet --eval \"%s\"", + d.Mongo, d.ConfParams.AdminUsername, d.ConfParams.AdminPassword, d.PrimaryIP, + d.PrimaryPort, d.ScriptContent) + if _, err := util.RunBashCmd( + cmd, + "", nil, + 10*time.Second); err != nil { + d.runtime.Logger.Error(fmt.Sprintf("execute addUser script fail, error:%s", err)) + return fmt.Errorf("execute addUser script fail, error:%s", err) + } + + time.Sleep(2 * time.Second) + + // 检查 + flag, err = common.AuthCheckUser(d.Mongo, d.ConfParams.AdminUsername, d.ConfParams.AdminPassword, + d.PrimaryIP, d.PrimaryPort, d.ConfParams.AuthDb, d.ConfParams.Username) + if err != nil { + return err + } + if flag == true { + d.runtime.Logger.Error(fmt.Sprintf("delete user:%s fail, error:%s", d.ConfParams.Username, err)) + return fmt.Errorf("delete user:%s fail, error:%s", d.ConfParams.Username, err) + } + d.runtime.Logger.Info("execute deleteUser script successfully") + return nil +} diff --git a/dbm-services/mongo/db-tools/dbactuator/pkg/atomjobs/atommongodb/initiate_replicaset.go b/dbm-services/mongo/db-tools/dbactuator/pkg/atomjobs/atommongodb/initiate_replicaset.go new file mode 100644 index 0000000000..f4c3587fae --- /dev/null +++ b/dbm-services/mongo/db-tools/dbactuator/pkg/atomjobs/atommongodb/initiate_replicaset.go @@ -0,0 +1,278 @@ +package atommongodb + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + "time" + + "dbm-services/mongo/db-tools/dbactuator/pkg/common" + "dbm-services/mongo/db-tools/dbactuator/pkg/consts" + "dbm-services/mongo/db-tools/dbactuator/pkg/jobruntime" + "dbm-services/mongo/db-tools/dbactuator/pkg/util" + + "github.com/go-playground/validator/v10" +) + +// InitConfParams 参数 +type InitConfParams struct { + IP string `json:"ip" validate:"required"` + Port int `json:"port" validate:"required"` + App string `json:"app" validate:"required"` + SetId string `json:"setId" validate:"required"` + ConfigSvr bool `json:"configSvr"` // shardsvr configsvr + Ips []string `json:"ips" validate:"required"` // ip:port + Priority map[string]int `json:"priority" validate:"required"` // key->ip:port,value->priority + Hidden map[string]bool `json:"hidden" validate:"required"` // key->ip:port,value->hidden(true or false) +} + +// InitiateReplicaset 复制集初始化 +type InitiateReplicaset struct { + runtime *jobruntime.JobGenericRuntime + BinDir string + Mongo string + OsUser string + ConfFilePath string + ConfFileContent string + ConfParams *InitConfParams + ClusterId string + StatusChan chan int +} + +// NewInitiateReplicaset 实例化结构体 +func NewInitiateReplicaset() jobruntime.JobRunner { + return &InitiateReplicaset{} +} + +// Name 获取原子任务的名字 +func (i *InitiateReplicaset) Name() string { + return "init_replicaset" +} + +// Run 运行原子任务 +func (i *InitiateReplicaset) Run() error { + // 获取配置内容 + if err := i.makeConfContent(); err != nil { + return err + } + + // 生成js脚本 + if err := i.createInitiateReplicasetScript(); err != nil { + return err + } + + // 执行js脚本 + if err := i.execScript(); err != nil { + return err + } + + // 检查状态 + go i.checkStatus() + + // 获取状态 + if err := i.getStatus(); err != nil { + return err + } + + return nil +} + +// Retry 重试 +func (i *InitiateReplicaset) Retry() uint { + return 2 +} + +// Rollback 回滚 +func (i *InitiateReplicaset) Rollback() error { + return nil +} + +// Init 初始化 +func (i *InitiateReplicaset) Init(runtime *jobruntime.JobGenericRuntime) error { + // 获取安装参数 + i.runtime = runtime + i.runtime.Logger.Info("start to init") + i.BinDir = consts.UsrLocal + i.Mongo = filepath.Join(i.BinDir, "mongodb", "bin", "mongo") + i.OsUser = consts.GetProcessUser() + i.ConfFilePath = filepath.Join("/", "tmp", "initiateReplicaset.js") + i.StatusChan = make(chan int, 1) + + // 获取MongoDB配置文件参数 + if err := json.Unmarshal([]byte(i.runtime.PayloadDecoded), &i.ConfParams); err != nil { + i.runtime.Logger.Error(fmt.Sprintf( + "get parameters of initiateReplicaset fail by json.Unmarshal, error:%s", err)) + return fmt.Errorf("get parameters of initiateReplicaset fail by json.Unmarshal, error:%s", err) + } + i.ClusterId = strings.Join([]string{i.ConfParams.App, i.ConfParams.SetId}, "-") + i.runtime.Logger.Info("init successfully") + + // 进行校验 + if err := i.checkParams(); err != nil { + return err + } + + return nil +} + +// checkParams 校验参数 +func (i *InitiateReplicaset) checkParams() error { + // 校验重启配置参数 + validate := validator.New() + i.runtime.Logger.Info("start to validate parameters of initiateReplicaset") + if err := validate.Struct(i.ConfParams); err != nil { + i.runtime.Logger.Error(fmt.Sprintf("validate parameters of initiateReplicaset fail, error:%s", err)) + return fmt.Errorf("validate parameters of initiateReplicaset fail, error:%s", err) + } + i.runtime.Logger.Info("validate parameters of initiateReplicaset successfully") + return nil +} + +// makeConfContent 获取配置内容 +func (i *InitiateReplicaset) makeConfContent() error { + i.runtime.Logger.Info("start to make config content of initiateReplicaset") + jsonConfReplicaset := common.NewJsonConfReplicaset() + jsonConfReplicaset.Id = i.ClusterId + for index, value := range i.ConfParams.Ips { + member := common.NewMember() + member.Id = index + member.Host = i.ConfParams.Ips[index] + member.Priority = i.ConfParams.Priority[value] + member.Hidden = i.ConfParams.Hidden[value] + jsonConfReplicaset.Members = append(jsonConfReplicaset.Members, member) + } + jsonConfReplicaset.ConfigSvr = i.ConfParams.ConfigSvr + + var err error + confJson, err := json.Marshal(jsonConfReplicaset) + if err != nil { + i.runtime.Logger.Error( + fmt.Sprintf("config content of initiateReplicaset json Marshal fial, error:%s", err)) + return fmt.Errorf("config content of initiateReplicaset json Marshal fial, error:%s", err) + } + i.ConfFileContent = strings.Join([]string{"var config=", + string(confJson), "\n", "rs.initiate(config)\n"}, "") + i.runtime.Logger.Info("make config content of initiateReplicaset successfully") + return nil +} + +// createInitiateReplicasetScript 生成js脚本 +func (i *InitiateReplicaset) createInitiateReplicasetScript() error { + i.runtime.Logger.Info("start to create initiateReplicaset script") + confFile, err := os.OpenFile(i.ConfFilePath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, DefaultPerm) + defer confFile.Close() + if err != nil { + i.runtime.Logger.Error( + fmt.Sprintf("create script file of initiateReplicaset json Marshal fail, error:%s", err)) + return fmt.Errorf("create script file of initiateReplicaset json Marshal fail, error:%s", err) + } + + if _, err = confFile.WriteString(i.ConfFileContent); err != nil { + i.runtime.Logger.Error( + fmt.Sprintf("create script file of initiateReplicaset write content fail, error:%s", + err)) + return fmt.Errorf("create script file of initiateReplicaset write content fail, error:%s", + err) + } + i.runtime.Logger.Info("create initiateReplicaset script successfully") + return nil +} + +// getPrimaryInfo 检查状态 +func (i *InitiateReplicaset) getPrimaryInfo() (bool, error) { + i.runtime.Logger.Info("start to check replicaset status") + result, err := common.InitiateReplicasetGetPrimaryInfo(i.Mongo, i.ConfParams.IP, i.ConfParams.Port) + if err != nil { + i.runtime.Logger.Error(fmt.Sprintf("get initiateReplicaset primary info fail, error:%s", err)) + return false, fmt.Errorf("get initiateReplicaset primary info fail, error:%s", err) + } + i.runtime.Logger.Info("check replicaset status successfully") + for _, v := range i.ConfParams.Ips { + if v == result { + return true, nil + } + } + + return false, nil +} + +// checkStatus 检查复制集状态 +func (i *InitiateReplicaset) checkStatus() { + for { + result, err := common.NoAuthGetPrimaryInfo(i.Mongo, i.ConfParams.IP, i.ConfParams.Port) + if err != nil { + i.runtime.Logger.Error("check replicaset status fail, error:%s", err) + fmt.Sprintf("check replicaset status fail, error:%s\n", err) + panic(fmt.Sprintf("check replicaset status fail, error:%s\n", err.Error())) + } + if result != "" { + i.StatusChan <- 1 + } + time.Sleep(2 * time.Second) + } +} + +// execScript 执行脚本 +func (i *InitiateReplicaset) execScript() error { + // 检查 + flag, err := i.getPrimaryInfo() + if err != nil { + return err + } + if flag == true { + i.runtime.Logger.Info("replicaset has been initiated") + if err = i.removeScript(); err != nil { + return err + } + + return nil + } + + // 执行脚本 + i.runtime.Logger.Info("start to execute initiateReplicaset script") + cmd := fmt.Sprintf("%s --host %s --port %d --quiet %s", + i.Mongo, "127.0.0.1", i.ConfParams.Port, i.ConfFilePath) + if _, err = util.RunBashCmd( + cmd, + "", nil, + 10*time.Second); err != nil { + i.runtime.Logger.Error("execute initiateReplicaset script fail, error:%s", err) + return fmt.Errorf("execute initiateReplicaset script fail, error:%s", err) + } + i.runtime.Logger.Info("execute initiateReplicaset script successfully") + return nil +} + +// getStatus 检查复制集状态，是否创建成功 +func (i *InitiateReplicaset) getStatus() error { + for { + select { + case status := <-i.StatusChan: + if status == 1 { + i.runtime.Logger.Info("initiate replicaset successfully") + // 删除脚本 + if err := i.removeScript(); err != nil { + return err + } + return nil + } + default: + + } + } +} + +// removeScript 删除脚本 +func (i *InitiateReplicaset) removeScript() error { + // 删除脚本 + i.runtime.Logger.Info("start to remove initiateReplicaset script") + if err := common.RemoveFile(i.ConfFilePath); err != nil { + i.runtime.Logger.Error(fmt.Sprintf("remove initiateReplicaset script fail, error:%s", err)) + return fmt.Errorf("remove initiateReplicaset script fail, error:%s", err) + } + i.runtime.Logger.Info("remove initiateReplicaset script successfully") + + return nil +} diff --git a/dbm-services/mongo/db-tools/dbactuator/pkg/atomjobs/atommongodb/mongo_deinstall.go b/dbm-services/mongo/db-tools/dbactuator/pkg/atomjobs/atommongodb/mongo_deinstall.go new file mode 100644 index 0000000000..a9e795fd95 --- /dev/null +++ b/dbm-services/mongo/db-tools/dbactuator/pkg/atomjobs/atommongodb/mongo_deinstall.go @@ -0,0 +1,230 @@ +package atommongodb + +import ( + "encoding/json" + "fmt" + "path/filepath" + "strconv" + "strings" + "time" + + "dbm-services/mongo/db-tools/dbactuator/pkg/common" + "dbm-services/mongo/db-tools/dbactuator/pkg/consts" + "dbm-services/mongo/db-tools/dbactuator/pkg/jobruntime" + "dbm-services/mongo/db-tools/dbactuator/pkg/util" + + "github.com/go-playground/validator/v10" +) + +// DeInstallConfParams 参数 +type DeInstallConfParams struct { + IP string `json:"ip" validate:"required"` + Port int `json:"port" validate:"required"` + App string `json:"app" validate:"required"` + SetId string `json:"setId" validate:"required"` + NodeInfo []string `json:"nodeInfo" validate:"required"` // []string ip,ip 如果为复制集节点，则为复制集所有节点的ip；如果为mongos，则为mongos的ip + InstanceType string `json:"instanceType" validate:"required"` // mongod mongos +} + +// DeInstall 添加分片到集群 +type DeInstall struct { + runtime *jobruntime.JobGenericRuntime + BinDir string + DataDir string + BackupDir string + DbpathDir string + InstallPath string + PortDir string + LogPortDir string + DbPathRenameDir string + LogPathRenameDir string + Mongo string + OsUser string + ServiceStatus bool + IPInfo string + ConfParams *DeInstallConfParams +} + +// NewDeInstall 实例化结构体 +func NewDeInstall() jobruntime.JobRunner { + return &DeInstall{} +} + +// Name 获取原子任务的名字 +func (d *DeInstall) Name() string { + return "mongo_deinstall" +} + +// Run 运行原子任务 +func (d *DeInstall) Run() error { + // 检查实例状态 + if err := d.checkMongoService(); err != nil { + return err + } + + // 关闭进程 + if err := d.shutdownProcess(); err != nil { + return err + } + + // rename目录 + if err := d.DirRename(); err != nil { + return err + } + + return nil +} + +// Retry 重试 +func (d *DeInstall) Retry() uint { + return 2 +} + +// Rollback 回滚 +func (d *DeInstall) Rollback() error { + return nil +} + +// Init 初始化 +func (d *DeInstall) Init(runtime *jobruntime.JobGenericRuntime) error { + // 获取安装参数 + d.runtime = runtime + d.runtime.Logger.Info("start to init") + d.BinDir = consts.UsrLocal + d.DataDir = consts.GetMongoDataDir() + d.BackupDir = consts.GetMongoBackupDir() + + d.OsUser = consts.GetProcessUser() + + // 获取MongoDB配置文件参数 + if err := json.Unmarshal([]byte(d.runtime.PayloadDecoded), &d.ConfParams); err != nil { + d.runtime.Logger.Error( + "get parameters of deInstall fail by json.Unmarshal, error:%s", err) + return fmt.Errorf("get parameters of deInstall fail by json.Unmarshal, error:%s", err) + } + + // 获取各种目录 + d.InstallPath = filepath.Join(d.BinDir, "mongodb") + d.Mongo = filepath.Join(d.BinDir, "mongodb", "bin", "mongo") + strPort := strconv.Itoa(d.ConfParams.Port) + d.PortDir = filepath.Join(d.DataDir, "mongodata", strPort) + d.DbpathDir = filepath.Join(d.DataDir, "mongodata", strPort, "db") + d.DbPathRenameDir = filepath.Join(d.DataDir, "mongodata", fmt.Sprintf("%s_%s_%s_%d", + d.ConfParams.InstanceType, d.ConfParams.App, d.ConfParams.SetId, d.ConfParams.Port)) + d.IPInfo = strings.Join(d.ConfParams.NodeInfo, "|") + d.LogPortDir = filepath.Join(d.BackupDir, "mongolog", strPort) + d.LogPathRenameDir = filepath.Join(d.BackupDir, "mongolog", fmt.Sprintf("%s_%s_%s_%d", + d.ConfParams.InstanceType, d.ConfParams.App, d.ConfParams.SetId, d.ConfParams.Port)) + + // 进行校验 + if err := d.checkParams(); err != nil { + return err + } + + return nil +} + +// checkParams 校验参数 +func (d *DeInstall) checkParams() error { + // 校验配置参数 + d.runtime.Logger.Info("start to validate parameters") + validate := validator.New() + d.runtime.Logger.Info("start to validate parameters of deInstall") + if err := validate.Struct(d.ConfParams); err != nil { + d.runtime.Logger.Error("validate parameters of deInstall fail, error:%s", err) + return fmt.Errorf("validate parameters of deInstall fail, error:%s", err) + } + return nil +} + +// checkMongoService 检查mongo服务 +func (d *DeInstall) checkMongoService() error { + d.runtime.Logger.Info("start to check process status") + flag, _, err := common.CheckMongoService(d.ConfParams.Port) + if err != nil { + d.runtime.Logger.Error("get mongo service status fail, error:%s", err) + return fmt.Errorf("get mongo service status fail, error:%s", err) + } + d.ServiceStatus = flag + return nil +} + +// checkConnection 检查连接 +func (d *DeInstall) checkConnection() error { + d.runtime.Logger.Info("start to check connection") + cmd := fmt.Sprintf( + "source /etc/profile;netstat -nat | grep %d |awk '{print $5}'|awk -F: '{print $1}'|sort|uniq -c|sort -nr |grep -Ewv '0.0.0.0|127.0.0.1|%s' || true", + d.ConfParams.Port, d.IPInfo) + + result, err := util.RunBashCmd( + cmd, + "", nil, + 10*time.Second) + if err != nil { + d.runtime.Logger.Error("check connection fail, error:%s", err) + return fmt.Errorf("check connection fail, error:%s", err) + } + result = strings.Replace(result, "\n", "", -1) + if result != "" { + d.runtime.Logger.Error("check connection fail, there are some connections") + return fmt.Errorf("check connection fail, there are some connections") + } + return nil +} + +// shutdownProcess 关闭进程 +func (d *DeInstall) shutdownProcess() error { + if d.ServiceStatus == true { + d.runtime.Logger.Info("start to shutdown service") + // 检查连接 + if err := d.checkConnection(); err != nil { + return err + } + + // 关闭进程 + if err := common.ShutdownMongoProcess(d.OsUser, d.ConfParams.InstanceType, d.BinDir, d.DbpathDir, + d.ConfParams.Port); err != nil { + d.runtime.Logger.Error("shutdown mongo service fail, error:%s", err) + return fmt.Errorf("shutdown mongo service fail, error:%s", err) + } + } + + return nil +} + +// DirRename 打包数据目录 +func (d *DeInstall) DirRename() error { + // renameDb数据目录 + flag := util.FileExists(d.PortDir) + if flag == true { + d.runtime.Logger.Info("start to rename db directory") + cmd := fmt.Sprintf( + "mv %s %s", + d.PortDir, d.DbPathRenameDir) + if _, err := util.RunBashCmd( + cmd, + "", nil, + 10*time.Second); err != nil { + d.runtime.Logger.Error("rename db directory fail, error:%s", err) + return fmt.Errorf("rename db directory fail, error:%s", err) + } + } + + // renameDb日志目录 + flag = util.FileExists(d.LogPortDir) + if flag == true { + d.runtime.Logger.Info("start to rename log directory") + cmd := fmt.Sprintf( + "mv %s %s", + d.LogPortDir, d.LogPathRenameDir) + if _, err := util.RunBashCmd( + cmd, + "", nil, + 10*time.Second); err != nil { + d.runtime.Logger.Error("rename log directory fail, error:%s", err) + return fmt.Errorf("rename log directory fail, error:%s", err) + } + } + + return nil +} diff --git a/dbm-services/mongo/db-tools/dbactuator/pkg/atomjobs/atommongodb/mongo_execute_script.go b/dbm-services/mongo/db-tools/dbactuator/pkg/atomjobs/atommongodb/mongo_execute_script.go new file mode 100644 index 0000000000..2f1da8b259 --- /dev/null +++ b/dbm-services/mongo/db-tools/dbactuator/pkg/atomjobs/atommongodb/mongo_execute_script.go @@ -0,0 +1,331 @@ +package atommongodb + +import ( + "encoding/base64" + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + "os" + "path/filepath" + "strconv" + "strings" + "time" + + "dbm-services/mongo/db-tools/dbactuator/pkg/common" + "dbm-services/mongo/db-tools/dbactuator/pkg/consts" + "dbm-services/mongo/db-tools/dbactuator/pkg/jobruntime" + "dbm-services/mongo/db-tools/dbactuator/pkg/util" + + "github.com/go-playground/validator/v10" +) + +// ExecScriptConfParams 参数 +type ExecScriptConfParams struct { + IP string `json:"ip" validate:"required"` + Port int `json:"port" validate:"required"` + Script string `json:"script" validate:"required"` + Type string `json:"type" validate:"required"` // cluster：执行脚本为传入的mongos replicaset：执行脚本为指定节点 + Secondary bool `json:"secondary"` // 复制集是否在secondary节点执行script + AdminUsername string `json:"adminUsername" validate:"required"` + AdminPassword string `json:"adminPassword" validate:"required"` + RepoUrl string `json:"repoUrl"` // 制品库url + RepoUsername string `json:"repoUsername"` // 制品库用户名 + RepoToken string `json:"repoToken"` // 制品库token + RepoProject string `json:"repoProject"` // 制品库project + RepoRepo string `json:"repoRepo"` // 制品库repo + RepoPath string `json:"repoPath"` // 制品库路径 +} + +// ExecScript 添加分片到集群 +type ExecScript struct { + runtime *jobruntime.JobGenericRuntime + BinDir string + Mongo string + OsUser string + OsGroup string + execIP string + execPort int + ScriptDir string + ScriptContent string + ScriptFilePath string + ResultFilePath string + ConfParams *ExecScriptConfParams +} + +// NewExecScript 实例化结构体 +func NewExecScript() jobruntime.JobRunner { + return &ExecScript{} +} + +// Name 获取原子任务的名字 +func (e *ExecScript) Name() string { + return "mongo_execute_script" +} + +// Run 运行原子任务 +func (e *ExecScript) Run() error { + // 生成script内容 + if err := e.makeScriptContent(); err != nil { + return err + } + + // 创建script文件 + if err := e.creatScriptFile(); err != nil { + return err + } + + // 执行脚本生成结果文件 + if err := e.execScript(); err != nil { + return err + } + + // 上传结果文件到制品库 + if err := e.uploadFile(); err != nil { + return err + } + + return nil +} + +// Retry 重试 +func (e *ExecScript) Retry() uint { + return 2 +} + +// Rollback 回滚 +func (e *ExecScript) Rollback() error { + return nil +} + +// Init 初始化 +func (e *ExecScript) Init(runtime *jobruntime.JobGenericRuntime) error { + // 获取安装参数 + e.runtime = runtime + e.runtime.Logger.Info("start to init") + e.BinDir = consts.UsrLocal + e.OsUser = consts.GetProcessUser() + e.OsGroup = consts.GetProcessUserGroup() + + // 获取MongoDB配置文件参数 + if err := json.Unmarshal([]byte(e.runtime.PayloadDecoded), &e.ConfParams); err != nil { + e.runtime.Logger.Error( + "get parameters of execScript fail by json.Unmarshal, error:%s", err) + return fmt.Errorf("get parameters of execScript fail by json.Unmarshal, error:%s", err) + } + + // 获取各种目录 + e.Mongo = filepath.Join(e.BinDir, "mongodb", "bin", "mongo") + e.ScriptDir = filepath.Join("/", "home", e.OsUser, e.runtime.UID) + e.ScriptFilePath = filepath.Join(e.ScriptDir, strings.Join([]string{"script", "js"}, ".")) + e.ResultFilePath = filepath.Join(e.ScriptDir, strings.Join([]string{"result", "txt"}, ".")) + e.runtime.Logger.Info("init successfully") + + // 复制集获取执行脚本的IP端口 默认为primary节点 可以指定secondary节点 + if e.ConfParams.Type == "cluster" { + e.execIP = e.ConfParams.IP + e.execPort = e.ConfParams.Port + } + if e.ConfParams.Type == "replicaset" { + primaryInfo, err := common.AuthGetPrimaryInfo(e.Mongo, e.ConfParams.AdminUsername, + e.ConfParams.AdminPassword, + e.ConfParams.IP, e.ConfParams.Port) + if err != nil { + e.runtime.Logger.Error("init get primary info fail, error:%s", err) + return fmt.Errorf("init get primary info fail, error:%s", err) + } + e.execIP = strings.Split(primaryInfo, ":")[0] + e.execPort, _ = strconv.Atoi(strings.Split(primaryInfo, ":")[1]) + if e.ConfParams.Secondary == true { + _, _, _, _, _, memberInfo, err := common.GetNodeInfo(e.Mongo, e.ConfParams.IP, e.ConfParams.Port, + e.ConfParams.AdminUsername, e.ConfParams.AdminPassword, e.ConfParams.IP, e.ConfParams.Port) + if err != nil { + e.runtime.Logger.Error("init get member info fail, error:%s", err) + return fmt.Errorf("init get member info fail, error:%s", err) + } + for _, v := range memberInfo { + if v["state"] == "2" && v["hidden"] == "false" { + e.execIP = strings.Split(v["name"], ":")[0] + e.execPort, _ = strconv.Atoi(strings.Split(v["name"], ":")[1]) + } + } + } + } + + // 进行校验 + if err := e.checkParams(); err != nil { + return err + } + + return nil +} + +// checkParams 校验参数 +func (e *ExecScript) checkParams() error { + // 校验配置参数 + e.runtime.Logger.Info("start to validate parameters") + validate := validator.New() + e.runtime.Logger.Info("start to validate parameters of deInstall") + if err := validate.Struct(e.ConfParams); err != nil { + e.runtime.Logger.Error("validate parameters of execScript fail, error:%s", err) + return fmt.Errorf("validate parameters of execScript fail, error:%s", err) + } + e.runtime.Logger.Info("validate parameters successfully") + return nil +} + +// makeScriptContent 生成script内容 +func (e *ExecScript) makeScriptContent() error { + // 复制集，判断在primary节点还是在secondary节点执行脚本 + e.runtime.Logger.Info("start to make script content") + if e.ConfParams.Type == "replicaset" && e.ConfParams.Secondary == true { + // 获取mongo版本呢 + mongoName := "mongod" + version, err := common.CheckMongoVersion(e.BinDir, mongoName) + if err != nil { + e.runtime.Logger.Error("get mongo service version fail, error:%s", err) + return fmt.Errorf("get mongo service version fail, error:%s", err) + } + splitVersion := strings.Split(version, ".") + mainVersion, _ := strconv.ParseFloat(strings.Join([]string{splitVersion[0], splitVersion[1]}, "."), 32) + + // secondary执行script + secondaryOk := "rs.slaveOk()\n" + if mainVersion >= 3.6 { + secondaryOk = "rs.secondaryOk()\n" + } + e.ScriptContent = strings.Join([]string{secondaryOk, e.ConfParams.Script}, "") + e.runtime.Logger.Info("make script content successfully") + return nil + } + e.ScriptContent = e.ConfParams.Script + e.runtime.Logger.Info("make script content successfully") + return nil +} + +// creatScriptFile 创建script文件 +func (e *ExecScript) creatScriptFile() error { + // 创建目录 + e.runtime.Logger.Info("start to make script directory") + if err := util.MkDirsIfNotExists([]string{e.ScriptDir}); err != nil { + e.runtime.Logger.Error("create script directory:%s fail, error:%s", e.ScriptDir, err) + return fmt.Errorf("create script directory:%s fail, error:%s", e.ScriptDir, err) + } + + // 创建文件 + e.runtime.Logger.Info("start to create script file") + script, err := os.OpenFile(e.ScriptFilePath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, DefaultPerm) + defer script.Close() + if err != nil { + e.runtime.Logger.Error( + fmt.Sprintf("create script file fail, error:%s", err)) + return fmt.Errorf("create script file fail, error:%s", err) + } + if _, err = script.WriteString(e.ScriptContent); err != nil { + e.runtime.Logger.Error( + fmt.Sprintf("script file write content fail, error:%s", + err)) + return fmt.Errorf("script file write content fail, error:%s", + err) + } + e.runtime.Logger.Info("create script file successfully") + // 修改配置文件属主 + e.runtime.Logger.Info("start to execute chown command for script file") + if _, err = util.RunBashCmd( + fmt.Sprintf("chown -R %s.%s %s", e.OsUser, e.OsGroup, e.ScriptDir), + "", nil, + 10*time.Second); err != nil { + e.runtime.Logger.Error(fmt.Sprintf("chown script file fail, error:%s", err)) + return fmt.Errorf("chown script file fail, error:%s", err) + } + e.runtime.Logger.Info("execute chown command for script file successfully") + return nil +} + +// execScript 执行脚本 +func (e *ExecScript) execScript() error { + e.runtime.Logger.Info("start to execute script") + cmd := fmt.Sprintf( + "%s -u %s -p '%s' --host %s --port %d --authenticationDatabase=admin --quiet %s > %s", + e.Mongo, e.ConfParams.AdminUsername, e.ConfParams.AdminPassword, e.execIP, e.execPort, + e.ScriptFilePath, e.ResultFilePath) + cmdX := fmt.Sprintf( + "%s -u %s -p '%s' --host %s --port %d --authenticationDatabase=admin --quiet %s > %s", + e.Mongo, e.ConfParams.AdminUsername, "xxx", e.execIP, e.execPort, + e.ScriptFilePath, e.ResultFilePath) + if _, err := util.RunBashCmd( + cmd, + "", nil, + 10*time.Second); err != nil { + e.runtime.Logger.Error("execute script:%s fail, error:%s", cmdX, err) + return fmt.Errorf("execute script:%s fail, error:%s", cmdX, err) + } + e.runtime.Logger.Info("execute script:%s successfully", cmdX) + return nil +} + +// Output 请求响应结构体 +type Output struct { + Code int `json:"code"` + Message string `json:"message"` +} + +// uploadFile 上传结果文件 +func (e *ExecScript) uploadFile() error { + if e.ConfParams.RepoUrl == "" { + return nil + } + e.runtime.Logger.Info("start to upload result file") + // url + url := strings.Join([]string{e.ConfParams.RepoUrl, e.ConfParams.RepoProject, e.ConfParams.RepoRepo, + e.ConfParams.RepoPath, e.runtime.UID, "result.txt"}, "/") + + // 生成请求body内容 + file, err := ioutil.ReadFile(e.ResultFilePath) + if err != nil { + e.runtime.Logger.Error("get result file content fail, error:%s", err) + return fmt.Errorf("get result file content fail, error:%s", err) + } + + // 生成请求 + request, err := http.NewRequest("PUT", url, strings.NewReader(string(file))) + if err != nil { + e.runtime.Logger.Error("create request for uploading result file fail, error:%s", err) + return fmt.Errorf("create request for uploading result file fail, error:%s", err) + } + + // 设置请求头 + auth := base64.StdEncoding.EncodeToString([]byte(strings.Join([]string{e.ConfParams.RepoUsername, + e.ConfParams.RepoToken}, ":"))) + request.Header.Set("Authorization", "Basic "+auth) + request.Header.Set("X-BKREPO-EXPIRES", "30") + request.Header.Set("X-BKREPO-OVERWRITE", "true") + request.Header.Set("Content-Type", "multipart/form-data") + if err != nil { + e.runtime.Logger.Error("set request head for uploading result file fail, error:%s", err) + return fmt.Errorf("set request head for uploading result file fail, error:%s", err) + } + + // 执行请求 + response, err := http.DefaultClient.Do(request) + defer response.Body.Close() + if err != nil { + e.runtime.Logger.Error("request server for uploading result file fail, error:%s", err) + return fmt.Errorf("request server for uploading result file fail, error:%s", err) + } + + // 解析响应 + resp, err := ioutil.ReadAll(response.Body) + if err != nil { + e.runtime.Logger.Error("read data from response fail, error:%s", err) + return fmt.Errorf("read data from response fail, error:%s", err) + } + output := Output{} + _ = json.Unmarshal(resp, &output) + if output.Code != 0 && output.Message == "" { + e.runtime.Logger.Error("upload file fail, error:%s", output.Message) + return fmt.Errorf("upload file fail, error:%s", output.Message) + } + e.runtime.Logger.Info("upload result file successfully") + return nil +} diff --git a/dbm-services/mongo/db-tools/dbactuator/pkg/atomjobs/atommongodb/mongo_process_restart.go b/dbm-services/mongo/db-tools/dbactuator/pkg/atomjobs/atommongodb/mongo_process_restart.go new file mode 100644 index 0000000000..b802e1fc19 --- /dev/null +++ b/dbm-services/mongo/db-tools/dbactuator/pkg/atomjobs/atommongodb/mongo_process_restart.go @@ -0,0 +1,399 @@ +package atommongodb + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "strconv" + "strings" + + "dbm-services/mongo/db-tools/dbactuator/pkg/common" + "dbm-services/mongo/db-tools/dbactuator/pkg/consts" + "dbm-services/mongo/db-tools/dbactuator/pkg/jobruntime" + + "github.com/go-playground/validator/v10" + "gopkg.in/yaml.v2" +) + +// RestartConfParams 重启进程参数 +type RestartConfParams struct { + IP string `json:"ip" validate:"required"` + Port int `json:"port" validate:"required"` + InstanceType string `json:"instanceType" validate:"required"` // mongos mongod + SingleNodeInstallRestart bool `json:"singleNodeInstallRestart"` // mongod替换节点安装后重启 + Auth bool `json:"auth"` // true->auth false->noauth + CacheSizeGB int `json:"cacheSizeGB"` // 可选，重启mongod的参数 + MongoSConfDbOld string `json:"mongoSConfDbOld"` // 可选，ip:port + MongoSConfDbNew string `json:"MongoSConfDbNew"` // 可选，ip:port + AdminUsername string `json:"adminUsername"` + AdminPassword string `json:"adminPassword"` +} + +// MongoRestart 重启mongo进程 +type MongoRestart struct { + runtime *jobruntime.JobGenericRuntime + BinDir string + DataDir string + DbpathDir string + Mongo string + OsUser string // MongoDB安装在哪个用户下 + OsGroup string + ConfParams *RestartConfParams + AuthConfFilePath string + NoAuthConfFilePath string +} + +// NewMongoRestart 实例化结构体 +func NewMongoRestart() jobruntime.JobRunner { + return &MongoRestart{} +} + +// Name 获取原子任务的名字 +func (r *MongoRestart) Name() string { + return "mongo_restart" +} + +// Run 运行原子任务 +func (r *MongoRestart) Run() error { + // 修改配置文件参数 + if err := r.changeParam(); err != nil { + return err + } + + // mongod的primary进行主备切换 + if err := r.RsStepDown(); err != nil { + return err + } + + // 关闭服务 + if err := r.shutdown(); err != nil { + return err + } + + // 启动服务 + if err := r.startup(); err != nil { + return err + } + + return nil +} + +// Retry 重试 +func (r *MongoRestart) Retry() uint { + return 2 +} + +// Rollback 回滚 +func (r *MongoRestart) Rollback() error { + return nil +} + +// Init 初始化 +func (r *MongoRestart) Init(runtime *jobruntime.JobGenericRuntime) error { + // 获取安装参数 + r.runtime = runtime + r.runtime.Logger.Info("start to init") + r.BinDir = consts.UsrLocal + r.DataDir = consts.GetRedisDataDir() + r.OsUser = consts.GetProcessUser() + r.OsGroup = consts.GetProcessUserGroup() + r.Mongo = filepath.Join(r.BinDir, "mongodb", "bin", "mongo") + + // 获取MongoDB配置文件参数 + if err := json.Unmarshal([]byte(r.runtime.PayloadDecoded), &r.ConfParams); err != nil { + r.runtime.Logger.Error(fmt.Sprintf( + "get parameters of mongo restart fail by json.Unmarshal, error:%s", err)) + return fmt.Errorf("get parameters of mongo restart fail by json.Unmarshal, error:%s", err) + } + + // 设置各种路径 + strPort := strconv.Itoa(r.ConfParams.Port) + r.DbpathDir = filepath.Join(r.DataDir, "mongodata", strPort, "db") + r.AuthConfFilePath = filepath.Join(r.DataDir, "mongodata", strPort, "mongo.conf") + r.NoAuthConfFilePath = filepath.Join(r.DataDir, "mongodata", strPort, "noauth.conf") + r.runtime.Logger.Info("init successfully") + + // 安装前进行校验 + if err := r.checkParams(); err != nil { + return err + } + + return nil +} + +// checkParams 校验参数 +func (r *MongoRestart) checkParams() error { + // 校验重启配置参数 + validate := validator.New() + r.runtime.Logger.Info("start to validate parameters of restart") + if err := validate.Struct(r.ConfParams); err != nil { + r.runtime.Logger.Error(fmt.Sprintf("validate parameters of restart fail, error:%s", err)) + return fmt.Errorf("validate parameters of restart fail, error:%s", err) + } + r.runtime.Logger.Info("validate parameters of restart successfully") + return nil +} + +// changeParam 修改参数 +func (r *MongoRestart) changeParam() error { + if r.ConfParams.InstanceType == "mongos" && + r.ConfParams.MongoSConfDbOld != "" && r.ConfParams.MongoSConfDbNew != "" { + if err := r.changeConfigDb(); err != nil { + return err + } + return nil + } + if err := r.changeCacheSizeGB(); err != nil { + return err + } + return nil +} + +// changeConfigDb 修改mongoS的ConfigDb参数 +func (r *MongoRestart) changeConfigDb() error { + r.runtime.Logger.Info("start to change configDB value of config file") + // 获取配置文件内容 + readAuthConfFileContent, _ := ioutil.ReadFile(r.AuthConfFilePath) + readNoAuthConfFileContent, _ := ioutil.ReadFile(r.NoAuthConfFilePath) + + // 修改configDB配置 + yamlAuthMongoSConf := common.NewYamlMongoSConf() + yamlNoAuthMongoSConf := common.NewYamlMongoSConf() + _ = yaml.Unmarshal(readAuthConfFileContent, yamlAuthMongoSConf) + _ = yaml.Unmarshal(readNoAuthConfFileContent, yamlNoAuthMongoSConf) + yamlAuthMongoSConf.Sharding.ConfigDB = strings.Replace(yamlAuthMongoSConf.Sharding.ConfigDB, + r.ConfParams.MongoSConfDbOld, r.ConfParams.MongoSConfDbNew, -1) + yamlNoAuthMongoSConf.Sharding.ConfigDB = strings.Replace(yamlNoAuthMongoSConf.Sharding.ConfigDB, + r.ConfParams.MongoSConfDbOld, r.ConfParams.MongoSConfDbNew, -1) + authConfFileContent, _ := yamlAuthMongoSConf.GetConfContent() + noAuthConfFileContent, _ := yamlNoAuthMongoSConf.GetConfContent() + + // 修改authConfFile + authConfFile, err := os.OpenFile(r.AuthConfFilePath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, DefaultPerm) + defer authConfFile.Close() + if err != nil { + r.runtime.Logger.Error( + fmt.Sprintf("create auth config file fail, error:%s", err)) + return fmt.Errorf("create auth config file fail, error:%s", err) + } + if _, err = authConfFile.WriteString(string(authConfFileContent)); err != nil { + r.runtime.Logger.Error( + fmt.Sprintf("change configDB value of auth config file write content fail, error:%s", + err)) + return fmt.Errorf("change configDB value of auth config file write content fail, error:%s", + err) + } + + // 修改noAuthConfFile + noAuthConfFile, err := os.OpenFile(r.NoAuthConfFilePath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, DefaultPerm) + defer noAuthConfFile.Close() + if err != nil { + r.runtime.Logger.Error(fmt.Sprintf("create no auth config file fail, error:%s", err)) + return fmt.Errorf("create no auth config file fail, error:%s", err) + } + if _, err = noAuthConfFile.WriteString(string(noAuthConfFileContent)); err != nil { + r.runtime.Logger.Error( + fmt.Sprintf("change configDB value of no auth config file write content fail, error:%s", + err)) + return fmt.Errorf("change configDB value of no auth config file write content fail, error:%s", + err) + } + r.runtime.Logger.Info("change configDB value of config file successfully") + + return nil +} + +// changeCacheSizeGB 修改CacheSizeGB +func (r *MongoRestart) changeCacheSizeGB() error { + if r.ConfParams.CacheSizeGB == 0 { + return nil + } + + // 检查mongo版本 + r.runtime.Logger.Info("start to check mongo version") + version, err := common.CheckMongoVersion(r.BinDir, "mongod") + if err != nil { + r.runtime.Logger.Error(fmt.Sprintf("check mongo version fail, error:%s", err)) + return fmt.Errorf("check mongo version fail, error:%s", err) + } + mainVersion, _ := strconv.Atoi(strings.Split(version, ".")[0]) + r.runtime.Logger.Info("check mongo version successfully") + + if mainVersion >= 3 { + r.runtime.Logger.Info("start to change CacheSizeGB value of config file") + // 获取配置文件内容 + readAuthConfFileContent, _ := ioutil.ReadFile(r.AuthConfFilePath) + readNoAuthConfFileContent, _ := ioutil.ReadFile(r.NoAuthConfFilePath) + + // 修改CacheSizeGB大小并写入文件 + yamlAuthConfFile := common.NewYamlMongoDBConf() + yamlNoAuthConfFile := common.NewYamlMongoDBConf() + _ = yaml.Unmarshal(readAuthConfFileContent, &yamlAuthConfFile) + _ = yaml.Unmarshal(readNoAuthConfFileContent, &yamlNoAuthConfFile) + if r.ConfParams.CacheSizeGB == 0 { + return nil + } + if r.ConfParams.CacheSizeGB != yamlAuthConfFile.Storage.WiredTiger.EngineConfig.CacheSizeGB { + yamlAuthConfFile.Storage.WiredTiger.EngineConfig.CacheSizeGB = r.ConfParams.CacheSizeGB + yamlNoAuthConfFile.Storage.WiredTiger.EngineConfig.CacheSizeGB = r.ConfParams.CacheSizeGB + authConfFileContent, _ := yamlAuthConfFile.GetConfContent() + noAuthConfFileContent, _ := yamlNoAuthConfFile.GetConfContent() + + // 修改authConfFile + authConfFile, err := os.OpenFile(r.AuthConfFilePath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, DefaultPerm) + defer authConfFile.Close() + if err != nil { + r.runtime.Logger.Error( + fmt.Sprintf("create auth config file fail, error:%s", err)) + return fmt.Errorf("create auth config file fail, error:%s", err) + } + if _, err = authConfFile.WriteString(string(authConfFileContent)); err != nil { + r.runtime.Logger.Error( + fmt.Sprintf("change CacheSizeGB value of auth config file write content fail, error:%s", + err)) + return fmt.Errorf("change CacheSizeGB value of auth config file write content fail, error:%s", + err) + } + + // 修改noAuthConfFile + noAuthConfFile, err := os.OpenFile(r.NoAuthConfFilePath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, DefaultPerm) + defer noAuthConfFile.Close() + if err != nil { + r.runtime.Logger.Error(fmt.Sprintf("create no auth config file fail, error:%s", err)) + return fmt.Errorf("create no auth config file fail, error:%s", err) + } + if _, err = noAuthConfFile.WriteString(string(noAuthConfFileContent)); err != nil { + r.runtime.Logger.Error( + fmt.Sprintf("change CacheSizeGB value of no auth config file write content fail, error:%s", + err)) + return fmt.Errorf("change CacheSizeGB value of no auth config file write content fail, error:%s", + err) + } + } + r.runtime.Logger.Info("change CacheSizeGB value of config file successfully") + } + return nil +} + +// checkPrimary 检查该节点是否是primary +func (r *MongoRestart) checkPrimary() (bool, error) { + r.runtime.Logger.Info("start to check if it is primary") + var info string + var err error + // 安装时无需密码验证。安装成功后需要密码验证 + if r.ConfParams.AdminUsername != "" && r.ConfParams.AdminPassword != "" { + info, err = common.AuthGetPrimaryInfo(r.Mongo, r.ConfParams.AdminUsername, + r.ConfParams.AdminPassword, r.ConfParams.IP, r.ConfParams.Port) + } else { + info, err = common.NoAuthGetPrimaryInfo(r.Mongo, + r.ConfParams.IP, r.ConfParams.Port) + } + if err != nil { + r.runtime.Logger.Error("get primary info fail, error:%s", err) + return false, fmt.Errorf("get primary info fail, error:%s", err) + } + if info == fmt.Sprintf("%s:%d", r.ConfParams.IP, r.ConfParams.Port) { + return true, nil + } + r.runtime.Logger.Info("check if it is primary successfully") + return false, nil +} + +// RsStepDown 主备切换 +func (r *MongoRestart) RsStepDown() error { + if r.ConfParams.InstanceType != "mongos" { + if r.ConfParams.SingleNodeInstallRestart == true { + return nil + } + r.runtime.Logger.Info("start to check mongod service before rsStepDown") + flag, _, err := common.CheckMongoService(r.ConfParams.Port) + if err != nil { + r.runtime.Logger.Error("check mongod service fail, error:%s", err) + return fmt.Errorf("check mongod service fail, error:%s", err) + } + r.runtime.Logger.Info("check mongod service before rsStepDown successfully") + if flag == false { + return nil + } + + // 检查是否是primary + flag1, err := r.checkPrimary() + if err != nil { + return err + } + if flag1 == true { + r.runtime.Logger.Info("start to convert primary secondary db") + // 安装时无需密码验证。安装成功后需要密码验证 + var flag2 bool + if r.ConfParams.AdminUsername != "" && r.ConfParams.AdminPassword != "" { + flag2, err = common.AuthRsStepDown(r.Mongo, r.ConfParams.IP, r.ConfParams.Port, + r.ConfParams.AdminUsername, r.ConfParams.AdminPassword) + } else { + flag2, err = common.NoAuthRsStepDown(r.Mongo, r.ConfParams.IP, r.ConfParams.Port) + } + if err != nil { + r.runtime.Logger.Error("convert primary secondary db fail, error:%s", err) + return fmt.Errorf("convert primary secondary db fail, error:%s", err) + } + if flag2 == true { + r.runtime.Logger.Info("convert primary secondary db successfully") + return nil + } + } + } + + return nil +} + +// shutdown 关闭服务 +func (r *MongoRestart) shutdown() error { + // 检查服务是否存在 + r.runtime.Logger.Info("start to check %s service", r.ConfParams.InstanceType) + result, _, err := common.CheckMongoService(r.ConfParams.Port) + if err != nil { + r.runtime.Logger.Error("check %s service fail, error:%s", r.ConfParams.InstanceType, err) + return fmt.Errorf("check %s service fail, error:%s", r.ConfParams.InstanceType, err) + } + if result != true { + r.runtime.Logger.Info("%s service has been close", r.ConfParams.InstanceType) + return nil + } + r.runtime.Logger.Info("check %s service successfully", r.ConfParams.InstanceType) + + // 关闭服务 + r.runtime.Logger.Info("start to shutdown %s", r.ConfParams.InstanceType) + if err = common.ShutdownMongoProcess(r.OsUser, r.ConfParams.InstanceType, r.BinDir, r.DbpathDir, + r.ConfParams.Port); err != nil { + r.runtime.Logger.Error(fmt.Sprintf("shutdown %s fail, error:%s", r.ConfParams.InstanceType, err)) + return fmt.Errorf("shutdown %s fail, error:%s", r.ConfParams.InstanceType, err) + } + r.runtime.Logger.Info("shutdown %s successfully", r.ConfParams.InstanceType) + return nil +} + +// startup 开启服务 +func (r *MongoRestart) startup() error { + // 检查服务是否存在 + r.runtime.Logger.Info("start to check %s service", r.ConfParams.InstanceType) + result, _, err := common.CheckMongoService(r.ConfParams.Port) + if err != nil { + r.runtime.Logger.Error("check %s service fail, error:%s", r.ConfParams.InstanceType, err) + return fmt.Errorf("check %s service fail, error:%s", r.ConfParams.InstanceType, err) + } + if result == true { + r.runtime.Logger.Info("%s service has been open", r.ConfParams.InstanceType) + return nil + } + r.runtime.Logger.Info("check %s service successfully", r.ConfParams.InstanceType) + + // 开启服务 + r.runtime.Logger.Info("start to startup %s", r.ConfParams.InstanceType) + if err = common.StartMongoProcess(r.BinDir, r.ConfParams.Port, r.OsUser, r.ConfParams.Auth); err != nil { + r.runtime.Logger.Error("startup %s fail, error:%s", r.ConfParams.InstanceType, err) + return fmt.Errorf("startup %s fail, error:%s", r.ConfParams.InstanceType, err) + } + r.runtime.Logger.Info("startup %s successfully", r.ConfParams.InstanceType) + return nil +} diff --git a/dbm-services/mongo/db-tools/dbactuator/pkg/atomjobs/atommongodb/mongo_set_profiler.go b/dbm-services/mongo/db-tools/dbactuator/pkg/atomjobs/atommongodb/mongo_set_profiler.go new file mode 100644 index 0000000000..f53466fddb --- /dev/null +++ b/dbm-services/mongo/db-tools/dbactuator/pkg/atomjobs/atommongodb/mongo_set_profiler.go @@ -0,0 +1,186 @@ +package atommongodb + +import ( + "encoding/json" + "fmt" + "path/filepath" + "strconv" + "strings" + "time" + + "dbm-services/mongo/db-tools/dbactuator/pkg/common" + "dbm-services/mongo/db-tools/dbactuator/pkg/consts" + "dbm-services/mongo/db-tools/dbactuator/pkg/jobruntime" + "dbm-services/mongo/db-tools/dbactuator/pkg/util" + + "github.com/go-playground/validator/v10" +) + +// SetProfilerConfParams 参数 +type SetProfilerConfParams struct { + IP string `json:"ip" validate:"required"` + Port int `json:"port" validate:"required"` + DbName string `json:"dbName" validate:"required"` + Level int `json:"level" validate:"required"` + ProfileSize int `json:"profileSize"` // 单位：GB + AdminUsername string `json:"adminUsername" validate:"required"` + AdminPassword string `json:"adminPassword" validate:"required"` +} + +// SetProfiler 添加分片到集群 +type SetProfiler struct { + runtime *jobruntime.JobGenericRuntime + BinDir string + Mongo string + OsUser string + PrimaryIP string + PrimaryPort int + ConfParams *SetProfilerConfParams +} + +// NewSetProfiler 实例化结构体 +func NewSetProfiler() jobruntime.JobRunner { + return &SetProfiler{} +} + +// Name 获取原子任务的名字 +func (s *SetProfiler) Name() string { + return "mongo_set_profiler" +} + +// Run 运行原子任务 +func (s *SetProfiler) Run() error { + // 生成script内容 + if err := s.setProfileSize(); err != nil { + return err + } + + // 执行脚本生成结果文件 + if err := s.setProfileLevel(); err != nil { + return err + } + + return nil +} + +// Retry 重试 +func (s *SetProfiler) Retry() uint { + return 2 +} + +// Rollback 回滚 +func (s *SetProfiler) Rollback() error { + return nil +} + +// Init 初始化 +func (s *SetProfiler) Init(runtime *jobruntime.JobGenericRuntime) error { + // 获取安装参数 + s.runtime = runtime + s.runtime.Logger.Info("start to init") + s.BinDir = consts.UsrLocal + s.OsUser = consts.GetProcessUser() + + // 获取MongoDB配置文件参数 + if err := json.Unmarshal([]byte(s.runtime.PayloadDecoded), &s.ConfParams); err != nil { + s.runtime.Logger.Error( + "get parameters of setProfiler fail by json.Unmarshal, error:%s", err) + return fmt.Errorf("get parameters of setProfiler fail by json.Unmarshal, error:%s", err) + } + + // 获取primary信息 + info, err := common.AuthGetPrimaryInfo(s.Mongo, s.ConfParams.AdminUsername, s.ConfParams.AdminPassword, + s.ConfParams.IP, s.ConfParams.Port) + if err != nil { + s.runtime.Logger.Error("get primary db info fail, error:%s", err) + return fmt.Errorf("get primary db info fail, error:%s", err) + } + sliceInfo := strings.Split(info, ":") + s.PrimaryIP = sliceInfo[0] + s.PrimaryPort, _ = strconv.Atoi(sliceInfo[1]) + + // 获取各种目录 + s.Mongo = filepath.Join(s.BinDir, "mongodb", "bin", "mongo") + s.runtime.Logger.Info("init successfully") + + // 进行校验 + if err = s.checkParams(); err != nil { + return err + } + + return nil +} + +// checkParams 校验参数 +func (s *SetProfiler) checkParams() error { + // 校验配置参数 + s.runtime.Logger.Info("start to validate parameters") + validate := validator.New() + s.runtime.Logger.Info("start to validate parameters of deInstall") + if err := validate.Struct(s.ConfParams); err != nil { + s.runtime.Logger.Error("validate parameters of setProfiler fail, error:%s", err) + return fmt.Errorf("validate parameters of setProfiler fail, error:%s", err) + } + s.runtime.Logger.Info("validate parameters successfully") + return nil +} + +// setProfileSize 设置profile大小 +func (s *SetProfiler) setProfileSize() error { + // 获取profile级别 + status, err := common.GetProfilingLevel(s.Mongo, s.ConfParams.IP, s.ConfParams.Port, + s.ConfParams.AdminUsername, s.ConfParams.AdminPassword, s.ConfParams.DbName) + if err != nil { + s.runtime.Logger.Error("get profile level fail, error:%s", err) + return fmt.Errorf("get profile level fail, error:%s", err) + } + if status != 0 { + if err = common.SetProfilingLevel(s.Mongo, s.ConfParams.IP, s.ConfParams.Port, s.ConfParams.AdminUsername, + s.ConfParams.AdminPassword, s.ConfParams.DbName, 0); err != nil { + s.runtime.Logger.Error("set profile level 0 fail, error:%s", err) + return fmt.Errorf("set profile level 0 fail, error:%s", err) + } + } + + // 删除profile.system + cmd := fmt.Sprintf( + "%s -u %s -p '%s' --host %s --port %d --authenticationDatabase=admin --quiet --eval \"db.getMongo().getDB('%s').system.profile.drop()\"", + s.Mongo, s.ConfParams.AdminUsername, s.ConfParams.AdminPassword, s.ConfParams.IP, s.ConfParams.Port, + s.ConfParams.DbName) + if _, err = util.RunBashCmd( + cmd, + "", nil, + 10*time.Second); err != nil { + s.runtime.Logger.Error("delete system.profile fail, error:%s", err) + return fmt.Errorf("set system.profile fail, error:%s", err) + } + + // 设置profile.system + s.runtime.Logger.Info("start to set system.profile size") + profileSizeBytes := s.ConfParams.ProfileSize * 1024 * 1024 * 1024 + cmd = fmt.Sprintf( + "%s -u %s -p '%s' --host %s --port %d --authenticationDatabase=admin --quiet --eval \"db.getMongo().getDB('%s').createCollection('system.profile',{ capped: true, size:%d })\"", + s.Mongo, s.ConfParams.AdminUsername, s.ConfParams.AdminPassword, s.ConfParams.IP, s.ConfParams.Port, + s.ConfParams.DbName, profileSizeBytes) + if _, err = util.RunBashCmd( + cmd, + "", nil, + 10*time.Second); err != nil { + s.runtime.Logger.Error("set system.profile size fail, error:%s", err) + return fmt.Errorf("set system.profile size fail, error:%s", err) + } + s.runtime.Logger.Info("set system.profile size successfully") + return nil +} + +// setProfileLevel 生成脚本内容 +func (s *SetProfiler) setProfileLevel() error { + s.runtime.Logger.Info("start to set profile level") + if err := common.SetProfilingLevel(s.Mongo, s.ConfParams.IP, s.ConfParams.Port, s.ConfParams.AdminUsername, + s.ConfParams.AdminPassword, s.ConfParams.DbName, s.ConfParams.Level); err != nil { + s.runtime.Logger.Error("set profile level fail, error:%s", err) + return fmt.Errorf("set profile level fail, error:%s", err) + } + s.runtime.Logger.Info("set profile level successfully") + return nil +} diff --git a/dbm-services/mongo/db-tools/dbactuator/pkg/atomjobs/atommongodb/mongod_change_oplogsize.go b/dbm-services/mongo/db-tools/dbactuator/pkg/atomjobs/atommongodb/mongod_change_oplogsize.go new file mode 100644 index 0000000000..7948ac8191 --- /dev/null +++ b/dbm-services/mongo/db-tools/dbactuator/pkg/atomjobs/atommongodb/mongod_change_oplogsize.go @@ -0,0 +1,501 @@ +package atommongodb + +import ( + "dbm-services/mongo/db-tools/dbactuator/pkg/common" + "dbm-services/mongo/db-tools/dbactuator/pkg/consts" + "dbm-services/mongo/db-tools/dbactuator/pkg/jobruntime" + "dbm-services/mongo/db-tools/dbactuator/pkg/util" + "encoding/json" + "fmt" + "os" + "path/filepath" + "strconv" + "strings" + "time" + + "github.com/go-playground/validator/v10" +) + +// MongoDChangeOplogSizeConfParams 参数 // 修改oplog大小 +type MongoDChangeOplogSizeConfParams struct { + IP string `json:"ip" validate:"required"` // 执行节点 + Port int `json:"port" validate:"required"` + AdminUsername string `json:"adminUsername" validate:"required"` + AdminPassword string `json:"adminPassword" validate:"required"` + NewOplogSize int `json:"newOplogSize"` // 单位：GB +} + +// MongoDChangeOplogSize 修改oplog大小 +type MongoDChangeOplogSize struct { + runtime *jobruntime.JobGenericRuntime + BinDir string + Mongo string + MongoD string + OsUser string + OsGroup string + DataDir string + DbpathDir string + ConfParams *MongoDChangeOplogSizeConfParams + PrimaryIP string + PrimaryPort int + MainVersion int + Version float64 + AuthConfFilePath string + NoAuthConfFilePath string + OplogSizeMB int + NewOplogSizeMB int + ScriptDir string + ScriptFilePath string + NewPort int +} + +// NewMongoDChangeOplogSize 实例化结构体 +func NewMongoDChangeOplogSize() jobruntime.JobRunner { + return &MongoDChangeOplogSize{} +} + +// Name 获取原子任务的名字 +func (c *MongoDChangeOplogSize) Name() string { + return "mongod_change_oplogsize" +} + +// Run 运行原子任务 +func (c *MongoDChangeOplogSize) Run() error { + // 检查现有oplog大小 + if err := c.checkOplogSizeAndFreeSpace(); err != nil { + return err + } + + // 修改配置文件 + if err := c.changeConfigFile(); err != nil { + return err + } + + // 修改oplog大小 + if err := c.changeOplogSize(); err != nil { + return err + } + + return nil +} + +// Retry 重试 +func (c *MongoDChangeOplogSize) Retry() uint { + return 2 +} + +// Rollback 回滚 +func (c *MongoDChangeOplogSize) Rollback() error { + return nil +} + +// Init 初始化 +func (c *MongoDChangeOplogSize) Init(runtime *jobruntime.JobGenericRuntime) error { + // 获取安装参数 + c.runtime = runtime + c.runtime.Logger.Info("start to init") + c.BinDir = consts.UsrLocal + c.Mongo = filepath.Join(c.BinDir, "mongodb", "bin", "mongo") + c.MongoD = filepath.Join(c.BinDir, "mongodb", "bin", "mongod") + c.OsUser = consts.GetProcessUser() + c.OsGroup = consts.GetProcessUserGroup() + c.DataDir = consts.GetMongoDataDir() + + // 获取MongoDB配置文件参数 + if err := json.Unmarshal([]byte(c.runtime.PayloadDecoded), &c.ConfParams); err != nil { + c.runtime.Logger.Error(fmt.Sprintf( + "get parameters of mongoDChangeOplogSize fail by json.Unmarshal, error:%s", err)) + return fmt.Errorf("get parameters of mongoDChangeOplogSize fail by json.Unmarshal, error:%s", err) + } + // 获取路径 + strPort := strconv.Itoa(c.ConfParams.Port) + c.AuthConfFilePath = filepath.Join(c.DataDir, "mongodata", strPort, "mongo.conf") + c.NoAuthConfFilePath = filepath.Join(c.DataDir, "mongodata", strPort, "noauth.conf") + c.DbpathDir = filepath.Join(c.DataDir, "mongodata", strPort, "db") + c.ScriptDir = filepath.Join("/", "home", c.OsUser, c.runtime.UID) + c.ScriptFilePath = filepath.Join(c.ScriptDir, strings.Join([]string{"script", "js"}, ".")) + c.NewOplogSizeMB = c.ConfParams.NewOplogSize * 1024 + c.NewPort = c.ConfParams.Port + 1000 + + // 获取primary信息 + info, err := common.AuthGetPrimaryInfo(c.Mongo, c.ConfParams.AdminUsername, c.ConfParams.AdminPassword, + c.ConfParams.IP, c.ConfParams.Port) + if err != nil { + c.runtime.Logger.Error(fmt.Sprintf( + "get primary db info of mongoDChangeOplogSize fail, error:%s", err)) + return fmt.Errorf("get primary db info of mongoDChangeOplogSize fail, error:%s", err) + } + // 判断info是否为null + if info == "" { + c.runtime.Logger.Error(fmt.Sprintf( + "get primary db info of mongoDChangeOplogSize fail, error:%s", err)) + return fmt.Errorf("get primary db info of mongoDChangeOplogSize fail, error:%s", err) + } + getInfo := strings.Split(info, ":") + c.PrimaryIP = getInfo[0] + c.PrimaryPort, _ = strconv.Atoi(getInfo[1]) + + // 获取mongo版本 + version, err := common.CheckMongoVersion(c.BinDir, "mongod") + if err != nil { + c.runtime.Logger.Error(fmt.Sprintf("check mongo version fail, error:%s", err)) + return fmt.Errorf("check mongo version fail, error:%s", err) + } + c.MainVersion, _ = strconv.Atoi(strings.Split(version, ".")[0]) + c.Version, _ = strconv.ParseFloat(version[0:3], 64) + + c.runtime.Logger.Info("init successfully") + + // 进行校验 + if err = c.checkParams(); err != nil { + return err + } + + return nil +} + +// checkParams 校验参数 +func (c *MongoDChangeOplogSize) checkParams() error { + // 校验重启配置参数 + validate := validator.New() + c.runtime.Logger.Info("start to validate parameters of mongoDChangeOplogSize") + if err := validate.Struct(c.ConfParams); err != nil { + c.runtime.Logger.Error("validate parameters of mongoDChangeOplogSize fail, error:%s", err) + return fmt.Errorf("validate parameters of mongoDChangeOplogSize fail, error:%s", err) + } + c.runtime.Logger.Info("validate parameters of mongoDChangeOplogSize successfully") + return nil +} + +// checkOplogSizeAndFreeSpace 检查现有oplog大小及挂载点的剩余空间 +func (c *MongoDChangeOplogSize) checkOplogSizeAndFreeSpace() error { + // 检查现有oplog大小 + c.runtime.Logger.Info("start to check current oplogSize") + cmd := fmt.Sprintf( + "%s --host %s --port %d --authenticationDatabase=admin --quiet --eval \"%s\" %s", + c.Mongo, c.ConfParams.IP, c.ConfParams.Port, + "db.getSiblingDB('local').oplog.rs.stats(1024*1024*1024).maxSize", "admin") + oplogSize, err := util.RunBashCmd( + cmd, + "", nil, + 10*time.Second) + if err != nil { + c.runtime.Logger.Error("get current oplogSize fail, error:%s", err) + return fmt.Errorf("get current oplogSize fail, error:%s", err) + } + oplogSizeInt, _ := strconv.Atoi(strings.Replace(oplogSize, "\n", "", -1)) + c.OplogSizeMB = oplogSizeInt * 1024 + if oplogSizeInt == c.ConfParams.NewOplogSize { + c.runtime.Logger.Error("newOplogSize:%dGB is same as current oplogSize:%dGB", c.ConfParams.NewOplogSize, oplogSizeInt) + return fmt.Errorf("newOplogSize:%dGB is same as current oplogSize:%dGB", c.ConfParams.NewOplogSize, oplogSizeInt) + } else if oplogSizeInt > c.ConfParams.NewOplogSize { + c.runtime.Logger.Error("newOplogSize:%dGB is less than current oplogSize:%dGB", c.ConfParams.NewOplogSize, + oplogSizeInt) + return fmt.Errorf("newOplogSize:%dGB is less than current oplogSize:%dGB", c.ConfParams.NewOplogSize, oplogSizeInt) + } + c.runtime.Logger.Info("check current oplogSize successfully") + + // + c.runtime.Logger.Info("start to check free space about mountPoint") + cmd = fmt.Sprintf("df -m |grep %s | awk '{print $4}'", c.OsUser) + result, err := util.RunBashCmd( + cmd, + "", nil, + 10*time.Second) + if err != nil { + c.runtime.Logger.Error("check free space about mountPoint, error:%s", err) + return fmt.Errorf("check free space about mountPoint, error:%s", err) + } + result = strings.Replace(result, "\n", "", -1) + resultInt, _ := strconv.Atoi(result) + if resultInt < c.NewOplogSizeMB-c.OplogSizeMB+5120 { + c.runtime.Logger.Error("free space is not enough about mountPoint") + return fmt.Errorf("free space is not enough about mountPoint") + } + c.runtime.Logger.Info("check free space about mountPoint successfully") + return nil +} + +// changeConfigFile 修改配置文件 +func (c *MongoDChangeOplogSize) changeConfigFile() error { + c.runtime.Logger.Info("start to change config file content") + var authCmd string + var noAuthCmd string + var checkAuthCmd string + var checkNoAuthcmd string + // 修改oplog大小参数及 + if c.MainVersion >= 3 { + authCmd = fmt.Sprintf("sed -i \"s/oplogSizeMB: %d/oplogSizeMB: %d/g\" %s", c.OplogSizeMB, c.NewOplogSizeMB, + c.AuthConfFilePath) + noAuthCmd = fmt.Sprintf("sed -i \"s/oplogSizeMB: %d/oplogSizeMB: %d/g\" %s", c.OplogSizeMB, c.NewOplogSizeMB, + c.NoAuthConfFilePath) + } else { + authCmd = fmt.Sprintf("sed -i \"soplogSize=%d/oplogSize=%d/g\" %s", c.OplogSizeMB, c.NewOplogSizeMB, + c.AuthConfFilePath) + noAuthCmd = fmt.Sprintf("sed -i \"s/oplogSize=%d/oplogSize=%d/g\" %s", c.OplogSizeMB, c.NewOplogSizeMB, + c.NoAuthConfFilePath) + } + if _, err := util.RunBashCmd( + authCmd, + "", nil, + 10*time.Second); err != nil { + c.runtime.Logger.Error("change auth config file fail, error:%s", err) + return fmt.Errorf("change auth config file fail, error:%s", err) + } + if _, err := util.RunBashCmd( + noAuthCmd, + "", nil, + 10*time.Second); err != nil { + c.runtime.Logger.Error("change noAuth config file fail, error:%s", err) + return fmt.Errorf("change noAuth config file fail, error:%s", err) + } + // 检查oplog大小参数 + if c.MainVersion >= 3 { + checkAuthCmd = fmt.Sprintf("cat %s |grep oplogSizeMB", c.AuthConfFilePath) + checkNoAuthcmd = fmt.Sprintf("cat %s |grep oplogSizeMB", c.NoAuthConfFilePath) + } else { + checkAuthCmd = fmt.Sprintf("cat %s |grep oplogSize", c.AuthConfFilePath) + checkNoAuthcmd = fmt.Sprintf("cat %s |grep oplogSize", c.NoAuthConfFilePath) + } + result, err := util.RunBashCmd( + checkAuthCmd, + "", nil, + 10*time.Second) + if err != nil { + c.runtime.Logger.Error("check oplogSize parameter of auth config file fail, error:%s", err) + return fmt.Errorf("check oplogSize parameter of auth config file fail, error:%s", err) + } + if strings.Contains(result, strconv.Itoa(c.NewOplogSizeMB)) { + c.runtime.Logger.Info("change oplogSize parameter of auth config file successfully") + } + result1, err := util.RunBashCmd( + checkNoAuthcmd, + "", nil, + 10*time.Second) + if err != nil { + c.runtime.Logger.Error("check oplogSize parameter of noAuth config file fail, error:%s", err) + return fmt.Errorf("check oplogSize parameter of noAuth config file fail, error:%s", err) + } + if strings.Contains(result1, strconv.Itoa(c.NewOplogSizeMB)) { + c.runtime.Logger.Info("change oplogSize parameter of noAuth config file successfully") + } + c.runtime.Logger.Info("change config file content successfully") + return nil +} + +// createScript db版本小于3.6 创建修改脚本 +func (c *MongoDChangeOplogSize) createScript() error { + c.runtime.Logger.Info("create change oplogSize script") + scriptContent := `db = db.getSiblingDB('local'); +db.temp.drop(); +db.temp.save( db.oplog.rs.find( { }, { ts: 1, h: 1 } ).sort( {$natural : -1} ).limit(1).next() ); +db.oplog.rs.drop(); +db.runCommand( { create: "oplog.rs", capped: true, size: ({{oplogSizeGB}} * 1024 * 1024 * 1024) } ); +db.oplog.rs.save( db.temp.findOne() ); +db.temp.drop();` + scriptContent = strings.Replace(scriptContent, "{{oplogSizeGB}}", strconv.Itoa(c.ConfParams.NewOplogSize), -1) + // 创建目录 + if err := util.MkDirsIfNotExists([]string{c.ScriptDir}); err != nil { + c.runtime.Logger.Error("create script directory:%s fail, error:%s", c.ScriptDir, err) + return fmt.Errorf("create script directory:%s fail, error:%s", c.ScriptDir, err) + } + // 创建文件 + c.runtime.Logger.Info("start to create script file") + script, err := os.OpenFile(c.ScriptFilePath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, DefaultPerm) + defer script.Close() + if err != nil { + c.runtime.Logger.Error( + fmt.Sprintf("create script file fail, error:%s", err)) + return fmt.Errorf("create script file fail, error:%s", err) + } + if _, err = script.WriteString(scriptContent); err != nil { + c.runtime.Logger.Error( + fmt.Sprintf("script file write content fail, error:%s", + err)) + return fmt.Errorf("script file write content fail, error:%s", + err) + } + // 修改配置文件属主 + if _, err = util.RunBashCmd( + fmt.Sprintf("chown -R %s.%s %s", c.OsUser, c.OsGroup, c.ScriptDir), + "", nil, + 10*time.Second); err != nil { + c.runtime.Logger.Error("chown script file fail, error:%s", err) + return fmt.Errorf("chown script file fail, error:%s", err) + } + return nil +} + +// standaloneStart 单机形式启动 +func (c *MongoDChangeOplogSize) standaloneStart() error { + if err := common.ShutdownMongoProcess(c.OsUser, "mongod", c.BinDir, c.DbpathDir, + c.ConfParams.Port); err != nil { + c.runtime.Logger.Error("shutdown mongod fail, error:%s", err) + return fmt.Errorf("shutdown mongod fail, error:%s", err) + } + // 检查NewPort是否使用 + for { + flag, _ := util.CheckPortIsInUse(c.ConfParams.IP, strconv.Itoa(c.NewPort)) + if flag { + c.NewPort += 1 + } + if !flag { + break + } + } + // 以单机形式启动 + cmd := fmt.Sprintf("%s --port %d --dbpath %s --fork", c.MongoD, c.NewPort, c.DbpathDir) + if _, err := util.RunBashCmd( + cmd, + "", nil, + 10*time.Second); err != nil { + c.runtime.Logger.Error("startup mongod by port:%d fail, error:%s", c.NewPort, err) + return fmt.Errorf("startup mongod by port:%d fail, error:%s", c.NewPort, err) + } + // 检查是否启动成功 + flag, _, err := common.CheckMongoService(c.NewPort) + if err != nil || flag == false { + c.runtime.Logger.Error("check mongod fail by port:%d fail, error:%s", c.NewPort, err) + return fmt.Errorf("check mongod fail by port:%d fail, error:%s", c.NewPort, err) + } + return nil +} + +// normalStart 正常启动 +func (c *MongoDChangeOplogSize) normalStart() error { + if err := common.ShutdownMongoProcess(c.OsUser, "mongod", c.BinDir, c.DbpathDir, + c.NewPort); err != nil { + c.runtime.Logger.Error("shutdown mongod about port:%d fail, error:%s", c.NewPort, err) + return fmt.Errorf("shutdown mongod about port:%d fail, error:%s", c.NewPort, err) + } + if err := common.StartMongoProcess(c.BinDir, c.ConfParams.Port, c.OsUser, true); err != nil { + c.runtime.Logger.Error("startup mongod about port:%d fail, error:%s", c.ConfParams.Port, err) + return fmt.Errorf("startup mongod about port:%d fail, error:%s", c.ConfParams.Port, err) + } + // 检查是否启动成功 + flag, _, err := common.CheckMongoService(c.NewPort) + if err != nil || flag == false { + c.runtime.Logger.Error("check mongod fail about port:%d fail, error:%s", c.ConfParams.Port, err) + return fmt.Errorf("check mongod fail about port:%d fail, error:%s", c.ConfParams.Port, err) + } + return nil +} + +// setOplog db版本小于3.6修改oplog +func (c *MongoDChangeOplogSize) setOplog() error { + // 关闭dbmon + c.runtime.Logger.Info("stop dbmon") + cmd := fmt.Sprintf("/home/%s/dbmon/stop.sh", c.OsUser) + if _, err := util.RunBashCmd( + cmd, + "", nil, + 10*time.Second); err != nil { + c.runtime.Logger.Error("stop dbmon fail, error:%s", err) + return fmt.Errorf("stop dbmon fail, error:%s", err) + } + + // 如果是主库进行主备切换 + if c.ConfParams.IP == c.PrimaryIP && c.ConfParams.Port == c.PrimaryPort { + c.runtime.Logger.Info("change primary to secondary") + flag, err := common.AuthRsStepDown(c.Mongo, c.ConfParams.IP, c.ConfParams.Port, c.ConfParams.AdminUsername, + c.ConfParams.AdminPassword) + if err != nil { + c.runtime.Logger.Error("change primary to secondary fail, error:%s", err) + return fmt.Errorf("change primary to secondary fail, error:%s", err) + } + if !flag { + c.runtime.Logger.Error("change primary to secondary fail") + return fmt.Errorf("change primary to secondary fail") + } + } + + // 创建修改oplog脚本 + if err := c.createScript(); err != nil { + return err + } + + // 关闭进程以单机形式启动 + if err := c.standaloneStart(); err != nil { + return err + } + + // 执行修改oplog脚本脚本 + cmd = fmt.Sprintf( + "%s -u %s -p '%s' --host %s --port %d --authenticationDatabase=admin --quiet %s", + c.Mongo, c.ConfParams.AdminUsername, c.ConfParams.AdminPassword, c.ConfParams.IP, c.ConfParams.Port, + c.ScriptFilePath) + if _, err := util.RunBashCmd( + cmd, + "", nil, + 10*time.Second); err != nil { + c.runtime.Logger.Error("execute script fail, error:%s", err) + return fmt.Errorf("execute script fail, error:%s", err) + } + // 检查新建oplog文档数量，需要等于1 + cmd = fmt.Sprintf( + "%s --host %s --port %d --authenticationDatabase=admin --quiet --eval \"%s\" %s", + c.Mongo, c.ConfParams.IP, c.ConfParams.Port, "db.getSiblingDB('local').oplog.rs.count()", "admin") + result, err := util.RunBashCmd( + cmd, + "", nil, + 10*time.Second) + if err != nil { + c.runtime.Logger.Error("check the number of new oplog document fail, error:%s", err) + return fmt.Errorf("check the number of new oplog document fail, error:%s", err) + } + result = strings.Replace(result, "\n", "", -1) + resultInt, _ := strconv.Atoi(result) + if resultInt != 1 { + c.runtime.Logger.Error("number of new oplog document is not equal 1, please check") + return fmt.Errorf("number of new oplog document is not equal 1, please check") + } + + // 关闭进程正常重启进程 + if err = c.normalStart(); err != nil { + return err + } + + // 开启dbmon + c.runtime.Logger.Info("start dbmon") + cmd = fmt.Sprintf("/home/%s/dbmon/start.sh", c.OsUser) + if _, err = util.RunBashCmd( + cmd, + "", nil, + 10*time.Second); err != nil { + c.runtime.Logger.Error("start dbmon fail, error:%s", err) + return fmt.Errorf("start dbmon fail, error:%s", err) + } + return nil +} + +// setOplog3 db版本大于等于3.6修改oplog +func (c *MongoDChangeOplogSize) setOplog3() error { + script := fmt.Sprintf("db.adminCommand({replSetResizeOplog: 1, size: %d})", c.NewOplogSizeMB) + cmd := fmt.Sprintf( + "%s --host %s --port %d --authenticationDatabase=admin --quiet --eval \"%s\" %s", + c.Mongo, c.ConfParams.IP, c.ConfParams.Port, script, "admin") + if _, err := util.RunBashCmd( + cmd, + "", nil, + 10*time.Second); err != nil { + c.runtime.Logger.Error("change oplogSize fail, error:%s", err) + return fmt.Errorf("change oplogSize fail, error:%s", err) + } + return nil +} + +// changeOplogSize 修改oplog大小 +func (c *MongoDChangeOplogSize) changeOplogSize() error { + c.runtime.Logger.Info("start to change oplog size") + if c.Version >= 3.6 { + if err := c.setOplog3(); err != nil { + return err + } + } else { + if err := c.setOplog(); err != nil { + return err + } + } + c.runtime.Logger.Info("change oplog size successfully") + return nil +} diff --git a/dbm-services/mongo/db-tools/dbactuator/pkg/atomjobs/atommongodb/mongod_install.go b/dbm-services/mongo/db-tools/dbactuator/pkg/atomjobs/atommongodb/mongod_install.go new file mode 100644 index 0000000000..1a444924a7 --- /dev/null +++ b/dbm-services/mongo/db-tools/dbactuator/pkg/atomjobs/atommongodb/mongod_install.go @@ -0,0 +1,492 @@ +package atommongodb + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "path/filepath" + "strconv" + "strings" + "time" + + "dbm-services/mongo/db-tools/dbactuator/pkg/common" + "dbm-services/mongo/db-tools/dbactuator/pkg/consts" + "dbm-services/mongo/db-tools/dbactuator/pkg/jobruntime" + "dbm-services/mongo/db-tools/dbactuator/pkg/util" + + "github.com/go-playground/validator/v10" +) + +/* + +1.预检查 +检查输入的参数 检查端口是否合规 检查安装包 检查端口是否被使用（如果使用，则检查是否是mongodb服务） + +2.解压安装包 判断是否已经解压过，版本是否正确 +解压文件 做软链接 修改文件属主 + +3.安装 判断目录是否已被创建 +创建相关各级目录-判断目录是否已经创建过 修改目录属主 创建配置文件(noauth, auth) 创建dbtype文件 复制集创建key文件 + +4.启动服务 +以noauth启动服务 + +*/ + +// MongoDBPortMin MongoDB最小端口 +const MongoDBPortMin = 27000 + +// MongoDBPortMax MongoDB最大端口 +const MongoDBPortMax = 28999 + +// DefaultPerm 创建目录、文件的默认权限 +const DefaultPerm = 0755 + +// MongoDBConfParams 配置文件参数 +type MongoDBConfParams struct { + common.MediaPkg `json:"mediapkg"` + IP string `json:"ip" validate:"required"` + Port int `json:"port" validate:"required"` + DbVersion string `json:"dbVersion" validate:"required"` + InstanceType string `json:"instanceType" validate:"required"` // mongos mongod + App string `json:"app" validate:"required"` + SetId string `json:"setId" validate:"required"` // 复制集为集群id，cluster为集群id+序号 + KeyFile string `json:"keyFile" validate:"required"` // keyFile的内容 app-setId + Auth bool `json:"auth"` // true：以验证方式启动mongod false：以非验证方式启动mongod + ClusterRole string `json:"clusterRole"` // 部署cluster时填写，shardsvr configsvr；部署复制集时为空 + DbConfig struct { + SlowOpThresholdMs int `json:"slowOpThresholdMs"` + CacheSizeGB int `json:"cacheSizeGB"` + OplogSizeMB int `json:"oplogSizeMB" validate:"required"` + Destination string `json:"destination"` + } `json:"dbConfig" validate:"required"` +} + +// MongoDBInstall MongoDB安装 +type MongoDBInstall struct { + runtime *jobruntime.JobGenericRuntime + BinDir string + BackupDir string + DataDir string + OsUser string // MongoDB安装在哪个用户下 + OsGroup string + ConfParams *MongoDBConfParams + DbpathDir string + BackupPath string + AuthConfFilePath string + AuthConfFileContent []byte + NoAuthConfFilePath string + NoAuthConfFileContent []byte + DbTypeFilePath string + LogPath string + PidFilePath string + KeyFilePath string + InstallPackagePath string + LockFilePath string // 锁文件路径 +} + +// NewMongoDBInstall 实例化结构体 +func NewMongoDBInstall() jobruntime.JobRunner { + return &MongoDBInstall{} +} + +// Name 获取原子任务的名字 +func (m *MongoDBInstall) Name() string { + return "mongod_install" +} + +// Run 运行原子任务 +func (m *MongoDBInstall) Run() error { + // 进行校验 + status, err := m.checkParams() + if err != nil { + return err + } + if status { + return nil + } + + // 解压安装包并修改属主 + if err = m.unTarAndCreateSoftLink(); err != nil { + return err + } + + // 创建目录并修改属主 + if err = m.mkdir(); err != nil { + return err + } + + // 创建配置文件，key文件并修改属主 + if err = m.createFile(); err != nil { + return err + } + + // 启动服务 + if err = m.startup(); err != nil { + return err + } + + return nil +} + +// Retry 重试 +func (m *MongoDBInstall) Retry() uint { + return 2 +} + +// Rollback 回滚 +func (m *MongoDBInstall) Rollback() error { + return nil +} + +// Init 初始化 +func (m *MongoDBInstall) Init(runtime *jobruntime.JobGenericRuntime) error { + // 获取安装参数 + m.runtime = runtime + m.runtime.Logger.Info("start to init") + m.BinDir = consts.UsrLocal + m.BackupDir = consts.GetMongoBackupDir() + m.DataDir = consts.GetMongoDataDir() + m.OsUser = consts.GetProcessUser() + m.OsGroup = consts.GetProcessUserGroup() + + // 获取MongoDB配置文件参数 + if err := json.Unmarshal([]byte(m.runtime.PayloadDecoded), &m.ConfParams); err != nil { + m.runtime.Logger.Error(fmt.Sprintf( + "get parameters of mongodb config file fail by json.Unmarshal, error:%s", err)) + return fmt.Errorf("get parameters of mongodb config file fail by json.Unmarshal, error:%s", err) + } + + // 获取信息 + m.InstallPackagePath = m.ConfParams.MediaPkg.GetAbsolutePath() + + // 设置各种路径 + strPort := strconv.Itoa(m.ConfParams.Port) + m.DbpathDir = filepath.Join(m.DataDir, "mongodata", strPort, "db") + m.BackupPath = filepath.Join(m.BackupDir, "dbbak") + m.AuthConfFilePath = filepath.Join(m.DataDir, "mongodata", strPort, "mongo.conf") + m.NoAuthConfFilePath = filepath.Join(m.DataDir, "mongodata", strPort, "noauth.conf") + m.LogPath = filepath.Join(m.BackupDir, "mongolog", strPort, "mongo.log") + PidFileName := fmt.Sprintf("pid.%s", strPort) + m.PidFilePath = filepath.Join(m.DataDir, "mongodata", strPort, PidFileName) + m.KeyFilePath = filepath.Join(m.DataDir, "mongodata", strPort, "key_of_mongo") + m.DbTypeFilePath = filepath.Join(m.DataDir, "mongodata", strPort, "dbtype") + m.LockFilePath = filepath.Join(m.DataDir, "mongoinstall.lock") + + m.runtime.Logger.Info("init successfully") + + // 生成配置文件内容 + if err := m.makeConfContent(); err != nil { + return err + } + + return nil +} + +// makeConfContent 生成配置文件内容 +func (m *MongoDBInstall) makeConfContent() error { + mainVersion, err := strconv.Atoi(strings.Split(m.ConfParams.DbVersion, ".")[0]) + if err != nil { + return err + } + + // mongodb 3.0及以上得到配置文件内容 + if mainVersion >= 3 { + m.runtime.Logger.Info("start to make mongodb config file content") + conf := common.NewYamlMongoDBConf() + conf.Storage.DbPath = m.DbpathDir + conf.Storage.Engine = "wiredTiger" + conf.Storage.WiredTiger.EngineConfig.CacheSizeGB = m.ConfParams.DbConfig.CacheSizeGB + conf.Replication.OplogSizeMB = m.ConfParams.DbConfig.OplogSizeMB + conf.Replication.ReplSetName = strings.Join([]string{m.ConfParams.App, m.ConfParams.SetId}, + "-") + conf.SystemLog.LogAppend = true + conf.SystemLog.Path = m.LogPath + conf.SystemLog.Destination = m.ConfParams.DbConfig.Destination + conf.ProcessManagement.Fork = true + conf.ProcessManagement.PidFilePath = m.PidFilePath + conf.Net.Port = m.ConfParams.Port + conf.Net.BindIp = strings.Join([]string{"127.0.0.1", m.ConfParams.IP}, ",") + conf.Net.WireObjectCheck = false + conf.OperationProfiling.SlowOpThresholdMs = m.ConfParams.DbConfig.SlowOpThresholdMs + conf.Sharding.ClusterRole = m.ConfParams.ClusterRole + // 获取非验证配置文件内容 + m.NoAuthConfFileContent, err = conf.GetConfContent() + if err != nil { + m.runtime.Logger.Error(fmt.Sprintf( + "version:%s make mongodb no auth config file content fail, error:%s", m.ConfParams.DbVersion, err)) + return fmt.Errorf("version:%s make mongodb no auth config file content fail, error:%s", + m.ConfParams.DbVersion, err) + } + conf.Security.KeyFile = m.KeyFilePath + // 获取验证配置文件内容 + m.AuthConfFileContent, err = conf.GetConfContent() + if err != nil { + m.runtime.Logger.Error(fmt.Sprintf( + "version:%s make mongodb auth config file content fail, error:%s", + m.ConfParams.DbVersion, err)) + return fmt.Errorf("version:%s make mongodb auth config file content fail, error:%s", + m.ConfParams.DbVersion, err) + } + m.runtime.Logger.Info("make mongodb config file content successfully") + return nil + } + + // mongodb 3.0以下获取配置文件内容 + // 获取非验证配置文件内容 + m.runtime.Logger.Info("start to make mongodb config file content") + NoAuthConf := common.IniNoAuthMongoDBConf + AuthConf := common.IniAuthMongoDBConf + replSet := strings.Join([]string{m.ConfParams.App, m.ConfParams.SetId}, + "-") + NoAuthConf = strings.Replace(NoAuthConf, "{{replSet}}", replSet, -1) + AuthConf = strings.Replace(AuthConf, "{{replSet}}", replSet, -1) + NoAuthConf = strings.Replace(NoAuthConf, "{{dbpath}}", m.DbpathDir, -1) + AuthConf = strings.Replace(AuthConf, "{{dbpath}}", m.DbpathDir, -1) + NoAuthConf = strings.Replace(NoAuthConf, "{{logpath}}", m.LogPath, -1) + AuthConf = strings.Replace(AuthConf, "{{logpath}}", m.LogPath, -1) + NoAuthConf = strings.Replace(NoAuthConf, "{{pidfilepath}}", m.PidFilePath, -1) + AuthConf = strings.Replace(AuthConf, "{{pidfilepath}}", m.PidFilePath, -1) + strPort := strconv.Itoa(m.ConfParams.Port) + NoAuthConf = strings.Replace(NoAuthConf, "{{port}}", strPort, -1) + AuthConf = strings.Replace(AuthConf, "{{port}}", strPort, -1) + bindIP := strings.Join([]string{"127.0.0.1", m.ConfParams.IP}, ",") + NoAuthConf = strings.Replace(NoAuthConf, "{{bind_ip}}", bindIP, -1) + AuthConf = strings.Replace(AuthConf, "{{bind_ip}}", bindIP, -1) + strOplogSize := strconv.Itoa(m.ConfParams.DbConfig.OplogSizeMB) + NoAuthConf = strings.Replace(NoAuthConf, "{{oplogSize}}", strOplogSize, -1) + AuthConf = strings.Replace(AuthConf, "{{oplogSize}}", strOplogSize, -1) + NoAuthConf = strings.Replace(NoAuthConf, "{{instanceRole}}", m.ConfParams.ClusterRole, -1) + AuthConf = strings.Replace(AuthConf, "{{instanceRole}}", m.ConfParams.ClusterRole, -1) + AuthConf = strings.Replace(AuthConf, "{{keyFile}}", m.KeyFilePath, -1) + m.NoAuthConfFileContent = []byte(NoAuthConf) + m.AuthConfFileContent = []byte(AuthConf) + m.runtime.Logger.Info("make mongodb config file content successfully") + + return nil +} + +// checkParams 校验参数 检查输入的参数 检查端口是否合规 检查安装包 检查端口是否被使用（如果使用，则检查是否是mongodb服务） +func (m *MongoDBInstall) checkParams() (bool, error) { + // 校验MongoDB配置文件 + m.runtime.Logger.Info("start to validate parameters") + validate := validator.New() + m.runtime.Logger.Info("start to validate parameters of mongodb config file") + if err := validate.Struct(m.ConfParams); err != nil { + m.runtime.Logger.Error(fmt.Sprintf("validate parameters of mongodb config file fail, error:%s", err)) + return false, fmt.Errorf("validate parameters of mongodb config file fail, error:%s", err) + } + // 校验port是否合规 + m.runtime.Logger.Info("start to validate port if it is correct") + if m.ConfParams.Port < MongoDBPortMin || m.ConfParams.Port > MongoDBPortMax { + m.runtime.Logger.Error(fmt.Sprintf( + "validate port if it is correct, port is not within defalut range [%d,%d]", + MongoDBPortMin, MongoDBPortMax)) + return false, fmt.Errorf("validate port if it is correct, port is not within defalut range [%d,%d]", + MongoDBPortMin, MongoDBPortMax) + } + + // 校验安装包是否存在，md5值是否一致 + m.runtime.Logger.Info("start to validate install package") + if flag := util.FileExists(m.InstallPackagePath); !flag { + m.runtime.Logger.Error(fmt.Sprintf("validate install package, %s is not existed", + m.InstallPackagePath)) + return false, fmt.Errorf("validate install file, %s is not existed", + m.InstallPackagePath) + } + md5, _ := util.GetFileMd5(m.InstallPackagePath) + if m.ConfParams.MediaPkg.PkgMd5 != md5 { + m.runtime.Logger.Error(fmt.Sprintf("validate install package md5 fail, md5 is incorrect")) + return false, fmt.Errorf("validate install package md5 fail, md5 is incorrect") + } + + // 校验端口是否使用 + m.runtime.Logger.Info("start to validate port if it has been used") + flag, _ := util.CheckPortIsInUse(m.ConfParams.IP, strconv.Itoa(m.ConfParams.Port)) + if flag { + // 校验端口是否是mongod进程 + cmd := fmt.Sprintf("netstat -ntpl |grep %d | awk '{print $7}' |head -1", m.ConfParams.Port) + result, _ := util.RunBashCmd(cmd, "", nil, 10*time.Second) + if strings.Contains(result, "mongod") { + // 检查配置文件是否一致，读取已有配置文件与新生成的配置文件内容对比 + content, _ := ioutil.ReadFile(m.AuthConfFilePath) + if strings.Compare(string(content), string(m.AuthConfFileContent)) == 0 { + // 检查mongod版本 + version, err := common.CheckMongoVersion(m.BinDir, "mongod") + if err != nil { + m.runtime.Logger.Error( + fmt.Sprintf("mongod has been installed, port:%d, check mongod version fail. error:%s", + m.ConfParams.Port, version)) + return false, fmt.Errorf("mongod has been installed, port:%d, check mongod version fail. error:%s", + m.ConfParams.Port, version) + } + if version == m.ConfParams.DbVersion { + m.runtime.Logger.Info(fmt.Sprintf("mongod has been installed, port:%d, version:%s", + m.ConfParams.Port, version)) + return true, nil + } + m.runtime.Logger.Error(fmt.Sprintf("other mongod has been installed, port:%d, version:%s", + m.ConfParams.Port, version)) + return false, fmt.Errorf("other mongod has been installed, port:%d, version:%s", + m.ConfParams.Port, version) + } + + } + m.runtime.Logger.Error( + fmt.Sprintf("validate port if it has been used, port:%d is used by other process", + m.ConfParams.Port)) + return false, fmt.Errorf("validate port if it has been used, port:%d is used by other process", + m.ConfParams.Port) + } + m.runtime.Logger.Info("validate parameters successfully") + return false, nil +} + +// unTarAndCreateSoftLink 解压安装包，创建软链接并给目录授权 +func (m *MongoDBInstall) unTarAndCreateSoftLink() error { + // 解压目录 + unTarPath := filepath.Join(m.BinDir, m.ConfParams.MediaPkg.GePkgBaseName()) + + // soft link目录 + installPath := filepath.Join(m.BinDir, "mongodb") + + // 解压安装包并授权 + // 安装多实例并发执行添加文件锁 + m.runtime.Logger.Info("start to get install file lock") + fileLock := common.NewFileLock(m.LockFilePath) + // 获取锁 + err := fileLock.Lock() + if err != nil { + for { + err = fileLock.Lock() + if err != nil { + time.Sleep(1 * time.Second) + continue + } + m.runtime.Logger.Info("get install file lock successfully") + break + } + } else { + m.runtime.Logger.Info("get install file lock successfully") + } + + if err = common.UnTarAndCreateSoftLinkAndChown(m.runtime, m.BinDir, + m.InstallPackagePath, unTarPath, installPath, m.OsUser, m.OsGroup); err != nil { + return err + } + // 释放锁 + _ = fileLock.UnLock() + m.runtime.Logger.Info("release install file lock successfully") + + // 检查mongod版本 + m.runtime.Logger.Info("start to check mongod version") + version, err := common.CheckMongoVersion(m.BinDir, "mongod") + if err != nil { + m.runtime.Logger.Error(fmt.Sprintf("%s has been existed, check mongodb version, error:%s", + installPath, err)) + return fmt.Errorf("%s has been existed, check mongodb version, error:%s", + installPath, err) + } + if version != m.ConfParams.DbVersion { + m.runtime.Logger.Error( + fmt.Sprintf("%s has been existed, check mongodb version, version:%s is incorrect", + installPath, version)) + return fmt.Errorf("%s has been existed, check mongodb version, version:%s is incorrect", + installPath, version) + } + m.runtime.Logger.Info("check mongod version successfully") + return nil +} + +// mkdir 创建相关目录并给目录授权 +func (m *MongoDBInstall) mkdir() error { + // 创建日志文件目录 + logPathDir, _ := filepath.Split(m.LogPath) + m.runtime.Logger.Info("start to create log directory") + if err := util.MkDirsIfNotExistsWithPerm([]string{logPathDir}, DefaultPerm); err != nil { + m.runtime.Logger.Error(fmt.Sprintf("create log directory fail, error:%s", err)) + return fmt.Errorf("create log directory fail, error:%s", err) + } + m.runtime.Logger.Info("create log directory successfully") + + // 创建数据文件目录 + m.runtime.Logger.Info("start to create data directory") + if err := util.MkDirsIfNotExistsWithPerm([]string{m.DbpathDir}, DefaultPerm); err != nil { + m.runtime.Logger.Error(fmt.Sprintf("create data directory fail, error:%s", err)) + return fmt.Errorf("create data directory fail, error:%s", err) + } + m.runtime.Logger.Info("create data directory successfully") + + // 创建备份文件目录 + m.runtime.Logger.Info("start to create backup directory") + if err := util.MkDirsIfNotExistsWithPerm([]string{m.BackupDir}, DefaultPerm); err != nil { + m.runtime.Logger.Error(fmt.Sprintf("create backup directory fail, error:%s", err)) + return fmt.Errorf("create backup directory fail, error:%s", err) + } + m.runtime.Logger.Info("create backup directory successfully") + + // 修改目录属主 + m.runtime.Logger.Info("start to execute chown command for dbPath, logPath and backupPath") + if _, err := util.RunBashCmd( + fmt.Sprintf("chown -R %s.%s %s", m.OsUser, m.OsGroup, filepath.Join(logPathDir, "../")), + "", nil, + 10*time.Second); err != nil { + m.runtime.Logger.Error(fmt.Sprintf("chown log directory fail, error:%s", err)) + return fmt.Errorf("chown log directory fail, error:%s", err) + } + if _, err := util.RunBashCmd( + fmt.Sprintf("chown -R %s.%s %s", m.OsUser, m.OsGroup, filepath.Join(m.DbpathDir, "../../")), + "", nil, + 10*time.Second); err != nil { + m.runtime.Logger.Error(fmt.Sprintf("chown data directory fail, error:%s", err)) + return fmt.Errorf("chown data directory fail, error:%s", err) + } + if _, err := util.RunBashCmd( + fmt.Sprintf("chown -R %s.%s %s", m.OsUser, m.OsGroup, m.BackupDir), + "", nil, + 10*time.Second); err != nil { + m.runtime.Logger.Error(fmt.Sprintf("chown backup directory fail, error:%s", err)) + return fmt.Errorf("chown backup directory fail, error:%s", err) + } + m.runtime.Logger.Info("execute chown command for dbPath, logPath and backupPath successfully") + return nil +} + +// createFile 创建配置文件以及key文件 +func (m *MongoDBInstall) createFile() error { + if err := common.CreateConfFileAndKeyFileAndDbTypeFileAndChown( + m.runtime, m.AuthConfFilePath, m.AuthConfFileContent, m.OsUser, m.OsGroup, m.NoAuthConfFilePath, + m.NoAuthConfFileContent, m.KeyFilePath, m.ConfParams.KeyFile, m.DbTypeFilePath, + m.ConfParams.InstanceType, DefaultPerm); err != nil { + return err + } + return nil +} + +// startup 启动服务 +func (m *MongoDBInstall) startup() error { + // 声明mongod可执行文件路径，把路径写入/etc/profile + if err := common.AddPathToProfile(m.runtime, m.BinDir); err != nil { + return err + } + + // 启动服务 + m.runtime.Logger.Info("start to startup mongod") + if err := common.StartMongoProcess(m.BinDir, m.ConfParams.Port, + m.OsUser, m.ConfParams.Auth); err != nil { + m.runtime.Logger.Error("startup mongod fail, error:%s", err) + return fmt.Errorf("startup mongod fail, error:%s", err) + } + flag, service, err := common.CheckMongoService(m.ConfParams.Port) + if err != nil { + m.runtime.Logger.Error("check %s fail, error:%s", service, err) + return fmt.Errorf("check %s fail, error:%s", service, err) + } + if flag == false { + m.runtime.Logger.Error("startup %s fail", service) + return fmt.Errorf("startup %s fail", service) + } + m.runtime.Logger.Info("startup %s successfully", service) + + return nil +} diff --git a/dbm-services/mongo/db-tools/dbactuator/pkg/atomjobs/atommongodb/mongod_replace.go b/dbm-services/mongo/db-tools/dbactuator/pkg/atomjobs/atommongodb/mongod_replace.go new file mode 100644 index 0000000000..65e0e1083b --- /dev/null +++ b/dbm-services/mongo/db-tools/dbactuator/pkg/atomjobs/atommongodb/mongod_replace.go @@ -0,0 +1,379 @@ +package atommongodb + +import ( + "encoding/json" + "fmt" + "path/filepath" + "strconv" + "strings" + "time" + + "dbm-services/mongo/db-tools/dbactuator/pkg/common" + "dbm-services/mongo/db-tools/dbactuator/pkg/consts" + "dbm-services/mongo/db-tools/dbactuator/pkg/jobruntime" + "dbm-services/mongo/db-tools/dbactuator/pkg/util" + + "github.com/go-playground/validator/v10" +) + +// MongoDReplaceConfParams 参数 // 替换mongod +type MongoDReplaceConfParams struct { + IP string `json:"ip" validate:"required"` // 执行节点 + Port int `json:"port" validate:"required"` + SourceIP string `json:"sourceIP"` // 源节点，新加节点时可以为null + SourcePort int `json:"sourcePort"` // 源端口，新加节点时可以为null + SourceDown bool `json:"sourceDown"` // 源端已down机 true：已down false：未down + AdminUsername string `json:"adminUsername" validate:"required"` + AdminPassword string `json:"adminPassword" validate:"required"` + TargetIP string `json:"targetIP"` // 目标节点，移除节点时可以为null + TargetPort int `json:"targetPort"` // 目标端口，移除节点时可以为null + TargetPriority string `json:"targetPriority"` // 可选，默认为null，如果为null，则使用source端的Priority，取值：0-正无穷 + TargetHidden string `json:"targetHidden"` // 可选，默认为null，如果为null，则使用source端的Hidden，取值：null，0，1，0：显现 1：隐藏 +} + +// MongoDReplace 添加分片到集群 +type MongoDReplace struct { + runtime *jobruntime.JobGenericRuntime + BinDir string + Mongo string + OsUser string + DataDir string + DbpathDir string + PrimaryIP string + PrimaryPort int + AddTargetScript string + ConfParams *MongoDReplaceConfParams + TargetIPStatus int + TargetPriority int + TargetHidden bool + StatusCh chan int +} + +// NewMongoDReplace 实例化结构体 +func NewMongoDReplace() jobruntime.JobRunner { + return &MongoDReplace{} +} + +// Name 获取原子任务的名字 +func (r *MongoDReplace) Name() string { + return "mongod_replace" +} + +// Run 运行原子任务 +func (r *MongoDReplace) Run() error { + // 主节点进行切换 + if err := r.primaryStepDown(); err != nil { + return err + } + + // 生成添加新节点脚本 + if err := r.makeAddTargetScript(); err != nil { + return err + } + + // 执行添加新节点脚本 + if err := r.execAddTargetScript(); err != nil { + return err + } + + // 查看新节点状态 + go r.checkTargetStatus() + + // 执行删除老节点脚本 + if err := r.checkTargetStatusAndRemoveSource(); err != nil { + return err + } + + return nil +} + +// Retry 重试 +func (r *MongoDReplace) Retry() uint { + return 2 +} + +// Rollback 回滚 +func (r *MongoDReplace) Rollback() error { + return nil +} + +// Init 初始化 +func (r *MongoDReplace) Init(runtime *jobruntime.JobGenericRuntime) error { + // 获取安装参数 + r.runtime = runtime + r.runtime.Logger.Info("start to init") + r.BinDir = consts.UsrLocal + r.Mongo = filepath.Join(r.BinDir, "mongodb", "bin", "mongo") + r.OsUser = consts.GetProcessUser() + r.DataDir = consts.GetMongoDataDir() + + // 获取MongoDB配置文件参数 + if err := json.Unmarshal([]byte(r.runtime.PayloadDecoded), &r.ConfParams); err != nil { + r.runtime.Logger.Error(fmt.Sprintf( + "get parameters of mongodReplace fail by json.Unmarshal, error:%s", err)) + return fmt.Errorf("get parameters of mongodReplace fail by json.Unmarshal, error:%s", err) + } + + r.DbpathDir = filepath.Join(r.DataDir, "mongodata", strconv.Itoa(r.ConfParams.Port), "db") + + // 获取primary信息 + info, err := common.AuthGetPrimaryInfo(r.Mongo, r.ConfParams.AdminUsername, r.ConfParams.AdminPassword, + r.ConfParams.IP, r.ConfParams.Port) + if err != nil { + r.runtime.Logger.Error(fmt.Sprintf( + "get primary db info of mongodReplace fail, error:%s", err)) + return fmt.Errorf("get primary db info of mongodReplace fail, error:%s", err) + } + // 判断info是否为null + if info == "" { + r.runtime.Logger.Error(fmt.Sprintf( + "get primary db info of mongodReplace fail, error:%s", err)) + return fmt.Errorf("get primary db info of mongodReplace fail, error:%s", err) + } + getInfo := strings.Split(info, ":") + r.PrimaryIP = getInfo[0] + r.PrimaryPort, _ = strconv.Atoi(getInfo[1]) + r.StatusCh = make(chan int, 1) + + // 获取源端的配置信息 + _, _, _, hidden, priority, _, err := common.GetNodeInfo(r.Mongo, r.PrimaryIP, r.PrimaryPort, + r.ConfParams.AdminUsername, r.ConfParams.AdminPassword, r.ConfParams.SourceIP, r.ConfParams.SourcePort) + if err != nil { + return err + } + r.TargetHidden = hidden + if r.ConfParams.TargetHidden == "0" { + r.TargetHidden = false + } else if r.ConfParams.TargetHidden == "1" { + r.TargetHidden = true + } + + r.TargetPriority = priority + if r.ConfParams.TargetPriority != "" { + r.TargetPriority, _ = strconv.Atoi(r.ConfParams.TargetPriority) + } + + r.runtime.Logger.Info("init successfully") + + // 进行校验 + if err = r.checkParams(); err != nil { + return err + } + + return nil +} + +// checkParams 校验参数 +func (r *MongoDReplace) checkParams() error { + // 校验重启配置参数 + validate := validator.New() + r.runtime.Logger.Info("start to validate parameters of mongodReplace") + if err := validate.Struct(r.ConfParams); err != nil { + r.runtime.Logger.Error("validate parameters of mongodReplace fail, error:%s", err) + return fmt.Errorf("validate parameters of mongodReplace fail, error:%s", err) + } + r.runtime.Logger.Info("validate parameters of mongodReplace successfully") + return nil +} + +// makeAddTargetScript 创建添加脚本 +func (r *MongoDReplace) makeAddTargetScript() error { + if r.ConfParams.TargetIP == "" { + return nil + } + // 生成脚本内容 + r.runtime.Logger.Info("start to make addTarget script content") + addMember := common.NewReplicasetMemberAdd() + addMember.Host = strings.Join([]string{r.ConfParams.TargetIP, strconv.Itoa(r.ConfParams.TargetPort)}, ":") + addMember.Priority = r.TargetPriority + addMember.Hidden = r.TargetHidden + addMemberJson, err := addMember.GetJson() + if err != nil { + r.runtime.Logger.Error("get addMemberJson info fail, error:%s", err) + return fmt.Errorf("get addMemberJson info fail, error:%s", err) + } + addMemberJson = strings.Replace(addMemberJson, "\"", "\\\"", -1) + addTargetConfScript := strings.Join([]string{"rs.add(", addMemberJson, ")"}, "") + r.AddTargetScript = addTargetConfScript + r.runtime.Logger.Info("make addTarget script content successfully") + return nil +} + +// execAddTargetScript 执行添加脚本 +func (r *MongoDReplace) execAddTargetScript() error { + if r.ConfParams.TargetIP == "" { + return nil + } + // 检查target是否已经存在 + flag, _, _, _, _, _, _ := common.GetNodeInfo(r.Mongo, r.PrimaryIP, r.PrimaryPort, + r.ConfParams.AdminUsername, r.ConfParams.AdminPassword, r.ConfParams.TargetIP, r.ConfParams.TargetPort) + if flag == true { + r.runtime.Logger.Info("target:%s has been existed", strings.Join( + []string{r.ConfParams.TargetIP, strconv.Itoa(r.ConfParams.TargetPort)}, ":")) + return nil + } + + r.runtime.Logger.Info("start to execute addTarget script") + cmd := fmt.Sprintf( + "%s -u %s -p '%s' --host %s --port %d --authenticationDatabase=admin --quiet --eval \"%s\"", + r.Mongo, r.ConfParams.AdminUsername, r.ConfParams.AdminPassword, r.PrimaryIP, + r.PrimaryPort, r.AddTargetScript) + if _, err := util.RunBashCmd( + cmd, + "", nil, + 10*time.Second); err != nil { + r.runtime.Logger.Error("execute addTarget script fail, error:%s", err) + return fmt.Errorf("execute addTarget script fail, error:%s", err) + } + r.runtime.Logger.Info("execute addTarget script successfully") + return nil +} + +// checkTargetStatus 检查target状态 +func (r *MongoDReplace) checkTargetStatus() { + if r.ConfParams.TargetIP == "" { + return + } + r.runtime.Logger.Info("start to check Target status") + for { + _, _, status, _, _, _, err := common.GetNodeInfo(r.Mongo, r.PrimaryIP, r.PrimaryPort, + r.ConfParams.AdminUsername, + r.ConfParams.AdminPassword, r.ConfParams.TargetIP, r.ConfParams.TargetPort) + if err != nil { + r.runtime.Logger.Error("get target status fail, error:%s", err) + } + if status != 0 { + r.StatusCh <- status + if status == 2 { + r.runtime.Logger.Info("target status is %d", status) + return + } + } + time.Sleep(5 * time.Second) + } +} + +// primaryStepDown 主库切换 +func (r *MongoDReplace) primaryStepDown() error { + if r.ConfParams.SourceIP == r.PrimaryIP && r.ConfParams.SourcePort == r.PrimaryPort { + r.runtime.Logger.Info("start to convert primary secondary db") + flag, err := common.AuthRsStepDown(r.Mongo, r.PrimaryIP, r.PrimaryPort, r.ConfParams.AdminUsername, + r.ConfParams.AdminPassword) + if err != nil { + r.runtime.Logger.Error(fmt.Sprintf("convert primary secondary db fail, error:%s", err)) + return fmt.Errorf("convert primary secondary db fail, error:%s", err) + } + if flag == true { + info, err := common.AuthGetPrimaryInfo(r.Mongo, r.ConfParams.AdminUsername, r.ConfParams.AdminPassword, + r.ConfParams.IP, r.ConfParams.Port) + if err != nil { + r.runtime.Logger.Error(fmt.Sprintf("get new primary info fail, error:%s", err)) + return fmt.Errorf("get new primary info fail, error:%s", err) + } + if info != fmt.Sprintf("%s:%d", r.ConfParams.IP, r.ConfParams.Port) { + r.runtime.Logger.Info("convert primary secondary db successfully") + infoSlice := strings.Split(info, ":") + r.PrimaryIP = infoSlice[0] + r.PrimaryPort, _ = strconv.Atoi(infoSlice[1]) + return nil + } + } + } + return nil +} + +// shutdownSourceProcess 关闭源端mongod进程 +func (r *MongoDReplace) shutdownSourceProcess() error { + flag, _, _ := common.CheckMongoService(r.ConfParams.Port) + if flag == false { + r.runtime.Logger.Info("source mongod process has been shut") + return nil + } + r.runtime.Logger.Info("start to shutdown source mongod process") + if err := common.ShutdownMongoProcess(r.OsUser, "mongod", r.BinDir, r.DbpathDir, r.ConfParams.Port); err != nil { + source := fmt.Sprintf("%s:%d", r.ConfParams.IP, r.ConfParams.Port) + r.runtime.Logger.Error(fmt.Sprintf("shutdown source:%s fail, error:%s", source, err)) + return fmt.Errorf("shutdown source:%s fail, error:%s", source, err) + } + r.runtime.Logger.Info("shutdown source mongod process successfully") + return nil +} + +// removeSource 复制集中移除source +func (r *MongoDReplace) removeSource() error { + if r.ConfParams.SourceIP == "" { + return nil + } + // 检查source是否存在 + flag, _, _, _, _, _, _ := common.GetNodeInfo(r.Mongo, r.PrimaryIP, r.PrimaryPort, + r.ConfParams.AdminUsername, r.ConfParams.AdminPassword, r.ConfParams.SourceIP, r.ConfParams.SourcePort) + if flag == false { + r.runtime.Logger.Info("source:%s has been remove", strings.Join( + []string{r.ConfParams.SourceIP, strconv.Itoa(r.ConfParams.SourcePort)}, ":")) + return nil + } + r.runtime.Logger.Info("start to make remove source script content") + removeSourceConfScript := strings.Join([]string{ + "rs.remove(", + fmt.Sprintf("\\\"%s:%d\\\"", r.ConfParams.SourceIP, r.ConfParams.SourcePort), + ")"}, "") + r.runtime.Logger.Info("make remove source script content successfully") + r.runtime.Logger.Info("start to execute remove source script") + cmd := fmt.Sprintf( + "%s -u %s -p '%s' --host %s --port %d --authenticationDatabase=admin --quiet --eval \"%s\"", + r.Mongo, r.ConfParams.AdminUsername, r.ConfParams.AdminPassword, r.PrimaryIP, + r.PrimaryPort, removeSourceConfScript) + if _, err := util.RunBashCmd( + cmd, + "", nil, + 10*time.Second); err != nil { + r.runtime.Logger.Error(fmt.Sprintf("execute remove source script fail, error:%s", err)) + return fmt.Errorf("execute remove source script fail, error:%s", err) + } + r.runtime.Logger.Info("execute remove source script successfully") + return nil +} + +// checkTargetStatusAndRemoveSource 监控状态并移除 +func (r *MongoDReplace) checkTargetStatusAndRemoveSource() error { + // 下架老节点 + if r.ConfParams.TargetIP == "" && r.ConfParams.SourceDown == false { + if err := r.shutdownSourceProcess(); err != nil { + return err + } + if err := r.removeSource(); err != nil { + return err + } + return nil + } else if r.ConfParams.TargetIP == "" && r.ConfParams.SourceDown == true { + if err := r.removeSource(); err != nil { + return err + } + return nil + } + // 先添加新节点再移除老节点，或者添加新节点 + for { + select { + // 超时时间 + case <-time.After(50 * time.Second): + return fmt.Errorf("check target status timeout") + case status := <-r.StatusCh: + if status == 2 && r.ConfParams.SourceDown == false { + if err := r.shutdownSourceProcess(); err != nil { + return err + } + if err := r.removeSource(); err != nil { + return err + } + return nil + } else if status == 2 && r.ConfParams.SourceDown == true { + if err := r.removeSource(); err != nil { + return err + } + return nil + } else if status == 2 && r.ConfParams.SourceIP == "" { + return nil + } + } + } +} diff --git a/dbm-services/mongo/db-tools/dbactuator/pkg/atomjobs/atommongodb/mongos_install.go b/dbm-services/mongo/db-tools/dbactuator/pkg/atomjobs/atommongodb/mongos_install.go new file mode 100644 index 0000000000..868d754eaf --- /dev/null +++ b/dbm-services/mongo/db-tools/dbactuator/pkg/atomjobs/atommongodb/mongos_install.go @@ -0,0 +1,441 @@ +package atommongodb + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "path/filepath" + "strconv" + "strings" + "time" + + "dbm-services/mongo/db-tools/dbactuator/pkg/common" + "dbm-services/mongo/db-tools/dbactuator/pkg/consts" + "dbm-services/mongo/db-tools/dbactuator/pkg/jobruntime" + "dbm-services/mongo/db-tools/dbactuator/pkg/util" + + "github.com/go-playground/validator/v10" +) + +// MongoSConfParams 配置文件参数 +type MongoSConfParams struct { + common.MediaPkg `json:"mediapkg"` + IP string `json:"ip" validate:"required"` + Port int `json:"port" validate:"required"` + InstanceType string `json:"instanceType" validate:"required"` // mongos mongod + App string `json:"app" validate:"required"` + SetId string `json:"setId" validate:"required"` + KeyFile string `json:"keyFile" validate:"required"` // keyFile的内容 app-setId + Auth bool `json:"auth"` // true：以验证方式启动mongos false：以非验证方式启动mongos + ConfigDB []string `json:"configDB" validate:"required"` // ip:port + DbConfig struct { + SlowOpThresholdMs int `json:"slowOpThresholdMs"` + Destination string `json:"destination"` + } `json:"dbConfig" validate:"required"` +} + +// MongoSInstall MongoS安装 +type MongoSInstall struct { + runtime *jobruntime.JobGenericRuntime + BinDir string + DataDir string + OsUser string // MongoDB安装在哪个用户下 + OsGroup string + ConfParams *MongoSConfParams + DbVersion string + AuthConfFilePath string + AuthConfFileContent []byte + NoAuthConfFilePath string + NoAuthConfFileContent []byte + DbTypeFilePath string + LogPath string + PidFilePath string + KeyFilePath string + InstallPackagePath string + LockFilePath string // 锁文件路径 +} + +// NewMongoSInstall 实例化结构体 +func NewMongoSInstall() jobruntime.JobRunner { + return &MongoSInstall{} +} + +// Name 获取原子任务的名字 +func (s *MongoSInstall) Name() string { + return "mongos_install" +} + +// Run 运行原子任务 +func (s *MongoSInstall) Run() error { + // 进行校验 + status, err := s.checkParams() + if err != nil { + return err + } + if status { + return nil + } + + // 解压安装包并修改属主 + if err = s.unTarAndCreateSoftLink(); err != nil { + return err + } + + // 创建目录并修改属主 + if err = s.mkdir(); err != nil { + return err + } + + // 创建配置文件，key文件并修改属主 + if err = s.creatFile(); err != nil { + return err + } + + // 启动服务 + if err = s.startup(); err != nil { + return err + } + + return nil +} + +// Retry 重试 +func (s *MongoSInstall) Retry() uint { + return 2 +} + +// Rollback 回滚 +func (s *MongoSInstall) Rollback() error { + return nil +} + +// Init 初始化 +func (s *MongoSInstall) Init(runtime *jobruntime.JobGenericRuntime) error { + // 获取安装参数 + s.runtime = runtime + s.runtime.Logger.Info("start to init") + s.BinDir = consts.UsrLocal + s.DataDir = consts.GetMongoDataDir() + s.OsUser = consts.GetProcessUser() + s.OsGroup = consts.GetProcessUserGroup() + + // 获取MongoDB配置文件参数 + if err := json.Unmarshal([]byte(s.runtime.PayloadDecoded), &s.ConfParams); err != nil { + s.runtime.Logger.Error(fmt.Sprintf( + "get parameters of mongodb config file fail by json.Unmarshal, error:%s", err)) + return fmt.Errorf("get parameters of mongodb config file fail by json.Unmarshal, error:%s", err) + } + + // 获取信息 + s.InstallPackagePath = s.ConfParams.MediaPkg.GetAbsolutePath() + s.DbVersion = strings.Split(s.ConfParams.MediaPkg.GePkgBaseName(), "-")[3] + + // 设置各种路径 + strPort := strconv.Itoa(s.ConfParams.Port) + s.AuthConfFilePath = filepath.Join(s.DataDir, "mongodata", strPort, "mongo.conf") + s.NoAuthConfFilePath = filepath.Join(s.DataDir, "mongodata", strPort, "noauth.conf") + s.LogPath = filepath.Join(s.DataDir, "mongolog", strPort, "mongo.log") + PidFileName := fmt.Sprintf("pid.%s", strPort) + s.PidFilePath = filepath.Join(s.DataDir, "mongodata", strPort, PidFileName) + s.KeyFilePath = filepath.Join(s.DataDir, "mongodata", strPort, "key_of_mongo") + s.DbTypeFilePath = filepath.Join(s.DataDir, "mongodata", strPort, "dbtype") + s.LockFilePath = filepath.Join(s.DataDir, "mongoinstall.lock") + + // 生成配置文件内容 + s.runtime.Logger.Info("make mongos config file content") + if err := s.makeConfContent(); err != nil { + return err + } + + return nil +} + +// makeConfContent 生成配置文件内容 +func (s *MongoSInstall) makeConfContent() error { + // 只支持mongos 3.0及以上得到配置文件内容 + // 判断mongos版本 + s.runtime.Logger.Info("start to make config file content") + mainVersion, err := strconv.Atoi(strings.Split(s.DbVersion, ".")[0]) + if err != nil { + s.runtime.Logger.Error( + "get %s version fail, error:%s", s.ConfParams.InstanceType, err) + return fmt.Errorf("get %s version fail, error:%s", s.ConfParams.InstanceType, err) + } + clusterId := strings.Join([]string{s.ConfParams.App, s.ConfParams.SetId, "conf"}, "-") + IpConfigDB := strings.Join(s.ConfParams.ConfigDB, ",") + configDB := strings.Join([]string{clusterId, IpConfigDB}, "/") + + // 生成mongos配置文件 + conf := common.NewYamlMongoSConf() + conf.Sharding.ConfigDB = configDB + conf.SystemLog.LogAppend = true + conf.SystemLog.Path = s.LogPath + conf.SystemLog.Destination = s.ConfParams.DbConfig.Destination + conf.ProcessManagement.Fork = true + conf.ProcessManagement.PidFilePath = s.PidFilePath + conf.Net.Port = s.ConfParams.Port + conf.Net.BindIp = strings.Join([]string{"127.0.0.1", s.ConfParams.IP}, ",") + conf.Net.WireObjectCheck = false + // mongos版本小于4获取配置文件内容 + if mainVersion < 4 { + s.NoAuthConfFileContent, err = conf.GetConfContent() + if err != nil { + s.runtime.Logger.Error( + "version:%s make mongos no auth config file content fail, error:%s", s.DbVersion, err) + return fmt.Errorf("version:%s make mongos no auth config file content fail, error:%s", + s.DbVersion, err) + } + conf.Security.KeyFile = s.KeyFilePath + // 获取验证配置文件内容 + s.AuthConfFileContent, err = conf.GetConfContent() + if err != nil { + s.runtime.Logger.Error(fmt.Sprintf( + "version:%s make mongos auth config file content fail, error:%s", + s.DbVersion, err)) + return fmt.Errorf("version:%s make mongos auth config file content fail, error:%s", + s.DbVersion, err) + } + s.runtime.Logger.Info("make config file content successfully") + return nil + } + + // mongos版本4及以上获取配置文件内容 + conf.OperationProfiling.SlowOpThresholdMs = s.ConfParams.DbConfig.SlowOpThresholdMs + conf.OperationProfiling.SlowOpThresholdMs = s.ConfParams.DbConfig.SlowOpThresholdMs + // 获取非验证配置文件内容 + s.NoAuthConfFileContent, err = conf.GetConfContent() + if err != nil { + s.runtime.Logger.Error( + "version:%s make mongos no auth config file content fail, error:%s", s.DbVersion, err) + return fmt.Errorf("version:%s make mongos no auth config file content fail, error:%s", + s.DbVersion, err) + } + conf.Security.KeyFile = s.KeyFilePath + // 获取验证配置文件内容 + s.AuthConfFileContent, err = conf.GetConfContent() + if err != nil { + s.runtime.Logger.Error(fmt.Sprintf( + "version:%s make mongos auth config file content fail, error:%s", + s.DbVersion, err)) + return fmt.Errorf("version:%s make mongos auth config file content fail, error:%s", + s.DbVersion, err) + } + s.runtime.Logger.Info("make config file content successfully") + return nil +} + +// checkParams 校验参数 检查输入的参数 检查端口是否合规 检查安装包 检查端口是否被使用（如果使用，则检查是否是mongodb服务） +func (s *MongoSInstall) checkParams() (bool, error) { + // 校验Mongo配置文件 + s.runtime.Logger.Info("start to validate parameters") + validate := validator.New() + s.runtime.Logger.Info("start to validate parameters of mongos config file") + if err := validate.Struct(s.ConfParams); err != nil { + s.runtime.Logger.Error(fmt.Sprintf("validate parameters of mongos config file fail, error:%s", err)) + return false, fmt.Errorf("validate parameters of mongos config file fail, error:%s", err) + } + s.runtime.Logger.Info("= validate parameters of mongos config file successfully") + + // 校验port是否合规 + s.runtime.Logger.Info("start to validate port if it is correct") + if s.ConfParams.Port < MongoDBPortMin || s.ConfParams.Port > MongoDBPortMax { + s.runtime.Logger.Error(fmt.Sprintf( + "validate port if it is correct, port is not within defalut range [%d,%d]", + MongoDBPortMin, MongoDBPortMax)) + return false, fmt.Errorf("validate port if it is correct, port is not within defalut range [%d,%d]", + MongoDBPortMin, MongoDBPortMax) + } + s.runtime.Logger.Info("validate port if it is correct successfully") + + // 校验安装包是否存在，md5值是否一致 + s.runtime.Logger.Info("start to validate install package") + if flag := util.FileExists(s.InstallPackagePath); !flag { + s.runtime.Logger.Error(fmt.Sprintf("validate install package, %s is not existed", + s.InstallPackagePath)) + return false, fmt.Errorf("validate install file, %s is not existed", + s.InstallPackagePath) + } + md5, _ := util.GetFileMd5(s.InstallPackagePath) + if s.ConfParams.MediaPkg.PkgMd5 != md5 { + s.runtime.Logger.Error(fmt.Sprintf("validate install package md5 fail, md5 is incorrect")) + return false, fmt.Errorf("validate install package md5 fail, md5 is incorrect") + } + s.runtime.Logger.Info("validate install package md5 successfully") + + // 校验端口是否使用 + s.runtime.Logger.Info("start to validate port if it has been used") + flag, _ := util.CheckPortIsInUse(s.ConfParams.IP, strconv.Itoa(s.ConfParams.Port)) + if flag { + // 校验端口是否是mongod进程 + cmd := fmt.Sprintf("netstat -ntpl |grep %d | awk '{print $7}' |head -1", s.ConfParams.Port) + result, _ := util.RunBashCmd(cmd, "", nil, 10*time.Second) + if strings.Contains(result, "mongos") { + // 检查配置文件是否一致，读取已有配置文件与新生成的配置文件内容对比 + content, _ := ioutil.ReadFile(s.AuthConfFilePath) + if strings.Compare(string(content), string(s.AuthConfFileContent)) == 0 { + // 检查mongodb版本 + version, err := common.CheckMongoVersion(s.BinDir, "mongos") + if err != nil { + s.runtime.Logger.Error( + fmt.Sprintf("mongos has been installed, port:%d, check mongos version fail. error:%s", + s.ConfParams.Port, version)) + return false, fmt.Errorf("mongos has been installed, port:%d, check mongos version fail. error:%s", + s.ConfParams.Port, version) + } + if version == s.DbVersion { + s.runtime.Logger.Info(fmt.Sprintf("mongos has been installed, port:%d, version:%s", + s.ConfParams.Port, version)) + return true, nil + } + s.runtime.Logger.Error(fmt.Sprintf("other mongos has been installed, port:%d, version:%s", + s.ConfParams.Port, version)) + return false, fmt.Errorf("other mongos has been installed, port:%d, version:%s", + s.ConfParams.Port, version) + } + + } + s.runtime.Logger.Error( + fmt.Sprintf("validate port if it has been used, port:%d is used by other process", + s.ConfParams.Port)) + return false, fmt.Errorf("validate port if it has been used, port:%d is used by other process", + s.ConfParams.Port) + } + s.runtime.Logger.Info("validate port if it has been used successfully") + s.runtime.Logger.Info("validate parameters successfully") + return false, nil +} + +// unTarAndCreateSoftLink 解压安装包，创建软链接并给目录授权 +func (s *MongoSInstall) unTarAndCreateSoftLink() error { + // 判断解压目录是否存在 + unTarPath := filepath.Join(s.BinDir, s.ConfParams.MediaPkg.GePkgBaseName()) + + // soft link目录 + installPath := filepath.Join(s.BinDir, "mongodb") + + // 解压安装包并授权 + // 安装多实例并发执行添加文件锁 + s.runtime.Logger.Info("start to get install file lock") + fileLock := common.NewFileLock(s.LockFilePath) + // 获取锁 + err := fileLock.Lock() + if err != nil { + for { + err = fileLock.Lock() + if err != nil { + time.Sleep(1 * time.Second) + continue + } + s.runtime.Logger.Info("get install file lock successfully") + break + } + } else { + s.runtime.Logger.Info("get install file lock successfully") + } + if err = common.UnTarAndCreateSoftLinkAndChown(s.runtime, s.BinDir, + s.InstallPackagePath, unTarPath, installPath, s.OsUser, s.OsGroup); err != nil { + return err + } + // 释放锁 + s.runtime.Logger.Info("release install file lock successfully") + _ = fileLock.UnLock() + + // 检查mongos版本 + s.runtime.Logger.Info("start to check mongos version") + version, err := common.CheckMongoVersion(s.BinDir, "mongos") + if err != nil { + s.runtime.Logger.Error(fmt.Sprintf("%s has been existed, check mongodb version, error:%s", + installPath, err)) + return fmt.Errorf("%s has been existed, check mongodb version, error:%s", + installPath, err) + } + if version != s.DbVersion { + s.runtime.Logger.Error( + fmt.Sprintf("%s has been existed, check mongodb version, version:%s is incorrect", + installPath, version)) + return fmt.Errorf("%s has been existed, check mongodb version, version:%s is incorrect", + installPath, version) + } + s.runtime.Logger.Info("check mongos version successfully") + return nil +} + +// mkdir 创建相关目录并给目录授权 +func (s *MongoSInstall) mkdir() error { + // 创建日志文件目录 + logPathDir, _ := filepath.Split(s.LogPath) + s.runtime.Logger.Info("start to create log directory") + if err := util.MkDirsIfNotExistsWithPerm([]string{logPathDir}, DefaultPerm); err != nil { + s.runtime.Logger.Error(fmt.Sprintf("create log directory fail, error:%s", err)) + return fmt.Errorf("create log directory fail, error:%s", err) + } + s.runtime.Logger.Info("create log directory successfully") + + // 创建配置文件目录 + confFilePathDir, _ := filepath.Split(s.AuthConfFilePath) + s.runtime.Logger.Info("start to create data directory") + if err := util.MkDirsIfNotExistsWithPerm([]string{confFilePathDir}, DefaultPerm); err != nil { + s.runtime.Logger.Error(fmt.Sprintf("create data directory fail, error:%s", err)) + return fmt.Errorf("create data directory fail, error:%s", err) + } + s.runtime.Logger.Info("create data directory successfully") + + // 修改目录属主 + s.runtime.Logger.Info("start to execute chown command for dbPath, logPath and backupPath") + if _, err := util.RunBashCmd( + fmt.Sprintf("chown -R %s.%s %s", s.OsUser, s.OsGroup, filepath.Join(logPathDir, "../")), + "", nil, + 10*time.Second); err != nil { + s.runtime.Logger.Error(fmt.Sprintf("chown log directory fail, error:%s", err)) + return fmt.Errorf("chown log directory fail, error:%s", err) + } + if _, err := util.RunBashCmd( + fmt.Sprintf("chown -R %s.%s %s", s.OsUser, s.OsGroup, filepath.Join(confFilePathDir, "../")), + "", nil, + 10*time.Second); err != nil { + s.runtime.Logger.Error(fmt.Sprintf("chown data directory fail, error:%s", err)) + return fmt.Errorf("chown data directory fail, error:%s", err) + } + s.runtime.Logger.Info("execute chown command for dbPath, logPath and backupPath successfully") + return nil +} + +// createFile 创建配置文件以及key文件 +func (s *MongoSInstall) creatFile() error { + // 创建配置文件，key文件，dbType文件并授权 + if err := common.CreateConfFileAndKeyFileAndDbTypeFileAndChown( + s.runtime, s.AuthConfFilePath, s.AuthConfFileContent, s.OsUser, s.OsGroup, s.NoAuthConfFilePath, + s.NoAuthConfFileContent, s.KeyFilePath, s.ConfParams.KeyFile, s.DbTypeFilePath, + s.ConfParams.InstanceType, DefaultPerm); err != nil { + return err + } + return nil +} + +// startup 启动服务 +func (s *MongoSInstall) startup() error { + // 申明mongos可执行文件路径，把路径写入/etc/profile + if err := common.AddPathToProfile(s.runtime, s.BinDir); err != nil { + return err + } + + // 启动服务 + s.runtime.Logger.Info("start to startup mongos") + if err := common.StartMongoProcess(s.BinDir, s.ConfParams.Port, + s.OsUser, s.ConfParams.Auth); err != nil { + s.runtime.Logger.Error(fmt.Sprintf("startup mongos fail, error:%s", err)) + return fmt.Errorf("shutdown mongos fail, error:%s", err) + } + flag, service, err := common.CheckMongoService(s.ConfParams.Port) + if err != nil { + s.runtime.Logger.Error("check %s fail, error:%s", service, err) + return fmt.Errorf("check %s fail, error:%s", service, err) + } + if flag == false { + s.runtime.Logger.Error("startup %s fail", service) + return fmt.Errorf("startup %s fail", service) + } + s.runtime.Logger.Info("startup %s successfully", service) + + return nil +} diff --git a/dbm-services/mongo/db-tools/dbactuator/pkg/atomjobs/atommongodb/replicaset_stepdown.go b/dbm-services/mongo/db-tools/dbactuator/pkg/atomjobs/atommongodb/replicaset_stepdown.go new file mode 100644 index 0000000000..631bbd2f33 --- /dev/null +++ b/dbm-services/mongo/db-tools/dbactuator/pkg/atomjobs/atommongodb/replicaset_stepdown.go @@ -0,0 +1,130 @@ +package atommongodb + +import ( + "encoding/json" + "fmt" + "path/filepath" + "strconv" + "strings" + + "dbm-services/mongo/db-tools/dbactuator/pkg/common" + "dbm-services/mongo/db-tools/dbactuator/pkg/consts" + "dbm-services/mongo/db-tools/dbactuator/pkg/jobruntime" + + "github.com/go-playground/validator/v10" +) + +// StepDownConfParams 参数 +type StepDownConfParams struct { + IP string `json:"ip" validate:"required"` + Port int `json:"port" validate:"required"` + AdminUsername string `json:"adminUsername" validate:"required"` + AdminPassword string `json:"adminPassword" validate:"required"` +} + +// StepDown 添加分片到集群 +type StepDown struct { + runtime *jobruntime.JobGenericRuntime + BinDir string + Mongo string + OsUser string + PrimaryIP string + PrimaryPort int + ConfParams *StepDownConfParams +} + +// NewStepDown 实例化结构体 +func NewStepDown() jobruntime.JobRunner { + return &StepDown{} +} + +// Name 获取原子任务的名字 +func (s *StepDown) Name() string { + return "replicaset_stepdown" +} + +// Run 运行原子任务 +func (s *StepDown) Run() error { + // 执行主备切换 + if err := s.execStepDown(); err != nil { + return err + } + + return nil +} + +// Retry 重试 +func (s *StepDown) Retry() uint { + return 2 +} + +// Rollback 回滚 +func (s *StepDown) Rollback() error { + return nil +} + +// Init 初始化 +func (s *StepDown) Init(runtime *jobruntime.JobGenericRuntime) error { + // 获取安装参数 + s.runtime = runtime + s.runtime.Logger.Info("start to init") + s.BinDir = consts.UsrLocal + s.Mongo = filepath.Join(s.BinDir, "mongodb", "bin", "mongo") + s.OsUser = consts.GetProcessUser() + + // 获取MongoDB配置文件参数 + if err := json.Unmarshal([]byte(s.runtime.PayloadDecoded), &s.ConfParams); err != nil { + s.runtime.Logger.Error(fmt.Sprintf( + "get parameters of stepDown fail by json.Unmarshal, error:%s", err)) + return fmt.Errorf("get parameters of stepDown fail by json.Unmarshal, error:%s", err) + } + + // 获取primary信息 + info, err := common.AuthGetPrimaryInfo(s.Mongo, s.ConfParams.AdminUsername, s.ConfParams.AdminPassword, + s.ConfParams.IP, s.ConfParams.Port) + if err != nil { + s.runtime.Logger.Error(fmt.Sprintf( + "get primary db info of stepDown fail, error:%s", err)) + return fmt.Errorf("get primary db info of stepDown fail, error:%s", err) + } + getInfo := strings.Split(info, ":") + s.PrimaryIP = getInfo[0] + s.PrimaryPort, _ = strconv.Atoi(getInfo[1]) + + // 进行校验 + s.runtime.Logger.Info("start to validate parameters") + if err = s.checkParams(); err != nil { + return err + } + + return nil +} + +// checkParams 校验参数 +func (s *StepDown) checkParams() error { + // 校验配置参数 + validate := validator.New() + s.runtime.Logger.Info("start to validate parameters of deleteUser") + if err := validate.Struct(s.ConfParams); err != nil { + s.runtime.Logger.Error(fmt.Sprintf("validate parameters of deleteUser fail, error:%s", err)) + return fmt.Errorf("validate parameters of deleteUser fail, error:%s", err) + } + return nil +} + +// execStepDown 执行切换 +func (s *StepDown) execStepDown() error { + s.runtime.Logger.Info("start to convert primary secondary db") + flag, err := common.AuthRsStepDown(s.Mongo, s.PrimaryIP, s.PrimaryPort, s.ConfParams.AdminUsername, + s.ConfParams.AdminPassword) + if err != nil { + s.runtime.Logger.Error("convert primary secondary db fail, error:%s", err) + return fmt.Errorf("convert primary secondary db fail, error:%s", err) + } + if flag == true { + s.runtime.Logger.Info("convert primary secondary db successfully") + return nil + } + + return nil +} diff --git a/dbm-services/mongo/db-tools/dbactuator/pkg/atomjobs/atomsys/atomsys.go b/dbm-services/mongo/db-tools/dbactuator/pkg/atomjobs/atomsys/atomsys.go new file mode 100644 index 0000000000..bb529bb5f4 --- /dev/null +++ b/dbm-services/mongo/db-tools/dbactuator/pkg/atomjobs/atomsys/atomsys.go @@ -0,0 +1,2 @@ +// Package atomsys os系统原子任务 +package atomsys diff --git a/dbm-services/mongo/db-tools/dbactuator/pkg/atomjobs/atomsys/os_mongo_init.go b/dbm-services/mongo/db-tools/dbactuator/pkg/atomjobs/atomsys/os_mongo_init.go new file mode 100644 index 0000000000..0b0935fc3c --- /dev/null +++ b/dbm-services/mongo/db-tools/dbactuator/pkg/atomjobs/atomsys/os_mongo_init.go @@ -0,0 +1,123 @@ +package atomsys + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "strings" + "time" + + "dbm-services/mongo/db-tools/dbactuator/pkg/common" + "dbm-services/mongo/db-tools/dbactuator/pkg/consts" + "dbm-services/mongo/db-tools/dbactuator/pkg/jobruntime" + "dbm-services/mongo/db-tools/dbactuator/pkg/util" + + "github.com/go-playground/validator/v10" +) + +// OsMongoInitConfParams 系统初始化参数 +type OsMongoInitConfParams struct { + User string `json:"user" validate:"required"` + Password string `json:"password" validate:"required"` +} + +// OsMongoInit 系统初始化原子任务 +type OsMongoInit struct { + runtime *jobruntime.JobGenericRuntime + ConfParams *OsMongoInitConfParams + OsUser string + OsGroup string +} + +// NewOsMongoInit new +func NewOsMongoInit() jobruntime.JobRunner { + return &OsMongoInit{} +} + +// Init 初始化 +func (o *OsMongoInit) Init(runtime *jobruntime.JobGenericRuntime) error { + // 获取安装参数 + o.runtime = runtime + o.runtime.Logger.Info("start to init") + o.OsUser = consts.GetProcessUser() + o.OsGroup = consts.GetProcessUserGroup() + // 获取MongoDB配置文件参数 + if err := json.Unmarshal([]byte(o.runtime.PayloadDecoded), &o.ConfParams); err != nil { + o.runtime.Logger.Error( + "get parameters of mongoOsInit fail by json.Unmarshal, error:%s", err) + return fmt.Errorf("get parameters of mongoOsInit fail by json.Unmarshal, error:%s", err) + } + o.runtime.Logger.Info("init successfully") + + // 进行校验 + if err := o.checkParams(); err != nil { + return err + } + + return nil +} + +// checkParams 校验参数 +func (o *OsMongoInit) checkParams() error { + // 校验配置参数 + o.runtime.Logger.Info("start to validate parameters") + validate := validator.New() + o.runtime.Logger.Info("start to validate parameters of deInstall") + if err := validate.Struct(o.ConfParams); err != nil { + o.runtime.Logger.Error("validate parameters of mongoOsInit fail, error:%s", err) + return fmt.Errorf("validate parameters of mongoOsInit fail, error:%s", err) + } + o.runtime.Logger.Info("validate parameters successfully") + return nil +} + +// Name 名字 +func (o *OsMongoInit) Name() string { + return "os_mongo_init" +} + +// Run 执行函数 +func (o *OsMongoInit) Run() error { + // 获取初始化脚本 + o.runtime.Logger.Info("start to make init script content") + data := common.MongoShellInit + data = strings.Replace(data, "{{user}}", o.OsUser, -1) + data = strings.Replace(data, "{{group}}", o.OsGroup, -1) + o.runtime.Logger.Info("make init script content successfully") + + // 创建脚本文件 + o.runtime.Logger.Info("start to create init script file") + tmpScriptName := "/tmp/sysinit.sh" + if err := ioutil.WriteFile(tmpScriptName, []byte(data), 07555); err != nil { + o.runtime.Logger.Error("write tmp script failed %s", err.Error()) + return err + } + o.runtime.Logger.Info("create init script file successfully") + + // 执行脚本 + o.runtime.Logger.Info("start to execute init script") + _, err := util.RunBashCmd(tmpScriptName, "", nil, 30*time.Second) + if err != nil { + o.runtime.Logger.Error("execute init script fail, error:%s", err) + return fmt.Errorf("execute init script fail, error:%s", err) + } + o.runtime.Logger.Info("execute init script successfully") + // 设置用户名密码 + o.runtime.Logger.Info("start to set user:%s password", o.OsUser) + err = util.SetOSUserPassword(o.ConfParams.User, o.ConfParams.Password) + o.runtime.Logger.Info("set user:%s password successfully", o.OsUser) + if err != nil { + return err + } + return nil +} + +// Retry times +func (o *OsMongoInit) Retry() uint { + return 2 +} + +// Rollback rollback +func (o *OsMongoInit) Rollback() error { + return nil +} diff --git a/dbm-services/mongo/db-tools/dbactuator/pkg/backupsys/backupsys.go b/dbm-services/mongo/db-tools/dbactuator/pkg/backupsys/backupsys.go new file mode 100644 index 0000000000..1ef15f9f0a --- /dev/null +++ b/dbm-services/mongo/db-tools/dbactuator/pkg/backupsys/backupsys.go @@ -0,0 +1,231 @@ +// Package backupsys 备份系统 +package backupsys + +import ( + "bufio" + "encoding/json" + "fmt" + "strconv" + "strings" + "time" + + "dbm-services/mongo/db-tools/dbactuator/mylog" + "dbm-services/mongo/db-tools/dbactuator/pkg/consts" + "dbm-services/mongo/db-tools/dbactuator/pkg/util" +) + +// UploadTask 操作备份系统 +type UploadTask struct { + Files []string `json:"files"` // 全路径 + TaskIDs []uint64 `json:"taskids"` + Tag string `json:"tag"` +} + +// UploadFiles 上传文件 +func (task *UploadTask) UploadFiles() (err error) { + var taskIDStr string + var taskIDNum uint64 + if len(task.Files) == 0 { + return + } + if task.Tag == "" { + err = fmt.Errorf("BackupSystem uploadFiles tag(%s) cannot be empty", task.Tag) + mylog.Logger.Error(err.Error()) + return + } + for _, file := range task.Files { + if !util.FileExists(file) { + err = fmt.Errorf("BackupSystem uploadFiles %s not exists", file) + mylog.Logger.Error(err.Error()) + return + } + } + for _, bkfile := range task.Files { + bkCmd := fmt.Sprintf("%s -n -f %s --with-md5 -t %s|grep 'taskid'|awk -F: '{print $2}'", + consts.BackupClient, bkfile, task.Tag) + mylog.Logger.Info(bkCmd) + taskIDStr, err = util.RunBashCmd(bkCmd, "", nil, 10*time.Minute) + if err != nil { + return + } + taskIDNum, err = strconv.ParseUint(taskIDStr, 10, 64) + if err != nil { + err = fmt.Errorf("%s ParseUint failed,err:%v", taskIDStr, err) + mylog.Logger.Error(err.Error()) + return + } + task.TaskIDs = append(task.TaskIDs, taskIDNum) + } + return +} + +// CheckTasksStatus 检查tasks状态 +func (task *UploadTask) CheckTasksStatus() (runningTaskIDs, failTaskIDs, succTaskIDs []uint64, + runningFiles, failedFiles, succFiles []string, failMsgs []string, err error) { + var status TaskStatus + for idx, taskID := range task.TaskIDs { + status, err = GetTaskStatus(taskID) + if err != nil { + return + } + if status.Status > 4 { + // err = fmt.Errorf("ToBackupSystem %s failed,err:%s,taskid:%d", + // status.File, status.StatusInfo, taskID) + // mylog.Logger.Error(err.Error()) + failMsgs = append(failMsgs, fmt.Sprintf("taskid:%d,failMsg:%s", taskID, status.StatusInfo)) + failedFiles = append(failedFiles, task.Files[idx]) + failTaskIDs = append(failTaskIDs, task.TaskIDs[idx]) + } else if status.Status == 4 { + succFiles = append(succFiles, task.Files[idx]) + succTaskIDs = append(succTaskIDs, task.TaskIDs[idx]) + } else if status.Status < 4 { + runningFiles = append(runningFiles, task.Files[idx]) + runningTaskIDs = append(runningTaskIDs, task.TaskIDs[idx]) + } + } + return +} + +// WaitForUploadFinish 等待所有files上传成功 +func (task *UploadTask) WaitForUploadFinish() (err error) { + var times int64 + var msg string + var runningFiles, failFiles, succFiles, failMsgs []string + for { + times++ + _, _, _, runningFiles, failFiles, succFiles, failMsgs, err = task.CheckTasksStatus() + if err != nil { + return + } + // 只要有running的task,则继续等待 + if len(runningFiles) > 0 { + if times%6 == 0 { + // 每分钟打印一次日志 + msg = fmt.Sprintf("files[%+v] cnt:%d upload to backupSystem still running", runningFiles, len(runningFiles)) + mylog.Logger.Info(msg) + } + time.Sleep(10 * time.Second) + continue + } + if len(failMsgs) > 0 { + err = fmt.Errorf("failCnt:%d,failFiles:[%+v],err:%s", len(failFiles), failFiles, strings.Join(failFiles, ",")) + mylog.Logger.Error(err.Error()) + return + } + if len(succFiles) == len(task.Files) { + return nil + } + break + } + return +} + +// TaskStatus backup_client -q --taskid=xxxx 命令的结果 +type TaskStatus struct { + File string `json:"file"` + Host string `json:"host"` + SednupDateTime time.Time `json:"sendup_datetime"` + Status int `json:"status"` + StatusInfo string `json:"status_info"` + StartTime time.Time `json:"start_time"` + CompleteTime time.Time `json:"complete_time"` + ExpireTime time.Time `json:"expire_time"` +} + +// String 用于打印 +func (status *TaskStatus) String() string { + statusBytes, _ := json.Marshal(status) + return string(statusBytes) +} + +// GetTaskStatus 执行backup_client -q --taskid=xxxx 命令的结果并解析 +func GetTaskStatus(taskid uint64) (status TaskStatus, err error) { + var cmdRet string + bkCmd := fmt.Sprintf("%s -q --taskid=%d", consts.BackupClient, taskid) + cmdRet, err = util.RunBashCmd(bkCmd, "", nil, 30*time.Second) + if err != nil { + return + } + scanner := bufio.NewScanner(strings.NewReader(cmdRet)) + scanner.Split(bufio.ScanLines) + for scanner.Scan() { + line := scanner.Text() + line = strings.TrimSpace(line) + if line == "" { + continue + } + l01 := strings.SplitN(line, ":", 2) + if len(l01) != 2 { + err = fmt.Errorf("len()!=2,cmd:%s,result format not correct:%s", bkCmd, cmdRet) + mylog.Logger.Error(err.Error()) + return + } + first := strings.TrimSpace(l01[0]) + second := strings.TrimSpace(l01[1]) + switch first { + case "file": + status.File = second + case "host": + status.Host = second + case "sendup datetime": + if second == "0000-00-00 00:00:00" { + status.SednupDateTime = time.Time{} // "0000-01-00 00:00:00" + break + } + status.SednupDateTime, err = time.ParseInLocation(consts.UnixtimeLayout, second, time.Local) + if err != nil { + err = fmt.Errorf("time.Parse 'sendup datetime' failed,err:%v,value:%s,cmd:%s", err, second, bkCmd) + mylog.Logger.Error(err.Error()) + return + } + case "status": + status.Status, err = strconv.Atoi(second) + if err != nil { + err = fmt.Errorf("strconv.Atoi failed,err:%v,value:%s,cmd:%s", err, second, bkCmd) + mylog.Logger.Error(err.Error()) + return + } + case "status info": + status.StatusInfo = second + case "start_time": + if second == "0000-00-00 00:00:00" { + status.StartTime = time.Time{} // "0000-01-00 00:00:00" + break + } + status.StartTime, err = time.ParseInLocation(consts.UnixtimeLayout, second, time.Local) + if err != nil { + err = fmt.Errorf("time.Parse start_time failed,err:%v,value:%s,cmd:%s", err, second, bkCmd) + mylog.Logger.Error(err.Error()) + return + } + case "complete_time": + if second == "0000-00-00 00:00:00" { + status.CompleteTime = time.Time{} // "0000-01-00 00:00:00" + break + } + status.CompleteTime, err = time.ParseInLocation(consts.UnixtimeLayout, second, time.Local) + if err != nil { + err = fmt.Errorf("time.Parse complete_time failed,err:%v,value:%s,cmd:%s", err, second, bkCmd) + mylog.Logger.Error(err.Error()) + return + } + case "expire_time": + if second == "0000-00-00 00:00:00" { + status.ExpireTime = time.Time{} // "0000-01-00 00:00:00" + break + } + status.ExpireTime, err = time.ParseInLocation(consts.UnixtimeLayout, second, time.Local) + if err != nil { + err = fmt.Errorf("time.Parse expire_time failed,err:%v,value:%s,cmd:%s", err, second, bkCmd) + mylog.Logger.Error(err.Error()) + return + } + } + } + if err = scanner.Err(); err != nil { + err = fmt.Errorf("scanner.Scan failed,err:%v,cmd:%s", err, cmdRet) + mylog.Logger.Error(err.Error()) + return + } + return +} diff --git a/dbm-services/mongo/db-tools/dbactuator/pkg/common/exporter_conf.go b/dbm-services/mongo/db-tools/dbactuator/pkg/common/exporter_conf.go new file mode 100644 index 0000000000..d275658bdd --- /dev/null +++ b/dbm-services/mongo/db-tools/dbactuator/pkg/common/exporter_conf.go @@ -0,0 +1,46 @@ +package common + +import ( + "dbm-services/mongo/db-tools/dbactuator/pkg/consts" + "dbm-services/mongo/db-tools/dbactuator/pkg/util" + "encoding/json" + "fmt" + "io/ioutil" + "os" + "path/filepath" +) + +func getConfFileName(port int) string { + return filepath.Join(consts.ExporterConfDir, fmt.Sprintf("%d.conf", port)) +} + +// setExporterConfig 写入ExporterConfig文件 +// 目录固定:. consts.ExporterConfDir +// 文件名称:. $port.conf +// 文件已经存在, 覆盖. +// 文件写入失败，报错. + +// WriteExporterConfigFile 写exporter配置文件 +func WriteExporterConfigFile(port int, data interface{}) (err error) { + var fileData []byte + var confFile string + err = util.MkDirsIfNotExists([]string{consts.ExporterConfDir}) + if err != nil { + return err + } + confFile = getConfFileName(port) + fileData, _ = json.Marshal(data) + err = ioutil.WriteFile(confFile, fileData, 0755) + if err != nil { + return err + } + util.LocalDirChownMysql(consts.ExporterConfDir) + return nil +} + +// DeleteExporterConfigFile 删除Exporter配置文件. +func DeleteExporterConfigFile(port int) (err error) { + var confFile string + confFile = getConfFileName(port) + return os.Remove(confFile) +} diff --git a/dbm-services/mongo/db-tools/dbactuator/pkg/common/filelock.go b/dbm-services/mongo/db-tools/dbactuator/pkg/common/filelock.go new file mode 100644 index 0000000000..7c7d52a864 --- /dev/null +++ b/dbm-services/mongo/db-tools/dbactuator/pkg/common/filelock.go @@ -0,0 +1,34 @@ +package common + +import ( + "os" + "syscall" +) + +// FileLock 结构体 +type FileLock struct { + Path string + FD *os.File +} + +// NewFileLock 生成结构体 +func NewFileLock(path string) *FileLock { + fd, _ := os.Open(path) + return &FileLock{ + Path: path, + FD: fd, + } +} + +// Lock 加锁 +func (f *FileLock) Lock() error { + err := syscall.Flock(int(f.FD.Fd()), syscall.LOCK_EX|syscall.LOCK_NB) + return err +} + +// UnLock 解锁 +func (f *FileLock) UnLock() error { + defer f.FD.Close() + err := syscall.Flock(int(f.FD.Fd()), syscall.LOCK_UN) + return err +} diff --git a/dbm-services/mongo/db-tools/dbactuator/pkg/common/initiate_replicaset_conf.go b/dbm-services/mongo/db-tools/dbactuator/pkg/common/initiate_replicaset_conf.go new file mode 100644 index 0000000000..f4b9cd7902 --- /dev/null +++ b/dbm-services/mongo/db-tools/dbactuator/pkg/common/initiate_replicaset_conf.go @@ -0,0 +1,37 @@ +package common + +import "encoding/json" + +// JsonConfReplicaset 复制集配置 +type JsonConfReplicaset struct { + Id string `json:"_id"` + ConfigSvr bool `json:"configsvr"` + Members []*Member `json:"members"` +} + +// Member 成员 +type Member struct { + Id int `json:"_id"` + Host string `json:"host"` + Priority int `json:"priority"` + Hidden bool `json:"hidden"` +} + +// NewJsonConfReplicaset 获取结构体 +func NewJsonConfReplicaset() *JsonConfReplicaset { + return &JsonConfReplicaset{} +} + +// GetConfContent 获取配置内容 +func (j *JsonConfReplicaset) GetConfContent() ([]byte, error) { + confContent, err := json.Marshal(j) + if err != nil { + return nil, err + } + return confContent, nil +} + +// NewMember 获取结构体 +func NewMember() *Member { + return &Member{} +} diff --git a/dbm-services/mongo/db-tools/dbactuator/pkg/common/media_pkg.go b/dbm-services/mongo/db-tools/dbactuator/pkg/common/media_pkg.go new file mode 100644 index 0000000000..3ae6d84ce1 --- /dev/null +++ b/dbm-services/mongo/db-tools/dbactuator/pkg/common/media_pkg.go @@ -0,0 +1,130 @@ +package common + +import ( + "fmt" + "os" + "path/filepath" + "regexp" + "time" + + "dbm-services/mongo/db-tools/dbactuator/mylog" + "dbm-services/mongo/db-tools/dbactuator/pkg/consts" + "dbm-services/mongo/db-tools/dbactuator/pkg/util" +) + +// MediaPkg 通用介质包处理 +type MediaPkg struct { + Pkg string `json:"pkg" validate:"required"` // 安装包名 + PkgMd5 string `json:"pkg_md5" validate:"required,md5"` // 安装包MD5 +} + +// GetAbsolutePath 返回介质存放的绝对路径 +func (m *MediaPkg) GetAbsolutePath() string { + return filepath.Join(consts.PackageSavePath, m.Pkg) +} + +// GePkgBaseName 例如将 mysql-5.7.20-linux-x86_64-tmysql-3.1.5-gcs.tar.gz +// 解析出 mysql-5.7.20-linux-x86_64-tmysql-3.1.5-gcs +// 用于做软连接使用 +func (m *MediaPkg) GePkgBaseName() string { + pkgFullName := filepath.Base(m.GetAbsolutePath()) + return regexp.MustCompile("(.tar.gz|.tgz)$").ReplaceAllString(pkgFullName, "") +} + +// Check 检查介质包 +func (m *MediaPkg) Check() (err error) { + var fileMd5 string + // 判断安装包是否存在 + pkgAbPath := m.GetAbsolutePath() + if !util.FileExists(pkgAbPath) { + return fmt.Errorf("%s不存在", pkgAbPath) + } + if fileMd5, err = util.GetFileMd5(pkgAbPath); err != nil { + return fmt.Errorf("获取[%s]md5失败, %v", m.Pkg, err.Error()) + } + // 校验md5 + if fileMd5 != m.PkgMd5 { + return fmt.Errorf("安装包的md5不匹配,[%s]文件的md5[%s]不正确", fileMd5, m.PkgMd5) + } + return +} + +// DbToolsMediaPkg db工具包 +type DbToolsMediaPkg struct { + MediaPkg +} + +// Install 安装dbtools +// 1. 确保本地 /data/install/dbtool.tar.gz 存在,且md5校验ok; +// 2. 检查 {REDIS_BACKUP_DIR}/dbbak/dbatool.tar.gz 与 /data/install/dbtool.tar.gz 是否一致; +// - md5一致,则忽略更新; +// - /data/install/dbtool.tar.gz 不存在 or md5不一致 则用最新 /data/install/dbtool.tar.gz 工具覆盖 {REDIS_BACKUP_DIR}/dbbak/dbatool +// 3. 创建 /home/mysql/dbtools -> /data/dbbak/dbtools 软链接 +// 4. cp /data/install/dbtool.tar.gz {REDIS_BACKUP_DIR}/dbbak/dbatool.tar.gz +func (pkg *DbToolsMediaPkg) Install() (err error) { + var fileMd5 string + var overrideLocal bool = true + var newMysqlHomeLink bool = true + var realLink string + err = pkg.Check() + if err != nil { + return + } + toolsName := filepath.Base(consts.DbToolsPath) + backupDir := filepath.Join(consts.GetRedisBackupDir(), "dbbak") // 如 /data/dbbak + bakdirToolsTar := filepath.Join(backupDir, toolsName+".tar.gz") // 如 /data/dbbak/dbtools.tar.gz + installToolTar := pkg.GetAbsolutePath() + if util.FileExists(bakdirToolsTar) { + fileMd5, err = util.GetFileMd5(bakdirToolsTar) + if err != nil { + return + } + if fileMd5 == pkg.PkgMd5 { + overrideLocal = false + } + } + if overrideLocal { + // 最新介质覆盖本地 + untarCmd := fmt.Sprintf("tar -zxf %s -C %s", installToolTar, backupDir) + mylog.Logger.Info(untarCmd) + _, err = util.RunBashCmd(untarCmd, "", nil, 10*time.Minute) + if err != nil { + return + } + } + if !util.FileExists(filepath.Join(backupDir, toolsName)) { // 如 /data/dbbak/dbtools 目录不存在 + err = fmt.Errorf("dir:%s not exists", filepath.Join(backupDir, toolsName)) + mylog.Logger.Error(err.Error()) + return + } + if util.FileExists(consts.DbToolsPath) { + realLink, err = filepath.EvalSymlinks(consts.DbToolsPath) + if err != nil { + err = fmt.Errorf("filepath.EvalSymlinks %s fail,err:%v", consts.DbToolsPath, err) + mylog.Logger.Error(err.Error()) + return err + } + if realLink == filepath.Join(backupDir, toolsName) { // /home/mysql/dbtools 已经是指向 /data/dbbak/dbtools 的软连接 + newMysqlHomeLink = false + } + } + if newMysqlHomeLink { + // 需创建 /home/mysql/dbtools -> /data/dbbak/dbtools 软链接 + err = os.Symlink(filepath.Join(backupDir, toolsName), consts.DbToolsPath) + if err != nil { + err = fmt.Errorf("os.Symlink %s -> %s fail,err:%s", consts.DbToolsPath, filepath.Join(backupDir, toolsName), err) + mylog.Logger.Error(err.Error()) + return + } + mylog.Logger.Info("create softLink success,%s -> %s", consts.DbToolsPath, filepath.Join(backupDir, toolsName)) + } + cpCmd := fmt.Sprintf("cp %s %s", installToolTar, bakdirToolsTar) + mylog.Logger.Info(cpCmd) + _, err = util.RunBashCmd(cpCmd, "", nil, 10*time.Minute) + if err != nil { + return + } + util.LocalDirChownMysql(consts.DbToolsPath) + util.LocalDirChownMysql(backupDir) + return nil +} diff --git a/dbm-services/mongo/db-tools/dbactuator/pkg/common/mongo_common.go b/dbm-services/mongo/db-tools/dbactuator/pkg/common/mongo_common.go new file mode 100644 index 0000000000..ca0db039e2 --- /dev/null +++ b/dbm-services/mongo/db-tools/dbactuator/pkg/common/mongo_common.go @@ -0,0 +1,575 @@ +package common + +import ( + "crypto/md5" + "encoding/hex" + "encoding/json" + "fmt" + "os" + "path/filepath" + "strconv" + "strings" + "time" + + "dbm-services/mongo/db-tools/dbactuator/pkg/jobruntime" + "dbm-services/mongo/db-tools/dbactuator/pkg/util" +) + +// UnTarAndCreateSoftLinkAndChown 解压目录，创建软链接并修改属主 +func UnTarAndCreateSoftLinkAndChown(runtime *jobruntime.JobGenericRuntime, binDir string, installPackagePath string, + unTarPath string, + installPath string, user string, group string) error { + // 解压安装包 + if !util.FileExists(unTarPath) { + // 解压到/usr/local目录下 + runtime.Logger.Info("start to unTar install package") + tarCmd := fmt.Sprintf("tar -zxf %s -C %s", installPackagePath, binDir) + if _, err := util.RunBashCmd(tarCmd, "", nil, 10*time.Second); err != nil { + runtime.Logger.Error(fmt.Sprintf("untar install file fail, error:%s", err)) + return fmt.Errorf("untar install file fail, error:%s", err) + } + runtime.Logger.Info("unTar install package successfully") + // 修改属主 + runtime.Logger.Info("start to execute chown command for unTar directory") + if _, err := util.RunBashCmd( + fmt.Sprintf("chown -R %s.%s %s", user, group, unTarPath), + "", nil, + 10*time.Second); err != nil { + runtime.Logger.Error(fmt.Sprintf("chown untar directory fail, error:%s", err)) + return fmt.Errorf("chown untar directory fail, error:%s", err) + } + runtime.Logger.Info("execute chown command for unTar directory successfully") + } + + // 创建软链接 + if !util.FileExists(installPath) { + // 创建软链接 + runtime.Logger.Info("start to create soft link") + softLink := fmt.Sprintf("ln -s %s %s", unTarPath, installPath) + if _, err := util.RunBashCmd(softLink, "", nil, 10*time.Second); err != nil { + runtime.Logger.Error( + fmt.Sprintf("install directory create softLink fail, error:%s", err)) + return fmt.Errorf("install directory create softLink fail, error:%s", err) + } + runtime.Logger.Info("create soft link successfully") + + // 修改属主 + runtime.Logger.Info("start to execute chown command for softLink directory") + if _, err := util.RunBashCmd( + fmt.Sprintf("chown -R %s.%s %s", user, group, installPath), + "", nil, + 10*time.Second); err != nil { + runtime.Logger.Error(fmt.Sprintf("chown softlink directory fail, error:%s", err)) + return fmt.Errorf("chown softlink directory fail, error:%s", err) + } + runtime.Logger.Info("execute chown command for softLink directory successfully") + + } + + return nil +} + +// GetMd5 获取md5值 +func GetMd5(str string) string { + h := md5.New() + h.Write([]byte(str)) + return hex.EncodeToString(h.Sum(nil)) +} + +// CheckMongoVersion 检查mongo版本 +func CheckMongoVersion(binDir string, mongoName string) (string, error) { + cmd := fmt.Sprintf("%s -version |grep -E 'db version|mongos version'| awk -F \" \" '{print $3}' |sed 's/v//g'", + filepath.Join(binDir, "mongodb", "bin", mongoName)) + getVersion, err := util.RunBashCmd(cmd, "", nil, 10*time.Second) + getVersion = strings.Replace(getVersion, "\n", "", -1) + if err != nil { + return "", err + } + return getVersion, nil +} + +// CheckMongoService 检查mongo服务是否存在 +func CheckMongoService(port int) (bool, string, error) { + cmd := fmt.Sprintf("netstat -ntpl |grep %d | awk '{print $7}' |head -1", port) + result, err := util.RunBashCmd(cmd, "", nil, 10*time.Second) + if err != nil { + return false, "", err + } + if strings.Contains(result, "mongos") { + return true, "mongos", nil + } + if strings.Contains(result, "mongod") { + return true, "mongod", nil + } + return false, "", nil +} + +// CreateConfFileAndKeyFileAndDbTypeFileAndChown 创建配置文件，key文件，dbType文件并授权 +func CreateConfFileAndKeyFileAndDbTypeFileAndChown(runtime *jobruntime.JobGenericRuntime, authConfFilePath string, + authConfFileContent []byte, user string, group string, noAuthConfFilePath string, noAuthConfFileContent []byte, + keyFilePath string, keyFileContent string, dbTypeFilePath string, instanceType string, + defaultPerm os.FileMode) error { + // 创建Auth配置文件 + runtime.Logger.Info("start to create auth config file") + authConfFile, err := os.OpenFile(authConfFilePath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, defaultPerm) + defer authConfFile.Close() + if err != nil { + runtime.Logger.Error(fmt.Sprintf("create auth config file fail, error:%s", err)) + return fmt.Errorf("create auth config file fail, error:%s", err) + } + if _, err = authConfFile.WriteString(string(authConfFileContent)); err != nil { + runtime.Logger.Error(fmt.Sprintf("auth config file write content fail, error:%s", err)) + return fmt.Errorf("auth config file write content fail, error:%s", err) + } + runtime.Logger.Info("create auth config file successfully") + + // 修改配置文件属主 + runtime.Logger.Info("start to execute chown command for auth config file") + if _, err = util.RunBashCmd( + fmt.Sprintf("chown -R %s.%s %s", user, group, authConfFilePath), + "", nil, + 10*time.Second); err != nil { + runtime.Logger.Error(fmt.Sprintf("chown auth config file fail, error:%s", err)) + return fmt.Errorf("chown auth config file fail, error:%s", err) + } + runtime.Logger.Info("start to execute chown command for auth config file successfully") + + // 创建NoAuth配置文件 + runtime.Logger.Info("start to create no auth config file") + noAuthConfFile, err := os.OpenFile(noAuthConfFilePath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, defaultPerm) + defer noAuthConfFile.Close() + if err != nil { + runtime.Logger.Error(fmt.Sprintf("create no auth config file fail, error:%s", err)) + return fmt.Errorf("create no auth config file fail, error:%s", err) + } + if _, err = noAuthConfFile.WriteString(string(noAuthConfFileContent)); err != nil { + runtime.Logger.Error(fmt.Sprintf("auth no config file write content fail, error:%s", err)) + return fmt.Errorf("auth no config file write content fail, error:%s", err) + } + runtime.Logger.Info("create no auth config file successfully") + + // 修改配置文件属主 + runtime.Logger.Info("start to execute chown command for no auth config file") + if _, err = util.RunBashCmd( + fmt.Sprintf("chown -R %s.%s %s", user, group, noAuthConfFilePath), + "", nil, + 10*time.Second); err != nil { + runtime.Logger.Error(fmt.Sprintf("chown no auth config file fail, error:%s", err)) + return fmt.Errorf("chown no auth config file fail, error:%s", err) + } + runtime.Logger.Info("execute chown command for no auth config file successfully") + + // 创建key文件 + runtime.Logger.Info("start to create key file") + keyFile, err := os.OpenFile(keyFilePath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0600) + defer keyFile.Close() + if err != nil { + runtime.Logger.Error(fmt.Sprintf("create key file fail, error:%s", err)) + return fmt.Errorf("create key file fail, error:%s", err) + } + key := GetMd5(keyFileContent) + if _, err = keyFile.WriteString(key); err != nil { + runtime.Logger.Error(fmt.Sprintf("key file write content fail, error:%s", err)) + return fmt.Errorf("key file write content fail, error:%s", err) + } + runtime.Logger.Info("create key file successfully") + + // 修改key文件属主 + runtime.Logger.Info("start to execute chown command for key file") + if _, err = util.RunBashCmd( + fmt.Sprintf("chown -R %s.%s %s", user, group, keyFilePath), + "", nil, + 10*time.Second); err != nil { + runtime.Logger.Error(fmt.Sprintf("chown key file fail, error:%s", err)) + return fmt.Errorf("chown key file fail, error:%s", err) + } + runtime.Logger.Info("execute chown command for key file successfully") + + // 创建dbType文件 + runtime.Logger.Info("start to create dbType file") + dbTypeFile, err := os.OpenFile(dbTypeFilePath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, defaultPerm) + defer dbTypeFile.Close() + if err != nil { + runtime.Logger.Error(fmt.Sprintf("create dbType file fail, error:%s", err)) + return fmt.Errorf("create dbType file fail, error:%s", err) + } + if _, err = dbTypeFile.WriteString(instanceType); err != nil { + runtime.Logger.Error(fmt.Sprintf("dbType file write content fail, error:%s", err)) + return fmt.Errorf("dbType file write content fail, error:%s", err) + } + runtime.Logger.Info("create dbType file successfully") + + // 修改dbType文件属主 + runtime.Logger.Info("start to execute chown command for dbType file") + if _, err = util.RunBashCmd( + fmt.Sprintf("chown -R %s.%s %s", user, group, dbTypeFilePath), + "", nil, + 10*time.Second); err != nil { + runtime.Logger.Error(fmt.Sprintf("chown dbType file fail, error:%s", err)) + return fmt.Errorf("chown dbType file fail, error:%s", err) + } + runtime.Logger.Info("execute chown command for dbType file successfully") + + return nil + +} + +// StartMongoProcess 启动进程 +func StartMongoProcess(binDir string, port int, user string, auth bool) error { + // 启动服务 + var cmd string + cmd = fmt.Sprintf("su %s -c \"%s %d %s\"", user, + filepath.Join(binDir, "mongodb", "bin", "start_mongo.sh"), + port, "noauth") + if auth == true { + cmd = fmt.Sprintf("su %s -c \"%s %d\"", user, + filepath.Join(binDir, "mongodb", "bin", "start_mongo.sh"), + port) + } + if _, err := util.RunBashCmd( + cmd, + "", nil, + 10*time.Second); err != nil { + return err + } + return nil +} + +// ShutdownMongoProcess 关闭进程 +func ShutdownMongoProcess(user string, instanceType string, binDir string, dbpathDir string, port int) error { + var cmd string + cmd = fmt.Sprintf("su %s -c \"%s --shutdown --dbpath %s\"", + user, filepath.Join(binDir, "mongodb", "bin", "mongod"), dbpathDir) + if instanceType == "mongos" { + cmd = fmt.Sprintf("ps -ef|grep mongos |grep -v grep|grep %d|awk '{print $2}' | xargs kill -2", port) + } + if _, err := util.RunBashCmd( + cmd, + "", nil, + 10*time.Second); err != nil { + return err + } + return nil +} + +// AddPathToProfile 把可执行文件路径写入/etc/profile +func AddPathToProfile(runtime *jobruntime.JobGenericRuntime, binDir string) error { + runtime.Logger.Info("start to add binary path in /etc/profile") + etcProfilePath := "/etc/profile" + addEtcProfile := fmt.Sprintf(` +if ! grep -i %s: %s; +then +echo "export PATH=%s:\$PATH" >> %s +fi`, filepath.Join(binDir, "mongodb", "bin"), etcProfilePath, filepath.Join(binDir, "mongodb", "bin"), etcProfilePath) + runtime.Logger.Info(addEtcProfile) + if _, err := util.RunBashCmd(addEtcProfile, "", nil, 10*time.Second); err != nil { + runtime.Logger.Error(fmt.Sprintf("binary path add in /etc/profile, error:%s", err)) + return fmt.Errorf("binary path add in /etc/profile, error:%s", err) + } + runtime.Logger.Info("add binary path in /etc/profile successfully") + return nil +} + +// AuthGetPrimaryInfo 获取primary节点信息 +func AuthGetPrimaryInfo(mongoBin string, username string, password string, ip string, port int) (string, + error) { + // 超时时间 + timeout := time.After(20 * time.Second) + for { + select { + case <-timeout: + return "", fmt.Errorf("get primary info timeout") + default: + cmd := fmt.Sprintf( + "%s -u %s -p '%s' --host %s --port %d --authenticationDatabase=admin --quiet --eval \"rs.isMaster().primary\"", + mongoBin, username, password, ip, port) + result, err := util.RunBashCmd( + cmd, + "", nil, + 10*time.Second) + if err != nil { + return "", err + } + if strings.Replace(result, "\n", "", -1) == "" { + time.Sleep(1 * time.Second) + continue + } + primaryInfo := strings.Replace(result, "\n", "", -1) + return primaryInfo, nil + } + } +} + +// NoAuthGetPrimaryInfo 获取primary节点信息 +func NoAuthGetPrimaryInfo(mongoBin string, ip string, port int) (string, error) { + // 超时时间 + timeout := time.After(20 * time.Second) + for { + select { + case <-timeout: + return "", fmt.Errorf("get primary info timeout") + default: + cmd := fmt.Sprintf( + "%s --host %s --port %d --quiet --eval \"rs.isMaster().primary\"", + mongoBin, ip, port) + result, err := util.RunBashCmd( + cmd, + "", nil, + 10*time.Second) + if err != nil { + return "", err + } + if strings.Replace(result, "\n", "", -1) == "" { + time.Sleep(1 * time.Second) + continue + } + primaryInfo := strings.Replace(result, "\n", "", -1) + return primaryInfo, nil + } + + } +} + +// InitiateReplicasetGetPrimaryInfo 复制集初始化时判断 +func InitiateReplicasetGetPrimaryInfo(mongoBin string, ip string, port int) (string, error) { + cmd := fmt.Sprintf( + "%s --host %s --port %d --quiet --eval \"rs.isMaster().primary\"", + mongoBin, ip, port) + result, err := util.RunBashCmd( + cmd, + "", nil, + 10*time.Second) + if err != nil { + return "", err + } + primaryInfo := strings.Replace(result, "\n", "", -1) + return primaryInfo, nil +} + +// RemoveFile 删除文件 +func RemoveFile(filePath string) error { + cmd := fmt.Sprintf("rm -rf %s", filePath) + if _, err := util.RunBashCmd( + cmd, + "", nil, + 10*time.Second); err != nil { + return err + } + return nil +} + +// CreateFile 创建文件 +func CreateFile(path string) error { + installLockFile, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY, 0600) + if err != nil { + return err + } + defer installLockFile.Close() + return nil +} + +// AuthCheckUser 检查user是否存在 +func AuthCheckUser(mongoBin string, username string, password string, ip string, port int, authDb string, + checkUsername string) (bool, error) { + cmd := fmt.Sprintf( + "%s -u %s -p '%s' --host %s --port %d --authenticationDatabase=admin --quiet --eval \"db.getMongo().getDB('%s').getUser('%s')\"", + mongoBin, username, password, ip, port, authDb, checkUsername) + result, err := util.RunBashCmd( + cmd, + "", nil, + 10*time.Second) + if err != nil { + return false, fmt.Errorf("get user info fail, error:%s", err) + } + if strings.Contains(result, checkUsername) == true { + return true, nil + } + + return false, nil +} + +// GetNodeInfo 获取mongod节点信息 _id int state int hidden bool priority int +func GetNodeInfo(mongoBin string, ip string, port int, username string, password string, + sourceIP string, sourcePort int) (bool, int, int, bool, int, []map[string]string, error) { + source := strings.Join([]string{sourceIP, strconv.Itoa(sourcePort)}, ":") + cmdStatus := fmt.Sprintf( + "%s -u %s -p '%s' --host %s --port %d --authenticationDatabase=admin --quiet --eval \"rs.status().members\"", + mongoBin, username, password, ip, port) + cmdConf := fmt.Sprintf( + "%s -u %s -p '%s' --host %s --port %d --authenticationDatabase=admin --quiet --eval \"rs.conf().members\"", + mongoBin, username, password, ip, port) + + // 获取状态 + result1, err := util.RunBashCmd( + cmdStatus, + "", nil, + 10*time.Second) + if err != nil { + return false, 0, 0, false, 0, nil, fmt.Errorf("get members status info fail, error:%s", err) + } + result1 = strings.Replace(result1, " ", "", -1) + result1 = strings.Replace(result1, "\n", "", -1) + result1 = strings.Replace(result1, "NumberLong(", "", -1) + result1 = strings.Replace(result1, "Timestamp(", "", -1) + result1 = strings.Replace(result1, "ISODate(", "", -1) + result1 = strings.Replace(result1, ",1)", "", -1) + result1 = strings.Replace(result1, ",3)", "", -1) + result1 = strings.Replace(result1, ",2)", "", -1) + result1 = strings.Replace(result1, ",6)", "", -1) + result1 = strings.Replace(result1, ",0)", "", -1) + result1 = strings.Replace(result1, ")", "", -1) + + // 获取配置 + result2, err := util.RunBashCmd( + cmdConf, + "", nil, + 10*time.Second) + if err != nil { + return false, 0, 0, false, 0, nil, fmt.Errorf("get members conf info fail, error:%s", err) + } + result2 = strings.Replace(result2, " ", "", -1) + result2 = strings.Replace(result2, "\n", "", -1) + result2 = strings.Replace(result2, "NumberLong(", "", -1) + result2 = strings.Replace(result2, "Timestamp(", "", -1) + result2 = strings.Replace(result2, "ISODate(", "", -1) + result2 = strings.Replace(result2, ",1)", "", -1) + result2 = strings.Replace(result2, ")", "", -1) + + var statusSlice []map[string]interface{} + var confSlice []map[string]interface{} + if err = json.Unmarshal([]byte(result1), &statusSlice); err != nil { + return false, 0, 0, false, 0, nil, fmt.Errorf("get members status info json.Unmarshal fail, error:%s", err) + } + if err = json.Unmarshal([]byte(result2), &confSlice); err != nil { + return false, 0, 0, false, 0, nil, fmt.Errorf("get members conf info json.Unmarshal fail, error:%s", err) + } + + // 格式化配置信息 + var memberInfo []map[string]string + for _, v := range statusSlice { + member := make(map[string]string) + member["name"] = v["name"].(string) + member["state"] = fmt.Sprintf("%1.0f", v["state"]) + for _, k := range confSlice { + if k["host"].(string) == member["name"] { + member["hidden"] = strconv.FormatBool(k["hidden"].(bool)) + break + } + } + memberInfo = append(memberInfo, member) + } + + var id int + var state int + var hidden bool + var priority int + flag := false + for _, key := range statusSlice { + if key["name"].(string) == source { + id, _ = strconv.Atoi(fmt.Sprintf("%1.0f", key["_id"])) + state, _ = strconv.Atoi(fmt.Sprintf("%1.0f", key["state"])) + flag = true + break + } + } + for _, key := range confSlice { + if key["host"].(string) == source { + hidden = key["hidden"].(bool) + priority, _ = strconv.Atoi(fmt.Sprintf("%1.0f", key["priority"])) + break + } + } + return flag, id, state, hidden, priority, memberInfo, nil + +} + +// AuthRsStepDown 主备切换 +func AuthRsStepDown(mongoBin string, ip string, port int, username string, password string) (bool, error) { + cmd := fmt.Sprintf( + "%s -u %s -p '%s' --host %s --port %d --authenticationDatabase=admin --quiet --eval \"rs.stepDown()\"", + mongoBin, username, password, ip, port) + _, _ = util.RunBashCmd( + cmd, + "", nil, + 10*time.Second) + time.Sleep(time.Second * 3) + primaryInfo, err := AuthGetPrimaryInfo(mongoBin, username, password, ip, port) + if err != nil { + return false, err + } + if primaryInfo == strings.Join([]string{ip, strconv.Itoa(port)}, ":") { + return false, nil + } + + return true, nil +} + +// NoAuthRsStepDown 主备切换 +func NoAuthRsStepDown(mongoBin string, ip string, port int) (bool, error) { + cmd := fmt.Sprintf( + "%s --host %s --port %d --authenticationDatabase=admin --quiet --eval \"rs.stepDown()\"", + mongoBin, ip, port) + _, _ = util.RunBashCmd( + cmd, + "", nil, + 10*time.Second) + time.Sleep(time.Second * 3) + primaryInfo, err := NoAuthGetPrimaryInfo(mongoBin, ip, port) + if err != nil { + return false, err + } + if primaryInfo == strings.Join([]string{ip, strconv.Itoa(port)}, ":") { + return false, nil + } + return true, nil +} + +// CheckBalancer 检查balancer的值 +func CheckBalancer(mongoBin string, ip string, port int, username string, password string) (string, + error) { + cmd := fmt.Sprintf( + "%s -u %s -p '%s' --host %s --port %d --authenticationDatabase=admin --quiet --eval \"sh.getBalancerState()\"", + mongoBin, username, password, ip, port) + result, err := util.RunBashCmd( + cmd, + "", nil, + 10*time.Second) + if err != nil { + return "", err + } + result = strings.Replace(result, "\n", "", -1) + return result, nil +} + +// GetProfilingLevel 获取profile级别 +func GetProfilingLevel(mongoBin string, ip string, port int, username string, password string, + dbName string) (int, error) { + cmd := fmt.Sprintf( + "%s -u %s -p '%s' --host %s --port %d --authenticationDatabase=admin --quiet --eval \"db.getMongo().getDB('%s').getProfilingLevel()\"", + mongoBin, username, password, ip, port, dbName) + result, err := util.RunBashCmd( + cmd, + "", nil, + 10*time.Second) + if err != nil { + return -1, err + } + intResult, _ := strconv.Atoi(result) + return intResult, nil +} + +// SetProfilingLevel 设置profile级别 +func SetProfilingLevel(mongoBin string, ip string, port int, username string, password string, + dbName string, level int) error { + cmd := fmt.Sprintf( + "%s -u %s -p '%s' --host %s --port %d --authenticationDatabase=admin --quiet --eval \"db.getMongo().getDB('%s').setProfilingLevel(%d)\"", + mongoBin, username, password, ip, port, dbName, level) + _, err := util.RunBashCmd( + cmd, + "", nil, + 10*time.Second) + if err != nil { + return err + } + return nil +} diff --git a/dbm-services/mongo/db-tools/dbactuator/pkg/common/mongo_init_shell.go b/dbm-services/mongo/db-tools/dbactuator/pkg/common/mongo_init_shell.go new file mode 100644 index 0000000000..6ed75a98f5 --- /dev/null +++ b/dbm-services/mongo/db-tools/dbactuator/pkg/common/mongo_init_shell.go @@ -0,0 +1,165 @@ +package common + +// MongoShellInit 初始化os的shell脚本 +var MongoShellInit = `#!/bin/sh +# 新建用户 + +function _exit() { + rm $0 + exit +} +#handler nscd restart 默认使用mysql用户 +#如果存在mysql用户组就groupadd mysql -g 202 +egrep "^{{group}}" /etc/group >& /dev/null +if [ $? -ne 0 ] +then +groupadd {{group}} -g 2000 +fi +#考虑到可能上架已运行的机器，userdel有风险，不采用这种方法 +#如果存在user用户就删掉（因为有可能1）id不为30019，2）不存在home目录） +id {{user}} >& /dev/null +if [ $? -ne 0 ] +then + useradd -m -d /home/{{user}} -g 2000 -G users -u 2000 {{user}} + chage -M 99999 {{user}} + if [ ! -d /home/{{user}} ]; + then + mkdir -p /home/{{user}} + fi + chmod 755 /home/{{user}} + usermod -d /home/{{user}} {{user}} 2>/dev/null +fi +if [[ -z "$MONGO_DATA_DIR" ]] +then + echo "env MONGO_DATA_DIR cannot be empty" >&2 + exit -1 +fi +if [[ -z "$MONGO_BACKUP_DIR" ]] +then + echo "env MONGO_BACKUP_DIR cannot be empty" >&2 + exit -1 +fi + +if [ ! -d $MONGO_DATA_DIR ] +then + mkdir -p $MONGO_DATA_DIR +fi + +if [ ! -d $MONGO_BACKUP_DIR ] +then + mkdir -p $RMONGO_BACKUP_DIR +fi + +#添加mongo安装锁文件 +if [ ! -f $MONGO_DATA_DIR/mongoinstall.lock ] +then + touch $MONGO_DATA_DIR/mongoinstall.lock +fi + +#如果存在mysql用户,上面那一步会报错，也不会创建/home/mysql，所以判断下并创建/home/mysql +if [ ! -d /data ]; +then + ln -s $MONGO_BACKUP_DIR /data +fi +if [ ! -d /data1 ]; +then + ln -s $MONGO_DATA_DIR /data1 +fi +if [[ ! -d /data1/dbha ]] +then + mkdir -p /data1/dbha +fi +chown -R {{user}} /data1/dbha +if [[ ! -d /data/dbha ]] +then + mkdir -p /data/dbha +fi +chown -R {{user}} /data/dbha +if [[ ! -d /data/install ]] +then + mkdir -p /data/install + chown -R {{user}} /data/install +fi +if [[ ! -d $MONGO_BACKUP_DIR/dbbak ]] +then + mkdir -p $MONGO_BACKUP_DIR/dbbak + chown -R {{user}} $MONGO_BACKUP_DIR/dbbak +fi +chown -R {{user}} /home/{{user}} +chmod -R a+rwx /data/install +rm -rf /home/{{user}}/install +ln -s /data/install /home/{{user}}/install +chown -R {{user}} /home/{{user}}/install +#password="$2" +#password=$(echo "$2" | /home/mysql/install/lib/tools/base64 -d) +#echo "mysql:$password" | chpasswd +FOUND=$(grep 'ulimit -n 204800' /etc/profile) +if [ -z "$FOUND" ]; then + echo 'ulimit -n 204800' >> /etc/profile +fi +FOUND=$(grep 'export LC_ALL=en_US' /etc/profile) +if [ -z "$FOUND" ]; then + echo 'export LC_ALL=en_US' >> /etc/profile +fi +#FOUND=$(grep 'export PATH=/usr/local/mongodb/bin/:$PATH' /etc/profile) +#if [ -z "$FOUND" ]; then +# echo 'export PATH=/usr/local/mongodb/bin/:$PATH' >> /etc/profile +#fi +FOUND_umask=$(grep '^umask 022' /etc/profile) +if [ -z "$FOUND_umask" ]; then + echo 'umask 022' >> /etc/profile +fi +FOUND=$(grep 'vm.swappiness = 0' /etc/sysctl.conf) +if [ -z "$FOUND" ];then +echo "vm.swappiness = 0" >> /etc/sysctl.conf +fi +FOUND=$(grep 'kernel.pid_max = 200000' /etc/sysctl.conf) +if [ -z "$FOUND" ];then +echo "kernel.pid_max = 200000" >> /etc/sysctl.conf +fi + +FOUND=$(grep '{{user}} soft nproc 64000' /etc/security/limits.conf) +if [ -z "$FOUND" ];then +echo "{{user}} soft nproc 64000" >> /etc/security/limits.conf +fi +FOUND=$(grep '{{user}} hard nproc 64000' /etc/security/limits.conf) +if [ -z "$FOUND" ];then +echo "{{user}} hard nproc 64000" >> /etc/security/limits.conf +fi +FOUND=$(grep '{{user}} soft fsize unlimited' /etc/security/limits.conf) +if [ -z "$FOUND" ];then +echo "{{user}} soft fsize unlimited" >> /etc/security/limits.conf +fi +FOUND=$(grep '{{user}} hard fsize unlimited' /etc/security/limits.conf) +if [ -z "$FOUND" ];then +echo "{{user}} hard fsize unlimited" >> /etc/security/limits.conf +fi +FOUND=$(grep '{{user}} soft memlock unlimited' /etc/security/limits.conf) +if [ -z "$FOUND" ];then +echo "{{user}} soft memlock unlimited" >> /etc/security/limits.conf +fi +FOUND=$(grep '{{user}} hard memlock unlimited' /etc/security/limits.conf) +if [ -z "$FOUND" ];then +echo "{{user}} hard memlock unlimited" >> /etc/security/limits.conf +fi +FOUND=$(grep '{{user}} soft as unlimited' /etc/security/limits.conf) +if [ -z "$FOUND" ];then +echo "{{user}} soft as unlimited" >> /etc/security/limits.conf +fi +FOUND=$(grep '{{user}} hard as unlimited' /etc/security/limits.conf) +if [ -z "$FOUND" ];then +echo "{{user}} hard as unlimited" >> /etc/security/limits.conf +fi + +FOUND=$(grep 'session required pam_limits.so' /etc/pam.d/login) +if [ -z "$FOUND" ];then +echo "session required pam_limits.so" >> /etc/pam.d/login +fi + +FOUND=$(grep 'session required pam_limits.so' /etc/pam.d/su) +if [ -z "$FOUND" ];then +echo "session required pam_limits.so" >> /etc/pam.d/su +fi + +/sbin/sysctl -p +_exit` diff --git a/dbm-services/mongo/db-tools/dbactuator/pkg/common/mongo_user_conf.go b/dbm-services/mongo/db-tools/dbactuator/pkg/common/mongo_user_conf.go new file mode 100644 index 0000000000..add72f1a02 --- /dev/null +++ b/dbm-services/mongo/db-tools/dbactuator/pkg/common/mongo_user_conf.go @@ -0,0 +1,35 @@ +package common + +import "encoding/json" + +// MongoRole 角色 +type MongoRole struct { + Role string `json:"role"` + Db string `json:"db"` +} + +// MongoUser 用户 +type MongoUser struct { + User string `json:"user"` + Pwd string `json:"pwd"` + Roles []*MongoRole `json:"roles"` +} + +// NewMongoUser 生成结构体 +func NewMongoUser() *MongoUser { + return &MongoUser{} +} + +// GetContent 转成json +func (m *MongoUser) GetContent() (string, error) { + content, err := json.Marshal(m) + if err != nil { + return "", err + } + return string(content), nil +} + +// NewMongoRole 生成结构体 +func NewMongoRole() *MongoRole { + return &MongoRole{} +} diff --git a/dbm-services/mongo/db-tools/dbactuator/pkg/common/mongod_conf.go b/dbm-services/mongo/db-tools/dbactuator/pkg/common/mongod_conf.go new file mode 100644 index 0000000000..7cc3629df2 --- /dev/null +++ b/dbm-services/mongo/db-tools/dbactuator/pkg/common/mongod_conf.go @@ -0,0 +1,87 @@ +package common + +import ( + "gopkg.in/yaml.v2" +) + +// YamlMongoDBConf 3.0及以上配置文件 +type YamlMongoDBConf struct { + Storage struct { + DbPath string `yaml:"dbPath"` + Engine string `yaml:"engine"` + WiredTiger struct { + EngineConfig struct { + CacheSizeGB int `yaml:"cacheSizeGB"` + } `yaml:"engineConfig"` + } `yaml:"wiredTiger"` + } `yaml:"storage"` + Replication struct { + OplogSizeMB int `yaml:"oplogSizeMB"` + ReplSetName string `yaml:"replSetName"` + } `yaml:"replication"` + SystemLog struct { + LogAppend bool `yaml:"logAppend"` + Path string `yaml:"path"` + Destination string `yaml:"destination"` + } `yaml:"systemLog"` + ProcessManagement struct { + Fork bool `yaml:"fork"` + PidFilePath string `yaml:"pidFilePath"` + } `yaml:"processManagement"` + Net struct { + Port int `yaml:"port"` + BindIp string `yaml:"bindIp"` + WireObjectCheck bool `yaml:"wireObjectCheck"` + } `yaml:"net"` + OperationProfiling struct { + SlowOpThresholdMs int `yaml:"slowOpThresholdMs"` + } `yaml:"operationProfiling"` + Sharding struct { + ClusterRole string `yaml:"clusterRole,omitempty"` + } `yaml:"sharding,omitempty"` + Security struct { + KeyFile string `yaml:"keyFile,omitempty"` + } `yaml:"security,omitempty"` +} + +// NewYamlMongoDBConf 生成结构体 +func NewYamlMongoDBConf() *YamlMongoDBConf { + return &YamlMongoDBConf{} +} + +// GetConfContent 获取配置文件内容 +func (y *YamlMongoDBConf) GetConfContent() ([]byte, error) { + out, err := yaml.Marshal(y) + if err != nil { + return nil, err + } + return out, nil +} + +// IniNoAuthMongoDBConf 3.0以下配置文件 +var IniNoAuthMongoDBConf = `replSet={{replSet}} +dbpath={{dbpath}} +logpath={{logpath}} +pidfilepath={{pidfilepath}} +logappend=true +port={{port}} +bind_ip={{bind_ip}} +fork=true +nssize=16 +oplogSize={{oplogSize}} +{{instanceRole}} = true` + +// IniAuthMongoDBConf 3.0以下配置文件 +var IniAuthMongoDBConf = `replSet={{replSet}} +dbpath={{dbpath}} +logpath={{logpath}} +pidfilepath={{pidfilepath}} +logappend=true +port={{port}} +bind_ip={{bind_ip}} +keyFile={{keyFile}} +fork=true +nssize=16 +oplogSize={{oplogSize}} +{{instanceRole}} = true +` diff --git a/dbm-services/mongo/db-tools/dbactuator/pkg/common/mongos_conf.go b/dbm-services/mongo/db-tools/dbactuator/pkg/common/mongos_conf.go new file mode 100644 index 0000000000..66f791a1ca --- /dev/null +++ b/dbm-services/mongo/db-tools/dbactuator/pkg/common/mongos_conf.go @@ -0,0 +1,46 @@ +package common + +import ( + "gopkg.in/yaml.v2" +) + +// YamlMongoSConf 4.0及以上配置文件 +type YamlMongoSConf struct { + Sharding struct { + ConfigDB string `yaml:"configDB"` + } `yaml:"sharding"` + SystemLog struct { + LogAppend bool `yaml:"logAppend"` + Path string `yaml:"path"` + Destination string `yaml:"destination"` + } `yaml:"systemLog"` + ProcessManagement struct { + Fork bool `yaml:"fork"` + PidFilePath string `yaml:"pidFilePath"` + } `yaml:"processManagement"` + Net struct { + Port int `yaml:"port"` + BindIp string `yaml:"bindIp"` + WireObjectCheck bool `yaml:"wireObjectCheck"` + } `yaml:"net"` + OperationProfiling struct { + SlowOpThresholdMs int `yaml:"slowOpThresholdMs,omitempty"` + } `yaml:"operationProfiling,omitempty"` + Security struct { + KeyFile string `yaml:"keyFile,omitempty"` + } `yaml:"security,omitempty"` +} + +// NewYamlMongoSConf 生成结构体 +func NewYamlMongoSConf() *YamlMongoSConf { + return &YamlMongoSConf{} +} + +// GetConfContent 获取配置文件内容 +func (y *YamlMongoSConf) GetConfContent() ([]byte, error) { + out, err := yaml.Marshal(y) + if err != nil { + return nil, err + } + return out, nil +} diff --git a/dbm-services/mongo/db-tools/dbactuator/pkg/common/repliccaset_member_conf.go b/dbm-services/mongo/db-tools/dbactuator/pkg/common/repliccaset_member_conf.go new file mode 100644 index 0000000000..731c652038 --- /dev/null +++ b/dbm-services/mongo/db-tools/dbactuator/pkg/common/repliccaset_member_conf.go @@ -0,0 +1,24 @@ +package common + +import "encoding/json" + +// ReplicasetMemberAdd 复制集状态 +type ReplicasetMemberAdd struct { + Host string `json:"host"` // ip:port + Hidden bool `json:"hidden"` + Priority int `json:"priority"` +} + +// NewReplicasetMemberAdd 生成结构体 +func NewReplicasetMemberAdd() *ReplicasetMemberAdd { + return &ReplicasetMemberAdd{} +} + +// GetJson 获取json格式 +func (t *ReplicasetMemberAdd) GetJson() (string, error) { + byteInfo, err := json.Marshal(t) + if err != nil { + return "", err + } + return string(byteInfo), nil +} diff --git a/dbm-services/mongo/db-tools/dbactuator/pkg/consts/consts.go b/dbm-services/mongo/db-tools/dbactuator/pkg/consts/consts.go new file mode 100644 index 0000000000..8f1e4f0b98 --- /dev/null +++ b/dbm-services/mongo/db-tools/dbactuator/pkg/consts/consts.go @@ -0,0 +1,277 @@ +// Package consts 常量 +package consts + +const ( + // TendisTypePredixyRedisCluster predixy + RedisCluster架构 + TendisTypePredixyRedisCluster = "PredixyRedisCluster" + // TendisTypePredixyTendisplusCluster predixy + TendisplusCluster架构 + TendisTypePredixyTendisplusCluster = "PredixyTendisplusCluster" + // TendisTypeTwemproxyRedisInstance twemproxy + RedisInstance架构 + TendisTypeTwemproxyRedisInstance = "TwemproxyRedisInstance" + // TendisTypeTwemproxyTendisplusInstance twemproxy+ TendisplusInstance架构 + TendisTypeTwemproxyTendisplusInstance = "TwemproxyTendisplusInstance" + // TendisTypeTwemproxyTendisSSDInstance twemproxy+ TendisSSDInstance架构 + TendisTypeTwemproxyTendisSSDInstance = "TwemproxyTendisSSDInstance" + // TendisTypeRedisInstance RedisCache 主从版 + TendisTypeRedisInstance = "RedisInstance" + // TendisTypeTendisplusInsance Tendisplus 主从版 + TendisTypeTendisplusInsance = "TendisplusInstance" + // TendisTypeTendisSSDInsance TendisSSD 主从版 + TendisTypeTendisSSDInsance = "TendisSSDInstance" + // TendisTypeRedisCluster 原生RedisCluster 架构 + TendisTypeRedisCluster = "RedisCluster" + // TendisTypeTendisplusCluster TendisplusCluster架构 + TendisTypeTendisplusCluster = "TendisplusCluster" +) + +// kibis of bits +const ( + Byte = 1 << (iota * 10) + KiByte + MiByte + GiByte + TiByte + EiByte +) + +const ( + // RedisMasterRole redis role master + RedisMasterRole = "master" + // RedisSlaveRole redis role slave + RedisSlaveRole = "slave" + + // RedisNoneRole none role + RedisNoneRole = "none" + + // MasterLinkStatusUP up status + MasterLinkStatusUP = "up" + // MasterLinkStatusDown down status + MasterLinkStatusDown = "down" + + // TendisSSDIncrSyncState IncrSync state + TendisSSDIncrSyncState = "IncrSync" + // TendisSSDReplFollowtate REPL_FOLLOW state + TendisSSDReplFollowtate = "REPL_FOLLOW" +) + +const ( + // RedisLinkStateConnected redis connection status connected + RedisLinkStateConnected = "connected" + // RedisLinkStateDisconnected redis connection status disconnected + RedisLinkStateDisconnected = "disconnected" +) + +const ( + // NodeStatusPFail Node is in PFAIL state. Not reachable for the node you are contacting, but still logically reachable + NodeStatusPFail = "fail?" + // NodeStatusFail Node is in FAIL state. It was not reachable for multiple nodes that promoted the PFAIL state to FAIL + NodeStatusFail = "fail" + // NodeStatusHandshake Untrusted node, we are handshaking. + NodeStatusHandshake = "handshake" + // NodeStatusNoAddr No address known for this node + NodeStatusNoAddr = "noaddr" + // NodeStatusNoFlags no flags at all + NodeStatusNoFlags = "noflags" +) + +const ( + // ClusterStateOK command 'cluster info',cluster_state + ClusterStateOK = "ok" + // ClusterStateFail command 'cluster info',cluster_state + ClusterStateFail = "fail" +) +const ( + // DefaultMinSlots 0 + DefaultMinSlots = 0 + // DefaultMaxSlots 16383 + DefaultMaxSlots = 16383 + + // TwemproxyMaxSegment twemproxy max segment + TwemproxyMaxSegment = 419999 + // TotalSlots 集群总槽数 + TotalSlots = 16384 +) + +// time layout +const ( + UnixtimeLayout = "2006-01-02 15:04:05" + FilenameTimeLayout = "20060102-150405" + FilenameDayLayout = "20060102" +) + +// account +const ( + MysqlAaccount = "mysql" + MysqlGroup = "mysql" + OSAccount = "mysql" + OSGroup = "mysql" +) + +// path dirs +const ( + UsrLocal = "/usr/local" + PackageSavePath = "/data/install" + Data1Path = "/data1" + DataPath = "/data" + DbaReportSaveDir = "/home/mysql/dbareport/" + RedisReportSaveDir = "/home/mysql/dbareport/redis/" + ExporterConfDir = "/home/mysql/.exporter" + RedisReportLeftDay = 15 +) + +// tool path +const ( + DbToolsPath = "/home/mysql/dbtools" + RedisShakeBin = "/home/mysql/dbtools/redis-shake" + RedisSafeDeleteToolBin = "/home/mysql/dbtools/redisSafeDeleteTool" + LdbTendisplusBin = "/home/mysql/dbtools/ldb_tendisplus" + TredisverifyBin = "/home/mysql/dbtools/tredisverify" + TredisBinlogBin = "/home/mysql/dbtools/tredisbinlog" + TredisDumpBin = "/home/mysql/dbtools/tredisdump" + NetCatBin = "/home/mysql/dbtools/netcat" + TendisKeyLifecycleBin = "/home/mysql/dbtools/tendis-key-lifecycle" + ZkWatchBin = "/home/mysql/dbtools/zkwatch" + ZstdBin = "/home/mysql/dbtools/zstd" + LzopBin = "/home/mysql/dbtools/lzop" + LdbWithV38Bin = "/home/mysql/dbtools/ldb_with_len.3.8" + LdbWithV513Bin = "/home/mysql/dbtools/ldb_with_len.5.13" + MyRedisCaptureBin = "/home/mysql/dbtools/myRedisCapture" + BinlogToolTendisplusBin = "/home/mysql/dbtools/binlogtool_tendisplus" + RedisCliBin = "/home/mysql/dbtools/redis-cli" + TendisDataCheckBin = "/home/mysql/dbtools/tendisDataCheck" + RedisDiffKeysRepairerBin = "/home/mysql/dbtools/redisDiffKeysRepairer" +) + +// bk-dbmon path +const ( + BkDbmonPath = "/home/mysql/bk-dbmon" + BkDbmonBin = "/home/mysql/bk-dbmon/bk-dbmon" + BkDbmonConfFile = "/home/mysql/bk-dbmon/dbmon-config.yaml" + BkDbmonPort = 6677 + BkDbmonHTTPAddress = "127.0.0.1:6677" +) + +// backup +const ( + NormalBackupType = "normal_backup" + ForeverBackupType = "forever_backup" + BackupClient = "/usr/local/bin/backup_client" + BackupTarSplitSize = "8G" + RedisFullBackupTAG = "REDIS_FULL" + RedisBinlogTAG = "REDIS_BINLOG" + RedisForeverBackupTAG = "DBFILE" + RedisFullBackupReportType = "redis_fullbackup" + RedisBinlogBackupReportType = "redis_binlogbackup" + DoingRedisFullBackFileList = "redis_backup_file_list_%d_doing" + DoneRedisFullBackFileList = "redis_backup_file_list_%d_done" + DoingRedisBinlogFileList = "redis_binlog_file_list_%d_doing" + DoneRedisBinlogFileList = "redis_binlog_file_list_%d_done" + RedisFullbackupRepoter = "redis_fullbackup_%s.log" + RedisBinlogRepoter = "redis_binlog_%s.log" + BackupStatusStart = "start" + BackupStatusRunning = "running" + BackupStatusToBakSystemStart = "to_backup_system_start" + BackupStatusToBakSystemFailed = "to_backup_system_failed" + BackupStatusToBakSysSuccess = "to_backup_system_success" + BackupStatusFailed = "failed" + BackupStatusLocalSuccess = "local_success" +) + +// meta role +const ( + MetaRoleRedisMaster = "redis_master" + MetaRoleRedisSlave = "redis_slave" +) + +// proxy operations +const ( + ProxyStart = "proxy_open" + ProxyStop = "proxy_close" + ProxyRestart = "proxy_restart" + ProxyShutdown = "proxy_shutdown" +) + +const ( + // FlushDBRename .. + FlushDBRename = "cleandb" + // CacheFlushAllRename .. + CacheFlushAllRename = "cleanall" + // SSDFlushAllRename .. + SSDFlushAllRename = "flushalldisk" + // KeysRename .. + KeysRename = "mykeys" + // ConfigRename .. + ConfigRename = "confxx" +) + +// IsClusterDbType 存储端是否是cluster类型 +func IsClusterDbType(dbType string) bool { + if dbType == TendisTypePredixyRedisCluster || + dbType == TendisTypePredixyTendisplusCluster || + dbType == TendisTypeRedisCluster || + dbType == TendisTypeTendisplusCluster { + return true + } + return false +} + +// IsRedisInstanceDbType 存储端是否是cache类型 +func IsRedisInstanceDbType(dbType string) bool { + if dbType == TendisTypePredixyRedisCluster || + dbType == TendisTypeTwemproxyRedisInstance || + dbType == TendisTypeRedisInstance || + dbType == TendisTypeRedisCluster { + return true + } + return false +} + +// IsTwemproxyClusterType 检查proxy是否为Twemproxy +func IsTwemproxyClusterType(dbType string) bool { + if dbType == TendisTypeTwemproxyRedisInstance || + dbType == TendisTypeTwemproxyTendisSSDInstance || + dbType == TendisTypeTwemproxyTendisplusInstance { + return true + } + return false +} + +// IsTendisplusInstanceDbType 存储端是否是tendisplus类型 +func IsTendisplusInstanceDbType(dbType string) bool { + if dbType == TendisTypePredixyTendisplusCluster || + dbType == TendisTypeTwemproxyTendisplusInstance || + dbType == TendisTypeTendisplusInsance || + dbType == TendisTypeTendisplusCluster { + return true + } + return false +} + +// IsTendisSSDInstanceDbType 存储端是否是tendisSSD类型 +func IsTendisSSDInstanceDbType(dbType string) bool { + if dbType == TendisTypeTwemproxyTendisSSDInstance || + dbType == TendisTypeTendisSSDInsance { + return true + } + return false +} + +// IsAllowFlushMoreDB 是否支持flush 多DB +func IsAllowFlushMoreDB(dbType string) bool { + if dbType == TendisTypeRedisInstance || + dbType == TendisTypeTendisplusInsance { + return true + } + return false +} + +// IsAllowRandomkey 是否支持randomkey命令 +func IsAllowRandomkey(dbType string) bool { + if dbType == TendisTypePredixyTendisplusCluster || + dbType == TendisTypeTwemproxyTendisplusInstance || + dbType == TendisTypeTendisplusInsance || + dbType == TendisTypeTendisplusCluster { + return false + } + return true +} diff --git a/dbm-services/mongo/db-tools/dbactuator/pkg/consts/data_dir.go b/dbm-services/mongo/db-tools/dbactuator/pkg/consts/data_dir.go new file mode 100644 index 0000000000..c8c7ec6c0f --- /dev/null +++ b/dbm-services/mongo/db-tools/dbactuator/pkg/consts/data_dir.go @@ -0,0 +1,325 @@ +package consts + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "syscall" +) + +// fileExists 检查目录是否已经存在 +func fileExists(path string) bool { + _, err := os.Stat(path) + if err != nil { + return os.IsExist(err) + } + return true +} + +// IsMountPoint2 Determine if a directory is a mountpoint, by comparing the device for the directory +// with the device for it's parent. If they are the same, it's not a mountpoint, if they're +// different, it is. +// reference: https://github.com/cnaize/kubernetes/blob/master/pkg/util/mount/mountpoint_unix.go#L29 +// 该函数与util/util.go 中 IsMountPoint()相同,但package consts 不建议依赖其他模块故拷贝了实现 +func IsMountPoint2(file string) bool { + stat, err := os.Stat(file) + if err != nil { + return false + } + rootStat, err := os.Lstat(file + "/..") + if err != nil { + return false + } + // If the directory has the same device as parent, then it's not a mountpoint. + return stat.Sys().(*syscall.Stat_t).Dev != rootStat.Sys().(*syscall.Stat_t).Dev +} + +// SetRedisDataDir 设置环境变量 REDIS_DATA_DIR,并持久化到/etc/profile中 +// 如果函数参数 dataDir 不为空,则 REDIS_DATA_DIR = {dataDir} +// 否则,如果环境变量 REDIS_DATA_DIR 不为空,则直接读取; +// 否则,如果 /data1/redis 存在, 则 REDIS_DATA_DIR=/data1 +// 否则,如果 /data/redis, 则 REDIS_DATA_DIR=/data +// 否则,如果 /data1 是挂载点, 则 REDIS_DATA_DIR=/data1 +// 否则,如果 /data 是挂载点, 则 REDIS_DATA_DIR=/data +// 否则,REDIS_DATA_DIR=/data1 +func SetRedisDataDir(dataDir string) (err error) { + if dataDir == "" { + envDir := os.Getenv("REDIS_DATA_DIR") + if envDir != "" { // 环境变量 REDIS_DATA_DIR 不为空 + dataDir = envDir + } else { + if fileExists(filepath.Join(Data1Path, "redis")) { + // /data1/redis 存在 + dataDir = Data1Path + } else if fileExists(filepath.Join(DataPath, "redis")) { + // /data/redis 存在 + dataDir = DataPath + } else if IsMountPoint2(Data1Path) { + // /data1是挂载点 + dataDir = Data1Path + } else if IsMountPoint2(DataPath) { + // /data是挂载点 + dataDir = DataPath + } else { + // 函数参数 dataDir为空, 环境变量 REDIS_DATA_DIR 为空 + // /data1 和 /data 均不是挂载点 + // 强制指定 REDIS_DATA_DIR=/data1 + dataDir = Data1Path + } + } + } + dataDir = strings.TrimSpace(dataDir) + var ret []byte + shCmd := fmt.Sprintf(` +ret=$(grep '^export REDIS_DATA_DIR=' /etc/profile) +if [[ -z $ret ]] +then +echo "export REDIS_DATA_DIR=%s">>/etc/profile +fi + `, dataDir) + ret, err = exec.Command("bash", "-c", shCmd).Output() + if err != nil { + err = fmt.Errorf("SetRedisDataDir failed,err:%v,ret:%s,shCmd:%s", err, string(ret), shCmd) + return + } + os.Setenv("REDIS_DATA_DIR", dataDir) + return nil +} + +// GetRedisDataDir 获取环境变量 REDIS_DATA_DIR,不为空直接返回, +// 否则,如果目录 /data1/redis存在,返回 /data1; +// 否则,如果目录 /data/redis存在,返回 /data; +// 否则,返回 /data1 +func GetRedisDataDir() string { + dataDir := os.Getenv("REDIS_DATA_DIR") + if dataDir == "" { + if fileExists(filepath.Join(Data1Path, "redis")) { + // /data1/redis 存在 + dataDir = Data1Path + } else if fileExists(filepath.Join(DataPath, "redis")) { + // /data/redis 存在 + dataDir = DataPath + } else { + dataDir = Data1Path + } + } + return dataDir +} + +// SetRedisBakcupDir 设置环境变量 REDIS_BACKUP_DIR ,并持久化到/etc/profile中 +// 如果函数参数 backupDir 不为空,则 REDIS_BACKUP_DIR = {backupDir} +// 否则,如果环境变量 REDIS_BACKUP_DIR 不为空,则直接读取; +// 否则,如果 /data/dbbak 存在, 则 REDIS_BACKUP_DIR=/data +// 否则,如果 /data1/dbbak 存在, 则 REDIS_BACKUP_DIR=/data1 +// 否则,如果 /data 是挂载点, 则 REDIS_BACKUP_DIR=/data +// 否则,如果 /data1 是挂载点, 则 REDIS_BACKUP_DIR=/data1 +// 否则,REDIS_BACKUP_DIR=/data +func SetRedisBakcupDir(backupDir string) (err error) { + if backupDir == "" { + envDir := os.Getenv("REDIS_BACKUP_DIR") + if envDir != "" { + backupDir = envDir + } else { + if fileExists(filepath.Join(DataPath, "dbbak")) { + // /data/dbbak 存在 + backupDir = DataPath + } else if fileExists(filepath.Join(Data1Path, "dbbak")) { + // /data1/dbbak 存在 + backupDir = Data1Path + } else if IsMountPoint2(DataPath) { + // /data是挂载点 + backupDir = DataPath + } else if IsMountPoint2(Data1Path) { + // /data1是挂载点 + backupDir = Data1Path + } else { + // 函数参数 backupDir 为空, 环境变量 REDIS_BACKUP_DIR 为空 + // /data1 和 /data 均不是挂载点 + // 强制指定 REDIS_BACKUP_DIR=/data + backupDir = DataPath + } + } + } + backupDir = strings.TrimSpace(backupDir) + var ret []byte + shCmd := fmt.Sprintf(` +ret=$(grep '^export REDIS_BACKUP_DIR=' /etc/profile) +if [[ -z $ret ]] +then +echo "export REDIS_BACKUP_DIR=%s">>/etc/profile +fi + `, backupDir) + ret, err = exec.Command("bash", "-c", shCmd).Output() + if err != nil { + err = fmt.Errorf("SetRedisBakcupDir failed,err:%v,ret:%s", err, string(ret)) + return + } + os.Setenv("REDIS_BACKUP_DIR", backupDir) + return nil +} + +// GetRedisBackupDir 获取环境变量 REDIS_BACKUP_DIR,默认值 /data +// 否则,如果目录 /data/dbbak 存在,返回 /data; +// 否则,如果目录 /data1/dbbak 存在,返回 /data1; +// 否则,返回 /data +func GetRedisBackupDir() string { + dataDir := os.Getenv("REDIS_BACKUP_DIR") + if dataDir == "" { + if fileExists(filepath.Join(DataPath, "dbbak")) { + // /data/dbbak 存在 + dataDir = DataPath + } else if fileExists(filepath.Join(Data1Path, "dbbak")) { + // /data1/dbbak 存在 + dataDir = Data1Path + } else { + dataDir = DataPath + } + } + return dataDir +} + +// SetMongoDataDir 设置环境变量 MONGO_DATA_DIR,并持久化到/etc/profile中 +// 如果函数参数 dataDir 不为空,则 MONGO_DATA_DIR = {dataDir} +// 否则,如果环境变量 MONGO_DATA_DIR 不为空,则直接读取; +// 否则,如果 /data1/redis 存在, 则 MONGO_DATA_DIR=/data1 +// 否则,如果 /data/redis, 则 MONGO_DATA_DIR=/data +// 否则,如果 /data1 是挂载点, 则 MONGO_DATA_DIR=/data1 +// 否则,如果 /data 是挂载点, 则 MONGO_DATA_DIR=/data +// 否则,MONGO_DATA_DIR=/data1 +func SetMongoDataDir(dataDir string) (err error) { + if dataDir == "" { + envDir := os.Getenv("MONGO_DATA_DIR") + if envDir != "" { // 环境变量 MONGO_DATA_DIR 不为空 + dataDir = envDir + } else { + if fileExists(filepath.Join(Data1Path, "mongodata")) { + // /data1/mongodata 存在 + dataDir = Data1Path + } else if fileExists(filepath.Join(DataPath, "mongodata")) { + // /data/mongodata 存在 + dataDir = DataPath + } else if IsMountPoint2(Data1Path) { + // /data1是挂载点 + dataDir = Data1Path + } else if IsMountPoint2(DataPath) { + // /data是挂载点 + dataDir = DataPath + } else { + // 函数参数 dataDir为空, 环境变量 MONGO_DATA_DIR 为空 + // /data1 和 /data 均不是挂载点 + // 强制指定 MONGO_DATA_DIR=/data1 + dataDir = Data1Path + } + } + } + dataDir = strings.TrimSpace(dataDir) + var ret []byte + shCmd := fmt.Sprintf(` +ret=$(grep '^export MONGO_DATA_DIR=' /etc/profile) +if [[ -z $ret ]] +then +echo "export MONGO_DATA_DIR=%s">>/etc/profile +fi + `, dataDir) + ret, err = exec.Command("bash", "-c", shCmd).Output() + if err != nil { + err = fmt.Errorf("SetMongoDataDir failed,err:%v,ret:%s,shCmd:%s", err, string(ret), shCmd) + return + } + os.Setenv("MONGO_DATA_DIR", dataDir) + return nil +} + +// GetMongoDataDir 获取环境变量 MONGO_DATA_DIR,不为空直接返回, +// 否则,如果目录 /data1/mongodata存在,返回 /data1; +// 否则,如果目录 /data/mongodata存在,返回 /data; +// 否则,返回 /data1 +func GetMongoDataDir() string { + dataDir := os.Getenv("MONGO_DATA_DIR") + if dataDir == "" { + if fileExists(filepath.Join(Data1Path, "mongodata")) { + // /data1/mongodata 存在 + dataDir = Data1Path + } else if fileExists(filepath.Join(DataPath, "mongodata")) { + // /data/mongodata 存在 + dataDir = DataPath + } else { + dataDir = Data1Path + } + } + return dataDir +} + +// SetMongoBackupDir 设置环境变量 MONGO_BACKUP_DIR ,并持久化到/etc/profile中 +// 如果函数参数 backupDir 不为空,则 MONGO_BACKUP_DIR = {backupDir} +// 否则,如果环境变量 MONGO_BACKUP_DIR 不为空,则直接读取; +// 否则,如果 /data/dbbak 存在, 则 MONGO_BACKUP_DIR=/data +// 否则,如果 /data1/dbbak 存在, 则 MONGO_BACKUP_DIR=/data1 +// 否则,如果 /data 是挂载点, 则 MONGO_BACKUP_DIR=/data +// 否则,如果 /data1 是挂载点, 则 MONGO_BACKUP_DIR=/data1 +// 否则,MONGO_BACKUP_DIR=/data +func SetMongoBackupDir(backupDir string) (err error) { + if backupDir == "" { + envDir := os.Getenv("MONGO_BACKUP_DIR") + if envDir != "" { + backupDir = envDir + } else { + if fileExists(filepath.Join(DataPath, "dbbak")) { + // /data/dbbak 存在 + backupDir = DataPath + } else if fileExists(filepath.Join(Data1Path, "dbbak")) { + // /data1/dbbak 存在 + backupDir = Data1Path + } else if IsMountPoint2(DataPath) { + // /data是挂载点 + backupDir = DataPath + } else if IsMountPoint2(Data1Path) { + // /data1是挂载点 + backupDir = Data1Path + } else { + // 函数参数 backupDir 为空, 环境变量 MONGO_BACKUP_DIR 为空 + // /data1 和 /data 均不是挂载点 + // 强制指定 MONGO_BACKUP_DIR=/data + backupDir = DataPath + } + } + } + backupDir = strings.TrimSpace(backupDir) + var ret []byte + shCmd := fmt.Sprintf(` +ret=$(grep '^export MONGO_BACKUP_DIR=' /etc/profile) +if [[ -z $ret ]] +then +echo "export MONGO_BACKUP_DIR=%s">>/etc/profile +fi + `, backupDir) + ret, err = exec.Command("bash", "-c", shCmd).Output() + if err != nil { + err = fmt.Errorf("SetMongoBakcupDir failed,err:%v,ret:%s", err, string(ret)) + return + } + os.Setenv("MONGO_BACKUP_DIR", backupDir) + return nil +} + +// GetMongoBackupDir 获取环境变量 MONGO_BACKUP_DIR,默认值 /data +// 否则,如果目录 /data/dbbak 存在,返回 /data; +// 否则,如果目录 /data1/dbbak 存在,返回 /data1; +// 否则,返回 /data +func GetMongoBackupDir() string { + dataDir := os.Getenv("MONGO_BACKUP_DIR") + if dataDir == "" { + if fileExists(filepath.Join(DataPath, "dbbak")) { + // /data/dbbak 存在 + dataDir = DataPath + } else if fileExists(filepath.Join(Data1Path, "dbbak")) { + // /data1/dbbak 存在 + dataDir = Data1Path + } else { + dataDir = DataPath + } + } + return dataDir +} diff --git a/dbm-services/mongo/db-tools/dbactuator/pkg/consts/dts.go b/dbm-services/mongo/db-tools/dbactuator/pkg/consts/dts.go new file mode 100644 index 0000000000..342f035d3d --- /dev/null +++ b/dbm-services/mongo/db-tools/dbactuator/pkg/consts/dts.go @@ -0,0 +1,25 @@ +package consts + +// dts type +const ( + DtsTypeOneAppDiffCluster = "one_app_diff_cluster" // 一个业务下的不同集群 + DtsTypeDiffAppDiffCluster = "diff_app_diff_cluster" // 不同业务下的不同集群 + DtsTypeSyncToOtherSystem = "sync_to_other_system" // 同步到其他系统,如迁移到腾讯云 + DtsTypeUserBuiltToDbm = "user_built_to_dbm" // 用户自建redis到dbm系统 +) + +// IsDtsTypeSrcClusterBelongDbm (该dst类型中)源集群是否属于dbm系统 +func IsDtsTypeSrcClusterBelongDbm(dtsType string) bool { + if dtsType == DtsTypeOneAppDiffCluster || + dtsType == DtsTypeDiffAppDiffCluster || + dtsType == DtsTypeSyncToOtherSystem { + return true + } + return false +} + +// dts datacheck mode +const ( + DtsDataCheckByKeysFileMode = "bykeysfile" // 基于key提取结果,做数据校验 + DtsDataCheckByScanMode = "byscan" // 通过scan命令获取key名,做数据校验 +) diff --git a/dbm-services/mongo/db-tools/dbactuator/pkg/consts/test.go b/dbm-services/mongo/db-tools/dbactuator/pkg/consts/test.go new file mode 100644 index 0000000000..8dd5594c3e --- /dev/null +++ b/dbm-services/mongo/db-tools/dbactuator/pkg/consts/test.go @@ -0,0 +1,94 @@ +package consts + +import "fmt" + +// test consts +const ( + // ----- tendisplus master指定的端口范围 [11000,11999] ------ + // ----- tendisplus slave指定的端口范围 [12000,12999] ------ + // TestTendisPlusMasterStartPort master start port + TestTendisPlusMasterStartPort = 11000 + // TestTendisPlusSlaveStartPort slave start port + TestTendisPlusSlaveStartPort = 12000 + + // ExpansionTestTendisPlusMasterStartPort master start port + ExpansionTestTendisPlusMasterStartPort = 11100 + // ExpansionTestTendisPlusSlaveStartPort slave start port + ExpansionTestTendisPlusSlaveStartPort = 12100 + + // SlotTestTendisPlusMasterPort master start port + SlotTestTendisPlusMasterPort = 11200 + // SLotTestTendisPlusSlaveStart slave start port + SLotTestTendisPlusSlaveStart = 12200 + // SlotsMigrateTest 指定迁移slot + SlotsMigrateTest = "0-100" + + // TestSyncTendisPlusMasterStartPort make sync /redo slave + TestSyncTendisPlusMasterStartPort = 11300 + // TestSyncTendisPlusSlaveStartPort make sync / + TestSyncTendisPlusSlaveStartPort = 12300 + + // ----- cache redis master指定的端口范围 [13000,13999] ------ + // ----- cache redis slave指定的端口范围 [14000,14999] ------ + + // TestRedisMasterStartPort master start port + TestRedisMasterStartPort = 13000 + // TestRedisSlaveStartPort slave start port + TestRedisSlaveStartPort = 14000 + + // TestSyncRedisMasterStartPort make sync /redo slave + TestSyncRedisMasterStartPort = 13300 + // TestSyncRedisSlaveStartPort make sync / + TestSyncRedisSlaveStartPort = 14300 + + // ----- tendisssd master指定的端口范围 [14000,14999] ------ + // ----- tendisssd slave指定的端口范围 [15000,15999] ------ + + // TestTendisSSDMasterStartPort master start port + TestTendisSSDMasterStartPort = 15000 + // TestTendisSSDSlaveStartPort slave start port + TestTendisSSDSlaveStartPort = 16000 + + // TestTwemproxyPort twemproxy port + TestTwemproxyPort = 50100 + // TestPredixyPort predixy port + TestPredixyPort = 50200 + // TestSSDClusterTwemproxyPort twemproxy port + TestSSDClusterTwemproxyPort = 50300 + + // TestRedisInstanceNum instance number + TestRedisInstanceNum = 4 + + // ExpansionTestRedisInstanceNum instance number + ExpansionTestRedisInstanceNum = 2 + // SLotTestRedisInstanceNum instance number + SLotTestRedisInstanceNum = 1 +) +const ( + // RedisTestPasswd redis test password + RedisTestPasswd = "redisPassTest" + // ProxyTestPasswd proxy test password + ProxyTestPasswd = "proxyPassTest" +) + +// test uid/rootid/nodeid +const ( + TestUID = 1111 + TestRootID = 2222 + TestNodeID = 3333 +) + +var ( + // ActuatorTestCmd actuator测试命令 + ActuatorTestCmd = fmt.Sprintf( + // NOCC:tosa/linelength(设计如此) + "cd %s && ./dbactuator_redis --uid=%d --root_id=%d --node_id=%d --version_id=v1 --atom-job-list=%%q --payload=%%q --payload-format=raw", + PackageSavePath, TestUID, TestRootID, TestNodeID) +) + +const ( + // PayloadFormatRaw raw + PayloadFormatRaw = "raw" + // PayloadFormatBase64 base64 + PayloadFormatBase64 = "base64" +) diff --git a/dbm-services/mongo/db-tools/dbactuator/pkg/consts/user.go b/dbm-services/mongo/db-tools/dbactuator/pkg/consts/user.go new file mode 100644 index 0000000000..f19fa9bb06 --- /dev/null +++ b/dbm-services/mongo/db-tools/dbactuator/pkg/consts/user.go @@ -0,0 +1,80 @@ +package consts + +import ( + "fmt" + "os" + "os/exec" +) + +// SetProcessUser 设置os用户 +func SetProcessUser(user string) error { + // 如果有user参数，设置环境变量 + if user != "" { + envUser := os.Getenv("PROCESS_EXEC_USER") + if envUser == user { + return nil + } + envUser = user + var ret []byte + shCmd := fmt.Sprintf(` +ret=$(grep '^export PROCESS_EXEC_USER=' /etc/profile) +if [[ -z $ret ]] +then +echo "export PROCESS_EXEC_USER=%s">>/etc/profile +fi + `, envUser) + ret, err := exec.Command("bash", "-c", shCmd).Output() + if err != nil { + err = fmt.Errorf("SetProcessUser failed,err:%v,ret:%s", err, string(ret)) + return err + } + os.Setenv("PROCESS_EXEC_USER", envUser) + } + return nil +} + +// GetProcessUser 获取os用户 +func GetProcessUser() string { + envUser := os.Getenv("PROCESS_EXEC_USER") + if envUser == "" { + return OSAccount + } + return envUser +} + +// SetProcessUserGroup 设置os用户Group +func SetProcessUserGroup(group string) error { + // 如果有user参数，设置环境变量 + if group != "" { + envGroup := os.Getenv("PROCESS_EXEC_USER_GROUP") + if envGroup == group { + return nil + } + envGroup = group + var ret []byte + shCmd := fmt.Sprintf(` +ret=$(grep '^export PROCESS_EXEC_USER_GROUP=' /etc/profile) +if [[ -z $ret ]] +then +echo "export PROCESS_EXEC_USER_GROUP=%s">>/etc/profile +fi + `, envGroup) + ret, err := exec.Command("bash", "-c", shCmd).Output() + if err != nil { + err = fmt.Errorf("SetProcessUserGroup failed,err:%v,ret:%s", err, string(ret)) + return err + } + os.Setenv("PROCESS_EXEC_USER_GROUP", envGroup) + + } + return nil +} + +// GetProcessUserGroup 获取os用户group +func GetProcessUserGroup() string { + envGroup := os.Getenv("PROCESS_EXEC_USER_GROUP") + if envGroup == "" { + return OSGroup + } + return envGroup +} diff --git a/dbm-services/mongo/db-tools/dbactuator/pkg/customtime/customtime.go b/dbm-services/mongo/db-tools/dbactuator/pkg/customtime/customtime.go new file mode 100644 index 0000000000..3e9c9f8150 --- /dev/null +++ b/dbm-services/mongo/db-tools/dbactuator/pkg/customtime/customtime.go @@ -0,0 +1,76 @@ +// Package customtime 自定义time +package customtime + +import ( + "database/sql/driver" + "fmt" + "strings" + "time" +) + +// CustomTime 自定义时间类型 +type CustomTime struct { + time.Time +} + +const ctLayout = "2006-01-02 15:04:05" + +var nilTime = (time.Time{}).UnixNano() + +// UnmarshalJSON .. +func (ct *CustomTime) UnmarshalJSON(b []byte) (err error) { + s := strings.Trim(string(b), "\"") + if s == "null" || s == "" { + ct.Time = time.Time{} + return + } + ct.Time, err = time.ParseInLocation(ctLayout, s, time.Local) + return +} + +// MarshalJSON .. +func (ct CustomTime) MarshalJSON() ([]byte, error) { + if ct.Time.UnixNano() == nilTime { + return []byte("null"), nil + } + return []byte(fmt.Sprintf("\"%s\"", ct.Time.Format(ctLayout))), nil +} + +// Scan scan +func (ct *CustomTime) Scan(value interface{}) error { + switch v := value.(type) { + case []byte: + return ct.UnmarshalText(string(v)) + case string: + return ct.UnmarshalText(v) + case time.Time: + ct.Time = v + case nil: + ct.Time = time.Time{} + default: + return fmt.Errorf("cannot sql.Scan() CustomTime from: %#v", v) + } + return nil +} + +// UnmarshalText unmarshal ... +func (ct *CustomTime) UnmarshalText(value string) error { + dd, err := time.ParseInLocation(ctLayout, value, time.Local) + if err != nil { + return err + } + ct.Time = dd + return nil +} + +// Value .. +// 注意这里ct不能是指针 +// 参考文章:https://www.codenong.com/44638610/ +func (ct CustomTime) Value() (driver.Value, error) { + return driver.Value(ct.Local().Format(ctLayout)), nil +} + +// IsSet .. +func (ct *CustomTime) IsSet() bool { + return ct.UnixNano() != nilTime +} diff --git a/dbm-services/mongo/db-tools/dbactuator/pkg/jobmanager/jobmanager.go b/dbm-services/mongo/db-tools/dbactuator/pkg/jobmanager/jobmanager.go new file mode 100644 index 0000000000..1e35d3ee31 --- /dev/null +++ b/dbm-services/mongo/db-tools/dbactuator/pkg/jobmanager/jobmanager.go @@ -0,0 +1,153 @@ +// Package jobmanager 原子任务工厂类 与 管理类 +package jobmanager + +import ( + "fmt" + "log" + "runtime/debug" + "strings" + "sync" + "time" + + "dbm-services/mongo/db-tools/dbactuator/pkg/atomjobs/atommongodb" + "dbm-services/mongo/db-tools/dbactuator/pkg/atomjobs/atomsys" + "dbm-services/mongo/db-tools/dbactuator/pkg/jobruntime" + "dbm-services/mongo/db-tools/dbactuator/pkg/util" +) + +// AtomJobCreatorFunc 原子任务创建接口 +type AtomJobCreatorFunc func() jobruntime.JobRunner + +// JobGenericManager 原子任务管理者 +type JobGenericManager struct { + Runners []jobruntime.JobRunner `json:"runners"` + atomJobMapper map[string]AtomJobCreatorFunc + once sync.Once + runtime *jobruntime.JobGenericRuntime +} + +// NewJobGenericManager new +func NewJobGenericManager(uid, rootID, nodeID, versionID, payload, payloadFormat, atomJobs, baseDir string) ( + ret *JobGenericManager, err error) { + runtime, err := jobruntime.NewJobGenericRuntime(uid, rootID, nodeID, versionID, + payload, payloadFormat, atomJobs, baseDir) + if err != nil { + log.Panicf(err.Error()) + } + ret = &JobGenericManager{ + runtime: runtime, + } + return +} + +// LoadAtomJobs 加载子任务 +func (m *JobGenericManager) LoadAtomJobs() (err error) { + defer func() { + // err最后输出到标准错误 + if err != nil { + m.runtime.PrintToStderr(err.Error()) + } + }() + defer func() { + if r := recover(); r != nil { + err = fmt.Errorf("%s", (debug.Stack())) + } + }() + m.runtime.AtomJobList = strings.TrimSpace(m.runtime.AtomJobList) + if m.runtime.AtomJobList == "" { + err = fmt.Errorf("atomJobList(%s) cannot be empty", m.runtime.AtomJobList) + m.runtime.Logger.Error(err.Error()) + return + } + jobList := strings.Split(m.runtime.AtomJobList, ",") + for _, atomName := range jobList { + atomName = strings.TrimSpace(atomName) + if atomName == "" { + continue + } + atom := m.GetAtomJobInstance(atomName) + if atom == nil { + err = fmt.Errorf("atomJob(%s) not found", atomName) + m.runtime.Logger.Error(err.Error()) + return + } + m.Runners = append(m.Runners, atom) + m.runtime.Logger.Info(fmt.Sprintf("atomJob:%s instance load success", atomName)) + } + return +} + +// RunAtomJobs 顺序执行原子任务 +func (m *JobGenericManager) RunAtomJobs() (err error) { + defer func() { + // err最后输出到标准错误 + if err != nil { + m.runtime.PrintToStderr(err.Error() + "\n") + } + }() + defer func() { + if r := recover(); r != nil { + err = fmt.Errorf("%s", string(debug.Stack())) + } + }() + + m.runtime.StartHeartbeat(10 * time.Second) + + defer m.runtime.StopHeartbeat() + + for _, runner := range m.Runners { + name := util.GetTypeName(runner) + m.runtime.Logger.Info(fmt.Sprintf("begin to run %s init", name)) + if err = runner.Init(m.runtime); err != nil { + return + } + m.runtime.Logger.Info(fmt.Sprintf("begin to run %s", name)) + err = runner.Run() + if err != nil { + m.runtime.Logger.Info(fmt.Sprintf("runner %s run failed,err:%s", name, err)) + // err = runner.Rollback() + // if err != nil { + // err = fmt.Errorf("runner %s rollback failed,err:%+v", name, err) + // m.runtime.Logger.Error(err.Error()) + // return + // } + // m.runtime.Logger.Info(fmt.Sprintf("runner %s rollback success!!!", name)) + return + } + m.runtime.Logger.Info(fmt.Sprintf("finished run %s", name)) + } + m.runtime.Logger.Info(fmt.Sprintf("run all atomJobList:%s success", m.runtime.AtomJobList)) + + m.runtime.OutputPipeContextData() + return +} + +// GetAtomJobInstance 根据atomJobName,从m.atomJobMapper中获取其creator函数,执行creator函数 +func (m *JobGenericManager) GetAtomJobInstance(atomJob string) jobruntime.JobRunner { + m.once.Do(func() { + m.atomJobMapper = make(map[string]AtomJobCreatorFunc) + + // os初始化 + m.atomJobMapper[atomsys.NewOsMongoInit().Name()] = atomsys.NewOsMongoInit + // mongo atom jobs + m.atomJobMapper[atommongodb.NewMongoDBInstall().Name()] = atommongodb.NewMongoDBInstall + m.atomJobMapper[atommongodb.NewMongoSInstall().Name()] = atommongodb.NewMongoSInstall + m.atomJobMapper[atommongodb.NewInitiateReplicaset().Name()] = atommongodb.NewInitiateReplicaset + m.atomJobMapper[atommongodb.NewAddShardToCluster().Name()] = atommongodb.NewAddShardToCluster + m.atomJobMapper[atommongodb.NewAddUser().Name()] = atommongodb.NewAddUser + m.atomJobMapper[atommongodb.NewDelUser().Name()] = atommongodb.NewDelUser + m.atomJobMapper[atommongodb.NewMongoDReplace().Name()] = atommongodb.NewMongoDReplace + m.atomJobMapper[atommongodb.NewMongoRestart().Name()] = atommongodb.NewMongoRestart + m.atomJobMapper[atommongodb.NewStepDown().Name()] = atommongodb.NewStepDown + m.atomJobMapper[atommongodb.NewBalancer().Name()] = atommongodb.NewBalancer + m.atomJobMapper[atommongodb.NewDeInstall().Name()] = atommongodb.NewDeInstall + m.atomJobMapper[atommongodb.NewExecScript().Name()] = atommongodb.NewExecScript + m.atomJobMapper[atommongodb.NewSetProfiler().Name()] = atommongodb.NewSetProfiler + m.atomJobMapper[atommongodb.NewMongoDChangeOplogSize().Name()] = atommongodb.NewMongoDChangeOplogSize + }) + creator, ok := m.atomJobMapper[strings.ToLower(atomJob)] + if ok { + return creator() + } + return nil +} diff --git a/dbm-services/mongo/db-tools/dbactuator/pkg/jobruntime/jobrunner.go b/dbm-services/mongo/db-tools/dbactuator/pkg/jobruntime/jobrunner.go new file mode 100644 index 0000000000..26c2eb74cc --- /dev/null +++ b/dbm-services/mongo/db-tools/dbactuator/pkg/jobruntime/jobrunner.go @@ -0,0 +1,19 @@ +package jobruntime + +// JobRunner defines a behavior of a job +type JobRunner interface { + // Init doing some operation before run a job + // such as reading parametes + Init(*JobGenericRuntime) error + + // Name return the name of the job + Name() string + + // Run run a job + Run() error + + Retry() uint + + // Rollback you can define some rollback logic here when job fails + Rollback() error +} diff --git a/dbm-services/mongo/db-tools/dbactuator/pkg/jobruntime/jobruntime.go b/dbm-services/mongo/db-tools/dbactuator/pkg/jobruntime/jobruntime.go new file mode 100644 index 0000000000..e4d885b4e3 --- /dev/null +++ b/dbm-services/mongo/db-tools/dbactuator/pkg/jobruntime/jobruntime.go @@ -0,0 +1,159 @@ +// Package jobruntime 全局操作、全局变量 +package jobruntime + +import ( + "context" + "encoding/base64" + "encoding/json" + "fmt" + "log" + "os" + "os/exec" + "path/filepath" + "time" + + "dbm-services/common/go-pubpkg/logger" + "dbm-services/mongo/db-tools/dbactuator/mylog" + "dbm-services/mongo/db-tools/dbactuator/pkg/consts" + "dbm-services/mongo/db-tools/dbactuator/pkg/util" +) + +const ( + logDir = "logs/" +) + +// JobGenericRuntime job manager +type JobGenericRuntime struct { + UID string `json:"uid"` // 单据ID + RootID string `json:"rootId"` // 流程ID + NodeID string `json:"nodeId"` // 节点ID + VersionID string `json:"versionId"` // 运行版本ID + PayloadEncoded string `json:"payloadEncoded"` // 参数encoded + PayloadDecoded string `json:"payloadDecoded"` // 参数decoded + PayLoadFormat string `json:"payloadFormat"` // payload的内容格式,raw/base64 + AtomJobList string `json:"atomJobList"` // 原子任务列表,逗号分割 + BaseDir string `json:"baseDir"` + // ShareData保存多个atomJob间的中间结果,前后atomJob可通过ShareData通信 + ShareData interface{} `json:"shareData"` + // PipeContextData保存流程调用,上下文结果 + // PipeContextData=>json.Marshal=>Base64=>标准输出打印{Result} + PipeContextData interface{} `json:"pipeContextData"` + Logger *logger.Logger `json:"-"` // 线程安全日志输出 + ctx context.Context `json:"-"` + cancelFunc context.CancelFunc `json:"-"` + Err error +} + +// NewJobGenericRuntime new +func NewJobGenericRuntime(uid, rootID string, + nodeID, versionID, payload, payloadFormat, atomJobs, baseDir string) (ret *JobGenericRuntime, err error) { + ret = &JobGenericRuntime{ + UID: uid, + RootID: rootID, + NodeID: nodeID, + VersionID: versionID, + PayloadEncoded: payload, + PayLoadFormat: payloadFormat, + AtomJobList: atomJobs, + BaseDir: baseDir, + ShareData: nil, + } + + if ret.PayLoadFormat == consts.PayloadFormatRaw { + ret.PayloadDecoded = ret.PayloadEncoded + } else { + var decodedStr []byte + decodedStr, err = base64.StdEncoding.DecodeString(ret.PayloadEncoded) + if err != nil { + log.Printf("Base64.DecodeString failed,err:%v,encodedString:%s", err, ret.PayloadEncoded) + os.Exit(0) + } + ret.PayloadDecoded = string(decodedStr) + // log.Printf("===========PayloadDecoded========") + // log.Printf(ret.PayloadDecoded) + } + ret.ctx, ret.cancelFunc = context.WithCancel(context.TODO()) + ret.SetLogger() + return +} + +// SetLogger set logger +func (r *JobGenericRuntime) SetLogger() { + var err error + logFile := fmt.Sprintf("redis_actuator_%s_%s.log", r.UID, r.NodeID) + err = util.MkDirsIfNotExists([]string{logDir}) + if err != nil { + panic(err) + } + + logFilePath := filepath.Join(logDir, logFile) + file, err := os.OpenFile(logFilePath, os.O_CREATE|os.O_APPEND|os.O_WRONLY, os.ModePerm) + if err != nil { + panic(err) + } + extMap := map[string]string{ + "uid": r.UID, + "node_id": r.NodeID, + "root_id": r.RootID, + "version_id": r.VersionID, + } + r.Logger = logger.New(file, true, logger.InfoLevel, extMap) + r.Logger.Sync() + mylog.SetDefaultLogger(r.Logger) + + // 修改日志目录owner + chownCmd := fmt.Sprintf("chown -R %s.%s %s", consts.MysqlAaccount, consts.MysqlGroup, logDir) + cmd := exec.Command("bash", "-c", chownCmd) + cmd.Run() +} + +// PrintToStdout 打印到标准输出 +func (r *JobGenericRuntime) PrintToStdout(format string, args ...interface{}) { + fmt.Fprintf(os.Stdout, format, args...) +} + +// PrintToStderr 打印到标准错误 +func (r *JobGenericRuntime) PrintToStderr(format string, args ...interface{}) { + fmt.Fprintf(os.Stderr, format, args...) +} + +// OutputPipeContextData PipeContextData=>json.Marshal=>Base64=>标准输出打印{Result} +func (r *JobGenericRuntime) OutputPipeContextData() { + if r.PipeContextData == nil { + r.Logger.Info("no PipeContextData to output") + return + } + tmpBytes, err := json.Marshal(r.PipeContextData) + if err != nil { + r.Err = fmt.Errorf("json.Marshal PipeContextData failed,err:%v", err) + r.Logger.Error(r.Err.Error()) + return + } + // decode函数: base64.StdEncoding.DecodeString + base64Ret := base64.StdEncoding.EncodeToString(tmpBytes) + r.PrintToStdout("" + base64Ret + "") +} + +// StartHeartbeat 开始心跳 +func (r *JobGenericRuntime) StartHeartbeat(period time.Duration) { + go func() { + ticker := time.NewTicker(period) + defer ticker.Stop() + var heartbeatTime string + for { + select { + case <-ticker.C: + heartbeatTime = time.Now().Local().Format(consts.UnixtimeLayout) + r.PrintToStdout("[" + heartbeatTime + "]heartbeat\n") + case <-r.ctx.Done(): + r.Logger.Info("stop heartbeat") + return + } + } + }() +} + +// StopHeartbeat 结束心跳 +func (r *JobGenericRuntime) StopHeartbeat() { + r.cancelFunc() +} diff --git a/dbm-services/mongo/db-tools/dbactuator/pkg/report/filereport.go b/dbm-services/mongo/db-tools/dbactuator/pkg/report/filereport.go new file mode 100644 index 0000000000..dbc50ce87c --- /dev/null +++ b/dbm-services/mongo/db-tools/dbactuator/pkg/report/filereport.go @@ -0,0 +1,101 @@ +// Package report (备份等)记录上报 +package report + +import ( + "bufio" + "fmt" + "os" + "sync" + + "dbm-services/mongo/db-tools/dbactuator/mylog" +) + +var _ Reporter = (*FileReport)(nil) + +// FileReport 文件上报 +type FileReport struct { + saveFile string + fileP *os.File + bufWriter *bufio.Writer + mux sync.Mutex // 并发安全写入 +} + +// NewFileReport new +func NewFileReport(savefile string) (ret *FileReport, err error) { + ret = &FileReport{} + err = ret.SetSaveFile(savefile) + return ret, err +} + +// AddRecord 新增记录 +func (f *FileReport) AddRecord(item string, flush bool) (err error) { + if f.saveFile == "" { + err = fmt.Errorf("saveFile(%s) can't be empty", f.saveFile) + mylog.Logger.Error(err.Error()) + return + } + _, err = f.bufWriter.WriteString(item) + if err != nil { + err = fmt.Errorf("bufio.Writer WriteString fail,err:%v,saveFile:%s", err, f.saveFile) + mylog.Logger.Error(err.Error()) + return + } + if flush == true { + f.bufWriter.Flush() + } + return nil +} + +// SaveFile .. +func (f *FileReport) SaveFile() string { + return f.saveFile +} + +// SetSaveFile set方法 +func (f *FileReport) SetSaveFile(savefile string) error { + var err error + err = f.Close() + if err != nil { + return err + } + if savefile == "" { + err = fmt.Errorf("saveFile(%s) cannot be empty", savefile) + mylog.Logger.Error(err.Error()) + return err + } + f.saveFile = savefile + f.fileP, err = os.OpenFile(savefile, os.O_APPEND|os.O_CREATE|os.O_RDWR, 0644) + if err != nil { + err = fmt.Errorf("open file:%s fail,err:%v", savefile, err) + mylog.Logger.Error(err.Error()) + return err + } + f.bufWriter = bufio.NewWriter(f.fileP) + return nil +} + +// Close file +func (f *FileReport) Close() error { + f.mux.Lock() + defer f.mux.Unlock() + + var err error + if f.saveFile == "" { + return nil + } + f.saveFile = "" + + err = f.bufWriter.Flush() + if err != nil { + err = fmt.Errorf("bufio flush fail.err:%v,file:%s", err, f.saveFile) + mylog.Logger.Error(err.Error()) + return nil + } + err = f.fileP.Close() + if err != nil { + err = fmt.Errorf("file close fail.err:%v,file:%s", err, f.saveFile) + mylog.Logger.Error(err.Error()) + return nil + } + return nil +} diff --git a/dbm-services/mongo/db-tools/dbactuator/pkg/report/reporter.go b/dbm-services/mongo/db-tools/dbactuator/pkg/report/reporter.go new file mode 100644 index 0000000000..ce13a8f5b4 --- /dev/null +++ b/dbm-services/mongo/db-tools/dbactuator/pkg/report/reporter.go @@ -0,0 +1,58 @@ +package report + +import ( + "fmt" + "os" + "path/filepath" + "time" + + "dbm-services/mongo/db-tools/dbactuator/mylog" + "dbm-services/mongo/db-tools/dbactuator/pkg/consts" + "dbm-services/mongo/db-tools/dbactuator/pkg/util" +) + +// Reporter 上报接口 +type Reporter interface { + AddRecord(item string, flush bool) error + Close() error +} + +// CreateReportDir 创建上报目录 /home/mysql/dbareport -> {REDIS_BACKUP_DIR}/dbbak/dbareport +func CreateReportDir() (err error) { + mylog.Logger.Info("begin to create reportDir(%s)", consts.DbaReportSaveDir) + var realLink string + realReportDir := filepath.Join(consts.GetRedisBackupDir(), "dbbak", "dbareport") // 如 /data/dbbak/dbareport + if !util.FileExists(realReportDir) { + err = util.MkDirsIfNotExists([]string{realReportDir}) + if err != nil { + mylog.Logger.Error(err.Error()) + return + } + } + util.LocalDirChownMysql(realReportDir) + if util.FileExists(consts.DbaReportSaveDir) { + realLink, err = filepath.EvalSymlinks(consts.DbaReportSaveDir) + if err != nil { + err = fmt.Errorf("filepath.EvalSymlinks %s fail,err:%v", consts.DbaReportSaveDir, err) + mylog.Logger.Error(err.Error()) + return err + } + // /home/mysql/dbareport -> /data/dbbak/dbareport ok,直接返回 + if realLink == realReportDir { + return nil + } + // 如果 /home/mysql/dbareport 不是指向 /data/dbbak/dbareport,先删除 + rmCmd := "rm -rf " + consts.DbaReportSaveDir + util.RunBashCmd(rmCmd, "", nil, 1*time.Minute) + } + err = os.Symlink(realReportDir, filepath.Dir(consts.DbaReportSaveDir)) + if err != nil { + err = fmt.Errorf("os.Symlink %s -> %s fail,err:%s", consts.DbaReportSaveDir, realReportDir, err) + mylog.Logger.Error(err.Error()) + return + } + mylog.Logger.Info("create softLink success,%s -> %s", consts.DbaReportSaveDir, realReportDir) + util.MkDirsIfNotExists([]string{consts.RedisReportSaveDir}) + util.LocalDirChownMysql(consts.DbaReportSaveDir) + return +} diff --git a/dbm-services/mongo/db-tools/dbactuator/pkg/util/bkrepo.go b/dbm-services/mongo/db-tools/dbactuator/pkg/util/bkrepo.go new file mode 100644 index 0000000000..f2695eeac9 --- /dev/null +++ b/dbm-services/mongo/db-tools/dbactuator/pkg/util/bkrepo.go @@ -0,0 +1,102 @@ +package util + +import ( + "bytes" + "encoding/base64" + "fmt" + "io" + "mime/multipart" + "net/http" + "os" + + "dbm-services/mongo/db-tools/dbactuator/mylog" +) + +// FileServerInfo 文件服务器 +type FileServerInfo struct { + URL string `json:"url"` // 制品库地址 + Bucket string `json:"bucket"` // 目标bucket + Password string `json:"password"` // 制品库 password + Username string `json:"username"` // 制品库username + Project string `json:"project"` // 制品库project +} + +// UploadFile 上传文件到蓝盾制品库 +// filepath: 本地需要上传文件的路径 +// targetURL: 仓库文件完整路径 +func UploadFile(filepath string, targetURL string, username string, password string) (*http.Response, error) { + + userMsg := fmt.Sprintf(username + ":" + password) + token := base64.StdEncoding.EncodeToString([]byte(userMsg)) + msg := fmt.Sprintf("start upload files from %s to %s", filepath, targetURL) + mylog.Logger.Info(msg) + bodyBuf := bytes.NewBufferString("") + bodyWriter := multipart.NewWriter(bodyBuf) + + fh, err := os.Open(filepath) + if err != nil { + mylog.Logger.Info("error opening file") + return nil, err + } + boundary := bodyWriter.Boundary() + closeBuf := bytes.NewBufferString("") + + requestReader := io.MultiReader(bodyBuf, fh, closeBuf) + fi, err := fh.Stat() + if err != nil { + fmt.Printf("Error Stating file: %s", filepath) + return nil, err + } + req, err := http.NewRequest("PUT", targetURL, requestReader) + if err != nil { + return nil, err + } + + // Set headers for multipart, and Content Length + req.Header.Set("Content-Type", "multipart/form-data; boundary="+boundary) + // 文件是否可以被覆盖，默认false + req.Header.Set("X-BKREPO-OVERWRITE", "True") + // 文件默认保留半年 + req.Header.Set("X-BKREPO-EXPIRES", "183") + req.Header.Set("Authorization", "Basic "+token) + req.ContentLength = fi.Size() + int64(bodyBuf.Len()) + int64(closeBuf.Len()) + return http.DefaultClient.Do(req) + // return response, err +} + +// DownloadFile 从蓝盾制品库下载文件 +// filepath: 本地保存文件压缩包名 +// targetURL: 仓库文件完整路径 +func DownloadFile(filepath string, targetURL string, username string, password string) (err error) { + msg := fmt.Sprintf("start download files from %s to %s", targetURL, filepath) + mylog.Logger.Info(msg) + userMsg := fmt.Sprintf(username + ":" + password) + token := base64.StdEncoding.EncodeToString([]byte(userMsg)) + outFile, err := os.Create(filepath) + if err != nil { + return err + } + defer outFile.Close() + + resp, err := http.Get(targetURL) + if err != nil { + return err + } + resp.Header.Set("Authorization", "Basic "+token) + defer resp.Body.Close() + + // Check server response + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("bad status: %s", resp.Status) + } + + // Writer the body to file + _, err = io.Copy(outFile, resp.Body) + if err != nil { + return err + } + mylog.Logger.Info("finish download files") + + return nil + +} diff --git a/dbm-services/mongo/db-tools/dbactuator/pkg/util/compress.go b/dbm-services/mongo/db-tools/dbactuator/pkg/util/compress.go new file mode 100644 index 0000000000..4d54e3b734 --- /dev/null +++ b/dbm-services/mongo/db-tools/dbactuator/pkg/util/compress.go @@ -0,0 +1,206 @@ +package util + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "regexp" + "strings" + "time" + + "dbm-services/mongo/db-tools/dbactuator/mylog" + "dbm-services/mongo/db-tools/dbactuator/pkg/consts" + + "github.com/dustin/go-humanize" +) + +// IsZstdExecutable 通过'zstd -V'命令确定本地zstd工具是能正常运行的 +func IsZstdExecutable() (ok bool) { + var err error + if !FileExists(consts.ZstdBin) { + return false + } + cmd := exec.Command(consts.ZstdBin, "-V") + if err = cmd.Start(); err != nil { + // err = fmt.Errorf("'%s -V' cmd.Start fail,err:%v", zstdBin, err) + return false + } + if err = cmd.Wait(); err != nil { + // err = fmt.Errorf("'%s -V' cmd.Wait fail,err:%v", zstdBin, err) + return false + } + return true +} + +// CompressFile 压缩文件 +// 优先使用zstd 做压缩,zstd无法使用则使用gzip +func CompressFile(file, targetDir string, rmOrigin bool) (retFile string, err error) { + var compressCmd string + fileDir := filepath.Dir(file) + filename := filepath.Base(file) + if targetDir == "" { + targetDir = fileDir + } + if IsZstdExecutable() { + retFile = filepath.Join(targetDir, filename+".zst") + if rmOrigin { + compressCmd = fmt.Sprintf(`cd %s && %s --rm -T4 %s -o %s`, fileDir, consts.ZstdBin, filename, retFile) + } else { + compressCmd = fmt.Sprintf(`cd %s && %s -T4 %s -o %s`, fileDir, consts.ZstdBin, filename, retFile) + } + _, err = RunBashCmd(compressCmd, "", nil, 6*time.Hour) + if err != nil { + return + } + } else { + retFile = filepath.Join(fileDir, filename+".gz") + if rmOrigin { + compressCmd = fmt.Sprintf(`gzip < %s >%s && rm -f %s`, file, retFile, file) + } else { + compressCmd = fmt.Sprintf(`gzip < %s >%s`, file, retFile) + } + _, err = RunBashCmd(compressCmd, "", nil, 6*time.Hour) + if err != nil { + return + } + } + return +} + +// SplitLargeFile 切割大文件为小文件,并返回切割后的结果 +// 参数file须是全路径; +// 如果file大小 小于 splitTargetSize,则返回值splitTargetSize只包含 file 一个元素 +func SplitLargeFile(file, splitTargetSize string, rmOrigin bool) (splitedFiles []string, err error) { + var fileSize int64 + var splitLimit uint64 + var cmdRet string + if file == "" { + return + } + fileSize, err = GetFileSize(file) + if err != nil { + return + } + splitLimit, err = humanize.ParseBytes(splitTargetSize) + if err != nil { + err = fmt.Errorf("humanize.ParseBytes fail,err:%v,splitTargetSize:%s", err, splitTargetSize) + return + } + if fileSize < int64(splitLimit) { + splitedFiles = append(splitedFiles, file) + return + } + fileDir := filepath.Dir(file) + fileBase := filepath.Base(file) + fileBase = strings.TrimSuffix(fileBase, ".tar") + fileBase = strings.TrimSuffix(fileBase, ".tar.gz") + fileBase = fileBase + ".split." + splitCmd := fmt.Sprintf(`cd %s && split --verbose -a 3 -b %s -d %s %s|grep -i --only-match -E "%s[0-9]+"`, + fileDir, splitTargetSize, file, fileBase, fileBase) + mylog.Logger.Info(splitCmd) + cmdRet, err = RunBashCmd(splitCmd, "", nil, 6*time.Hour) + if err != nil { + return + } + l01 := strings.Split(cmdRet, "\n") + for _, item := range l01 { + item = strings.TrimSpace(item) + if item == "" { + continue + } + splitedFiles = append(splitedFiles, filepath.Join(fileDir, item)) + } + if rmOrigin { + err = os.Remove(file) + mylog.Logger.Info(fmt.Sprintf("rm %s", file)) + if err != nil { + err = fmt.Errorf("os.Remove fail,err:%v,file:%s", err, file) + return + } + } + return +} + +// TarADir 对一个目录进行tar打包, +// 如打包 /data/dbbak/REDIS-FULL-rocksdb-1.1.1.1-30000 为 /tmp/REDIS-FULL-rocksdb-1.1.1.1-30000.tar +// 参数: originDir 为 /data/dbbak/REDIS-FULL-rocksdb-1.1.1.1-30000 +// 参数: tarSaveDir 为 /tmp/ +// 返回值: tarFile 为 /tmp/REDIS-FULL-rocksdb-1.1.1.1-30000.tar +func TarADir(originDir, tarSaveDir string, rmOrigin bool) (tarFile string, err error) { + var tarCmd string + basename := filepath.Base(originDir) + baseDir := filepath.Dir(originDir) + if tarSaveDir == "" { + tarSaveDir = filepath.Dir(originDir) + } + tarFile = filepath.Join(tarSaveDir, basename+".tar") + + if rmOrigin { + tarCmd = fmt.Sprintf(`tar --remove-files -cf %s -C %s %s`, tarFile, baseDir, basename) + } else { + tarCmd = fmt.Sprintf(`tar -cf %s -C %s %s`, tarFile, baseDir, basename) + } + mylog.Logger.Info(tarCmd) + _, err = RunBashCmd(tarCmd, "", nil, 6*time.Hour) + if err != nil { + return + } + return +} + +// TarAndSplitADir 对目录tar打包并执行split +func TarAndSplitADir(originDir, targetSaveDir, splitTargetSize string, rmOrigin bool) ( + splitedFiles []string, err error) { + var tarFile string + tarFile, err = TarADir(originDir, targetSaveDir, rmOrigin) + if err != nil { + return + } + splitedFiles, err = SplitLargeFile(tarFile, splitTargetSize, rmOrigin) + if err != nil { + return + } + return +} + +// UnionSplitFiles 合并多个split文件为一个tar文件 +func UnionSplitFiles(dir string, splitFiles []string) (tarfile string, err error) { + if len(splitFiles) == 0 { + err = fmt.Errorf("splitFiles:%+v empty list", splitFiles) + return + } + if len(splitFiles) == 1 && strings.HasSuffix(splitFiles[0], ".tar") { + return splitFiles[0], nil + } + var name string + var fullpath string + var cmd01 string + reg01 := regexp.MustCompile(`.split.\d+$`) + baseNames := make([]string, 0, len(splitFiles)) + for _, file01 := range splitFiles { + name = filepath.Base(file01) + baseNames = append(baseNames, name) + if !reg01.MatchString(file01) { + err = fmt.Errorf("%+v not split files?", splitFiles) + return + } + fullpath = filepath.Join(dir, name) + if !FileExists(fullpath) { + err = fmt.Errorf("%s not exists", fullpath) + return + } + } + + prefix := reg01.ReplaceAllString(baseNames[0], "") + tarfile = prefix + ".tar" + if len(baseNames) == 1 { + cmd01 = fmt.Sprintf("cd %s && mv %s %s", dir, baseNames[0], tarfile) + } else { + cmd01 = fmt.Sprintf("cd %s && cat %s.split* > %s", dir, prefix, tarfile) + } + mylog.Logger.Info(cmd01) + _, err = RunBashCmd(cmd01, "", nil, 2*time.Hour) + tarfile = filepath.Join(dir, tarfile) + return +} diff --git a/dbm-services/mongo/db-tools/dbactuator/pkg/util/file.go b/dbm-services/mongo/db-tools/dbactuator/pkg/util/file.go new file mode 100644 index 0000000000..3e708ee613 --- /dev/null +++ b/dbm-services/mongo/db-tools/dbactuator/pkg/util/file.go @@ -0,0 +1,66 @@ +package util + +import ( + "bufio" + "bytes" + "crypto/md5" + "fmt" + "io" + "os" +) + +// FileExists 检查目录是否已经存在 +func FileExists(path string) bool { + _, err := os.Stat(path) + if err != nil { + return os.IsExist(err) + } + return true +} + +// GetFileMd5 求文件md5sum值 +func GetFileMd5(fileAbPath string) (md5sum string, err error) { + rFile, err := os.Open(fileAbPath) + if err != nil { + return "", fmt.Errorf("GetFileMd5 fail,err:%v,file:%s", err, fileAbPath) + } + defer func(rFile *os.File) { + _ = rFile.Close() + }(rFile) + h := md5.New() + if _, err := io.Copy(h, rFile); err != nil { + return "", fmt.Errorf("GetFileMd5 io.Copy fail,err:%v,file:%s", err, fileAbPath) + } + return fmt.Sprintf("%x", h.Sum(nil)), nil +} + +// FileLineCounter 计算文件行数 +// 参考: https://stackoverflow.com/questions/24562942/golang-how-do-i-determine-the-number-of-lines-in-a-file-efficiently +func FileLineCounter(filename string) (lineCnt uint64, err error) { + _, err = os.Stat(filename) + if err != nil && os.IsNotExist(err) == true { + return 0, fmt.Errorf("file:%s not exists", filename) + } + file, err := os.Open(filename) + if err != nil { + return 0, fmt.Errorf("file:%s open fail,err:%v", filename, err) + } + defer file.Close() + reader01 := bufio.NewReader(file) + buf := make([]byte, 32*1024) + lineCnt = 0 + lineSep := []byte{'\n'} + + for { + c, err := reader01.Read(buf) + lineCnt += uint64(bytes.Count(buf[:c], lineSep)) + + switch { + case err == io.EOF: + return lineCnt, nil + + case err != nil: + return lineCnt, fmt.Errorf("file:%s read fail,err:%v", filename, err) + } + } +} diff --git a/dbm-services/mongo/db-tools/dbactuator/pkg/util/net.go b/dbm-services/mongo/db-tools/dbactuator/pkg/util/net.go new file mode 100644 index 0000000000..413be8aceb --- /dev/null +++ b/dbm-services/mongo/db-tools/dbactuator/pkg/util/net.go @@ -0,0 +1,67 @@ +package util + +import ( + "fmt" + "net" +) + +// GetIpv4InterfaceName 根据ipv4地址获取网络接口名 +// https://stackoverflow.com/questions/23529663/how-to-get-all-addresses-and-masks-from-local-interfaces-in-go +func GetIpv4InterfaceName(ipv4 string) (interName string, err error) { + var ifaces []net.Interface + var addrs []net.Addr + + ifaces, err = net.Interfaces() + if err != nil { + err = fmt.Errorf("net.Interfaces fail,err:%v", err) + return + } + for _, i := range ifaces { + addrs, err = i.Addrs() + if err != nil { + // err = fmt.Errorf("%s get addrs fail,err:%v", i.Name, err) + continue + } + for _, a := range addrs { + switch v := a.(type) { + case *net.IPAddr: + if v.IP.String() == ipv4 { + return i.Name, nil + } + + case *net.IPNet: + if v.IP.String() == ipv4 { + return i.Name, nil + } + } + } + } + err = fmt.Errorf("ipv4:%s not found interfacename", ipv4) + return +} + +// GetInterfaceIpv4Addr 获取网络接口对应的 ipv4地址 +// https://gist.github.com/schwarzeni/f25031a3123f895ff3785970921e962c +func GetInterfaceIpv4Addr(interfaceName string) (addr string, err error) { + var ( + ief *net.Interface + addrs []net.Addr + ipv4Addr net.IP + ) + if ief, err = net.InterfaceByName(interfaceName); err != nil { // get interface + err = fmt.Errorf("net.InterfaceByName %s fail,err:%v", interfaceName, err) + return + } + if addrs, err = ief.Addrs(); err != nil { // get addresses + return + } + for _, addr := range addrs { // get ipv4 address + if ipv4Addr = addr.(*net.IPNet).IP.To4(); ipv4Addr != nil { + break + } + } + if ipv4Addr == nil { + return "", fmt.Errorf("interface %s don't have an ipv4 address\n", interfaceName) + } + return ipv4Addr.String(), nil +} diff --git a/dbm-services/mongo/db-tools/dbactuator/pkg/util/osCmd.go b/dbm-services/mongo/db-tools/dbactuator/pkg/util/osCmd.go new file mode 100644 index 0000000000..62750bf58b --- /dev/null +++ b/dbm-services/mongo/db-tools/dbactuator/pkg/util/osCmd.go @@ -0,0 +1,107 @@ +package util + +import ( + "bytes" + "context" + "fmt" + "io" + "os" + "os/exec" + "strings" + "time" + + "dbm-services/mongo/db-tools/dbactuator/mylog" +) + +// DealLocalCmdPid 处理本地命令得到pid +type DealLocalCmdPid interface { + DealProcessPid(pid int) error +} + +// RunBashCmd bash -c "$cmd" 执行命令并得到命令结果 +func RunBashCmd(cmd, outFile string, dealPidMethod DealLocalCmdPid, + timeout time.Duration) (retStr string, err error) { + opts := []string{"-c", cmd} + return RunLocalCmd("bash", opts, outFile, dealPidMethod, timeout) +} + +// RunLocalCmd 运行本地命令并得到命令结果 +/* + *参数: + * outFile: 不为空,则将标准输出结果打印到outFile中; + * dealPidMethod: 不为空,则将命令pid传给dealPidMethod.DealProcessPid()函数; + * logger: 用于打印日志; + */ +func RunLocalCmd( + cmd string, opts []string, outFile string, + dealPidMethod DealLocalCmdPid, timeout time.Duration) (retStr string, err error) { + ctx, cancel := context.WithTimeout(context.TODO(), timeout) + defer cancel() + + cmdCtx := exec.CommandContext(ctx, cmd, opts...) + var retBuffer bytes.Buffer + var errBuffer bytes.Buffer + var outFileHandler *os.File + if len(strings.TrimSpace(outFile)) == 0 { + cmdCtx.Stdout = &retBuffer + } else { + outFileHandler, err = os.Create(outFile) + if err != nil { + mylog.Logger.Error("RunLocalCmd create outfile fail,err:%v,outFile:%s", err, outFile) + return "", fmt.Errorf("RunLocalCmd create outfile fail,err:%v,outFile:%s", err, outFile) + } + defer outFileHandler.Close() + mylog.Logger.Info("RunLocalCmd create outfile(%s) success ...", outFile) + cmdCtx.Stdout = outFileHandler + } + cmdCtx.Stderr = &errBuffer + mylog.Logger.Debug("Running a new local cmd:%s,opts:%+v", cmd, opts) + + if err = cmdCtx.Start(); err != nil { + mylog.Logger.Error("RunLocalCmd cmd Start fail,err:%v,cmd:%s,opts:%+v", err, cmd, opts) + return "", fmt.Errorf("RunLocalCmd cmd Start fail,err:%v", err) + } + if dealPidMethod != nil { + dealPidMethod.DealProcessPid(cmdCtx.Process.Pid) + } + if err = cmdCtx.Wait(); err != nil { + mylog.Logger.Error("RunLocalCmd cmd wait fail,err:%v,errBuffer:%s,retBuffer:%s,cmd:%s,opts:%+v", err, + errBuffer.String(), retBuffer.String(), cmd, opts) + return "", fmt.Errorf("RunLocalCmd cmd wait fail,err:%v,detail:%s", err, errBuffer.String()) + } + retStr = retBuffer.String() + + if strings.TrimSpace(errBuffer.String()) != "" { + mylog.Logger.Error("RunLocalCmd fail,err:%v,cmd:%s,opts:%+v", errBuffer.String(), cmd, opts) + err = fmt.Errorf("RunLocalCmd fail,err:%s", retBuffer.String()+"\n"+errBuffer.String()) + } else { + err = nil + } + retStr = strings.TrimSpace(retStr) + return +} + +// SetOSUserPassword run set user password by chpasswd +func SetOSUserPassword(user, password string) error { + exec.Command("/bin/bash", "-c", "") + cmd := exec.Command("chpasswd") + stdin, err := cmd.StdinPipe() + if err != nil { + return fmt.Errorf("new pipe failed, err:%w", err) + } + go func() { + _, err := io.WriteString(stdin, fmt.Sprintf("%s:%s", user, password)) + if err != nil { + mylog.Logger.Warn("write into pipe failed, err:%s", err.Error()) + } + if err := stdin.Close(); err != nil { + mylog.Logger.Warn("colse stdin failed, err:%s", err.Error()) + } + }() + if output, err := cmd.CombinedOutput(); err != nil { + err = fmt.Errorf("run chpasswd failed, output:%s, err:%w", string(output), err) + mylog.Logger.Error(err.Error()) + return err + } + return nil +} diff --git a/dbm-services/mongo/db-tools/dbactuator/pkg/util/proxy_tools.go b/dbm-services/mongo/db-tools/dbactuator/pkg/util/proxy_tools.go new file mode 100644 index 0000000000..8f4d60b17a --- /dev/null +++ b/dbm-services/mongo/db-tools/dbactuator/pkg/util/proxy_tools.go @@ -0,0 +1,103 @@ +// Package util here mybe something +package util + +import ( + "bufio" + "crypto/md5" + "encoding/json" + "fmt" + "io" + "math/rand" + "net" + "sort" + "strconv" + "strings" + "time" +) + +// NCInstance NCInstance +var NCInstance *NetCat + +// NetCat use tcp for nc +type NetCat struct { + AdminAddr string + ReadTimeOut time.Duration + Nc net.Conn +} + +func init() { + NCInstance = &NetCat{} + rand.Seed(time.Now().UnixNano()) +} + +// GetTwemProxyBackendsMd5Sum 获取MD5 sum +func GetTwemProxyBackendsMd5Sum(addr string) (string, error) { + pinfo := strings.Split(addr, ":") + port, _ := strconv.Atoi(pinfo[1]) + segsMap, err := GetTwemproxyBackends(pinfo[0], port) + if err != nil { + return "errFailed", err + } + segList := []string{} + for addr, seg := range segsMap { + segList = append(segList, fmt.Sprintf("%s|%s", addr, seg)) + } + sort.Slice(segList, func(i, j int) bool { + return segList[i] > segList[j] + }) + + x, _ := json.Marshal(segList) + return fmt.Sprintf("%x", md5.Sum(x)), nil +} + +// DoSwitchTwemproxyBackends "change nosqlproxy $mt:$mp $st:$sp" +func DoSwitchTwemproxyBackends(ip string, port int, from, to string) (rst string, err error) { + addr := fmt.Sprintf("%s:%d", ip, port+1000) + nc, err := net.DialTimeout("tcp", addr, time.Second) + if err != nil { + return "nil", err + } + _, err = nc.Write([]byte(fmt.Sprintf("change nosqlproxy %s %s", from, to))) + if err != nil { + return "nil", err + } + return bufio.NewReader(nc).ReadString('\n') +} + +// GetTwemproxyBackends get nosqlproxy servers +func GetTwemproxyBackends(ip string, port int) (segs map[string]string, err error) { + addr := fmt.Sprintf("%s:%d", ip, port+1000) + nc, err := net.DialTimeout("tcp", addr, time.Second) + if err != nil { + return nil, err + } + if segs, err = GetSegDetails(nc); err != nil { + return nil, err + } + return segs, nil +} + +// GetSegDetails echo stats |nc twempip port +func GetSegDetails(nc net.Conn) (map[string]string, error) { + _, err := nc.Write([]byte("stats")) + if err != nil { + return nil, err + } + reader := bufio.NewReader(nc) + segs := make(map[string]string) + for { + // rep, _, err := reader.ReadLine() + line, _, err := reader.ReadLine() + if err != nil { + if err == io.EOF { + break + } + return nil, err + } + strws := strings.Split(string(line), " ") + if len(strws) == 4 { + segs[strws[2]] = strws[0] + } + } + return segs, nil +} diff --git a/dbm-services/mongo/db-tools/dbactuator/pkg/util/redisutil.go b/dbm-services/mongo/db-tools/dbactuator/pkg/util/redisutil.go new file mode 100644 index 0000000000..f9c4e95fd5 --- /dev/null +++ b/dbm-services/mongo/db-tools/dbactuator/pkg/util/redisutil.go @@ -0,0 +1,102 @@ +package util + +import ( + "fmt" + "path/filepath" + "strings" + "time" + + "dbm-services/mongo/db-tools/dbactuator/mylog" + "dbm-services/mongo/db-tools/dbactuator/pkg/consts" + + "github.com/shirou/gopsutil/v3/mem" +) + +// GetTendisplusBlockcache 返回单位Mbyte +// 如果系统内存小于4GB,则 instBlockcache = 系统总内存 * 0.3 / 实例数 +// 否则 instBlockcache = 系统总内存 * 0.5 / 实例数 +func GetTendisplusBlockcache(instCount uint64) (instBlockcache uint64, err error) { + if instCount <= 0 { + err = fmt.Errorf("instCount==%d <=0", instCount) + return + } + var vMem *mem.VirtualMemoryStat + vMem, err = mem.VirtualMemory() + if err != nil { + err = fmt.Errorf("mem.VirtualMemory fail,err:%v", err) + return + } + if vMem.Total < 4*consts.GiByte { + instBlockcache = vMem.Total * 3 / (10 * instCount) + } else { + instBlockcache = vMem.Total * 5 / (10 * instCount) + } + if instBlockcache < 128*consts.MiByte { + instBlockcache = 128 * consts.MiByte + } + instBlockcache = instBlockcache / consts.MiByte + return +} + +// GetTendisplusWriteBufferSize 返回单位是Byte +// 如果系统内存小于8GB,则 writeBufferSize = 8MB,否则 writeBufferSize = 32MB +func GetTendisplusWriteBufferSize(instCount uint64) (writeBufferSize uint64, err error) { + if instCount <= 0 { + err = fmt.Errorf("instCount==%d <=0", instCount) + return + } + var vMem *mem.VirtualMemoryStat + vMem, err = mem.VirtualMemory() + if err != nil { + err = fmt.Errorf("mem.VirtualMemory fail,err:%v", err) + return + } + if vMem.Total <= 8*consts.GiByte { + writeBufferSize = 8 * consts.MiByte + } else { + writeBufferSize = 32 * consts.MiByte + } + return +} + +// StopBkDbmon 停止bk-dbmon +func StopBkDbmon() (err error) { + if FileExists(consts.BkDbmonBin) { + stopScript := filepath.Join(consts.BkDbmonPath, "stop.sh") + stopCmd := fmt.Sprintf("su %s -c '%s'", consts.MysqlAaccount, "sh "+stopScript) + mylog.Logger.Info(stopCmd) + _, err = RunLocalCmd("su", []string{consts.MysqlAaccount, "-c", "sh " + stopScript}, + "", nil, 1*time.Minute) + return + } + killCmd := ` +pid=$(ps aux|grep 'bk-dbmon --config'|grep -v dbactuator|grep -v grep|awk '{print $2}') +if [[ -n $pid ]] +then +kill $pid +fi +` + mylog.Logger.Info(killCmd) + _, err = RunBashCmd(killCmd, "", nil, 1*time.Minute) + return +} + +// StartBkDbmon start local bk-dbmon +func StartBkDbmon() (err error) { + startScript := filepath.Join(consts.BkDbmonPath, "start.sh") + if !FileExists(startScript) { + err = fmt.Errorf("%s not exists", startScript) + mylog.Logger.Error(err.Error()) + return + } + startCmd := fmt.Sprintf("su %s -c 'nohup %s &'", consts.MysqlAaccount, "sh "+startScript) + mylog.Logger.Info(startCmd) + _, err = RunLocalCmd("su", []string{consts.MysqlAaccount, "-c", "nohup sh " + startScript + " &"}, + "", nil, 1*time.Minute) + + if err != nil && strings.Contains(err.Error(), "no crontab for") { + return nil + } + + return +} diff --git a/dbm-services/mongo/db-tools/dbactuator/pkg/util/reflect.go b/dbm-services/mongo/db-tools/dbactuator/pkg/util/reflect.go new file mode 100644 index 0000000000..0698796aa2 --- /dev/null +++ b/dbm-services/mongo/db-tools/dbactuator/pkg/util/reflect.go @@ -0,0 +1,20 @@ +package util + +import ( + "reflect" + "runtime" +) + +// GetTypeName 获取接口类型名 +func GetTypeName(object interface{}) string { + t := reflect.TypeOf(object) + if t.Kind() == reflect.Ptr { + return "*" + t.Elem().Name() + } + return t.Name() +} + +// GetFunctionName 获取函数名 +func GetFunctionName(i interface{}) string { + return runtime.FuncForPC(reflect.ValueOf(i).Pointer()).Name() +} diff --git a/dbm-services/mongo/db-tools/dbactuator/pkg/util/util.go b/dbm-services/mongo/db-tools/dbactuator/pkg/util/util.go new file mode 100644 index 0000000000..836e907223 --- /dev/null +++ b/dbm-services/mongo/db-tools/dbactuator/pkg/util/util.go @@ -0,0 +1,254 @@ +// Package util 公共函数 +package util + +import ( + "errors" + "fmt" + "net" + "os" + "path/filepath" + "sort" + "strconv" + "strings" + "syscall" + "time" + + "dbm-services/mongo/db-tools/dbactuator/pkg/consts" + + "golang.org/x/sys/unix" +) + +// NotFound error +const NotFound = "not found" + +// NewNotFound .. +func NewNotFound() error { + return errors.New(NotFound) +} + +// IsNotFoundErr .. +func IsNotFoundErr(err error) bool { + if err.Error() == NotFound { + return true + } + return false +} + +// GetCurrentDirectory 获取当前二进制程序所在执行路径 +func GetCurrentDirectory() (string, error) { + dir, err := filepath.Abs(filepath.Dir(os.Args[0])) + if err != nil { + return dir, fmt.Errorf("convert absolute path failed, err: %+v", err) + } + dir = strings.Replace(dir, "\\", "/", -1) + return dir, nil +} + +// GetLocalIP 获得本地ip +func GetLocalIP() (string, error) { + var localIP string + var err error + addrs, err := net.InterfaceAddrs() + if err != nil { + return localIP, fmt.Errorf("GetLocalIP net.InterfaceAddrs fail,err:%v", err) + } + for _, addr := range addrs { + if ipnet, ok := addr.(*net.IPNet); ok && !ipnet.IP.IsLoopback() { + if ipnet.IP.To4() != nil { + localIP = ipnet.IP.String() + return localIP, nil + } + } + } + return localIP, fmt.Errorf("can't find local ip") +} + +// IsMountPoint Determine if a directory is a mountpoint, by comparing the device for the directory +// with the device for it's parent. If they are the same, it's not a mountpoint, if they're +// different, it is. +// reference: https://github.com/cnaize/kubernetes/blob/master/pkg/util/mount/mountpoint_unix.go#L29 +func IsMountPoint(file string) (bool, error) { + stat, err := os.Stat(file) + if err != nil { + return false, err + } + rootStat, err := os.Lstat(file + "/..") + if err != nil { + return false, err + } + // If the directory has the same device as parent, then it's not a mountpoint. + return stat.Sys().(*syscall.Stat_t).Dev != rootStat.Sys().(*syscall.Stat_t).Dev, nil +} + +// FindFirstMountPoint find first mountpoint in prefer order +func FindFirstMountPoint(paths ...string) (string, error) { + for _, path := range paths { + if _, err := os.Stat(path); err != nil { + if os.IsNotExist(err) { + continue + } + } + isMountPoint, err := IsMountPoint(path) + if err != nil { + return "", fmt.Errorf("check whether mountpoint failed, path: %s, err: %v", path, err) + } + if isMountPoint { + return path, nil + } + } + return "", fmt.Errorf("no available mountpoint found, choices: %#v", paths) +} + +// CheckPortIsInUse 检查端口是否被占用 +func CheckPortIsInUse(ip, port string) (inUse bool, err error) { + timeout := time.Second + conn, err := net.DialTimeout("tcp", net.JoinHostPort(ip, port), timeout) + if err != nil && strings.Contains(err.Error(), "connection refused") { + return false, nil + } else if err != nil { + return false, fmt.Errorf("net.DialTimeout fail,err:%v", err) + } + if conn != nil { + defer func(conn net.Conn) { + _ = conn.Close() + }(conn) + return true, nil + } + return false, nil +} + +// IsValidIP 判断字符串是否是一个有效IP +func IsValidIP(ipStr string) bool { + if net.ParseIP(ipStr) == nil { + return false + } + return true +} + +// MkDirsIfNotExists 如果目录不存在则创建 +func MkDirsIfNotExists(dirs []string) error { + return MkDirsIfNotExistsWithPerm(dirs, 0755) +} + +// MkDirsIfNotExistsWithPerm 如果目录不存在则创建，并指定文件Perm +func MkDirsIfNotExistsWithPerm(dirs []string, perm os.FileMode) error { + for _, dir := range dirs { + _, err := os.Stat(dir) + if err == nil { + continue + } + if os.IsNotExist(err) == true { + err = os.MkdirAll(dir, perm) + if err != nil { + return fmt.Errorf("MkdirAll fail,err:%v,dir:%s", err, dirs) + } + } + } + return nil +} + +// IsExecOwner owner是否可执行 +func IsExecOwner(mode os.FileMode) bool { + return mode&0100 != 0 +} + +// IsExecGroup grouper是否可执行 +func IsExecGroup(mode os.FileMode) bool { + return mode&0010 != 0 +} + +// IsExecOther other是否可执行 +func IsExecOther(mode os.FileMode) bool { + return mode&0001 != 0 +} + +// IsExecAny owner/grouper/other 任意一个可执行 +func IsExecAny(mode os.FileMode) bool { + return mode&0111 != 0 +} + +// IsExecAll owner/grouper/other 全部可执行 +func IsExecAll(mode os.FileMode) bool { + return mode&0111 == 0111 +} + +// LocalDirChownMysql 改变localDir的属主为mysql +func LocalDirChownMysql(localDir string) (err error) { + cmd := fmt.Sprintf("chown -R %s.%s %s", consts.MysqlAaccount, consts.MysqlGroup, localDir) + _, err = RunBashCmd(cmd, "", nil, 1*time.Hour) + return +} + +// HostDiskUsage 本地路径所在磁盘使用情况 +type HostDiskUsage struct { + TotalSize uint64 `json:"ToTalSize"` // bytes + UsedSize uint64 `json:"UsedSize"` // bytes + AvailSize uint64 `json:"AvailSize"` // bytes + UsageRatio int `json:"UsageRatio"` +} + +// String 用于打印 +func (disk *HostDiskUsage) String() string { + ret := fmt.Sprintf("total_size=%dMB,used_size=%d,avail_size=%d,Use=%d%%", + disk.TotalSize/1024/1024, + disk.UsedSize/1024/1024, + disk.AvailSize/1024/1024, + disk.UsageRatio) + return ret +} + +// GetLocalDirDiskUsg 获取本地路径所在磁盘使用情况 +// 参考: +// https://stackoverflow.com/questions/20108520/get-amount-of-free-disk-space-using-go +// http://evertrain.blogspot.com/2018/05/golang-disk-free.html +func GetLocalDirDiskUsg(localDir string) (diskUsg HostDiskUsage, err error) { + var stat unix.Statfs_t + if err = unix.Statfs(localDir, &stat); err != nil { + err = fmt.Errorf("unix.Statfs fail,err:%v,localDir:%s", err, localDir) + return + } + diskUsg.TotalSize = stat.Blocks * uint64(stat.Bsize) + diskUsg.AvailSize = stat.Bavail * uint64(stat.Bsize) + diskUsg.UsedSize = (stat.Blocks - stat.Bfree) * uint64(stat.Bsize) + diskUsg.UsageRatio = int(diskUsg.UsedSize * 100 / diskUsg.TotalSize) + return +} + +// GetFileSize 获取文件大小(单位byte) +func GetFileSize(filename string) (size int64, err error) { + fileInfo, err := os.Stat(filename) + if err != nil { + err = fmt.Errorf("file:%s os.Stat fail,err:%v", filename, err) + return + } + return fileInfo.Size(), nil +} + +// IntSliceInter 两个[]int 求交集 +func IntSliceInter(list01, list02 []int) []int { + m01 := make(map[int]bool) + m02 := make(map[int]bool) + for _, item01 := range list01 { + m01[item01] = true + } + for _, item02 := range list02 { + m02[item02] = true + } + ret := []int{} + for item01 := range m01 { + if _, ok := m02[item01]; ok == true { + ret = append(ret, item01) + } + } + sort.Ints(ret) + return ret +} + +// IntSliceToString 将[]int join,返回一个字符串 +func IntSliceToString(src []int, seq string) string { + strList := []string{} + for _, item := range src { + strList = append(strList, strconv.Itoa(item)) + } + return strings.Join(strList, seq) +} diff --git a/dbm-services/mongo/db-tools/dbactuator/pkg/util/version.go b/dbm-services/mongo/db-tools/dbactuator/pkg/util/version.go new file mode 100644 index 0000000000..8e50739841 --- /dev/null +++ b/dbm-services/mongo/db-tools/dbactuator/pkg/util/version.go @@ -0,0 +1,119 @@ +package util + +import ( + "fmt" + "regexp" + "strconv" + "strings" + "time" + + "dbm-services/mongo/db-tools/dbactuator/mylog" +) + +func convertVersionToUint(version string) (total uint64, err error) { + version = strings.TrimSpace(version) + if version == "" { + return 0, nil + } + list01 := strings.Split(version, ".") + var billion string + var thousand string + var single string + if len(list01) == 0 { + err = fmt.Errorf("version:%s format not correct", version) + mylog.Logger.Error(err.Error()) + return 0, err + } + billion = list01[0] + if len(list01) >= 2 { + thousand = list01[1] + } + if len(list01) >= 3 { + single = list01[2] + } + + if billion != "" { + b, err := strconv.ParseUint(billion, 10, 64) + if err != nil { + err = fmt.Errorf("convertVersionToUint strconv.ParseUint fail,err:%v,billion:%s,version:%s", err, billion, version) + mylog.Logger.Error(err.Error()) + return 0, err + } + total += b * 1000000 + } + if thousand != "" { + t, err := strconv.ParseUint(thousand, 10, 64) + if err != nil { + err = fmt.Errorf("convertVersionToUint strconv.ParseUint fail,err:%v,thousand:%s,version:%s", err, thousand, version) + mylog.Logger.Error(err.Error()) + return 0, err + } + total += t * 1000 + } + if single != "" { + s, err := strconv.ParseUint(single, 10, 64) + if err != nil { + err = fmt.Errorf("convertVersionToUint strconv.ParseUint fail,err:%v,single:%s,version:%s", err, single, version) + mylog.Logger.Error(err.Error()) + return 0, err + } + total += s + } + return total, nil +} + +// VersionParse tendis版本解析 +/* + * VersionParse + * 2.8.17-TRedis-v1.2.20, baseVersion: 2008017,subVersion:1002020 + * 6.2.7,baseVersion: 6002007 + */ +func VersionParse(version string) (baseVersion, subVersion uint64, err error) { + reg01 := regexp.MustCompile(`[\d+.]+`) + rets := reg01.FindAllString(version, -1) + if len(rets) == 0 { + err = fmt.Errorf("TendisVersionParse version:%s format not correct", version) + mylog.Logger.Error(err.Error()) + return 0, 0, err + } + if len(rets) >= 1 { + baseVersion, err = convertVersionToUint(rets[0]) + if err != nil { + return 0, 0, err + } + } + if len(rets) >= 2 { + subVersion, err = convertVersionToUint(rets[1]) + if err != nil { + return 0, 0, err + } + } + + return baseVersion, subVersion, nil +} + +// RedisCliVersion redis-cli 的版本解析 +func RedisCliVersion(cliBin string) (baseVersion, subVersion uint64, err error) { + cmd := cliBin + " -v" + verRet, err := RunBashCmd(cmd, "", nil, 20*time.Second) + if err != nil { + return + } + baseVersion, subVersion, err = VersionParse(verRet) + if err != nil { + return + } + return +} + +// IsCliSupportedNoAuthWarning redis-cli 是否支持 --no-auth-warning参数 +func IsCliSupportedNoAuthWarning(cliBin string) bool { + bVer, _, err := RedisCliVersion(cliBin) + if err != nil { + return false + } + if bVer > 6000000 { + return true + } + return false +} diff --git a/dbm-services/mongo/db-tools/dbactuator/scripts/upload.sh b/dbm-services/mongo/db-tools/dbactuator/scripts/upload.sh new file mode 100644 index 0000000000..ff9e93e222 --- /dev/null +++ b/dbm-services/mongo/db-tools/dbactuator/scripts/upload.sh @@ -0,0 +1,182 @@ +#!/usr/bin/env bash + +# 安全模式 +set -euo pipefail + +# 重置PATH +PATH=/usr/local/sbin:/usr/sbin:/usr/bin:/sbin:/bin +export PATH + +# 通用脚本框架变量 +PROGRAM=$(basename "$0") +EXITCODE=0 + +BKREPO_USER= +BKREPO_PASSWORD= +BKREPO_API=http://127.0.0.1:8080 #介质库https地址 +BKREPO_PROJECT=generic # 项目代号 +BKREPO_NAME=bk-dbm # 仓库名字，默认自定义仓库 +DOWNLOAD_DIR=/tmp # 下载文件的默认路径：/tmp +BKREPO_METHOD=GET # 默认为下载 +BKREPO_PUT_OVERWRITE=true # 上传时是否覆盖仓库 +REMOTE_PATH= +declare -a REMOTE_FILE=() # 下载的文件列表 +declare -a UPLOAD_FILE=() # 上传的文件列表 + +trap 'rm -f /tmp/bkrepo_tool.*.log' EXIT +usage () { + cat < -p [ -d ] -r /devops/path1 -r /devops/path2 ... + $PROGRAM -u -p -X PUT -T local_file_path1 -T local_file_path2 -R remote_path + [ -u, --user [必填] "指定访问bkrepo的api用户名" ] + [ -p, --password [必填] "指定访问bkrepo的api密码" ] + [ -i, --url [必填] "指定访问bkrepo的url，默认是$BKREPO_API" ] + [ -r, --remote-file [必填] "指定下载的远程文件路径路径" ] + [ -n, --repo [选填] "指定项目的仓库名字，默认为$BKREPO_NAME" ] + [ -P, --project [选填] "指定项目名字，默认为blueking" ] + [ -d, --dir [选填] "指定下载制品库文件的存放文件夹，若不指定，则为/tmp" ] + [ -X, --method [选填] "默认为下载（GET），可选PUT，为上传" ] + + -X PUT时，以下参数生效: + [ -T, --upload-file [必填] "指定需要上传的本机文件路径" ] + [ -R, --remote-path [必填] "指定上传到的仓库目录的路径" ] + [ -O, --override [选填] "指定上传同名文件是否覆盖" ] + [ -h --help -? 查看帮助 ] +EOF +} + +usage_and_exit () { + usage + exit "$1" +} + +log () { + echo "$@" +} + +error () { + echo "$@" 1>&2 + usage_and_exit 1 +} + +warning () { + echo "$@" 1>&2 + EXITCODE=$((EXITCODE + 1)) +} + +# 解析命令行参数，长短混合模式 +(( $# == 0 )) && usage_and_exit 1 +while (( $# > 0 )); do + case "$1" in + -u | --user ) + shift + BKREPO_USER=$1 + ;; + -p | --password) + shift + BKREPO_PASSWORD=$1 + ;; + -i | --url) + shift + BKREPO_API=$1 + ;; + -d | --dir ) + shift + DOWNLOAD_DIR=$1 + ;; + -n | --name ) + shift + BKREPO_NAME=$1 + ;; + -P | --project ) + shift + BKREPO_PROJECT=$1 + ;; + -r | --remote-file ) + shift + REMOTE_FILE+=("$1") + ;; + -T | --upload-file ) + shift + UPLOAD_FILE+=("$1") + ;; + -O | --override) + BKREPO_PUT_OVERWRITE=true + ;; + -R | --remote-path ) + shift + REMOTE_PATH=$1 + ;; + -X | --method ) + shift + BKREPO_METHOD=$1 + ;; + --help | -h | '-?' ) + usage_and_exit 0 + ;; + -*) + error "不可识别的参数: $1" + ;; + *) + break + ;; + esac + shift +done + +if [[ -z "$BKREPO_USER" || -z "$BKREPO_PASSWORD" ]]; then + warning "-u, -p must not be empty" +fi + +if (( EXITCODE > 0 )); then + usage_and_exit "$EXITCODE" +fi + +case $BKREPO_METHOD in + GET ) + if ! [[ -d "$DOWNLOAD_DIR" ]]; then + mkdir -p "$DOWNLOAD_DIR" + fi + + cd "$DOWNLOAD_DIR" || { echo "can't change into $DOWNLOAD_DIR"; exit 1; } + + for remote_file in "${REMOTE_FILE[@]}"; do + echo "start downloading $remote_file ..." + curl -X "$BKREPO_METHOD" -sLO -u "$BKREPO_USER:$BKREPO_PASSWORD" "${BKREPO_API}/${BKREPO_PROJECT}/$BKREPO_NAME/$remote_file" + rt=$? + if [[ $rt -eq 0 ]]; then + echo "download $remote_file finished in $DOWNLOAD_DIR/${remote_file##*/}" + else + echo "download $remote_file with error code: <$rt>" + fi + done + ;; + PUT ) + for local_file in "${UPLOAD_FILE[@]}"; do + if [[ -r "$local_file" ]]; then + local_file_md5=$(md5sum "$local_file" | awk '{print $1}') + local_file_name=$(basename "$local_file") + http_code=$(curl -s -o /tmp/bkrepo_tool.$$.log -w "%{http_code}" \ + -u "$BKREPO_USER:$BKREPO_PASSWORD" "${BKREPO_API}/${BKREPO_PROJECT}/${BKREPO_NAME}/$REMOTE_PATH/$local_file_name" \ + -T "$local_file" \ + -H "X-BKREPO-OVERWRITE: $BKREPO_PUT_OVERWRITE" \ + -H "X-BKREPO-MD5: $local_file_md5" + ) + if [[ $http_code -eq 200 ]]; then + echo "upload $local_file to $REMOTE_PATH succeed" + else + echo "upload $local_file to $REMOTE_PATH failed" + echo "http response is: $( bool: name=kwargs["name"], immute_domain=kwargs["immute_domain"], alias=kwargs["alias"], + db_module_id=kwargs["db_module_id"], major_version=kwargs["major_version"], proxies=kwargs["proxies"], configs=kwargs["configs"], diff --git a/dbm-ui/backend/flow/plugins/components/collections/mongodb/exec_actuator_job.py b/dbm-ui/backend/flow/plugins/components/collections/mongodb/exec_actuator_job.py index 3912c8fe68..4b0e40a683 100644 --- a/dbm-ui/backend/flow/plugins/components/collections/mongodb/exec_actuator_job.py +++ b/dbm-ui/backend/flow/plugins/components/collections/mongodb/exec_actuator_job.py @@ -120,6 +120,12 @@ def _execute(self, data, parent_data) -> bool: kwargs["db_act_template"]["payload"]["adminUsername"] ] + # cluster添加shards，从上游流程节点获取密码 + if kwargs.get("add_shard_to_cluster", False): + kwargs["db_act_template"]["payload"]["adminPassword"] = trans_data[ + kwargs["db_act_template"]["payload"]["adminUsername"] + ] + # 拼接节点执行ip所需要的信息，ip信息统一用list处理拼接 if kwargs["get_trans_data_ip_var"]: exec_ips = self.splice_exec_ips_list( diff --git a/dbm-ui/backend/flow/utils/mongodb/calculate_cluster.py b/dbm-ui/backend/flow/utils/mongodb/calculate_cluster.py index 4c69a808f8..807069f113 100644 --- a/dbm-ui/backend/flow/utils/mongodb/calculate_cluster.py +++ b/dbm-ui/backend/flow/utils/mongodb/calculate_cluster.py @@ -12,7 +12,206 @@ from backend.configuration.constants import AffinityEnum from backend.db_meta.enums.cluster_type import ClusterType -from backend.flow.consts import MongoDBDomainPrefix, MongoDBTotalCache +from backend.flow.consts import MongoDBClusterDefaultPort, MongoDBDomainPrefix, MongoDBTotalCache + + +def machine_order_by_tolerance(disaster_tolerance_level: str, machine_set: list) -> list: + """通过容灾级别获取机器顺序""" + + machines = [] + # 主从节点分布在不同的机房 + if disaster_tolerance_level == AffinityEnum.CROS_SUBZONE: + mongo_machine_set = deepcopy(machine_set) + machines.append(mongo_machine_set[0]) + mongo_machine_set.remove(mongo_machine_set[0]) + for machine in mongo_machine_set: + if machine["sub_zone_id"] != machines[0]["sub_zone_id"]: + machines.append(machine) + break + mongo_machine_set.remove(machines[1]) + machines.extend(mongo_machine_set) + # 主从节点分布在相同的机房 + elif disaster_tolerance_level == AffinityEnum.SAME_SUBZONE: + machines = machine_set + return machines + + +def replicase_calc(payload: dict, payload_clusters: dict, app: str, domain_prefix: list) -> dict: + """replicase进行计算""" + + payload_clusters["spec_id"] = payload["spec_id"] + payload_clusters["spec_config"] = payload["infos"][0]["resource_spec"]["spec_config"] + # 获取全部主机 + hosts = [] + for info in payload["infos"]: + for machine in info["mongo_machine_set"]: + hosts.append({"ip": machine["ip"], "bk_cloud_id": machine["bk_cloud_id"]}) + payload_clusters["hosts"] = hosts + # 获取复制集实例 + sets = [] + node_replica_count = payload["node_replica_count"] + port = payload["start_port"] + oplog_percent = payload["oplog_percent"] / 100 + data_disk = "/data1" + # 计算cacheSizeGB和oplogSizeMB bk_mem:MB ["/data1"]["size"]:GB + avg_mem_size_gb = int( + payload["infos"][0]["mongo_machine_set"][0]["bk_mem"] + * MongoDBTotalCache.Cache_Percent + / node_replica_count + / 1024 + ) + if payload["infos"][0]["mongo_machine_set"][0]["storage"].get("/data1"): + data_disk = "/data1" + elif payload["infos"][0]["mongo_machine_set"][0]["storage"].get("/data"): + data_disk = "/data" + oplog_size_mb = int( + payload["infos"][0]["mongo_machine_set"][0]["storage"].get(data_disk)["size"] + * 1024 + * oplog_percent + / node_replica_count + ) + # 分配机器 + for index, info in enumerate(payload["infos"]): + # 通过容灾获取机器顺序 + machines = machine_order_by_tolerance(payload["disaster_tolerance_level"], info["mongo_machine_set"]) + # 获取机器对应的多个复制集 + replica_sets = payload["replica_sets"][index * node_replica_count : node_replica_count * (index + 1)] + + for replica_set_index, replica_set in enumerate(replica_sets): + skip_machine = True + if replica_set_index == 0: + skip_machine = False + nodes = [] + for machine_index, machine in enumerate(machines): + if machine_index == len(machines) - 1: + domain = "{}.{}.{}.db".format(domain_prefix[-1], replica_set["set_id"], app) + else: + domain = "{}.{}.{}.db".format(domain_prefix[machine_index], replica_set["set_id"], app) + nodes.append({"ip": machine["ip"], "bk_cloud_id": machine["bk_cloud_id"], "domain": domain}) + sets.append( + { + "set_id": replica_set["set_id"], + "alias": replica_set["name"], + "port": port, + "key_file": "{}-{}".format(app, replica_set["set_id"]), + "cacheSizeGB": avg_mem_size_gb, + "oplogSizeMB": oplog_size_mb, + "skip_machine": skip_machine, + "nodes": nodes, + } + ) + port += 1 + payload_clusters["sets"] = sets + return payload_clusters + + +def cluster_calc(payload: dict, payload_clusters: dict, app: str) -> dict: + """cluster进行计算""" + + payload_clusters["alias"] = payload["cluster_alias"] + payload_clusters["cluster_id"] = payload["cluster_name"] + payload_clusters["machine_specs"] = payload["machine_specs"] + oplog_percent = payload["oplog_percent"] / 100 + disaster_tolerance_level = payload["disaster_tolerance_level"] + node_replica_count = int(payload["shard_num"] / payload["shard_machine_group"]) + payload_clusters["key_file"] = "{}-{}".format(app, payload["cluster_name"]) + config_port = MongoDBClusterDefaultPort.CONFIG_PORT.value # 设置常量 + shard_port = MongoDBClusterDefaultPort.SHARD_START_PORT.value # 以这个27001开始 + shard_port_not_use = [payload["proxy_port"], config_port] + + # 计算configCacheSizeGB，shardCacheSizeGB，oplogSizeMB + shard_avg_mem_size_gb = int( + payload["nodes"]["mongodb"][0][0]["bk_mem"] * MongoDBTotalCache.Cache_Percent / node_replica_count / 1024 + ) + config_mem_size_gb = int( + payload["nodes"]["mongo_config"][0]["bk_mem"] * MongoDBTotalCache.Cache_Percent / node_replica_count / 1024 + ) + # shard oplogSizeMB + if payload["nodes"]["mongodb"][0][0]["storage"].get("/data1"): + data_disk = "/data1" + elif payload["nodes"]["mongodb"][0][0]["storage"].get("/data"): + data_disk = "/data" + shard_oplog_size_mb = int( + payload["nodes"]["mongodb"][0][0]["storage"].get(data_disk)["size"] * 1024 * oplog_percent / node_replica_count + ) + # config oplogSizeMB + if payload["nodes"]["mongo_config"][0]["storage"].get("/data1"): + data_disk = "/data1" + elif payload["nodes"]["mongo_config"][0]["storage"].get("/data"): + data_disk = "/data" + config_oplog_size_mb = int( + payload["nodes"]["mongo_config"][0]["storage"].get(data_disk)["size"] * 1024 * oplog_percent + ) + + # 获取全部主机 + hosts = [] + # mongo_config + for machine in payload["nodes"]["mongo_config"]: + hosts.append({"ip": machine["ip"], "bk_cloud_id": machine["bk_cloud_id"]}) + # mongodb + for machines in payload["nodes"]["mongodb"]: + for machine in machines: + hosts.append({"ip": machine["ip"], "bk_cloud_id": machine["bk_cloud_id"]}) + # mongos + for machine in payload["nodes"]["mongos"]: + hosts.append({"ip": machine["ip"], "bk_cloud_id": machine["bk_cloud_id"]}) + payload_clusters["hosts"] = hosts + + # 分配机器 + # mongo_config + config = {} + config["set_id"] = "{}-{}".format(payload["cluster_name"], "conf") # 设置常量 + config["port"] = config_port # 设置常量 + config["cacheSizeGB"] = config_mem_size_gb + config["oplogSizeMB"] = config_oplog_size_mb + machines = machine_order_by_tolerance(disaster_tolerance_level, payload["nodes"]["mongo_config"]) + config["nodes"] = [] + for machine in machines: + config["nodes"].append({"ip": machine["ip"], "bk_cloud_id": machine["bk_cloud_id"]}) + payload_clusters["config"] = config + # shards + # 获取shard的id，port + shard_info = [] + add_shards = {} + for i in range(payload["shard_num"]): + if shard_port in shard_port_not_use: + shard_port += 1 + shard_info.append( + { + "set_id": "{}-s{}".format(payload["cluster_name"], str(i + 1)), + "port": shard_port, + "cacheSizeGB": shard_avg_mem_size_gb, + "oplogSizeMB": shard_oplog_size_mb, + } + ) + shard_port += 1 + shards = [] + for index, machine_set in enumerate(payload["nodes"]["mongodb"]): + # 通过容灾获取机器顺序 + machines = machine_order_by_tolerance(payload["disaster_tolerance_level"], machine_set) + # 获取机器对应的多个复制集 + replica_sets = shard_info[index * node_replica_count : node_replica_count * (index + 1)] + for replica_set in replica_sets: + nodes = [{"ip": machine["ip"], "bk_cloud_id": machine["bk_cloud_id"]} for machine in machines] + replica_set["nodes"] = nodes + shards.append(replica_set) + add_shards["{}-{}".format(app, replica_set["set_id"])] = ",".join( + ["{}:{}".format(node["ip"], str(replica_set["port"])) for node in nodes[0:-1]] + ) + + payload_clusters["shards"] = shards + payload_clusters["add_shards"] = add_shards + + # mongos + mongos = {} + mongos["port"] = payload["proxy_port"] # 默认27021 + mongos["set_id"] = payload["cluster_name"] + mongos["domain"] = "mongos.{}.{}.db".format(payload["cluster_name"], app) + nodes = [{"ip": machine["ip"], "bk_cloud_id": machine["bk_cloud_id"]} for machine in payload["nodes"]["mongos"]] + mongos["nodes"] = nodes + payload_clusters["mongos"] = mongos + + return payload_clusters def calculate_cluster(payload: dict) -> dict: @@ -28,6 +227,8 @@ def calculate_cluster(payload: dict) -> dict: payload_clusters["app"] = payload["bk_app_abbr"] app = payload["bk_app_abbr"] payload_clusters["db_version"] = payload["db_version"] + cluster_type = payload["cluster_type"] + # 目前只支持11个节点 domain_prefix = [ MongoDBDomainPrefix.M1, @@ -43,80 +244,9 @@ def calculate_cluster(payload: dict) -> dict: MongoDBDomainPrefix.BACKUP, ] - if payload["cluster_type"] == ClusterType.MongoReplicaSet.value: - payload_clusters["spec_id"] = payload["spec_id"] - payload_clusters["spec_config"] = payload["infos"][0]["resource_spec"]["spec_config"] - # 获取全部主机 - hosts = [] - for info in payload["infos"]: - for machine in info["mongo_machine_set"]: - hosts.append({"ip": machine["ip"], "bk_cloud_id": machine["bk_cloud_id"]}) - payload_clusters["hosts"] = hosts - # 获取复制集实例 - sets = [] - node_replica_count = payload["node_replica_count"] - print("node_replica_count") - print(node_replica_count) - port = payload["start_port"] - oplog_percent = payload["oplog_percent"] / 100 - data_disk = "/data1" - # 计算cacheSizeGB和oplogSizeMB bk_mem:MB ["/data1"]["size"]:GB - avg_mem_size_gb = int( - payload["infos"][0]["mongo_machine_set"][0]["bk_mem"] - * MongoDBTotalCache.Cache_Percent - / node_replica_count - / 1024 - ) - if payload["infos"][0]["mongo_machine_set"][0]["storage"].get("/data1"): - data_disk = "/data1" - elif payload["infos"][0]["mongo_machine_set"][0]["storage"].get("/data"): - data_disk = "/data" - oplog_size_mb = int( - payload["infos"][0]["mongo_machine_set"][0]["storage"].get(data_disk)["size"] * 1024 * oplog_percent - ) - # 分配机器 - for index, info in enumerate(payload["infos"]): - machines = [] - # 主从节点分布在不同的机房 - if payload["disaster_tolerance_level"] == AffinityEnum.CROS_SUBZONE: - mongo_machine_set = deepcopy(info["mongo_machine_set"]) - if machines: - machines.clear() - machines.append(mongo_machine_set[0]) - mongo_machine_set.remove(mongo_machine_set[0]) - for machine in mongo_machine_set: - if machine["sub_zone_id"] != machines[0]["sub_zone_id"]: - machines.append(machine) - break - mongo_machine_set.remove(machines[1]) - machines.extend(mongo_machine_set) - elif payload["disaster_tolerance_level"] == AffinityEnum.SAME_SUBZONE: - machines = info["mongo_machine_set"] - replica_sets = payload["replica_sets"][index * node_replica_count : node_replica_count * (index + 1)] - for replica_set_index, replica_set in enumerate(replica_sets): - skip_machine = True - if replica_set_index == 0: - skip_machine = False - nodes = [] - for machine_index, machine in enumerate(machines): - if machine_index == len(machines) - 1: - domain = "{}.{}.{}.db".format(domain_prefix[-1], replica_set["set_id"], app) - else: - domain = "{}.{}.{}.db".format(domain_prefix[machine_index], replica_set["set_id"], app) - nodes.append({"ip": machine["ip"], "bk_cloud_id": machine["bk_cloud_id"], "domain": domain}) - sets.append( - { - "set_id": replica_set["set_id"], - "alias": replica_set["name"], - "port": port, - "cacheSizeGB": avg_mem_size_gb, - "oplogSizeMB": oplog_size_mb, - "skip_machine": skip_machine, - "nodes": nodes, - } - ) - port += 1 - payload_clusters["sets"] = sets - return payload_clusters - elif "cluster_type" == ClusterType.MongoShardedCluster.value: - pass + result = {} + if cluster_type == ClusterType.MongoReplicaSet.value: + result = replicase_calc(payload, payload_clusters, app, domain_prefix) + elif cluster_type == ClusterType.MongoShardedCluster.value: + result = cluster_calc(payload, payload_clusters, app) + return result diff --git a/dbm-ui/backend/flow/utils/mongodb/mongodb_dataclass.py b/dbm-ui/backend/flow/utils/mongodb/mongodb_dataclass.py index e9ef52b067..650942800b 100644 --- a/dbm-ui/backend/flow/utils/mongodb/mongodb_dataclass.py +++ b/dbm-ui/backend/flow/utils/mongodb/mongodb_dataclass.py @@ -60,9 +60,9 @@ def __get_define_config(self, namespace: str, conf_file: str, conf_type: str) -> data = DBConfigApi.query_conf_item( params={ - "bk_biz_id": self.payload["bk_biz_id"], + "bk_biz_id": str(self.payload["bk_biz_id"]), "level_name": LevelName.APP, - "level_value": self.payload["bk_biz_id"], + "level_value": str(self.payload["bk_biz_id"]), "conf_file": conf_file, "conf_type": conf_type, "namespace": namespace, @@ -185,6 +185,7 @@ def get_install_mongod_kwargs(self, node: dict, cluster_role: str) -> dict: "instanceType": MediumEnum.MongoD, "app": self.payload["app"], "setId": self.replicaset_info["set_id"], + "keyFile": self.payload["key_file"], "auth": True, "clusterRole": cluster_role, "dbConfig": db_config, @@ -227,6 +228,7 @@ def get_install_mongos_kwargs(self, node: dict) -> dict: "instanceType": MediumEnum.MongoS, "app": self.payload["app"], "setId": self.mongos_info["set_id"], + "keyFile": self.payload["key_file"], "auth": True, "configDB": config_db, "dbConfig": db_config, @@ -275,10 +277,11 @@ def get_add_relationship_to_meta_kwargs(self, replicaset_info: dict) -> dict: """添加replicaset关系到meta的kwargs""" info = { - "bk_biz_id": int(self.payload["bk_biz_id"]), + "bk_biz_id": self.payload["bk_biz_id"], "major_version": self.payload["db_version"], "creator": self.payload["created_by"], "region": self.payload["city"], + "db_module_id": 0, } instance_role = [ InstanceRole.MONGO_M1, @@ -297,12 +300,12 @@ def get_add_relationship_to_meta_kwargs(self, replicaset_info: dict) -> dict: info["cluster_type"] = ClusterType.MongoReplicaSet.value info["skip_machine"] = replicaset_info["skip_machine"] info["immute_domain"] = replicaset_info["nodes"][0]["domain"] - info["name"] = "{}-{}".format(self.payload["app"], replicaset_info["set_id"]) + info["name"] = replicaset_info["set_id"] info["alias"] = replicaset_info["alias"] info["spec_id"] = self.payload["spec_id"] info["spec_config"] = self.payload["spec_config"] info["bk_cloud_id"] = replicaset_info["nodes"][0]["bk_cloud_id"] - info["db_module_id"] = 0 + # 复制集节点 info["storages"] = [] if len(replicaset_info["nodes"]) <= 11: @@ -327,7 +330,7 @@ def get_add_relationship_to_meta_kwargs(self, replicaset_info: dict) -> dict: ) elif self.payload["cluster_type"] == ClusterType.MongoShardedCluster.value: info["cluster_type"] = ClusterType.MongoShardedCluster.value - info["name"] = "{}-{}".format(self.payload["app"], self.payload["config"]["set_id"]) + info["name"] = self.payload["cluster_id"] info["alias"] = self.payload["alias"] info["bk_cloud_id"] = self.payload["config"]["nodes"][0]["bk_cloud_id"] info["machine_specs"] = self.payload["machine_specs"] @@ -338,22 +341,27 @@ def get_add_relationship_to_meta_kwargs(self, replicaset_info: dict) -> dict: ] # config info["configs"] = [] - # TODO config name - for index, node in enumerate(self.payload["config"]["nodes"]): - if index == len(self.payload["config"]["nodes"]) - 1: - info["configs"].append( - {"ip": node["ip"], "port": self.payload["config"]["port"], "role": instance_role[-1]} - ) - else: - info["configs"].append( - {"ip": node["ip"], "port": self.payload["config"]["port"], "role": instance_role[index]} - ) + config = { + "shard": self.payload["config"]["set_id"], + "nodes": [], + } + if len(self.payload["config"]["nodes"]) <= 11: + for index, node in enumerate(self.payload["config"]["nodes"]): + if index == len(self.payload["config"]["nodes"]) - 1: + config["nodes"].append( + {"ip": node["ip"], "port": self.payload["config"]["port"], "role": instance_role[-1]} + ) + else: + config["nodes"].append( + {"ip": node["ip"], "port": self.payload["config"]["port"], "role": instance_role[index]} + ) + info["configs"].append(config) # shard info["storages"] = [] for shard in self.payload["shards"]: storage = { - "shard": "{}-{}".format(self.payload["app"], shard["set_id"]), + "shard": shard["set_id"], "nodes": [], } if len(shard["nodes"]) <= 11: @@ -371,6 +379,7 @@ def get_add_relationship_to_meta_kwargs(self, replicaset_info: dict) -> dict: def get_add_domain_to_dns_kwargs(self, cluster: bool) -> dict: """添加域名到dns的kwargs""" + if not cluster: domains = [ { @@ -398,6 +407,27 @@ def get_add_domain_to_dns_kwargs(self, cluster: bool) -> dict: "domains": domains, } + def get_add_shard_to_cluster_kwargs(self) -> dict: + """把shard添加到cluster的kwargs""" + + return { + "set_trans_data_dataclass": CommonContext.__name__, + "get_trans_data_ip_var": None, + "add_shard_to_cluster": True, + "bk_cloud_id": self.payload["mongos"]["nodes"][0]["bk_cloud_id"], + "exec_ip": self.payload["mongos"]["nodes"][0]["ip"], + "db_act_template": { + "action": MongoDBActuatorActionEnum.AddShardToCluster, + "file_path": self.file_path, + "payload": { + "ip": self.payload["mongos"]["nodes"][0]["ip"], + "port": self.payload["mongos"]["port"], + "adminUsername": MediumEnum.DbaUser, + "shards": self.payload["add_shards"], + }, + }, + } + def get_init_exec_script_kwargs(self, script_type: str) -> dict: """通过执行脚本"""