diff --git a/dbm-services/mongodb/db-tools/dbactuator/pkg/atomjobs/atommongodb/mongo_deinstall.go b/dbm-services/mongodb/db-tools/dbactuator/pkg/atomjobs/atommongodb/mongo_deinstall.go index ea66de1e32..42c4f89a45 100644 --- a/dbm-services/mongodb/db-tools/dbactuator/pkg/atomjobs/atommongodb/mongo_deinstall.go +++ b/dbm-services/mongodb/db-tools/dbactuator/pkg/atomjobs/atommongodb/mongo_deinstall.go @@ -20,7 +20,7 @@ import ( type DeInstallConfParams struct { IP string `json:"ip" validate:"required"` Port int `json:"port" validate:"required"` - SetId string `json:"setId" validate:"required"` + SetId string `json:"setId"` NodeInfo []string `json:"nodeInfo" validate:"required"` // []string ip,ip 如果为复制集节点，则为复制集所有节点的ip；如果为mongos，则为mongos的ip InstanceType string `json:"instanceType" validate:"required"` // mongod mongos Force bool `json:"force"` // 不检查连接，强制卸载 @@ -111,12 +111,19 @@ func (d *DeInstall) Init(runtime *jobruntime.JobGenericRuntime) error { strPort := strconv.Itoa(d.ConfParams.Port) d.PortDir = filepath.Join(d.DataDir, "mongodata", strPort) d.DbpathDir = filepath.Join(d.DataDir, "mongodata", strPort, "db") - d.DbPathRenameDir = filepath.Join(d.DataDir, "mongodata", fmt.Sprintf("%s_%s_%d", - d.ConfParams.InstanceType, d.ConfParams.SetId, d.ConfParams.Port)) + if d.ConfParams.SetId == "" { + d.DbPathRenameDir = filepath.Join(d.DataDir, "mongodata", fmt.Sprintf("%s_%d", + d.ConfParams.InstanceType, d.ConfParams.Port)) + d.LogPathRenameDir = filepath.Join(d.BackupDir, "mongolog", fmt.Sprintf("%s_%d", + d.ConfParams.InstanceType, d.ConfParams.Port)) + } else { + d.DbPathRenameDir = filepath.Join(d.DataDir, "mongodata", fmt.Sprintf("%s_%s_%d", + d.ConfParams.InstanceType, d.ConfParams.SetId, d.ConfParams.Port)) + d.LogPathRenameDir = filepath.Join(d.BackupDir, "mongolog", fmt.Sprintf("%s_%s_%d", + d.ConfParams.InstanceType, d.ConfParams.SetId, d.ConfParams.Port)) + } d.IPInfo = strings.Join(d.ConfParams.NodeInfo, "|") d.LogPortDir = filepath.Join(d.BackupDir, "mongolog", strPort) - d.LogPathRenameDir = filepath.Join(d.BackupDir, "mongolog", fmt.Sprintf("%s_%s_%d", - d.ConfParams.InstanceType, d.ConfParams.SetId, d.ConfParams.Port)) // 进行校验 if err := d.checkParams(); err != nil { diff --git a/dbm-ui/backend/db_services/mongodb/autofix/__init__.py b/dbm-ui/backend/db_services/mongodb/autofix/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/dbm-ui/backend/db_services/mongodb/autofix/mongodb_autofix_ticket.py b/dbm-ui/backend/db_services/mongodb/autofix/mongodb_autofix_ticket.py new file mode 100644 index 0000000000..943474732a --- /dev/null +++ b/dbm-ui/backend/db_services/mongodb/autofix/mongodb_autofix_ticket.py @@ -0,0 +1,108 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. +Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +import datetime +import logging + +from django.utils import timezone +from django.utils.crypto import get_random_string +from django.utils.translation import ugettext_lazy as _ + +from backend.configuration.constants import DBType +from backend.configuration.models.dba import DBAdministrator +from backend.db_services.dbbase.constants import IpSource +from backend.db_services.redis.autofix.enums import AutofixStatus +from backend.db_services.redis.autofix.models import RedisAutofixCore +from backend.ticket.constants import TicketType +from backend.ticket.models import Ticket +from backend.utils.time import datetime2str + +logger = logging.getLogger("root") + + +def get_resource_spec(mongos_list: list, mongod_list: list) -> dict: + """获取申请机器规格信息，采用故障机与新机器园区相对应""" + + resource_spec = {} + for mongos in mongos_list: + resource_spec.update( + { + mongos["ip"]: { + "spec_id": mongos["spec_id"], + "count": 1, + "spec_config": mongos["spec_config"], + "Location_spec": {"city": mongos["city"], "sub_zone_ids": [mongos["bk_sub_zone_id"]]}, + } + } + ) + # mongod自愈 + for mongod in mongod_list: + resource_spec.update( + { + mongod["ip"]: { + "spec_id": mongod["spec_id"], + "count": 1, + "spec_config": mongod["spec_config"], + "Location_spec": {"city": mongod["city"], "sub_zone_ids": [mongod["bk_sub_zone_id"]]}, + } + } + ) + return resource_spec + + +def mongo_create_ticket(cluster: RedisAutofixCore, cluster_ids: list, mongos_list: list, mongod_list: list): + """mongodb自愈创建单据""" + + # 获取dba + try: + mongodb_dba = DBAdministrator.objects.get(bk_biz_id=cluster.bk_biz_id, db_type=DBType.MongoDB.value).users + except DBAdministrator.DoesNotExist: + # 如果不存在，则取默认值 + mongodb_dba = DBAdministrator.objects.get(bk_biz_id=0, db_type=DBType.MongoDB.value).users + + # 申请机器规格信息 + resource_spec = get_resource_spec(mongos_list, mongod_list) + if not resource_spec: + return + # 集群类型 + if mongos_list: + cluster_type = mongos_list[0]["cluster_type"] + if mongod_list: + cluster_type = mongod_list[0]["cluster_type"] + # 单据信息 + details = { + "ip_source": IpSource.RESOURCE_POOL.value, + "infos": [ + { + "cluster_ids": cluster_ids, + "immute_domain": cluster.immute_domain, + "bk_cloud_id": cluster.bk_cloud_id, + "bk_biz_id": cluster.bk_biz_id, + "resource_spec": resource_spec, + "cluster_type": cluster_type, + "mongos_list": mongos_list, + "mongod_list": mongod_list, + } + ], + } + + # 创建单据 + ticket = Ticket.create_ticket( + ticket_type=TicketType.MONGODB_AUTOFIX.value, + creator=mongodb_dba[0], + bk_biz_id=cluster.bk_biz_id, + remark=_("自动发起-自愈任务-{}".format(cluster.immute_domain)), + details=details, + ) + cluster.ticket_id = ticket.id + cluster.status_version = get_random_string(12) + cluster.deal_status = AutofixStatus.AF_WFLOW.value + cluster.update_at = datetime2str(datetime.datetime.now(timezone.utc)) + cluster.save(update_fields=["ticket_id", "status_version", "deal_status", "update_at"]) diff --git a/dbm-ui/backend/db_services/plugin/nameservice/clb.py b/dbm-ui/backend/db_services/plugin/nameservice/clb.py index dbdce7d71d..ecd0d13dbd 100644 --- a/dbm-ui/backend/db_services/plugin/nameservice/clb.py +++ b/dbm-ui/backend/db_services/plugin/nameservice/clb.py @@ -384,3 +384,37 @@ def immute_domain_clb_ip(cluster_id: int, creator: str, bind: bool) -> Dict[str, return response_fail(code=3, message=message) return response_ok() return response_ok() + + +def operate_part_target(cluster_id: int, ips: list, bind: bool) -> dict: + """绑定或解绑部分后端主机""" + + # 获取信息 + cluster = get_cluster_info(cluster_id=cluster_id) + clb = cluster["clusterentry_set"]["clb"] + clb_id = clb["clb_id"] + listener_id = clb["listener_id"] + clb_region = clb["clb_region"] + + # 进行请求，得到返回结果 + if bind: + output = NameServiceApi.clb_register_part_target( + { + "region": clb_region, + "loadbalancerid": clb_id, + "listenerid": listener_id, + "ips": ips, + }, + raw=True, + ) + else: + output = NameServiceApi.clb_deregister_part_target( + { + "region": clb_region, + "loadbalancerid": clb_id, + "listenerid": listener_id, + "ips": ips, + }, + raw=True, + ) + return output diff --git a/dbm-ui/backend/db_services/redis/autofix/bill.py b/dbm-ui/backend/db_services/redis/autofix/bill.py index 246560f5f1..4aa4e52bd8 100644 --- a/dbm-ui/backend/db_services/redis/autofix/bill.py +++ b/dbm-ui/backend/db_services/redis/autofix/bill.py @@ -22,8 +22,10 @@ from backend.configuration.models.dba import DBAdministrator from backend.db_meta.api.cluster.apis import query_cluster_by_hosts from backend.db_meta.enums import ClusterType, MachineType +from backend.db_meta.enums.instance_role import InstanceRole from backend.db_meta.models import Machine from backend.db_services.dbbase.constants import IpSource +from backend.db_services.mongodb.autofix.mongodb_autofix_ticket import mongo_create_ticket from backend.db_services.redis.util import is_support_redis_auotfix from backend.ticket.builders import BuilderFactory from backend.ticket.constants import TicketStatus, TicketType @@ -38,6 +40,7 @@ def generate_autofix_ticket(fault_clusters: QuerySet): + """自愈创建单据""" for cluster in fault_clusters: # 目前仅支持这三种架构 if not is_support_redis_auotfix(cluster.cluster_type): @@ -53,13 +56,41 @@ def generate_autofix_ticket(fault_clusters: QuerySet): continue fault_machines = json.loads(cluster.fault_machines) - redis_proxies, redis_slaves, cluster_ids = [], [], [cluster.cluster_id] + mongos_list, mongod_list, redis_proxies, redis_slaves, cluster_ids = [], [], [], [], [cluster.cluster_id] for fault_machine in fault_machines: fault_ip = fault_machine["ip"] fault_obj = Machine.objects.filter(ip=fault_ip, bk_biz_id=cluster.bk_biz_id).get() - fault_info = {"ip": fault_ip, "spec_id": fault_obj.spec_id, "bk_sub_zone": fault_obj.bk_sub_zone} + fault_info = { + "ip": fault_ip, + "spec_id": fault_obj.spec_id, + "bk_sub_zone": fault_obj.bk_sub_zone, + "bk_sub_zone_id": fault_obj.bk_sub_zone_id, + "city": fault_obj.bk_city.logical_city.name, + "instance_type": fault_machine["instance_type"], + "spec_config": fault_obj.spec_config, + "cluster_type": cluster.cluster_type, + } if fault_machine["instance_type"] in [MachineType.TWEMPROXY.value, MachineType.PREDIXY.value]: redis_proxies.append(fault_info) + elif fault_machine["instance_type"] == MachineType.MONGOS.value: + mongos_list.append(fault_info) + elif fault_machine["instance_type"] in [ + InstanceRole.MONGO_M1.value, + InstanceRole.MONGO_M2.value, + InstanceRole.MONGO_M3.value, + InstanceRole.MONGO_M4.value, + InstanceRole.MONGO_M5.value, + InstanceRole.MONGO_M6.value, + InstanceRole.MONGO_M7.value, + InstanceRole.MONGO_M8.value, + InstanceRole.MONGO_M9.value, + InstanceRole.MONGO_M10.value, + InstanceRole.MONGO_BACKUP.value, + ]: + mongod_list.append(fault_info) + if cluster.cluster_type == ClusterType.MongoReplicaSet.value: + clusters = query_cluster_by_hosts(hosts=[fault_ip]) + cluster_ids = [cls_obj["cluster_id"] for cls_obj in clusters] else: if fault_obj.cluster_type == ClusterType.TendisRedisInstance.value: clusters = query_cluster_by_hosts(hosts=[fault_ip]) @@ -72,10 +103,14 @@ def generate_autofix_ticket(fault_clusters: QuerySet): cluster.immute_domain, redis_proxies, redis_slaves ) ) + if mongos_list or mongod_list: + mongo_create_ticket(cluster, cluster_ids, mongos_list, mongod_list) + return create_ticket(cluster, cluster_ids, redis_proxies, redis_slaves) def create_ticket(cluster: RedisAutofixCore, cluster_ids: list, redis_proxies: list, redis_slaves: list): + """redis自愈创建单据""" details = { "ip_source": IpSource.RESOURCE_POOL.value, "infos": [ diff --git a/dbm-ui/backend/db_services/redis/util.py b/dbm-ui/backend/db_services/redis/util.py index 3c51a1919a..86bc204429 100644 --- a/dbm-ui/backend/db_services/redis/util.py +++ b/dbm-ui/backend/db_services/redis/util.py @@ -103,4 +103,6 @@ def is_support_redis_auotfix(cluster_type: str) -> bool: ClusterType.TendisPredixyTendisplusCluster.value, ClusterType.TendisPredixyRedisCluster.value, ClusterType.TendisRedisInstance.value, + ClusterType.MongoShardedCluster.value, + ClusterType.MongoReplicaSet.value, ] diff --git a/dbm-ui/backend/flow/consts.py b/dbm-ui/backend/flow/consts.py index 85e6c04243..15b9d4589b 100644 --- a/dbm-ui/backend/flow/consts.py +++ b/dbm-ui/backend/flow/consts.py @@ -1441,7 +1441,7 @@ class MongoDBInstanceType(str, StructuredEnum): MongoS = EnumField("mongos", _("mongos")) -class MongoDBDfaultAuthDB(str, StructuredEnum): +class MongoDBDefaultAuthDB(str, StructuredEnum): """mongodb 默认验证db""" AuthDB = EnumField("admin", _("admin")) diff --git a/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/mongodb_autofix.py b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/mongodb_autofix.py new file mode 100644 index 0000000000..1cbbd33843 --- /dev/null +++ b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/mongodb_autofix.py @@ -0,0 +1,185 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. +Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +import logging.config +from typing import Dict, Optional + +from backend.db_meta.enums import ClusterType +from backend.db_meta.models import AppCache +from backend.flow.utils.mongodb.mongodb_repo import MongoRepository + +from . import mongodb_cluster_autofix, mongodb_replace + +logger = logging.getLogger("flow") + + +class MongoAutofixFlow(object): + """MongoDB自愈flow""" + + def __init__(self, root_id: str, data: Optional[Dict]): + """ + 传入参数 + @param root_id : 任务流程定义的root_id + @param data : 单据传递过来的参数列表，是dict格式 + """ + + self.root_id = root_id + self.data = data + self.autofix_info = self.data["infos"][0] + + def get_public_data(self) -> Dict: + """参数公共部分""" + + bk_biz_id = self.autofix_info["bk_biz_id"] + return { + "bk_biz_id": bk_biz_id, + "uid": self.data["uid"], + "bk_app_abbr": AppCache.objects.get(bk_biz_id=bk_biz_id).db_app_abbr, + "created_by": self.data["created_by"], + "ticket_type": self.data["ticket_type"], + "db_version": "", + "infos": {"MongoReplicaSet": [], "MongoShardedCluster": []}, + } + + def shard_get_data(self) -> Dict: + """分片集群获取参数""" + + flow_parameter = self.get_public_data() + bk_cloud_id = self.autofix_info["bk_cloud_id"] + cluster_id = self.autofix_info["cluster_ids"][0] + cluster_info = MongoRepository().fetch_one_cluster(withDomain=False, id=cluster_id) + flow_parameter["db_version"] = cluster_info.major_version + config = cluster_info.get_config() + shards = cluster_info.get_shards() + cluster = {} + mongos_nodes = [] + mongo_config = [] + mongodb = [] + # 获取mongos参数 + for mongos in self.autofix_info["mongos_list"]: + target = self.autofix_info[mongos["ip"]][0] + target["spec_id"] = mongos["spec_id"] + mongos_nodes.append( + { + "ip": mongos["ip"], + "bk_cloud_id": bk_cloud_id, + "spec_id": mongos["spec_id"], + "down": True, + "spec_config": mongos["spec_config"], + "target": target, + "instances": [ + { + "cluster_id": cluster_id, + "cluster_name": cluster_info.name, + "db_version": cluster_info.major_version, + "domain": self.autofix_info["immute_domain"], + "port": int(cluster_info.get_mongos()[0].port), + } + ], + } + ) + for mongod in self.autofix_info["mongod_list"]: + target = self.autofix_info[mongod["ip"]][0] + target["spec_id"] = mongod["spec_id"] + ip_info = { + "ip": mongod["ip"], + "bk_cloud_id": bk_cloud_id, + "spec_id": mongod["spec_id"], + "down": True, + "spec_config": mongod["spec_config"], + "target": target, + "instances": [], + } + instances = [] + # config + for member in config.members: + if member.ip == mongod["ip"]: + instances.append( + { + "cluster_id": cluster_id, + "cluster_name": cluster_info.name, + "seg_range": config.set_name, + "db_version": cluster_info.major_version, + "port": member.port, + } + ) + break + if instances: + ip_info["instances"] = instances + mongo_config.append(ip_info) + continue + # shard + for shard in shards: + for member in shard: + if member.ip == mongod["ip"]: + instances.append( + { + "cluster_id": cluster_id, + "cluster_name": cluster_info.name, + "seg_range": shard.set_name, + "db_version": cluster_info.major_version, + "port": member.port, + } + ) + break + ip_info["instances"] = instances + mongodb.append(ip_info) + cluster["mongos"] = mongos_nodes + cluster["mongo_config"] = mongo_config + cluster["mongodb"] = mongodb + flow_parameter["infos"]["MongoShardedCluster"].append(cluster) + return flow_parameter + + def rs_get_data(self) -> Dict: + """副本集获取参数""" + + flow_parameter = self.get_public_data() + bk_cloud_id = self.autofix_info["bk_cloud_id"] + cluster_ids = self.autofix_info["cluster_ids"] + for mongod in self.autofix_info["mongod_list"]: + instances = [] + for cluster_id in cluster_ids: + cluster_info = MongoRepository().fetch_one_cluster(withDomain=True, id=cluster_id) + flow_parameter["db_version"] = cluster_info.major_version + for member in cluster_info.get_shards()[0].members: + if mongod["ip"] == member.ip: + instances.append( + { + "cluster_id": cluster_id, + "cluster_name": cluster_info.name, + "db_version": cluster_info.major_version, + "domain": member.domain, + "port": member.port, + } + ) + target = self.autofix_info[mongod["ip"]][0] + target["spec_id"] = mongod["spec_id"] + flow_parameter["infos"]["MongoReplicaSet"].append( + { + "ip": mongod["ip"], + "bk_cloud_id": bk_cloud_id, + "spec_id": mongod["spec_id"], + "down": True, + "spec_config": mongod["spec_config"], + "target": target, + "instances": instances, + } + ) + return flow_parameter + + def autofix(self): + """进行自愈""" + + # 副本集 + if self.autofix_info["cluster_type"] == ClusterType.MongoReplicaSet.value: + mongodb_replace.MongoReplaceFlow(self.root_id, self.rs_get_data()).multi_host_replace_flow() + # 分片集群 + elif self.autofix_info["cluster_type"] == ClusterType.MongoShardedCluster.value: + mongodb_cluster_autofix.MongoClusterAutofixFlow(self.root_id, self.shard_get_data()).cluster_autofix_flow() diff --git a/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/mongodb_cluster_autofix.py b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/mongodb_cluster_autofix.py new file mode 100644 index 0000000000..4f31d61358 --- /dev/null +++ b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/mongodb_cluster_autofix.py @@ -0,0 +1,95 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. +Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +import logging.config +from typing import Dict, Optional + +from backend.db_meta.enums.cluster_type import ClusterType +from backend.flow.consts import MongoDBClusterRole +from backend.flow.engine.bamboo.scene.common.builder import Builder +from backend.flow.engine.bamboo.scene.mongodb.sub_task.cluster_monogs_autofix import mongos_autofix +from backend.flow.engine.bamboo.scene.mongodb.sub_task.cluster_shard_autofix import shard_autofix +from backend.flow.utils.mongodb.mongodb_dataclass import ActKwargs + +logger = logging.getLogger("flow") + + +class MongoClusterAutofixFlow(object): + """MongoDB分片集群自愈flow""" + + def __init__(self, root_id: str, data: Optional[Dict]): + """ + 传入参数 + @param root_id : 任务流程定义的root_id + @param data : 单据传递过来的参数列表，是dict格式 + """ + + self.root_id = root_id + self.data = data + self.get_kwargs = ActKwargs() + self.get_kwargs.payload = data + self.get_kwargs.get_file_path() + + def cluster_autofix_flow(self): + """ + 分片集群自愈流程 + """ + + # 创建流程实例 + pipeline = Builder(root_id=self.root_id, data=self.data) + + # 获取mongos mongo_config mongodb自愈信息 + autofix_info = self.data["infos"][ClusterType.MongoShardedCluster.value][0] + autofix_mongos = autofix_info["mongos"] + autofix_mongo_config = autofix_info["mongo_config"] + autofix_mongodb = autofix_info["mongodb"] + + # mongos自愈并行 + if autofix_mongos: + sub_pipelines = [] + for mongos_info_by_ip in autofix_mongos: + for mongos_instance in mongos_info_by_ip["instances"]: + self.get_kwargs.db_instance = mongos_instance + sub_pipeline = mongos_autofix( + root_id=self.root_id, + ticket_data=self.data, + sub_sub_kwargs=self.get_kwargs, + info=mongos_info_by_ip, + ) + sub_pipelines.append(sub_pipeline) + pipeline.add_parallel_sub_pipeline(sub_pipelines) + + # mongo_config mongodb自愈并行 + sub_pipelines = [] + # 替换config 以ip为维度 + for config_info_by_ip in autofix_mongo_config: + sub_pipeline = shard_autofix( + root_id=self.root_id, + ticket_data=self.data, + sub_kwargs=self.get_kwargs, + info=config_info_by_ip, + cluster_role=MongoDBClusterRole.ConfigSvr.value, + ) + sub_pipelines.append(sub_pipeline) + # 替换shard 以ip为维度 + for shard_info_by_ip in autofix_mongodb: + sub_pipeline = shard_autofix( + root_id=self.root_id, + ticket_data=self.data, + sub_kwargs=self.get_kwargs, + info=shard_info_by_ip, + cluster_role=MongoDBClusterRole.ShardSvr.value, + ) + sub_pipelines.append(sub_pipeline) + if sub_pipelines: + pipeline.add_parallel_sub_pipeline(sub_pipelines) + + # 运行流程 + pipeline.run_pipeline() diff --git a/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/mongodb_instance_deinstall.py b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/mongodb_instance_deinstall.py new file mode 100644 index 0000000000..c83eadd457 --- /dev/null +++ b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/mongodb_instance_deinstall.py @@ -0,0 +1,77 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. +Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +import logging.config +from typing import Dict, Optional + +from django.utils.translation import ugettext as _ + +from backend.flow.engine.bamboo.scene.common.builder import Builder +from backend.flow.engine.bamboo.scene.mongodb.sub_task.instance_deinstall import instance_deinstall +from backend.flow.plugins.components.collections.mongodb.exec_actuator_job import ExecuteDBActuatorJobComponent +from backend.flow.plugins.components.collections.mongodb.send_media import ExecSendMediaOperationComponent +from backend.flow.utils.mongodb.mongodb_dataclass import ActKwargs + +logger = logging.getLogger("flow") + + +class MongoInstanceDeInstallFlow(object): + """MongoDB实例卸载flow""" + + def __init__(self, root_id: str, data: Optional[Dict]): + """ + 传入参数 + @param root_id : 任务流程定义的root_id + @param data : 单据传递过来的参数列表，是dict格式 + """ + + self.root_id = root_id + self.data = data + self.get_kwargs = ActKwargs() + self.get_kwargs.payload = data + self.get_kwargs.get_file_path() + + def multi_instance_deinstall_flow(self): + """ + multi instance deinstall流程 + """ + + # 创建流程实例 + pipeline = Builder(root_id=self.root_id, data=self.data) + + # 获取参数 + self.get_kwargs.get_host_instance_deinstall() + + # 介质下发——job的api可以多个IP并行执行 + kwargs = self.get_kwargs.get_send_media_kwargs(media_type="actuator") + pipeline.add_act( + act_name=_("MongoDB-介质下发"), act_component_code=ExecSendMediaOperationComponent.code, kwargs=kwargs + ) + + # 创建原子任务执行目录 + kwargs = self.get_kwargs.get_create_dir_kwargs() + pipeline.add_act( + act_name=_("MongoDB-创建原子任务执行目录"), act_component_code=ExecuteDBActuatorJobComponent.code, kwargs=kwargs + ) + + # 复制集整机替换——子流程并行 + sub_pipelines = [] + for instance in self.data["infos"]: + sub_pipline = instance_deinstall( + root_id=self.root_id, + ticket_data=self.data, + sub_kwargs=self.get_kwargs, + info=instance, + ) + sub_pipelines.append(sub_pipline) + pipeline.add_parallel_sub_pipeline(sub_flow_list=sub_pipelines) + + # 运行流程 + pipeline.run_pipeline() diff --git a/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/mongodb_pitr_restore.py b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/mongodb_pitr_restore.py index 002784f287..2e15054ee4 100644 --- a/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/mongodb_pitr_restore.py +++ b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/mongodb_pitr_restore.py @@ -26,7 +26,7 @@ from backend.flow.engine.bamboo.scene.mongodb.sub_task.pitr_restore_sub import PitrRestoreSubTask from backend.flow.engine.bamboo.scene.mongodb.sub_task.send_media import SendMedia from backend.flow.plugins.components.collections.mongodb.exec_actuator_job2 import ExecJobComponent2 -from backend.flow.utils.mongodb.mongodb_repo import MongoRepository, MongoDBCluster, MongoNode +from backend.flow.utils.mongodb.mongodb_repo import MongoDBCluster, MongoNode, MongoRepository from backend.flow.utils.mongodb.mongodb_script_template import prepare_recover_dir_script from backend.flow.utils.mongodb.mongodb_util import MongoUtil diff --git a/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/__init__.py b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/__init__.py index d2c4c25eba..a3a8f5eaa9 100644 --- a/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/__init__.py +++ b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/__init__.py @@ -19,6 +19,7 @@ from .exec_script import exec_script from .increase_mongod import increase_mongod from .increase_mongos import increase_mongos +from .instance_deinstall import instance_deinstall from .instance_restart import instance_restart from .mongod_replace import mongod_replace from .mongos_install import mongos_install diff --git a/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/cluster_mongod_autofix.py b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/cluster_mongod_autofix.py new file mode 100644 index 0000000000..70a33c7c7d --- /dev/null +++ b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/cluster_mongod_autofix.py @@ -0,0 +1,171 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. +Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + +from copy import deepcopy +from typing import Dict, Optional + +from django.utils.translation import ugettext as _ + +from backend.db_meta.enums.cluster_type import ClusterType +from backend.flow.consts import MongoDBClusterRole, MongoDBInstanceType, MongoDBManagerUser +from backend.flow.engine.bamboo.scene.common.builder import SubBuilder +from backend.flow.plugins.components.collections.mongodb.add_password_to_db import ( + ExecAddPasswordToDBOperationComponent, +) +from backend.flow.plugins.components.collections.mongodb.delete_password_from_db import ( + ExecDeletePasswordFromDBOperationComponent, +) +from backend.flow.plugins.components.collections.mongodb.exec_actuator_job import ExecuteDBActuatorJobComponent +from backend.flow.plugins.components.collections.mongodb.instance_deinstall_ticket import ( + ExecInstanceDeInstallTicketOperationComponent, +) +from backend.flow.utils.mongodb.mongodb_dataclass import ActKwargs + + +def mongod_autofix( + root_id: str, + ticket_data: Optional[Dict], + sub_sub_kwargs: ActKwargs, + cluster_role: str, + info: dict, +) -> SubBuilder: + """ + mongod自愈流程 + """ + + # 获取变量 + sub_sub_get_kwargs = deepcopy(sub_sub_kwargs) + + # 创建子流程 + sub_sub_pipeline = SubBuilder(root_id=root_id, data=ticket_data) + + # 获取参数 + down = info.get("down") # 机器是否down + new_node = info["target"] + sub_sub_get_kwargs.payload["app"] = sub_sub_get_kwargs.payload["bk_app_abbr"] + sub_sub_get_kwargs.replicaset_info = {} + sub_sub_get_kwargs.replicaset_info["port"] = sub_sub_get_kwargs.db_instance["port"] + sub_sub_get_kwargs.cluster_type = ClusterType.MongoShardedCluster.value + cluster_name = sub_sub_get_kwargs.db_instance["seg_range"] + sub_sub_get_kwargs.payload["cluster_type"] = ClusterType.MongoShardedCluster.value + sub_sub_get_kwargs.payload["set_id"] = sub_sub_get_kwargs.db_instance["seg_range"] + sub_sub_get_kwargs.payload["key_file"] = sub_sub_get_kwargs.get_conf( + cluster_name=sub_sub_get_kwargs.db_instance["cluster_name"] + )["key_file"] + sub_sub_get_kwargs.payload["config_nodes"] = [] + sub_sub_get_kwargs.payload["shards_nodes"] = [] + sub_sub_get_kwargs.payload["mongos_nodes"] = [] + # 获取配置 + conf = sub_sub_get_kwargs.get_conf(cluster_name=sub_sub_get_kwargs.db_instance["cluster_name"]) + if cluster_role == MongoDBClusterRole.ConfigSvr.value: + sub_sub_get_kwargs.payload["config_nodes"] = [ + { + "ip": info["ip"], + "domain": sub_sub_get_kwargs.db_instance.get("domain", ""), + "port": sub_sub_get_kwargs.db_instance["port"], + "bk_cloud_id": info["bk_cloud_id"], + } + ] + sub_sub_get_kwargs.replicaset_info["cacheSizeGB"] = conf["config_cacheSizeGB"] + sub_sub_get_kwargs.replicaset_info["oplogSizeMB"] = conf["config_oplogSizeMB"] + elif cluster_role == MongoDBClusterRole.ShardSvr.value: + shard_nodes = { + "nodes": [ + { + "ip": info["ip"], + "domain": sub_sub_get_kwargs.db_instance.get("domain", ""), + "port": sub_sub_get_kwargs.db_instance["port"], + "bk_cloud_id": info["bk_cloud_id"], + } + ] + } + sub_sub_get_kwargs.payload["shards_nodes"].append(shard_nodes) + # shard直接获取配置 + sub_sub_get_kwargs.replicaset_info["cacheSizeGB"] = conf["cacheSizeGB"] + sub_sub_get_kwargs.replicaset_info["oplogSizeMB"] = conf["oplogSizeMB"] + sub_sub_get_kwargs.replicaset_info["set_id"] = cluster_name + sub_sub_get_kwargs.replicaset_info["nodes"] = [ + { + "ip": new_node["ip"], + "domain": sub_sub_get_kwargs.db_instance.get("domain", ""), + "bk_cloud_id": info["bk_cloud_id"], + "port": sub_sub_get_kwargs.db_instance["port"], + } + ] + sub_sub_get_kwargs.payload["nodes"] = [ + { + "ip": info["ip"], + "domain": sub_sub_get_kwargs.db_instance.get("domain", ""), + "port": sub_sub_get_kwargs.db_instance["port"], + "bk_cloud_id": info["bk_cloud_id"], + "role": MongoDBInstanceType.MongoD.value, + } + ] + sub_sub_get_kwargs.payload["bk_cloud_id"] = info["bk_cloud_id"] + + # mognod安装 + kwargs = sub_sub_get_kwargs.get_install_mongod_kwargs(node=new_node, cluster_role=cluster_role) + sub_sub_pipeline.add_act( + act_name=_("MongoDB-mongod安装-{}:{}".format(new_node["ip"], str(sub_sub_get_kwargs.db_instance["port"]))), + act_component_code=ExecuteDBActuatorJobComponent.code, + kwargs=kwargs, + ) + + # mognod替换 + kwargs = sub_sub_get_kwargs.get_instance_replace_kwargs(info=info, source_down=down) + sub_sub_pipeline.add_act( + act_name=_("MongoDB-mongod替换"), + act_component_code=ExecuteDBActuatorJobComponent.code, + kwargs=kwargs, + ) + + # 密码服务修改密码 + # 添加新实例密码 + kwargs = sub_sub_get_kwargs.get_add_password_to_db_kwargs( + usernames=[ + MongoDBManagerUser.DbaUser.value, + MongoDBManagerUser.AppDbaUser.value, + MongoDBManagerUser.MonitorUser.value, + MongoDBManagerUser.AppMonitorUser.value, + ], + info=sub_sub_get_kwargs.replicaset_info, + ) + # 用户获取密码 + kwargs = sub_sub_get_kwargs.get_password_from_db(info=kwargs) + # 是否是部署单据 + kwargs["create"] = False + sub_sub_pipeline.add_act( + act_name=_("MongoDB-保存新实例的dba用户及额外管理用户密码"), + act_component_code=ExecAddPasswordToDBOperationComponent.code, + kwargs=kwargs, + ) + # 删除老实例密码 + kwargs = sub_sub_get_kwargs.get_delete_pwd_kwargs() + sub_sub_pipeline.add_act( + act_name=_("MongoDB-删除老实例的dba用户及额外管理用户密码"), + act_component_code=ExecDeletePasswordFromDBOperationComponent.code, + kwargs=kwargs, + ) + + # 老实例提下架单 + kwargs = { + "infos": sub_sub_get_kwargs.payload["nodes"], + "creator": sub_sub_get_kwargs.payload["created_by"], + "bk_biz_id": sub_sub_get_kwargs.payload["bk_biz_id"], + } + sub_sub_pipeline.add_act( + act_name=_("MongoDB-实例下架提单"), + act_component_code=ExecInstanceDeInstallTicketOperationComponent.code, + kwargs=kwargs, + ) + return sub_sub_pipeline.build_sub_process( + sub_name=_("MongoDB--mongod自愈--{}:{}".format(info["ip"], str(sub_sub_get_kwargs.db_instance["port"]))) + ) diff --git a/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/cluster_monogs_autofix.py b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/cluster_monogs_autofix.py new file mode 100644 index 0000000000..90ecc32241 --- /dev/null +++ b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/cluster_monogs_autofix.py @@ -0,0 +1,246 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. +Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + +from copy import deepcopy +from typing import Dict, Optional + +from django.utils.translation import ugettext as _ + +from backend.db_meta import api +from backend.db_meta.enums import ClusterEntryType +from backend.db_meta.enums.cluster_type import ClusterType +from backend.flow.consts import MongoDBInstanceType, MongoDBManagerUser +from backend.flow.engine.bamboo.scene.common.builder import SubBuilder +from backend.flow.engine.bamboo.scene.mongodb.mongodb_install import install_plugin +from backend.flow.plugins.components.collections.mongodb.add_domain_to_dns import ExecAddDomainToDnsOperationComponent +from backend.flow.plugins.components.collections.mongodb.add_password_to_db import ( + ExecAddPasswordToDBOperationComponent, +) +from backend.flow.plugins.components.collections.mongodb.delete_password_from_db import ( + ExecDeletePasswordFromDBOperationComponent, +) +from backend.flow.plugins.components.collections.mongodb.exec_actuator_job import ExecuteDBActuatorJobComponent +from backend.flow.plugins.components.collections.mongodb.instance_deinstall_ticket import ( + ExecInstanceDeInstallTicketOperationComponent, +) +from backend.flow.plugins.components.collections.mongodb.mongodb_cmr_4_meta import CMRMongoDBMetaComponent +from backend.flow.plugins.components.collections.mongodb.send_media import ExecSendMediaOperationComponent +from backend.flow.plugins.components.collections.name_service.name_service import ExecNameServiceOperationComponent +from backend.flow.utils.mongodb.mongodb_dataclass import ActKwargs + + +def mongos_autofix(root_id: str, ticket_data: Optional[Dict], sub_sub_kwargs: ActKwargs, info: dict) -> SubBuilder: + """ + mongos自愈流程 + """ + + # 获取变量 + sub_sub_get_kwargs = deepcopy(sub_sub_kwargs) + + # 创建子流程 + sub_sub_pipeline = SubBuilder(root_id=root_id, data=ticket_data) + + # 获取参数 + sub_sub_get_kwargs.mongos_info = {} + sub_sub_get_kwargs.payload["config_nodes"] = [] + sub_sub_get_kwargs.payload["shards_nodes"] = [] + sub_sub_get_kwargs.payload["app"] = sub_sub_get_kwargs.payload["bk_app_abbr"] + sub_sub_get_kwargs.mongos_info["port"] = sub_sub_get_kwargs.db_instance["port"] + cluster_id = sub_sub_get_kwargs.db_instance["cluster_id"] + sub_sub_get_kwargs.mongos_info["conf_set_id"] = sub_sub_get_kwargs.get_config_set_name_replace( + cluster_id=cluster_id + ) + sub_sub_get_kwargs.cluster_type = ClusterType.MongoShardedCluster.value + # 获取db版本 + db_version = sub_sub_get_kwargs.payload["db_version"] + # db大版本 + sub_sub_get_kwargs.db_main_version = str(db_version.split("-")[1].split(".")[0]) + # db发行版本 + sub_sub_get_kwargs.db_release_version = db_version + # db发行 + sub_sub_get_kwargs.db_release = db_version.split("-")[0] + # db版本 + sub_sub_get_kwargs.payload["db_version"] = db_version.split("-")[1] + + sub_sub_get_kwargs.payload["key_file"] = sub_sub_get_kwargs.get_conf( + cluster_name=sub_sub_get_kwargs.db_instance["cluster_name"] + )["key_file"] + node = info["target"] + node["cluster_id"] = cluster_id + sub_sub_get_kwargs.payload["mongos"] = {} + sub_sub_get_kwargs.payload["mongos"]["nodes"] = [ + { + "ip": node["ip"], + "domain": sub_sub_get_kwargs.db_instance["domain"], + "bk_cloud_id": info["bk_cloud_id"], + "port": sub_sub_get_kwargs.db_instance["port"], + } + ] + sub_sub_get_kwargs.payload["mongos"]["port"] = sub_sub_get_kwargs.db_instance["port"] + sub_sub_get_kwargs.payload["mongos"]["domain"] = sub_sub_get_kwargs.db_instance["domain"] + sub_sub_get_kwargs.payload["cluster_type"] = ClusterType.MongoShardedCluster.value + sub_sub_get_kwargs.payload["mongos_nodes"] = [ + { + "ip": info["ip"], + "domain": sub_sub_get_kwargs.db_instance["domain"], + "port": sub_sub_get_kwargs.db_instance["port"], + "bk_cloud_id": info["bk_cloud_id"], + } + ] + sub_sub_get_kwargs.payload["nodes"] = [ + { + "ip": info["ip"], + "domain": sub_sub_get_kwargs.db_instance["domain"], + "port": sub_sub_get_kwargs.db_instance["port"], + "bk_cloud_id": info["bk_cloud_id"], + "role": MongoDBInstanceType.MongoS.value, + } + ] + sub_sub_get_kwargs.payload["bk_cloud_id"] = info["bk_cloud_id"] + sub_sub_get_kwargs.payload["set_id"] = sub_sub_get_kwargs.db_instance["cluster_name"] + + # 判断是否有clb + clb = False + if ( + ClusterEntryType.CLB.value + in api.cluster.nosqlcomm.other.get_cluster_detail(cluster_id=cluster_id)[0]["clusterentry_set"] + ): + clb = True + # dbha已做clb解绑，dns删除 + # # 删除clb中绑定的老ip + # if clb: + # kwargs = { + # "name_service_operation_type": "clb_deregister_part_target", + # "creator": sub_sub_get_kwargs.payload["creator"], + # "cluster_id": cluster_id, + # "ips": ["{}:{}".format(info["ip"], str(sub_sub_get_kwargs.db_instance["port"]))], + # } + # sub_sub_pipeline.add_act( + # act_name=_("MongoDB-clb解绑故障ip"), + # act_component_code=ExecNameServiceOperationComponent.code, + # kwargs=kwargs, + # ) + # + # # 删除老的dns + # kwargs = sub_sub_get_kwargs.get_delete_domain_kwargs() + # sub_sub_pipeline.add_act( + # act_name=_("MongoDB-删除老实例的domain指向"), + # act_component_code=ExecDeleteDomainFromDnsOperationComponent.code, + # kwargs=kwargs, + # ) + + # 获取信息 + sub_sub_get_kwargs.get_host_cluster_autofix(info=info) + + # 安装蓝鲸插件 + install_plugin(pipeline=sub_sub_pipeline, get_kwargs=sub_sub_get_kwargs, new_cluster=False) + + # 介质下发 + kwargs = sub_sub_get_kwargs.get_send_media_kwargs(media_type="all") + sub_sub_pipeline.add_act( + act_name=_("MongoDB-介质下发"), act_component_code=ExecSendMediaOperationComponent.code, kwargs=kwargs + ) + + # 创建原子任务执行目录 + kwargs = sub_sub_get_kwargs.get_create_dir_kwargs() + sub_sub_pipeline.add_act( + act_name=_("MongoDB-创建原子任务执行目录"), act_component_code=ExecuteDBActuatorJobComponent.code, kwargs=kwargs + ) + + # 机器初始化 + kwargs = sub_sub_get_kwargs.get_os_init_kwargs() + sub_sub_pipeline.add_act( + act_name=_("MongoDB-机器初始化"), act_component_code=ExecuteDBActuatorJobComponent.code, kwargs=kwargs + ) + + # mognos安装 + kwargs = sub_sub_get_kwargs.get_install_mongos_kwargs(node=node, replace=True) + sub_sub_pipeline.add_act( + act_name=_("MongoDB-mongos安装-{}:{}".format(node["ip"], str(sub_sub_get_kwargs.db_instance["port"]))), + act_component_code=ExecuteDBActuatorJobComponent.code, + kwargs=kwargs, + ) + + # mongos修改db_meta数据 + cluster_mongos = {} + cluster_mongos["mongos"] = [info] + cluster_mongos["db_type"] = "mongos" + cluster_mongos["created_by"] = sub_sub_get_kwargs.payload.get("created_by") + cluster_mongos["bk_biz_id"] = sub_sub_get_kwargs.payload.get("bk_biz_id") + kwargs = sub_sub_get_kwargs.get_change_meta_replace_kwargs(info=cluster_mongos, instance={}) + sub_sub_pipeline.add_act( + act_name=_("MongoDB-mongos修改meta"), act_component_code=CMRMongoDBMetaComponent.code, kwargs=kwargs + ) + + # clb绑定新ip + if clb: + kwargs = { + "name_service_operation_type": "clb_register_part_target", + "creator": sub_sub_get_kwargs.payload["created_by"], + "cluster_id": cluster_id, + "ips": ["{}:{}".format(node["ip"], str(sub_sub_get_kwargs.db_instance["port"]))], + } + sub_sub_pipeline.add_act( + act_name=_("MongoDB-clb绑定新ip"), + act_component_code=ExecNameServiceOperationComponent.code, + kwargs=kwargs, + ) + + # 添加新的dns + kwargs = sub_sub_get_kwargs.get_add_domain_to_dns_kwargs(cluster=True) + sub_sub_pipeline.add_act( + act_name=_("MongoDB-添加新实例的domain到dns"), + act_component_code=ExecAddDomainToDnsOperationComponent.code, + kwargs=kwargs, + ) + # 密码服务修改密码 + # 添加新实例密码 + kwargs = sub_sub_get_kwargs.get_add_password_to_db_kwargs( + usernames=[ + MongoDBManagerUser.DbaUser.value, + MongoDBManagerUser.AppDbaUser.value, + MongoDBManagerUser.MonitorUser.value, + MongoDBManagerUser.AppMonitorUser.value, + ], + info=sub_sub_get_kwargs.payload["mongos"], + ) + # 用户获取密码 + kwargs = sub_sub_get_kwargs.get_password_from_db(info=kwargs) + # 是否是部署单据 + kwargs["create"] = False + sub_sub_pipeline.add_act( + act_name=_("MongoDB-保存新实例的dba用户及额外管理用户密码"), + act_component_code=ExecAddPasswordToDBOperationComponent.code, + kwargs=kwargs, + ) + # 删除老实例密码 + kwargs = sub_sub_get_kwargs.get_delete_pwd_kwargs() + sub_sub_pipeline.add_act( + act_name=_("MongoDB-删除老实例的dba用户及额外管理用户密码"), + act_component_code=ExecDeletePasswordFromDBOperationComponent.code, + kwargs=kwargs, + ) + + # 老实例提下架单 + kwargs = { + "infos": sub_sub_get_kwargs.payload["nodes"], + "creator": sub_sub_get_kwargs.payload["created_by"], + "bk_biz_id": sub_sub_get_kwargs.payload["bk_biz_id"], + } + sub_sub_pipeline.add_act( + act_name=_("MongoDB-实例下架提单"), + act_component_code=ExecInstanceDeInstallTicketOperationComponent.code, + kwargs=kwargs, + ) + + return sub_sub_pipeline.build_sub_process( + sub_name=_("MongoDB--mongos自愈--{}:{}".format(info["ip"], str(sub_sub_get_kwargs.db_instance["port"]))) + ) diff --git a/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/cluster_shard_autofix.py b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/cluster_shard_autofix.py new file mode 100644 index 0000000000..9a11600250 --- /dev/null +++ b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/cluster_shard_autofix.py @@ -0,0 +1,100 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. +Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + +from copy import deepcopy +from typing import Dict, Optional + +from django.utils.translation import ugettext as _ + +from backend.flow.consts import MongoDBClusterRole +from backend.flow.engine.bamboo.scene.common.builder import SubBuilder +from backend.flow.engine.bamboo.scene.mongodb.mongodb_install import install_plugin +from backend.flow.plugins.components.collections.mongodb.exec_actuator_job import ExecuteDBActuatorJobComponent +from backend.flow.plugins.components.collections.mongodb.mongodb_cmr_4_meta import CMRMongoDBMetaComponent +from backend.flow.plugins.components.collections.mongodb.send_media import ExecSendMediaOperationComponent +from backend.flow.utils.mongodb.mongodb_dataclass import ActKwargs + +from .cluster_mongod_autofix import mongod_autofix + + +def shard_autofix( + root_id: str, ticket_data: Optional[Dict], sub_kwargs: ActKwargs, info: dict, cluster_role: str +) -> SubBuilder: + """ + shard自愈流程 + info 表示replicaset信息 + """ + + # 获取变量 + sub_get_kwargs = deepcopy(sub_kwargs) + + # 创建子流程 + sub_pipeline = SubBuilder(root_id=root_id, data=ticket_data) + + # 获取信息 + sub_get_kwargs.get_host_cluster_autofix(info=info) + + # 安装蓝鲸插件 + install_plugin(pipeline=sub_pipeline, get_kwargs=sub_get_kwargs, new_cluster=False) + + # 介质下发 + kwargs = sub_get_kwargs.get_send_media_kwargs(media_type="all") + sub_pipeline.add_act( + act_name=_("MongoDB-介质下发"), act_component_code=ExecSendMediaOperationComponent.code, kwargs=kwargs + ) + + # 创建原子任务执行目录 + kwargs = sub_get_kwargs.get_create_dir_kwargs() + sub_pipeline.add_act( + act_name=_("MongoDB-创建原子任务执行目录"), act_component_code=ExecuteDBActuatorJobComponent.code, kwargs=kwargs + ) + + # 机器初始化 + kwargs = sub_get_kwargs.get_os_init_kwargs() + sub_pipeline.add_act( + act_name=_("MongoDB-机器初始化"), act_component_code=ExecuteDBActuatorJobComponent.code, kwargs=kwargs + ) + + # 进行替换——并行 以ip为维度 + sub_sub_pipelines = [] + for mongodb_instance in info["instances"]: + sub_get_kwargs.db_instance = mongodb_instance + sub_sub_pipeline = mongod_autofix( + root_id=root_id, + ticket_data=ticket_data, + sub_sub_kwargs=sub_get_kwargs, + cluster_role=cluster_role, + info=info, + ) + sub_sub_pipelines.append(sub_sub_pipeline) + sub_pipeline.add_parallel_sub_pipeline(sub_sub_pipelines) + + # 修改db_meta数据 + info["created_by"] = sub_get_kwargs.payload.get("created_by") + info["bk_biz_id"] = sub_get_kwargs.payload.get("bk_biz_id") + if cluster_role == MongoDBClusterRole.ShardSvr.value: + info["db_type"] = "cluster_mongodb" + name = "shard" + elif cluster_role == MongoDBClusterRole.ConfigSvr.value: + info["db_type"] = "mongo_config" + name = "configDB" + # 修改所有mongos的configDB ip + kwargs = sub_get_kwargs.get_change_config_ip_kwargs(info=info) + sub_pipeline.add_act( + act_name=_("MongoDB--mongos修改configDB参数"), + act_component_code=ExecuteDBActuatorJobComponent.code, + kwargs=kwargs, + ) + kwargs = sub_get_kwargs.get_change_meta_replace_kwargs(info=info, instance={}) + sub_pipeline.add_act( + act_name=_("MongoDB-mongod修改meta"), act_component_code=CMRMongoDBMetaComponent.code, kwargs=kwargs + ) + return sub_pipeline.build_sub_process(sub_name=_("MongoDB--{}自愈--ip:{}".format(name, info["ip"]))) diff --git a/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/download_subtask.py b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/download_subtask.py index d9f4491cb0..e95dbb8498 100644 --- a/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/download_subtask.py +++ b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/download_subtask.py @@ -19,7 +19,7 @@ ) from backend.flow.utils.base.payload_handler import PayloadHandler from backend.flow.utils.mongodb.mongodb_dataclass import CommonContext -from backend.flow.utils.mongodb.mongodb_repo import MongoDBCluster, ReplicaSet, MongoNode +from backend.flow.utils.mongodb.mongodb_repo import MongoDBCluster, MongoNode, ReplicaSet # Prepare datafile 准备数据文件 diff --git a/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/hello_sub.py b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/hello_sub.py index 89511485fc..f18c304405 100644 --- a/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/hello_sub.py +++ b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/hello_sub.py @@ -17,7 +17,7 @@ from backend.flow.engine.bamboo.scene.mongodb.sub_task.base_subtask import BaseSubTask from backend.flow.plugins.components.collections.mongodb.exec_actuator_job2 import ExecJobComponent2 from backend.flow.utils.mongodb.mongodb_dataclass import CommonContext -from backend.flow.utils.mongodb.mongodb_repo import ReplicaSet, MongoNode +from backend.flow.utils.mongodb.mongodb_repo import MongoNode, ReplicaSet from backend.flow.utils.mongodb.mongodb_util import MongoUtil diff --git a/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/instance_deinstall.py b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/instance_deinstall.py new file mode 100644 index 0000000000..6f1991213e --- /dev/null +++ b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/instance_deinstall.py @@ -0,0 +1,56 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. +Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + +from copy import deepcopy +from typing import Dict, Optional + +from django.utils.translation import ugettext as _ + +from backend.flow.consts import MongoDBInstanceType +from backend.flow.engine.bamboo.scene.common.builder import SubBuilder +from backend.flow.plugins.components.collections.mongodb.exec_actuator_job import ExecuteDBActuatorJobComponent +from backend.flow.utils.mongodb.mongodb_dataclass import ActKwargs + + +def instance_deinstall(root_id: str, ticket_data: Optional[Dict], sub_kwargs: ActKwargs, info: dict) -> SubBuilder: + """ + instance deinstall流程 + """ + + # 获取变量 + sub_get_kwargs = deepcopy(sub_kwargs) + + # 创建子流程 + sub_sub_pipeline = SubBuilder(root_id=root_id, data=ticket_data) + + # 获取参数 + instance_type = "" + if info["role"] == MongoDBInstanceType.MongoD.value: + instance_type = MongoDBInstanceType.MongoD.value + elif info["role"] == MongoDBInstanceType.MongoS.value: + instance_type = MongoDBInstanceType.MongoS.value + sub_get_kwargs.payload["bk_cloud_id"] = info["bk_cloud_id"] + sub_get_kwargs.payload["set_id"] = "" + # 下架实例 + kwargs = sub_get_kwargs.get_mongo_deinstall_kwargs( + node_info=info, + instance_type=instance_type, + nodes_info=[info], + force=True, + rename_dir=True, + ) + sub_sub_pipeline.add_act( + act_name=_("实例下架-{}:{}".format(info["ip"], str(info["port"]))), + act_component_code=ExecuteDBActuatorJobComponent.code, + kwargs=kwargs, + ) + + return sub_sub_pipeline.build_sub_process(sub_name=_("MongoDB--实例下架--{}:{}".format(info["ip"], str(info["port"])))) diff --git a/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/mongod_replace.py b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/mongod_replace.py index ab6d09fc0e..c98cec73cd 100644 --- a/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/mongod_replace.py +++ b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/mongod_replace.py @@ -51,6 +51,7 @@ def mongod_replace( sub_sub_pipeline = SubBuilder(root_id=root_id, data=ticket_data) # 获取参数 + down = info.get("down") # 机器是否down new_node = info["target"] sub_sub_get_kwargs.payload["app"] = sub_sub_get_kwargs.payload["bk_app_abbr"] sub_sub_get_kwargs.replicaset_info = {} @@ -133,7 +134,7 @@ def mongod_replace( ) # mognod替换 - kwargs = sub_sub_get_kwargs.get_instance_replace_kwargs(info=info, source_down=False) + kwargs = sub_sub_get_kwargs.get_instance_replace_kwargs(info=info, source_down=down) sub_sub_pipeline.add_act( act_name=_("MongoDB-mongod替换"), act_component_code=ExecuteDBActuatorJobComponent.code, @@ -192,20 +193,21 @@ def mongod_replace( act_name=_("MongoDB-mongod修改meta"), act_component_code=MongoDBCapcityMetaComponent.code, kwargs=kwargs ) - # 下架老实例 - kwargs = sub_sub_get_kwargs.get_mongo_deinstall_kwargs( - node_info=sub_sub_get_kwargs.payload["nodes"][0], - instance_type=MongoDBInstanceType.MongoD.value, - nodes_info=sub_sub_get_kwargs.payload["nodes"], - force=force, - rename_dir=True, - ) - sub_sub_pipeline.add_act( - act_name=_("MongoDB-老实例下架-{}:{}".format(info["ip"], str(sub_sub_get_kwargs.db_instance["port"]))), - act_component_code=ExecuteDBActuatorJobComponent.code, - kwargs=kwargs, - ) - + if not down: + # 下架老实例 + kwargs = sub_sub_get_kwargs.get_mongo_deinstall_kwargs( + node_info=sub_sub_get_kwargs.payload["nodes"][0], + instance_type=MongoDBInstanceType.MongoD.value, + nodes_info=sub_sub_get_kwargs.payload["nodes"], + force=force, + rename_dir=True, + ) + sub_sub_pipeline.add_act( + act_name=_("MongoDB-老实例下架-{}:{}".format(info["ip"], str(sub_sub_get_kwargs.db_instance["port"]))), + act_component_code=ExecuteDBActuatorJobComponent.code, + kwargs=kwargs, + ) + # 老实例提下架单 TODO return sub_sub_pipeline.build_sub_process( sub_name=_("MongoDB--mongod替换--{}:{}".format(info["ip"], str(sub_sub_get_kwargs.db_instance["port"]))) ) diff --git a/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/mongos_replace.py b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/mongos_replace.py index 73748152f7..794eb97316 100644 --- a/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/mongos_replace.py +++ b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/mongos_replace.py @@ -14,6 +14,8 @@ from django.utils.translation import ugettext as _ +from backend.db_meta import api +from backend.db_meta.enums import ClusterEntryType from backend.db_meta.enums.cluster_type import ClusterType from backend.flow.consts import MongoDBInstanceType, MongoDBManagerUser from backend.flow.engine.bamboo.scene.common.builder import SubBuilder @@ -28,6 +30,7 @@ ExecDeletePasswordFromDBOperationComponent, ) from backend.flow.plugins.components.collections.mongodb.exec_actuator_job import ExecuteDBActuatorJobComponent +from backend.flow.plugins.components.collections.name_service.name_service import ExecNameServiceOperationComponent from backend.flow.utils.mongodb.mongodb_dataclass import ActKwargs @@ -39,7 +42,7 @@ def mongos_replace(root_id: str, ticket_data: Optional[Dict], sub_sub_kwargs: Ac # 获取变量 sub_sub_get_kwargs = deepcopy(sub_sub_kwargs) - # 创建子 + # 创建子流程 sub_sub_pipeline = SubBuilder(root_id=root_id, data=ticket_data) # 获取参数 @@ -86,9 +89,37 @@ def mongos_replace(root_id: str, ticket_data: Optional[Dict], sub_sub_kwargs: Ac "bk_cloud_id": info["bk_cloud_id"], } ] - sub_sub_get_kwargs.payload["app"] = sub_sub_get_kwargs.payload["bk_app_abbr"] sub_sub_get_kwargs.payload["bk_cloud_id"] = info["bk_cloud_id"] sub_sub_get_kwargs.payload["set_id"] = sub_sub_get_kwargs.db_instance["cluster_name"] + # 判断是否有clb + clb = False + if ( + ClusterEntryType.CLB.value + in api.cluster.nosqlcomm.other.get_cluster_detail(cluster_id=cluster_id)[0]["clusterentry_set"] + ): + clb = True + + # clb解绑老ip + if clb: + kwargs = { + "name_service_operation_type": "clb_deregister_part_target", + "creator": sub_sub_get_kwargs.payload["creator"], + "cluster_id": cluster_id, + "ips": ["{}:{}".format(info["ip"], str(sub_sub_get_kwargs.db_instance["port"]))], + } + sub_sub_pipeline.add_act( + act_name=_("MongoDB-clb解绑替换ip"), + act_component_code=ExecNameServiceOperationComponent.code, + kwargs=kwargs, + ) + + # 删除老的dns + kwargs = sub_sub_get_kwargs.get_delete_domain_kwargs() + sub_sub_pipeline.add_act( + act_name=_("MongoDB-删除老实例的domain指向"), + act_component_code=ExecDeleteDomainFromDnsOperationComponent.code, + kwargs=kwargs, + ) # mognos安装 kwargs = sub_sub_get_kwargs.get_install_mongos_kwargs(node=node, replace=True) @@ -98,7 +129,20 @@ def mongos_replace(root_id: str, ticket_data: Optional[Dict], sub_sub_kwargs: Ac kwargs=kwargs, ) - # 更改dns + # clb绑定新ip + if clb: + kwargs = { + "name_service_operation_type": "clb_register_part_target", + "creator": sub_sub_get_kwargs.payload["creator"], + "cluster_id": cluster_id, + "ips": ["{}:{}".format(node["ip"], str(sub_sub_get_kwargs.db_instance["port"]))], + } + sub_sub_pipeline.add_act( + act_name=_("MongoDB-clb绑定新ip"), + act_component_code=ExecNameServiceOperationComponent.code, + kwargs=kwargs, + ) + # 添加新的dns kwargs = sub_sub_get_kwargs.get_add_domain_to_dns_kwargs(cluster=True) sub_sub_pipeline.add_act( @@ -106,14 +150,6 @@ def mongos_replace(root_id: str, ticket_data: Optional[Dict], sub_sub_kwargs: Ac act_component_code=ExecAddDomainToDnsOperationComponent.code, kwargs=kwargs, ) - # 删除老的dns - kwargs = sub_sub_get_kwargs.get_delete_domain_kwargs() - sub_sub_pipeline.add_act( - act_name=_("MongoDB-删除老实例的domain指向"), - act_component_code=ExecDeleteDomainFromDnsOperationComponent.code, - kwargs=kwargs, - ) - # 密码服务修改密码 # 添加新实例密码 kwargs = sub_sub_get_kwargs.get_add_password_to_db_kwargs( @@ -141,19 +177,21 @@ def mongos_replace(root_id: str, ticket_data: Optional[Dict], sub_sub_kwargs: Ac act_component_code=ExecDeletePasswordFromDBOperationComponent.code, kwargs=kwargs, ) - # 下架老实例 - kwargs = sub_sub_get_kwargs.get_mongo_deinstall_kwargs( - node_info=sub_sub_get_kwargs.payload["mongos_nodes"][0], - instance_type=MongoDBInstanceType.MongoS.value, - nodes_info=sub_sub_get_kwargs.payload["mongos_nodes"], - force=True, - rename_dir=True, - ) - sub_sub_pipeline.add_act( - act_name=_("MongoDB-老实例下架-{}:{}".format(info["ip"], str(sub_sub_get_kwargs.db_instance["port"]))), - act_component_code=ExecuteDBActuatorJobComponent.code, - kwargs=kwargs, - ) + + if not info["down"]: + # 下架老实例 + kwargs = sub_sub_get_kwargs.get_mongo_deinstall_kwargs( + node_info=sub_sub_get_kwargs.payload["mongos_nodes"][0], + instance_type=MongoDBInstanceType.MongoS.value, + nodes_info=sub_sub_get_kwargs.payload["mongos_nodes"], + force=True, + rename_dir=True, + ) + sub_sub_pipeline.add_act( + act_name=_("MongoDB-老实例下架-{}:{}".format(info["ip"], str(sub_sub_get_kwargs.db_instance["port"]))), + act_component_code=ExecuteDBActuatorJobComponent.code, + kwargs=kwargs, + ) return sub_sub_pipeline.build_sub_process( sub_name=_("MongoDB--mongos替换--{}:{}".format(info["ip"], str(sub_sub_get_kwargs.db_instance["port"]))) ) diff --git a/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/pitr_restore_sub.py b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/pitr_restore_sub.py index fca394b881..cc6484282e 100644 --- a/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/pitr_restore_sub.py +++ b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/pitr_restore_sub.py @@ -17,7 +17,7 @@ from backend.flow.engine.bamboo.scene.mongodb.sub_task.base_subtask import BaseSubTask from backend.flow.plugins.components.collections.mongodb.exec_actuator_job2 import ExecJobComponent2 from backend.flow.utils.mongodb.mongodb_dataclass import CommonContext -from backend.flow.utils.mongodb.mongodb_repo import ReplicaSet, MongoNode +from backend.flow.utils.mongodb.mongodb_repo import MongoNode, ReplicaSet from backend.flow.utils.mongodb.mongodb_util import MongoUtil diff --git a/dbm-ui/backend/flow/engine/controller/mongodb.py b/dbm-ui/backend/flow/engine/controller/mongodb.py index eb50e3fb7c..655eeaa946 100644 --- a/dbm-ui/backend/flow/engine/controller/mongodb.py +++ b/dbm-ui/backend/flow/engine/controller/mongodb.py @@ -8,6 +8,7 @@ an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ +from backend.flow.engine.bamboo.scene.mongodb.mongodb_autofix import MongoAutofixFlow from backend.flow.engine.bamboo.scene.mongodb.mongodb_backup import MongoBackupFlow from backend.flow.engine.bamboo.scene.mongodb.mongodb_cluster_scale_mongos import ScaleMongoSFlow from backend.flow.engine.bamboo.scene.mongodb.mongodb_deinstall import MongoDBDeInstallFlow @@ -16,6 +17,7 @@ from backend.flow.engine.bamboo.scene.mongodb.mongodb_fake_install import MongoFakeInstallFlow from backend.flow.engine.bamboo.scene.mongodb.mongodb_install import MongoDBInstallFlow from backend.flow.engine.bamboo.scene.mongodb.mongodb_install_dbmon import MongoInstallDBMonFlow +from backend.flow.engine.bamboo.scene.mongodb.mongodb_instance_deinstall import MongoInstanceDeInstallFlow from backend.flow.engine.bamboo.scene.mongodb.mongodb_instance_restart import MongoRestartInstanceFlow from backend.flow.engine.bamboo.scene.mongodb.mongodb_migrate import MongoDBMigrateMetaFlow from backend.flow.engine.bamboo.scene.mongodb.mongodb_pitr_restore import MongoPitrRestoreFlow @@ -181,6 +183,22 @@ def disable_cluster(self): flow = MongoEnableDisableFlow(root_id=self.root_id, data=self.ticket_data) flow.multi_cluster_flow(enable=False) + def instance_deinstall(self): + """ + instance卸载 + """ + + flow = MongoInstanceDeInstallFlow(root_id=self.root_id, data=self.ticket_data) + flow.multi_instance_deinstall_flow() + + def mongo_autofix(self): + """ + mongodb自愈 + """ + + flow = MongoAutofixFlow(root_id=self.root_id, data=self.ticket_data) + flow.autofix() + def migrate_meta(self): """ 迁移元数据 diff --git a/dbm-ui/backend/flow/plugins/components/collections/mongodb/exec_actuator_job.py b/dbm-ui/backend/flow/plugins/components/collections/mongodb/exec_actuator_job.py index 8118febffd..ea14c12b96 100644 --- a/dbm-ui/backend/flow/plugins/components/collections/mongodb/exec_actuator_job.py +++ b/dbm-ui/backend/flow/plugins/components/collections/mongodb/exec_actuator_job.py @@ -125,15 +125,21 @@ def _execute(self, data, parent_data) -> bool: kwargs["db_act_template"]["payload"]["adminUsername"] ] - # 替换mongos安装新的mongos获取configDB配置 + # 替换mongos安装新的mongos获取configDB配置 所有的mongos修改configDB参数 mongos_replace_install = kwargs.get("mongos_replace_install", False) - if mongos_replace_install: + all_mongos_change_configdb = kwargs.get("all_mongos_change_configdb", False) + if mongos_replace_install or all_mongos_change_configdb: act_kwargs = ActKwargs() act_kwargs.payload = {} act_kwargs.get_cluster_info_deinstall(cluster_id=kwargs["cluster_id"]) - config_db = ["{}:{}".format(node["ip"], str(node["port"])) for node in act_kwargs.payload["config_nodes"]] if mongos_replace_install: + config_db = [ + "{}:{}".format(node["ip"], str(node["port"])) for node in act_kwargs.payload["config_nodes"] + ] kwargs["db_act_template"]["payload"]["configDB"] = config_db + elif all_mongos_change_configdb: + kwargs["exec_ip"] = [node["ip"] for node in act_kwargs.payload["mongos_nodes"]] + kwargs["db_act_template"]["payload"]["port"] = act_kwargs.payload["mongos_nodes"][0]["port"] # 拼接节点执行ip所需要的信息，ip信息统一用list处理拼接 if kwargs["get_trans_data_ip_var"]: diff --git a/dbm-ui/backend/flow/plugins/components/collections/mongodb/instance_deinstall_ticket.py b/dbm-ui/backend/flow/plugins/components/collections/mongodb/instance_deinstall_ticket.py new file mode 100644 index 0000000000..48a8d42642 --- /dev/null +++ b/dbm-ui/backend/flow/plugins/components/collections/mongodb/instance_deinstall_ticket.py @@ -0,0 +1,62 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. +Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +import logging +from typing import List + +from pipeline.component_framework.component import Component +from pipeline.core.flow.activity import Service + +from backend.flow.plugins.components.collections.common.base_service import BaseService +from backend.flow.utils.mongodb.instance_deinstall_ticket import instance_deinstall_ticket + +logger = logging.getLogger("json") + + +class ExecInstanceDeInstallTicketOperation(BaseService): + """ + InstanceDeInstallTicket服务 + """ + + def _execute(self, data, parent_data) -> bool: + """ + 实例下架提单功能的函数 + global_data 单据全局变量，格式字典 + kwargs 私有变量 + """ + + # 从流程节点中获取变量 + kwargs = data.get_one_of_inputs("kwargs") + + # 实例下架提单 + try: + instance_deinstall_ticket(infos=kwargs["infos"], creator=kwargs["creator"], bk_biz_id=kwargs["bk_biz_id"]) + except Exception as e: + self.log_error("create instance deinstall ticket fail, error:{}".format(e)) + return False + self.log_info("create instance deinstall ticket successfully") + return True + + # 流程节点输入参数 + def inputs_format(self) -> List: + return [ + Service.InputItem(name="kwargs", key="kwargs", type="dict", required=True), + Service.InputItem(name="global_data", key="global_data", type="dict", required=True), + ] + + +class ExecInstanceDeInstallTicketOperationComponent(Component): + """ + ExecInstanceDeInstallTicketOperation组件 + """ + + name = __name__ + code = "instance_deinstall_ticket" + bound_service = ExecInstanceDeInstallTicketOperation diff --git a/dbm-ui/backend/flow/plugins/components/collections/mongodb/prepare_instance_info.py b/dbm-ui/backend/flow/plugins/components/collections/mongodb/prepare_instance_info.py index dea7085cd3..c8abfffb87 100644 --- a/dbm-ui/backend/flow/plugins/components/collections/mongodb/prepare_instance_info.py +++ b/dbm-ui/backend/flow/plugins/components/collections/mongodb/prepare_instance_info.py @@ -35,12 +35,12 @@ def _execute(self, data, parent_data) -> bool: kwargs 私有变量 """ - trans_data = data.get_one_of_inputs("trans_data") + # trans_data = data.get_one_of_inputs("trans_data") kwargs = data.get_one_of_inputs("kwargs") - if trans_data is None or trans_data == "${trans_data}": - # 表示没有加载上下文内容，则在此添加 - trans_data = getattr(flow_context, kwargs["set_trans_data_dataclass"])() + # if trans_data is None or trans_data == "${trans_data}": + # 表示没有加载上下文内容，则在此添加 + trans_data = getattr(flow_context, kwargs["set_trans_data_dataclass"])() iplist = kwargs["trans_data_var"]["iplist"] bk_cloud_id = int(kwargs["trans_data_var"]["bk_cloud_id"]) diff --git a/dbm-ui/backend/flow/plugins/components/collections/name_service/name_service.py b/dbm-ui/backend/flow/plugins/components/collections/name_service/name_service.py index b4eceebf0f..2f714dca78 100644 --- a/dbm-ui/backend/flow/plugins/components/collections/name_service/name_service.py +++ b/dbm-ui/backend/flow/plugins/components/collections/name_service/name_service.py @@ -48,6 +48,12 @@ def _execute(self, data, parent_data) -> bool: # clb创建 if name_service_operation_type == "create_clb": res = clb.create_lb_and_register_target(cluster_id=cluster_id) + # clb绑定新ip + elif name_service_operation_type == "clb_register_part_target": + res = clb.operate_part_target(cluster_id=cluster_id, ips=kwargs["ips"], bind=True) + # clb解绑部分ip + elif name_service_operation_type == "clb_deregister_part_target": + res = clb.operate_part_target(cluster_id=cluster_id, ips=kwargs["ips"], bind=False) # polaris创建 elif name_service_operation_type == "create_polaris": res = polaris.create_service_alias_bind_targets(cluster_id=cluster_id) diff --git a/dbm-ui/backend/flow/urls.py b/dbm-ui/backend/flow/urls.py index cec16c0499..8c22497477 100644 --- a/dbm-ui/backend/flow/urls.py +++ b/dbm-ui/backend/flow/urls.py @@ -86,6 +86,7 @@ MongoDBExecScriptView, MongoDBIncreaseMongoSView, MongoDBIncreaseNodeView, + MongoDBInstanceDeInstallView, MongoDBInstanceRestartView, MongoDBReduceMongoSView, MongoDBReduceNodeView, @@ -335,6 +336,7 @@ url(r"^scene/multi_cluster_enable$", MongoDBEnableClusterView.as_view()), url(r"^scene/multi_cluster_disable$", MongoDBDisableClusterView.as_view()), url(r"^scene/multi_cluster_migrate_meta$", MongoDBClusterMigrateView.as_view()), + url(r"^scene/multi_instance_deinstall$", MongoDBInstanceDeInstallView.as_view()), # mongodb end # mysql upgrade url(r"^scene/upgrade_mysql_proxy$", UpgradeMySQLProxySceneApiView.as_view()), diff --git a/dbm-ui/backend/flow/utils/mongodb/instance_deinstall_ticket.py b/dbm-ui/backend/flow/utils/mongodb/instance_deinstall_ticket.py new file mode 100644 index 0000000000..b6084257b8 --- /dev/null +++ b/dbm-ui/backend/flow/utils/mongodb/instance_deinstall_ticket.py @@ -0,0 +1,31 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. +Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + +from django.utils.translation import ugettext_lazy as _ + +from backend.ticket.constants import TicketType +from backend.ticket.models import Ticket + + +def instance_deinstall_ticket(infos: list, creator: str, bk_biz_id: int): + """instance下架创建单据""" + + # 单据信息 + details = {"infos": infos} + + # 创建单据 + Ticket.create_ticket( + ticket_type=TicketType.MONGODB_INSTANCE_DEINSTALL.value, + creator=creator, + bk_biz_id=bk_biz_id, + remark=_("自动发起-实例下架-{}:{}".format(infos[0]["ip"], str(infos[0]["port"]))), + details=details, + ) diff --git a/dbm-ui/backend/flow/utils/mongodb/mongodb_dataclass.py b/dbm-ui/backend/flow/utils/mongodb/mongodb_dataclass.py index 4ff95f4cfb..33f9216fa6 100644 --- a/dbm-ui/backend/flow/utils/mongodb/mongodb_dataclass.py +++ b/dbm-ui/backend/flow/utils/mongodb/mongodb_dataclass.py @@ -30,7 +30,7 @@ ConfigTypeEnum, MediumEnum, MongoDBActuatorActionEnum, - MongoDBDfaultAuthDB, + MongoDBDefaultAuthDB, MongoDBInstanceType, MongoDBManagerUser, MongoDBTask, @@ -713,7 +713,7 @@ def get_add_manager_user_kwargs(self) -> dict: "password": "", "adminUsername": "", "adminPassword": "", - "authDb": MongoDBDfaultAuthDB.AuthDB, + "authDb": MongoDBDefaultAuthDB.AuthDB, "dbsPrivileges": [ { "db": "admin", @@ -1824,6 +1824,68 @@ def get_mongo_start_kwargs(self, node_info: dict, instance_type: str) -> dict: }, } + def get_host_instance_deinstall(self): + """instance卸载获取host""" + + hosts = [] + host_bk_cloud_id_set = set() + for instance in self.payload["infos"]: + host_bk_cloud_id_set.add("{}#{}".format(instance["ip"], instance["bk_cloud_id"])) + for host_bk_cloud_id in host_bk_cloud_id_set: + info = host_bk_cloud_id.split("#") + hosts.append( + { + "ip": info[0], + "bk_cloud_id": info[1], + } + ) + self.payload["hosts"] = hosts + + def get_host_cluster_autofix(self, info: dict): + """cluster自愈获取host信息""" + + plugin_hosts = [] + plugin_hosts.append({"ip": info["target"]["ip"], "bk_cloud_id": info["target"]["bk_cloud_id"]}) + # 获取参数 + self.payload["hosts"] = plugin_hosts + self.payload["plugin_hosts"] = plugin_hosts + + def get_change_config_ip_kwargs(self, info) -> dict: + """修改所有mongos的config的ip""" + + # 获取信息 + bk_cloud_id = info["target"]["bk_cloud_id"] + port = info["instances"][0]["port"] + username = MongoDBManagerUser.DbaUser.value + old_config_node = "{}:{}".format(info["ip"], port) + new_config_node = "{}:{}".format(info["target"]["ip"], port) + # 获取密码 + password = self.get_password(ip=info["ip"], port=port, bk_cloud_id=bk_cloud_id, username=username) + return { + "set_trans_data_dataclass": CommonContext.__name__, + "get_trans_data_ip_var": None, + "bk_cloud_id": bk_cloud_id, + "exec_ip": [], + "all_mongos_change_configdb": True, + "cluster_id": info["instances"][0]["cluster_id"], + "db_act_template": { + "action": MongoDBActuatorActionEnum.MongoRestart, + "file_path": self.file_path, + "payload": { + "ip": "0.0.0.0", + "port": 0, + "instanceType": MongoDBInstanceType.MongoS.value, + "auth": True, + "cacheSizeGB": 0, + "mongoSConfDbOld": old_config_node, + "mongoSConfDbNew": new_config_node, + "adminUsername": username, + "adminPassword": password, + "onlyChangeParam": True, + }, + }, + } + @dataclass() class CommonContext: diff --git a/dbm-ui/backend/flow/utils/mongodb/mongodb_repo.py b/dbm-ui/backend/flow/utils/mongodb/mongodb_repo.py index ad6a2e6834..7fc11038e2 100644 --- a/dbm-ui/backend/flow/utils/mongodb/mongodb_repo.py +++ b/dbm-ui/backend/flow/utils/mongodb/mongodb_repo.py @@ -12,7 +12,6 @@ from backend.flow.utils.mongodb import mongodb_password from backend.ticket.constants import InstanceType - # entities # Node -> ReplicaSet -> Cluster[Rs,ShardedCluster] # MongoNodeWithLabel diff --git a/dbm-ui/backend/flow/views/mongodb_scene.py b/dbm-ui/backend/flow/views/mongodb_scene.py index 63de125bb6..e4578bc34e 100644 --- a/dbm-ui/backend/flow/views/mongodb_scene.py +++ b/dbm-ui/backend/flow/views/mongodb_scene.py @@ -274,6 +274,18 @@ def post(request): return Response({"root_id": root_id}) +class MongoDBInstanceDeInstallView(FlowTestView): + """ + instance卸载 + """ + + @staticmethod + def post(request): + root_id = uuid.uuid1().hex + MongoDBController(root_id=root_id, ticket_data=request.data).instance_deinstall() + return Response({"root_id": root_id}) + + class MongoDBClusterMigrateView(FlowTestView): """ 迁移mongodb元数据 diff --git a/dbm-ui/backend/ticket/builders/mongodb/mongo_autofix.py b/dbm-ui/backend/ticket/builders/mongodb/mongo_autofix.py new file mode 100644 index 0000000000..a8a5481585 --- /dev/null +++ b/dbm-ui/backend/ticket/builders/mongodb/mongo_autofix.py @@ -0,0 +1,70 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. +Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +from django.utils.translation import ugettext_lazy as _ +from rest_framework import serializers + +from backend.db_meta.models import AppCache +from backend.db_services.dbbase.constants import IpSource +from backend.flow.engine.controller.mongodb import MongoDBController +from backend.ticket import builders +from backend.ticket.builders.mongodb.base import ( + BaseMongoDBOperateDetailSerializer, + BaseMongoDBOperateResourceParamBuilder, + BaseMongoShardedTicketFlowBuilder, +) +from backend.ticket.constants import TicketType + + +class MongoDBAutofixDetailSerializer(BaseMongoDBOperateDetailSerializer): + class AutofixDetailSerializer(serializers.Serializer): + class HostInfoSerializer(serializers.Serializer): + ip = serializers.IPAddressField(help_text=_("IP地址")) + spec_id = serializers.IntegerField(help_text=_("规格ID")) + + immute_domain = serializers.CharField(help_text=_("主域名")) + bk_cloud_id = serializers.IntegerField(help_text=_("云ID")) + bk_biz_id = serializers.IntegerField(help_text=_("业务ID")) + cluster_ids = serializers.ListField(help_text=_("集群ID")) + resource_spec = serializers.JSONField(help_text=_("资源规格")) + cluster_type = serializers.CharField(help_text=_("集群类型")) + mongos_list = serializers.ListSerializer(help_text=_("故障mongos列表"), child=HostInfoSerializer()) + mongod_list = serializers.ListSerializer(help_text=_("故障mongod列表"), child=HostInfoSerializer()) + + ip_source = serializers.ChoiceField( + help_text=_("主机来源"), choices=IpSource.get_choices(), default=IpSource.RESOURCE_POOL + ) + infos = serializers.ListSerializer(help_text=_("mongo自愈申请信息"), child=AutofixDetailSerializer()) + + +class MongoDBAutofixFlowParamBuilder(builders.FlowParamBuilder): + controller = MongoDBController.mongo_autofix + + def format_ticket_data(self): + bk_biz_id = self.ticket_data["bk_biz_id"] + self.ticket_data["bk_app_abbr"] = AppCache.objects.get(bk_biz_id=bk_biz_id).db_app_abbr + + +class MongoDBAutofixResourceParamBuilder(BaseMongoDBOperateResourceParamBuilder): + def format(self): + pass + + def post_callback(self): + pass + + +@builders.BuilderFactory.register(TicketType.MONGODB_AUTOFIX, is_apply=True) +class MongoDBAutofixFlowBuilder(BaseMongoShardedTicketFlowBuilder): + serializer = MongoDBAutofixDetailSerializer + inner_flow_builder = MongoDBAutofixFlowParamBuilder + inner_flow_name = _("MongoDB 故障自愈") + resource_batch_apply_builder = MongoDBAutofixResourceParamBuilder + default_need_itsm = False + default_need_manual_confirm = False diff --git a/dbm-ui/backend/ticket/builders/mongodb/mongo_instance_deinstall.py b/dbm-ui/backend/ticket/builders/mongodb/mongo_instance_deinstall.py new file mode 100644 index 0000000000..6e848e7ecd --- /dev/null +++ b/dbm-ui/backend/ticket/builders/mongodb/mongo_instance_deinstall.py @@ -0,0 +1,56 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. +Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +from django.utils.translation import ugettext_lazy as _ +from rest_framework import serializers + +from backend.db_meta.models import AppCache +from backend.flow.engine.controller.mongodb import MongoDBController +from backend.ticket import builders +from backend.ticket.builders.mongodb.base import ( + BaseMongoDBOperateDetailSerializer, + BaseMongoDBOperateResourceParamBuilder, + BaseMongoShardedTicketFlowBuilder, +) +from backend.ticket.constants import TicketType + + +class MongoDBInstanceDeInstallDetailSerializer(BaseMongoDBOperateDetailSerializer): + class InstanceDeInstallDetailSerializer(serializers.Serializer): + ip = serializers.CharField(help_text=_("IP地址")) + port = serializers.IntegerField(help_text=_("端口")) + role = serializers.ListField(help_text=_("实例角色")) + bk_cloud_id = serializers.IntegerField(help_text=_("云ID")) + + infos = serializers.ListSerializer(help_text=_("实例下架申请信息"), child=InstanceDeInstallDetailSerializer()) + + +class MongoDBInstanceDeInstallFlowParamBuilder(builders.FlowParamBuilder): + controller = MongoDBController.instance_deinstall + + def format_ticket_data(self): + bk_biz_id = self.ticket_data["bk_biz_id"] + self.ticket_data["bk_app_abbr"] = AppCache.objects.get(bk_biz_id=bk_biz_id).db_app_abbr + + +class MongoDBInstanceDeInstallResourceParamBuilder(BaseMongoDBOperateResourceParamBuilder): + def format(self): + pass + + def post_callback(self): + pass + + +@builders.BuilderFactory.register(TicketType.MONGODB_INSTANCE_DEINSTALL, is_apply=True) +class MongoDBInstanceDeInstallFlowBuilder(BaseMongoShardedTicketFlowBuilder): + serializer = MongoDBInstanceDeInstallDetailSerializer + inner_flow_builder = MongoDBInstanceDeInstallFlowParamBuilder + inner_flow_name = _("MongoDB 实例下架") + resource_batch_apply_builder = MongoDBInstanceDeInstallResourceParamBuilder diff --git a/dbm-ui/backend/ticket/constants.py b/dbm-ui/backend/ticket/constants.py index c09ab262ad..a44b941fa9 100644 --- a/dbm-ui/backend/ticket/constants.py +++ b/dbm-ui/backend/ticket/constants.py @@ -426,6 +426,8 @@ def get_cluster_type_by_ticket(cls, ticket_type): MONGODB_RESTORE = TicketEnumField("MONGODB_RESTORE", _("MongoDB 定点回档"), _("集群维护")) MONGODB_TEMPORARY_DESTROY = TicketEnumField("MONGODB_TEMPORARY_DESTROY", _("MongoDB 临时集群销毁"), _("集群维护")) MONGODB_INSTALL_DBMON = TicketEnumField("MONGODB_INSTALL_DBMON", _("MongoDB 安装DBMon"), _("集群维护")) + MONGODB_AUTOFIX = TicketEnumField("MONGODB_AUTOFIX", _("MongoDB 故障自愈"), _("集群维护")) + MONGODB_INSTANCE_DEINSTALL = TicketEnumField("MONGODB_INSTANCE_DEINSTALL", _("MongoDB 实例下架"), _("集群维护")) # 云区域组件 CLOUD_SERVICE_APPLY = EnumField("CLOUD_SERVICE_APPLY", _("云区域服务部署"))