diff --git a/dbm-ui/backend/db_services/mongodb/autofix/mongodb_autofix_ticket.py b/dbm-ui/backend/db_services/mongodb/autofix/mongodb_autofix_ticket.py index 37a1d9aa21..68f057338a 100644 --- a/dbm-ui/backend/db_services/mongodb/autofix/mongodb_autofix_ticket.py +++ b/dbm-ui/backend/db_services/mongodb/autofix/mongodb_autofix_ticket.py @@ -22,6 +22,7 @@ from backend.db_services.redis.autofix.models import RedisAutofixCore from backend.ticket.constants import TicketType from backend.ticket.models import Ticket +from backend.ticket.tasks.ticket_tasks import send_msg_for_flow from backend.utils.time import datetime2str logger = logging.getLogger("root") @@ -31,26 +32,14 @@ def get_resource_spec(mongos_list: list, mongod_list: list) -> dict: """获取申请机器规格信息，采用故障机与新机器园区相对应""" resource_spec = {} - for mongos in mongos_list: + for host in mongos_list + mongod_list: resource_spec.update( { - mongos["ip"]: { - "spec_id": mongos["spec_id"], + host["ip"]: { + "spec_id": host["spec_id"], "count": 1, - "spec_config": mongos["spec_config"], - "Location_spec": {"city": mongos["city"], "sub_zone_ids": [mongos["bk_sub_zone_id"]]}, - } - } - ) - # mongod自愈 - for mongod in mongod_list: - resource_spec.update( - { - mongod["ip"]: { - "spec_id": mongod["spec_id"], - "count": 1, - "spec_config": mongod["spec_config"], - "Location_spec": {"city": mongod["city"], "sub_zone_ids": [mongod["bk_sub_zone_id"]]}, + "spec_config": host["spec_config"], + "Location_spec": {"city": host["city"], "sub_zone_ids": [host["bk_sub_zone_id"]]}, } } ) @@ -97,6 +86,23 @@ def mongo_create_ticket(cluster: RedisAutofixCore, cluster_ids: list, mongos_lis remark=_("自动发起-自愈任务-{}".format(cluster.immute_domain)), details=details, ) + + # 发送自愈消息提醒 + ip_list = [] + for host in mongos_list + mongod_list: + ip_list.append(host["ip"]) + send_msg_for_flow.apply_async( + kwargs={ + "flow_id": ticket.id, + "flow_msg_type": "通知", + "flow_status": "开始执行", + "processor": ",".join(mongodb_dba), + "receiver": ",".join(mongodb_dba), + "detail_address": "自愈ip:[{}]".format(",".join(ip_list)), + } + ) + + # 回写tb_tendis_autofix_core表 cluster.ticket_id = ticket.id cluster.status_version = get_random_string(12) cluster.deal_status = AutofixStatus.AF_WFLOW.value diff --git a/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/mongodb_install.py b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/mongodb_install.py index f248cdb22a..e6b5069356 100644 --- a/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/mongodb_install.py +++ b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/mongodb_install.py @@ -76,6 +76,9 @@ def prepare_job(self, pipeline: Builder): 准备工作 """ + # 获取appdba和appmonitor密码 + self.get_kwargs.get_app_dba_monitor_pwd() + # 安装蓝鲸插件 install_plugin(pipeline=pipeline, get_kwargs=self.get_kwargs, new_cluster=True) diff --git a/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/cluster_enable_disable.py b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/cluster_enable_disable.py index a3b499e8f1..bdd70e63c4 100644 --- a/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/cluster_enable_disable.py +++ b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/cluster_enable_disable.py @@ -17,6 +17,9 @@ from backend.db_meta.enums.cluster_type import ClusterType from backend.flow.consts import MongoDBInstanceType from backend.flow.engine.bamboo.scene.common.builder import SubBuilder +from backend.flow.plugins.components.collections.mongodb.change_instance_status import ( + ChangeInstanceStatusOperationComponent, +) from backend.flow.plugins.components.collections.mongodb.enable_disable_mongodb import ( EnableDisableMongoDBOperationComponent, ) @@ -39,16 +42,26 @@ def cluster_enable_disable( # 设置参数 sub_get_kwargs.payload["app"] = sub_get_kwargs.payload["bk_app_abbr"] + cluster_type = sub_get_kwargs.payload["cluster_type"] # 获取集群信息 sub_get_kwargs.get_cluster_info_deinstall(cluster_id=cluster_id) + # 修改实例状态 + kwargs = { + "cluster_id": cluster_id, + "cluster_type": cluster_type, + "enable": enable, + } + sub_pipeline.add_act( + act_name=_("MongoDB--修改实例状态"), act_component_code=ChangeInstanceStatusOperationComponent.code, kwargs=kwargs + ) + acts_list = [] - name = "" # 启用 if enable: name = "enable" - if sub_get_kwargs.payload["cluster_type"] == ClusterType.MongoReplicaSet.value: + if cluster_type == ClusterType.MongoReplicaSet.value: for node in sub_get_kwargs.payload["nodes"]: kwargs = sub_get_kwargs.get_mongo_start_kwargs( node_info=node, @@ -61,7 +74,7 @@ def cluster_enable_disable( "kwargs": kwargs, } ) - elif sub_get_kwargs.payload["cluster_type"] == ClusterType.MongoShardedCluster.value: + elif cluster_type == ClusterType.MongoShardedCluster.value: for mongos in sub_get_kwargs.payload["mongos_nodes"]: kwargs = sub_get_kwargs.get_mongo_start_kwargs( node_info=mongos, @@ -77,7 +90,7 @@ def cluster_enable_disable( # 禁用 else: name = "disable" - if sub_get_kwargs.payload["cluster_type"] == ClusterType.MongoReplicaSet.value: + if cluster_type == ClusterType.MongoReplicaSet.value: for node in sub_get_kwargs.payload["nodes"]: kwargs = sub_get_kwargs.get_mongo_deinstall_kwargs( node_info=node, @@ -93,10 +106,8 @@ def cluster_enable_disable( "kwargs": kwargs, } ) - elif sub_get_kwargs.payload["cluster_type"] == ClusterType.MongoShardedCluster.value: + elif cluster_type == ClusterType.MongoShardedCluster.value: for mongos in sub_get_kwargs.payload["mongos_nodes"]: - print("=" * 100) - print(mongos) kwargs = sub_get_kwargs.get_mongo_deinstall_kwargs( node_info=mongos, instance_type=MongoDBInstanceType.MongoS.value, diff --git a/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/migrate_meta.py b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/migrate_meta.py index 14f311219e..2c9851112e 100644 --- a/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/migrate_meta.py +++ b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/migrate_meta.py @@ -27,7 +27,7 @@ def cluster_migrate( root_id: str, ticket_data: Optional[Dict], sub_kwargs: MigrateActKwargs, cluster: bool ) -> SubBuilder: """ - 单个replicaset迁移元数 + 集群迁移元数 """ # 获取变量 @@ -70,6 +70,12 @@ def cluster_migrate( kwargs=kwargs, ) + # 相同业务的appdba appmonitor是一致的，以业务为维度保存appdba appmonitor密码到密码服务 + kwargs = sub_get_kwargs.get_save_app_password_info() + sub_pipeline.add_act( + act_name=_("保存appdba appmonitor密码"), act_component_code=MongoDBMigrateMetaComponent.code, kwargs=kwargs + ) + # node保存密码到密码服务 kwargs = sub_get_kwargs.get_save_password_info() sub_pipeline.add_act(act_name=_("保存密码"), act_component_code=MongoDBMigrateMetaComponent.code, kwargs=kwargs) diff --git a/dbm-ui/backend/flow/plugins/components/collections/mongodb/change_instance_status.py b/dbm-ui/backend/flow/plugins/components/collections/mongodb/change_instance_status.py new file mode 100644 index 0000000000..c61003ca77 --- /dev/null +++ b/dbm-ui/backend/flow/plugins/components/collections/mongodb/change_instance_status.py @@ -0,0 +1,75 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. +Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +import logging +from typing import List + +from pipeline.component_framework.component import Component +from pipeline.core.flow.activity import Service + +from backend.db_meta.enums import InstanceStatus +from backend.db_meta.enums.cluster_type import ClusterType +from backend.db_meta.models import Cluster +from backend.flow.plugins.components.collections.common.base_service import BaseService + +logger = logging.getLogger("json") + + +class ChangeInstanceStatusOperation(BaseService): + """ + ChangeInstanceStatus服务 + """ + + def _execute(self, data, parent_data) -> bool: + """ + 执行创建名字服务功能的函数 + global_data 单据全局变量，格式字典 + kwargs 私有变量 + {"cluster_id": 1, "enable": True} + "enable": True 启用 False 禁用 + """ + + # 从流程节点中获取变量 + kwargs = data.get_one_of_inputs("kwargs") + + # 修改meta + if kwargs["enable"]: + status = InstanceStatus.RUNNING.value + else: + status = InstanceStatus.UNAVAILABLE.value + cluster_type = kwargs["cluster_type"] + cluster_id = kwargs["cluster_id"] + try: + if cluster_type == ClusterType.MongoReplicaSet.value: + Cluster.objects.get(id=cluster_id).storageinstance_set.all().update(status=status) + elif cluster_type == ClusterType.MongoShardedCluster.value: + Cluster.objects.get(id=cluster_id).proxyinstance_set.all().update(status=status) + except Exception as e: + logger.error("change cluster:{} instance status:{} fail, error:{}".format(str(cluster_id), status, e)) + return False + self.log_info("change cluster:{} instance status:{} successfully".format(str(cluster_id), status)) + return True + + # 流程节点输入参数 + def inputs_format(self) -> List: + return [ + Service.InputItem(name="kwargs", key="kwargs", type="dict", required=True), + Service.InputItem(name="global_data", key="global_data", type="dict", required=True), + ] + + +class ChangeInstanceStatusOperationComponent(Component): + """ + ChangeInstanceStatusOperation组件 + """ + + name = __name__ + code = "change_instance_status_operation" + bound_service = ChangeInstanceStatusOperation diff --git a/dbm-ui/backend/flow/plugins/components/collections/mongodb/get_manager_user_password.py b/dbm-ui/backend/flow/plugins/components/collections/mongodb/get_manager_user_password.py index facb9dd53c..0e8b3699f4 100644 --- a/dbm-ui/backend/flow/plugins/components/collections/mongodb/get_manager_user_password.py +++ b/dbm-ui/backend/flow/plugins/components/collections/mongodb/get_manager_user_password.py @@ -14,6 +14,7 @@ from pipeline.component_framework.component import Component from pipeline.core.flow.activity import Service +from backend.flow.consts import MongoDBManagerUser from backend.flow.plugins.components.collections.common.base_service import BaseService from backend.flow.utils.mongodb.mongodb_password import MongoDBPassword @@ -35,14 +36,15 @@ def _execute(self, data, parent_data) -> bool: # 从流程节点中获取变量 kwargs = data.get_one_of_inputs("kwargs") trans_data = data.get_one_of_inputs("trans_data") - # - # if trans_data is None or trans_data == "${trans_data}": - # # 表示没有加载上下文内容，则在此添加 - # trans_data = getattr(flow_context, kwargs["set_trans_data_dataclass"])() # 从密码服务获取密码 user_password = {} for user in kwargs["users"]: + # 同一业务所有集群的appdba与appmonitor一样 + if user in [MongoDBManagerUser.AppDbaUser.value, MongoDBManagerUser.AppMonitorUser.value]: + user_password[user] = kwargs[user] + continue + # 其他管理用户获取密码 result = MongoDBPassword().create_user_password() if result["password"] is None: self.log_error("user:{} get password fail, error:{}".format(user, result["info"])) diff --git a/dbm-ui/backend/flow/utils/mongodb/calculate_cluster.py b/dbm-ui/backend/flow/utils/mongodb/calculate_cluster.py index 2337b895dc..674c739ca8 100644 --- a/dbm-ui/backend/flow/utils/mongodb/calculate_cluster.py +++ b/dbm-ui/backend/flow/utils/mongodb/calculate_cluster.py @@ -66,6 +66,9 @@ def replicase_calc(payload: dict, payload_clusters: dict, app: str, domain_prefi # 获取复制集实例 sets = [] node_replica_count = payload["node_replica_count"] + node_count = payload["node_count"] + # 一个副本集的副本数量 + payload_clusters["node_count"] = node_count port = payload["start_port"] oplog_percent = payload["oplog_percent"] / 100 data_disk = "/data1" @@ -96,10 +99,15 @@ def replicase_calc(payload: dict, payload_clusters: dict, app: str, domain_prefi skip_machine = False nodes = [] for machine_index, machine in enumerate(machines): - if machine_index == len(machines) - 1: - domain = "{}.{}.{}.db".format(domain_prefix[-1], replica_set["set_id"], app) - else: + # 副本集node count等于1 + if node_count == 1: domain = "{}.{}.{}.db".format(domain_prefix[machine_index], replica_set["set_id"], app) + elif node_count > 1: + # 副本集node count大于1 + if machine_index == len(machines) - 1: + domain = "{}.{}.{}.db".format(domain_prefix[-1], replica_set["set_id"], app) + else: + domain = "{}.{}.{}.db".format(domain_prefix[machine_index], replica_set["set_id"], app) nodes.append({"ip": machine["ip"], "bk_cloud_id": machine["bk_cloud_id"], "domain": domain}) set_id_key_file = "{}-{}".format(app, replica_set["set_id"]) sets.append( @@ -132,6 +140,9 @@ def cluster_calc(payload: dict, payload_clusters: dict, app: str) -> dict: config_port = MongoDBClusterDefaultPort.CONFIG_PORT.value # 设置常量 shard_port = MongoDBClusterDefaultPort.SHARD_START_PORT.value # 以这个27001开始 shard_port_not_use = [payload["proxy_port"], config_port] + node_count = len(payload["nodes"]["mongodb"][0]) + # 一个副本集的副本数量 + payload_clusters["node_count"] = node_count # 计算configCacheSizeGB，shardCacheSizeGB，oplogSizeMB shard_avg_mem_size_gb = get_cache_size( diff --git a/dbm-ui/backend/flow/utils/mongodb/migrate_meta.py b/dbm-ui/backend/flow/utils/mongodb/migrate_meta.py index 7adc00f34e..bdcc08d427 100644 --- a/dbm-ui/backend/flow/utils/mongodb/migrate_meta.py +++ b/dbm-ui/backend/flow/utils/mongodb/migrate_meta.py @@ -19,7 +19,7 @@ from backend.db_meta.enums import ClusterEntryType from backend.db_meta.enums.cluster_type import ClusterType from backend.db_meta.models import CLBEntryDetail, Cluster, ClusterEntry -from backend.flow.consts import DEFAULT_CONFIG_CONFIRM, DEFAULT_DB_MODULE_ID +from backend.flow.consts import DEFAULT_CONFIG_CONFIRM, DEFAULT_DB_MODULE_ID, MongoDBManagerUser from backend.flow.utils import dns_manage from backend.flow.utils.mongodb.mongodb_password import MongoDBPassword @@ -109,6 +109,31 @@ def upsert_dba(self): DBAdministratorHandler.upsert_biz_admins(self.info["bk_biz_id"], self.info["db_admins"]) + def save_app_password(self): + """保存appdba appmonitor密码到密码服务""" + + for user in [MongoDBManagerUser.AppDbaUser.value, MongoDBManagerUser.AppMonitorUser.value]: + result = MongoDBPassword().get_password_from_db( + ip=str(self.info["bk_biz_id"]), port=0, bk_cloud_id=0, username=user + ) + if result["password"] is None: + logger.error("user:{} get password fail from db, error:{}".format(user, result["info"])) + if not result["password"] == "": + info = MongoDBPassword().save_password_to_db( + instances=[ + { + "ip": str(self.info["bk_biz_id"]), + "port": 0, + "bk_cloud_id": 0, + } + ], + username=user, + password=self.info[user], + operator="admin", + ) + if info != "": + logger.error("user:{} save password to db fail, error:{}".format(user, info)) + def save_password(self): """保存密码到密码服务""" diff --git a/dbm-ui/backend/flow/utils/mongodb/mongodb_dataclass.py b/dbm-ui/backend/flow/utils/mongodb/mongodb_dataclass.py index 33f9216fa6..856067a58d 100644 --- a/dbm-ui/backend/flow/utils/mongodb/mongodb_dataclass.py +++ b/dbm-ui/backend/flow/utils/mongodb/mongodb_dataclass.py @@ -267,6 +267,40 @@ def get_password(ip: str, port: int, bk_cloud_id: int, username: str) -> str: ) return result["password"] + def get_app_dba_monitor_pwd(self): + """获取appdba与appmonitor密码""" + + for user in [MongoDBManagerUser.AppDbaUser.value, MongoDBManagerUser.AppMonitorUser.value]: + result = MongoDBPassword().get_password_from_db( + ip=str(self.payload["bk_biz_id"]), port=0, bk_cloud_id=0, username=user + ) + if result["password"] is None: + raise ValueError("user:{} get password fail from db, error:{}".format(user, result["info"])) + # 不存在密码则获取密码并保存 + else: + if result["password"] == "": + get_password = MongoDBPassword().create_user_password() + if get_password["password"] is None: + raise ValueError("user:{} get password fail, error:{}".format(user, get_password["info"])) + self.payload[user] = get_password["password"] + # 保存密码 + info = MongoDBPassword().save_password_to_db( + instances=[ + { + "ip": str(self.payload["bk_biz_id"]), + "port": 0, + "bk_cloud_id": 0, + } + ], + username=user, + password=get_password["password"], + operator="admin", + ) + if info != "": + raise ValueError("user:{} save password to db fail, error:{}".format(user, info)) + else: + self.payload[user] = result["password"] + def get_send_media_kwargs(self, media_type: str) -> dict: """ 介质下发的kwargs @@ -455,14 +489,18 @@ def get_replicaset_init_kwargs(self, config_svr: bool) -> dict: instances = [ "{}:{}".format(node["ip"], str(self.replicaset_info["port"])) for node in self.replicaset_info["nodes"] ] + node_count = self.payload["node_count"] for index, instance in enumerate(instances): - if index == len(instances) - 1: - priority[instance] = 0 - hidden[instance] = True - else: + if node_count == 1: priority[instance] = 1 hidden[instance] = False - + elif node_count > 1: + if index == len(instances) - 1: + priority[instance] = 0 + hidden[instance] = True + else: + priority[instance] = 1 + hidden[instance] = False return { "set_trans_data_dataclass": CommonContext.__name__, "get_trans_data_ip_var": None, @@ -509,16 +547,8 @@ def get_add_relationship_to_meta_kwargs(self, replicaset_info: dict) -> dict: info["storages"] = [] if len(replicaset_info["nodes"]) <= 11: for index, node in enumerate(replicaset_info["nodes"]): - if index == len(replicaset_info["nodes"]) - 1: - info["storages"].append( - { - "role": self.instance_role[-1], - "ip": node["ip"], - "port": replicaset_info["port"], - "domain": node["domain"], - } - ) - else: + # 只有一个副本 + if self.payload["node_count"] == 1: info["storages"].append( { "role": self.instance_role[index], @@ -527,6 +557,25 @@ def get_add_relationship_to_meta_kwargs(self, replicaset_info: dict) -> dict: "domain": node["domain"], } ) + elif self.payload["node_count"] > 1: + if index == len(replicaset_info["nodes"]) - 1: + info["storages"].append( + { + "role": self.instance_role[-1], + "ip": node["ip"], + "port": replicaset_info["port"], + "domain": node["domain"], + } + ) + else: + info["storages"].append( + { + "role": self.instance_role[index], + "ip": node["ip"], + "port": replicaset_info["port"], + "domain": node["domain"], + } + ) elif self.payload["cluster_type"] == ClusterType.MongoShardedCluster.value: info["cluster_type"] = ClusterType.MongoShardedCluster.value info["name"] = self.payload["cluster_id"] @@ -731,7 +780,13 @@ def get_get_manager_password_kwargs(self) -> dict: set_name = self.replicaset_info["set_id"] else: set_name = "" - return {"set_trans_data_dataclass": CommonContext.__name__, "set_name": set_name, "users": self.manager_users} + return { + "set_trans_data_dataclass": CommonContext.__name__, + "set_name": set_name, + "users": self.manager_users, + "appdba": self.payload[MongoDBManagerUser.AppDbaUser.value], + "appmonitor": self.payload[MongoDBManagerUser.AppMonitorUser.value], + } def get_add_password_to_db_kwargs(self, usernames: list, info: dict) -> dict: """添加密码到db""" diff --git a/dbm-ui/backend/flow/utils/mongodb/mongodb_migrate_dataclass.py b/dbm-ui/backend/flow/utils/mongodb/mongodb_migrate_dataclass.py index 929a2538af..7ea09ff875 100644 --- a/dbm-ui/backend/flow/utils/mongodb/mongodb_migrate_dataclass.py +++ b/dbm-ui/backend/flow/utils/mongodb/mongodb_migrate_dataclass.py @@ -245,7 +245,9 @@ def get_migrate_info(self) -> dict: "region": self.source_cluster_info.get("region"), "db_module_id": DEFAULT_DB_MODULE_ID, "cluster_type": self.source_cluster_info.get("cluster_type"), - "disaster_tolerance_level": self.source_cluster_info.get("disaster_tolerance_level"), + # "disaster_tolerance_level": self.source_cluster_info.get("disaster_tolerance_level"), + # 亲和性默认为跨园区CROS_SUBZONE + "disaster_tolerance_level": "CROS_SUBZONE", } if self.source_cluster_info.get("cluster_type") == ClusterType.MongoReplicaSet.value: # role转换 @@ -315,6 +317,20 @@ def get_migrate_info(self) -> dict: ) return info + def get_save_app_password_info(self) -> dict: + """获取业务的appdba appmonitor密码信息""" + + info = { + "bk_biz_id": self.bk_biz_id, + "meta_func_name": MongoDBMigrateMeta.save_app_password.__name__, + } + if self.source_cluster_info.get("cluster_type") == ClusterType.MongoReplicaSet.value: + info["appdba"] = self.source_cluster_info.get("password").get("appdba") + info["appmonitor"] = self.source_cluster_info.get("password").get("appmonitor") + elif self.source_cluster_info.get("cluster_type") == ClusterType.MongoShardedCluster.value: + info["appdba"] = self.source_cluster_info.get("proxies_password").get("appdba") + info["appmonitor"] = self.source_cluster_info.get("proxies_password").get("appmonitor") + def get_save_password_info(self) -> dict: """获取保存密码信息""" diff --git a/dbm-ui/backend/flow/utils/mongodb/mongodb_password.py b/dbm-ui/backend/flow/utils/mongodb/mongodb_password.py index 7d2c9ee671..5c6dc9e345 100644 --- a/dbm-ui/backend/flow/utils/mongodb/mongodb_password.py +++ b/dbm-ui/backend/flow/utils/mongodb/mongodb_password.py @@ -93,6 +93,8 @@ def get_password_from_db(self, ip: str, port: int, bk_cloud_id: int, username: s result = self.get_nodes_password_from_db(nodes, username) if result["password"] is None: return {"password": None, "info": result["info"]} + if not result["password"]: + return {"password": "", "info": ""} return {"password": result["password"][0].get("password"), "info": None} def get_nodes_password_from_db(self, instances, username: str) -> list: