From 72c4ebd9356b0f68f47e4ece139c3cc65710a636 Mon Sep 17 00:00:00 2001 From: yyhenryyy Date: Mon, 19 Aug 2024 16:48:35 +0800 Subject: [PATCH] =?UTF-8?q?feat(mongodb):=20=E4=BB=8E=E6=97=A7=E7=B3=BB?= =?UTF-8?q?=E7=BB=9F=E8=BF=81=E7=A7=BB=E5=85=83=E6=95=B0=E6=8D=AE=E5=88=B0?= =?UTF-8?q?=E6=96=B0=E7=B3=BB=E7=BB=9F=20#5994?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dbm-ui/backend/components/dns/client.py | 5 + dbm-ui/backend/db_meta/models/spec.py | 12 +- dbm-ui/backend/flow/consts.py | 4 +- .../bamboo/scene/mongodb/mongodb_migrate.py | 70 +++ .../scene/mongodb/sub_task/migrate_meta.py | 103 +++++ .../scene/mongodb/sub_task/mongod_replace.py | 8 +- .../sub_task/multi_replicaset_migrate_meta.py | 47 ++ .../mongodb/sub_task/replicaset_replace.py | 4 +- .../backend/flow/engine/controller/mongodb.py | 9 + .../collections/mongodb/add_password_to_db.py | 2 +- .../collections/mongodb/migrate_meta.py | 56 +++ dbm-ui/backend/flow/urls.py | 2 + .../flow/utils/mongodb/migrate_meta.py | 197 ++++++++ .../flow/utils/mongodb/mongodb_dataclass.py | 21 +- .../mongodb/mongodb_migrate_dataclass.py | 423 ++++++++++++++++++ .../flow/utils/mongodb/mongodb_password.py | 2 + dbm-ui/backend/flow/views/mongodb_scene.py | 12 + 17 files changed, 959 insertions(+), 18 deletions(-) create mode 100644 dbm-ui/backend/flow/engine/bamboo/scene/mongodb/mongodb_migrate.py create mode 100644 dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/migrate_meta.py create mode 100644 dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/multi_replicaset_migrate_meta.py create mode 100644 dbm-ui/backend/flow/plugins/components/collections/mongodb/migrate_meta.py create mode 100644 dbm-ui/backend/flow/utils/mongodb/migrate_meta.py create mode 100644 dbm-ui/backend/flow/utils/mongodb/mongodb_migrate_dataclass.py diff --git a/dbm-ui/backend/components/dns/client.py b/dbm-ui/backend/components/dns/client.py index 388ac62fbc..6c7649ef50 100644 --- a/dbm-ui/backend/components/dns/client.py +++ b/dbm-ui/backend/components/dns/client.py @@ -50,6 +50,11 @@ def __init__(self): url="/api/v1/dns/domain/all", description=_("获取所有ip、域名关系"), ) + self.update_domain_belong_app = self.generate_data_api( + method="POST", + url="/api/v1/dns/domain/app", + description=_("更新域名所属业务关系"), + ) DnsApi = _DnsApi() diff --git a/dbm-ui/backend/db_meta/models/spec.py b/dbm-ui/backend/db_meta/models/spec.py index ce4ca61ced..5262a2e0d3 100644 --- a/dbm-ui/backend/db_meta/models/spec.py +++ b/dbm-ui/backend/db_meta/models/spec.py @@ -37,16 +37,18 @@ class Spec(AuditedModel): max_length=64, choices=ClusterType.get_choices(), help_text=_("集群类型:MySQL、Proxy、Spider") ) spec_machine_type = models.CharField(max_length=64, choices=MachineType.get_choices(), help_text=_("机器规格类型")) - cpu = models.JSONField(null=True, help_text=_("cpu规格描述:{'min':1,'max':10}")) - mem = models.JSONField(null=True, help_text=_("mem规格描述:{'min':100,'max':1000}")) - device_class = models.JSONField(null=True, help_text=_("实际机器机型: ['class1','class2'] ")) - storage_spec = models.JSONField(null=True, help_text=_("存储磁盘需求配置:{'mount_point':'/data','size':500,'type':'ssd'}")) + cpu = models.JSONField(null=True, help_text=_('cpu规格描述:{"min":1,"max":10}')) + mem = models.JSONField(null=True, help_text=_('mem规格描述:{"min":100,"max":1000}')) + device_class = models.JSONField(null=True, help_text=_('实际机器机型: ["class1","class2"]')) + storage_spec = models.JSONField( + null=True, help_text=_('存储磁盘需求配置:[{"mount_point":"/data","size":500,"type":"ssd"}]') + ) desc = models.TextField(help_text=_("资源规格描述"), null=True, blank=True) enable = models.BooleanField(help_text=_("是否启用"), default=True) # es专属 instance_num = models.IntegerField(default=0, help_text=_("实例数(es专属)")) # spider，redis集群专属 - qps = models.JSONField(default=dict, help_text=_("qps规格描述:{'min': 1, 'max': 100}")) + qps = models.JSONField(default=dict, help_text=_('qps规格描述:{"min": 1, "max": 100}')) class Meta: verbose_name = verbose_name_plural = _("资源规格(Spec)") diff --git a/dbm-ui/backend/flow/consts.py b/dbm-ui/backend/flow/consts.py index bb8e787e8a..94f30ba132 100644 --- a/dbm-ui/backend/flow/consts.py +++ b/dbm-ui/backend/flow/consts.py @@ -1137,10 +1137,10 @@ class RequestResultCode(int, StructuredEnum): class MongoDBPasswordRule(str, StructuredEnum): """ - mongodb密码规则 + mongodb密码规则名 """ - RULE = EnumField("password", _("密码规则")) + RULE = EnumField("mongo_password", _("MongoDB密码规则")) class MongoDBClusterRole(str, StructuredEnum): diff --git a/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/mongodb_migrate.py b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/mongodb_migrate.py new file mode 100644 index 0000000000..87e5e15209 --- /dev/null +++ b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/mongodb_migrate.py @@ -0,0 +1,70 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. +Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +import logging.config +from typing import Dict, Optional + +from backend.flow.engine.bamboo.scene.common.builder import Builder +from backend.flow.engine.bamboo.scene.mongodb.sub_task.migrate_meta import cluster_migrate +from backend.flow.engine.bamboo.scene.mongodb.sub_task.multi_replicaset_migrate_meta import multi_replicaset_migrate +from backend.flow.utils.mongodb.mongodb_migrate_dataclass import MigrateActKwargs + +logger = logging.getLogger("flow") + + +class MongoDBMigrateMetaFlow(object): + """元数据迁移flow""" + + def __init__(self, root_id: str, data: Optional[Dict]): + """ + 传入参数 + @param root_id : 任务流程定义的root_id + @param data : 单据传递过来的参数列表，是dict格式 + """ + + self.root_id = root_id + self.data = data + self.get_kwargs = MigrateActKwargs() + self.get_kwargs.payload = data + self.get_kwargs.bk_biz_id = data.get("bk_biz_id") + + def multi_cluster_migrate_flow(self): + """ + cluster migrate流程 + """ + + # 创建流程实例 + pipeline = Builder(root_id=self.root_id, data=self.data) + + # 副本集与分片集群并行 + # 副本集子流程串行 + sub_pipelines = [] + if self.data["MongoReplicaSet"]: + self.get_kwargs.multi_replicaset_info = self.data["MongoReplicaSet"] + sub_pipline = multi_replicaset_migrate( + root_id=self.root_id, + ticket_data=self.data, + sub_kwargs=self.get_kwargs, + ) + sub_pipelines.append(sub_pipline) + # 分片集群并行 + for cluster_info in self.data["MongoShardedCluster"]: + self.get_kwargs.source_cluster_info = cluster_info + sub_pipline = cluster_migrate( + root_id=self.root_id, + ticket_data=self.data, + sub_kwargs=self.get_kwargs, + cluster=True, + ) + sub_pipelines.append(sub_pipline) + pipeline.add_parallel_sub_pipeline(sub_flow_list=sub_pipelines) + + # 运行流程 + pipeline.run_pipeline() diff --git a/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/migrate_meta.py b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/migrate_meta.py new file mode 100644 index 0000000000..14f311219e --- /dev/null +++ b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/migrate_meta.py @@ -0,0 +1,103 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. +Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + +from copy import deepcopy +from typing import Dict, Optional + +from django.utils.translation import ugettext as _ + +from backend.db_meta.enums.cluster_type import ClusterType +from backend.flow.engine.bamboo.scene.common.builder import SubBuilder +from backend.flow.plugins.components.collections.mongodb.add_relationship_to_meta import ( + ExecAddRelationshipOperationComponent, +) +from backend.flow.plugins.components.collections.mongodb.migrate_meta import MongoDBMigrateMetaComponent +from backend.flow.utils.mongodb.mongodb_migrate_dataclass import MigrateActKwargs + + +def cluster_migrate( + root_id: str, ticket_data: Optional[Dict], sub_kwargs: MigrateActKwargs, cluster: bool +) -> SubBuilder: + """ + 单个replicaset迁移元数 + """ + + # 获取变量 + sub_get_kwargs = deepcopy(sub_kwargs) + # 创建子流程 + sub_pipeline = SubBuilder(root_id=root_id, data=ticket_data) + + # 获取副本集机器是否复用 + sub_get_kwargs.skip_machine() + + # 检查 是否已经迁移 从目标环境检查迁移ip是否复用 + kwargs = sub_get_kwargs.get_check_dest_cluster_info( + cluster_name=sub_get_kwargs.source_cluster_info.get("replsetname") + ) + sub_pipeline.add_act( + act_name=_("检查cluster目标端是否存在"), act_component_code=MongoDBMigrateMetaComponent.code, kwargs=kwargs + ) + + # 检查机器规格是否在目标端存在 + kwargs = sub_get_kwargs.get_check_spec_info() + sub_pipeline.add_act(act_name=_("检查目标端机器规格"), act_component_code=MongoDBMigrateMetaComponent.code, kwargs=kwargs) + + # dbconfig保存oplogsize cachesize + if cluster: + namespace = ClusterType.MongoShardedCluster.value + else: + namespace = ClusterType.MongoReplicaSet.value + kwargs = sub_get_kwargs.get_save_conf_info(namespace=namespace) + sub_pipeline.add_act(act_name=_("保存配置"), act_component_code=MongoDBMigrateMetaComponent.code, kwargs=kwargs) + + # 目标业务更新dba 检查目标业务的dba，不一致则更新 + kwargs = sub_get_kwargs.get_dba_info() + sub_pipeline.add_act(act_name=_("更新dba"), act_component_code=MongoDBMigrateMetaComponent.code, kwargs=kwargs) + + # 迁移数据 + kwargs = sub_get_kwargs.get_migrate_info() + sub_pipeline.add_act( + act_name=_("迁移meta"), + act_component_code=ExecAddRelationshipOperationComponent.code, + kwargs=kwargs, + ) + + # node保存密码到密码服务 + kwargs = sub_get_kwargs.get_save_password_info() + sub_pipeline.add_act(act_name=_("保存密码"), act_component_code=MongoDBMigrateMetaComponent.code, kwargs=kwargs) + + # 修改dns的app字段 + kwargs = sub_get_kwargs.get_change_dns_app_info() + sub_pipeline.add_act(act_name=_("更新dns的app"), act_component_code=MongoDBMigrateMetaComponent.code, kwargs=kwargs) + + if cluster: + name = "cluster" + # cluster删除shard的域名 迁移完，运行无误后再删 + # kwargs = sub_get_kwargs.get_shard_delete_doamin_info() + # sub_pipeline.add_act( + # act_name=_("删除shard的domain"), + # act_component_code=MongoDBMigrateMetaComponent.code, + # kwargs=kwargs + # ) + cluster_name = sub_get_kwargs.source_cluster_info["cluster_id"] + # 添加clb + if sub_get_kwargs.source_cluster_info["clb"]: + kwargs = sub_get_kwargs.get_clb_info() + sub_pipeline.add_act( + act_name=_("添加clb到meta"), act_component_code=MongoDBMigrateMetaComponent.code, kwargs=kwargs + ) + else: + name = "replicaset" + cluster_name = sub_get_kwargs.source_cluster_info["replsetname"] + + return sub_pipeline.build_sub_process( + sub_name=_("MongoDB--{}-meta迁移-{}-{}".format(name, sub_get_kwargs.payload["app"], cluster_name)) + ) diff --git a/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/mongod_replace.py b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/mongod_replace.py index d350964f27..ab6d09fc0e 100644 --- a/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/mongod_replace.py +++ b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/mongod_replace.py @@ -53,8 +53,7 @@ def mongod_replace( # 获取参数 new_node = info["target"] sub_sub_get_kwargs.payload["app"] = sub_sub_get_kwargs.payload["bk_app_abbr"] - if cluster_role != MongoDBClusterRole.ConfigSvr.value or not cluster_role: - sub_sub_get_kwargs.replicaset_info = {} + sub_sub_get_kwargs.replicaset_info = {} sub_sub_get_kwargs.replicaset_info["port"] = sub_sub_get_kwargs.db_instance["port"] force = True if cluster_role: @@ -68,6 +67,8 @@ def mongod_replace( sub_sub_get_kwargs.payload["config_nodes"] = [] sub_sub_get_kwargs.payload["shards_nodes"] = [] sub_sub_get_kwargs.payload["mongos_nodes"] = [] + # 获取配置 + conf = sub_sub_get_kwargs.get_conf(cluster_name=sub_sub_get_kwargs.db_instance["cluster_name"]) if cluster_role == MongoDBClusterRole.ConfigSvr.value: sub_sub_get_kwargs.payload["config_nodes"] = [ { @@ -77,6 +78,8 @@ def mongod_replace( "bk_cloud_id": info["bk_cloud_id"], } ] + sub_sub_get_kwargs.replicaset_info["cacheSizeGB"] = conf["config_cacheSizeGB"] + sub_sub_get_kwargs.replicaset_info["oplogSizeMB"] = conf["config_oplogSizeMB"] elif cluster_role == MongoDBClusterRole.ShardSvr.value: shard_nodes = { "nodes": [ @@ -90,7 +93,6 @@ def mongod_replace( } sub_sub_get_kwargs.payload["shards_nodes"].append(shard_nodes) # shard直接获取配置 - conf = sub_sub_get_kwargs.get_conf(cluster_name=sub_sub_get_kwargs.db_instance["cluster_name"]) sub_sub_get_kwargs.replicaset_info["cacheSizeGB"] = conf["cacheSizeGB"] sub_sub_get_kwargs.replicaset_info["oplogSizeMB"] = conf["oplogSizeMB"] else: diff --git a/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/multi_replicaset_migrate_meta.py b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/multi_replicaset_migrate_meta.py new file mode 100644 index 0000000000..b519553f9e --- /dev/null +++ b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/multi_replicaset_migrate_meta.py @@ -0,0 +1,47 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. +Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + +from copy import deepcopy +from typing import Dict, Optional + +from django.utils.translation import ugettext as _ + +from backend.flow.engine.bamboo.scene.common.builder import SubBuilder +from backend.flow.engine.bamboo.scene.mongodb.sub_task.migrate_meta import cluster_migrate +from backend.flow.utils.mongodb.mongodb_migrate_dataclass import MigrateActKwargs + + +def multi_replicaset_migrate( + root_id: str, + ticket_data: Optional[Dict], + sub_kwargs: MigrateActKwargs, +) -> SubBuilder: + """ + 多个replicaset迁移元数据 + """ + + # 获取变量 + sub_get_kwargs = deepcopy(sub_kwargs) + # 创建子流程 + sub_pipeline = SubBuilder(root_id=root_id, data=ticket_data) + + # 多个replicaset迁移元数据——串行 + for replicaset_info in sub_get_kwargs.multi_replicaset_info: + sub_get_kwargs.source_cluster_info = replicaset_info + flow = cluster_migrate( + root_id=root_id, + ticket_data=ticket_data, + sub_kwargs=sub_get_kwargs, + cluster=False, + ) + sub_pipeline.add_sub_pipeline(sub_flow=flow) + + return sub_pipeline.build_sub_process(sub_name=_("MongoDB--复制集迁移元数据")) diff --git a/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/replicaset_replace.py b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/replicaset_replace.py index f046f073df..9327db4279 100644 --- a/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/replicaset_replace.py +++ b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/replicaset_replace.py @@ -66,8 +66,8 @@ def replicaset_replace( ) # config计算参数 - if cluster_role == MongoDBClusterRole.ConfigSvr.value: - sub_get_kwargs.calc_param_replace(info=info, instance_num=0) + # if cluster_role == MongoDBClusterRole.ConfigSvr.value: + # sub_get_kwargs.calc_param_replace(info=info, instance_num=0) # 进行替换——并行 以ip为维度 sub_sub_pipelines = [] for mongodb_instance in info["instances"]: diff --git a/dbm-ui/backend/flow/engine/controller/mongodb.py b/dbm-ui/backend/flow/engine/controller/mongodb.py index 9ea1b21842..00f139b94b 100644 --- a/dbm-ui/backend/flow/engine/controller/mongodb.py +++ b/dbm-ui/backend/flow/engine/controller/mongodb.py @@ -17,6 +17,7 @@ from backend.flow.engine.bamboo.scene.mongodb.mongodb_install import MongoDBInstallFlow from backend.flow.engine.bamboo.scene.mongodb.mongodb_install_dbmon import MongoInstallDBMon from backend.flow.engine.bamboo.scene.mongodb.mongodb_instance_restart import MongoRestartInstanceFlow +from backend.flow.engine.bamboo.scene.mongodb.mongodb_migrate import MongoDBMigrateMetaFlow from backend.flow.engine.bamboo.scene.mongodb.mongodb_remove_ns import MongoRemoveNsFlow from backend.flow.engine.bamboo.scene.mongodb.mongodb_replace import MongoReplaceFlow from backend.flow.engine.bamboo.scene.mongodb.mongodb_restore import MongoRestoreFlow @@ -181,3 +182,11 @@ def disable_cluster(self): flow = MongoEnableDisableFlow(root_id=self.root_id, data=self.ticket_data) flow.multi_cluster_flow(enable=False) + + def migrate_meta(self): + """ + 迁移元数据 + """ + + flow = MongoDBMigrateMetaFlow(root_id=self.root_id, data=self.ticket_data) + flow.multi_cluster_migrate_flow() diff --git a/dbm-ui/backend/flow/plugins/components/collections/mongodb/add_password_to_db.py b/dbm-ui/backend/flow/plugins/components/collections/mongodb/add_password_to_db.py index 4881e36f82..0ec06a792a 100644 --- a/dbm-ui/backend/flow/plugins/components/collections/mongodb/add_password_to_db.py +++ b/dbm-ui/backend/flow/plugins/components/collections/mongodb/add_password_to_db.py @@ -54,7 +54,7 @@ def _execute(self, data, parent_data) -> bool: result = MongoDBPassword().save_password_to_db( instances=kwargs["nodes"], username=username, password=password, operator=kwargs["operator"] ) - if result is not None: + if result: self.log_error("add password of user:{} to db fail, error:{}".format(username, result)) return False self.log_info("add password of users:{} to db successfully".format(",".join(usernames))) diff --git a/dbm-ui/backend/flow/plugins/components/collections/mongodb/migrate_meta.py b/dbm-ui/backend/flow/plugins/components/collections/mongodb/migrate_meta.py new file mode 100644 index 0000000000..f0e7e931b8 --- /dev/null +++ b/dbm-ui/backend/flow/plugins/components/collections/mongodb/migrate_meta.py @@ -0,0 +1,56 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. +Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +import logging +from typing import List + +from pipeline.component_framework.component import Component +from pipeline.core.flow.activity import Service + +from backend.flow.plugins.components.collections.common.base_service import BaseService +from backend.flow.utils.mongodb.migrate_meta import MongoDBMigrateMeta + +logger = logging.getLogger("flow") + + +class MongoDBMigrateMetaService(BaseService): + """ + 根据单据类型执行相关功能 + """ + + def _execute(self, data, parent_data) -> bool: + kwargs = data.get_one_of_inputs("kwargs") + # global_data = data.get_one_of_inputs("global_data") + trans_data = data.get_one_of_inputs("trans_data") + + # if trans_data is None or trans_data == "${trans_data}": + # # 表示没有加载上下文内容，则在此添加 + # trans_data = getattr(flow_context, kwargs["set_trans_data_dataclass"])() + # + # if kwargs["is_update_trans_data"]: + # # 表示合并上下文的内容 + # cluster_info = {**asdict(trans_data), **kwargs["cluster"]} + # self.log_info(_("集群元信息:{}").format(cluster_info)) + result = MongoDBMigrateMeta(info=kwargs).action() + self.log_info("mongodb operate successfully") + data.outputs["trans_data"] = trans_data + return result + + def inputs_format(self) -> List: + return [ + Service.InputItem(name="kwargs", key="kwargs", type="dict", required=True), + Service.InputItem(name="global_data", key="global_data", type="dict", required=True), + ] + + +class MongoDBMigrateMetaComponent(Component): + name = __name__ + code = "mongodb_migrate_meta" + bound_service = MongoDBMigrateMetaService diff --git a/dbm-ui/backend/flow/urls.py b/dbm-ui/backend/flow/urls.py index c15dc5dc86..6b88a56f7a 100644 --- a/dbm-ui/backend/flow/urls.py +++ b/dbm-ui/backend/flow/urls.py @@ -77,6 +77,7 @@ from backend.flow.views.mongodb_scene import ( ClusterInstallApiView, MongoBackupApiView, + MongoDBClusterMigrateView, MongoDBCreateUserView, MongoDBDeInstallView, MongoDBDeleteUserView, @@ -329,6 +330,7 @@ url(r"^scene/multi_cluster_reduce_node$", MongoDBReduceNodeView.as_view()), url(r"^scene/multi_cluster_enable$", MongoDBEnableClusterView.as_view()), url(r"^scene/multi_cluster_disable$", MongoDBDisableClusterView.as_view()), + url(r"^scene/multi_cluster_migrate_meta$", MongoDBClusterMigrateView.as_view()), # mongodb end # mysql upgrade url(r"^scene/upgrade_mysql_proxy$", UpgradeMySQLProxySceneApiView.as_view()), diff --git a/dbm-ui/backend/flow/utils/mongodb/migrate_meta.py b/dbm-ui/backend/flow/utils/mongodb/migrate_meta.py new file mode 100644 index 0000000000..7adc00f34e --- /dev/null +++ b/dbm-ui/backend/flow/utils/mongodb/migrate_meta.py @@ -0,0 +1,197 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. +Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + +import logging + +from django.utils.translation import ugettext as _ + +from backend.components import DBConfigApi, DnsApi +from backend.components.dbconfig.constants import LevelName, ReqType +from backend.configuration.handlers.dba import DBAdministratorHandler +from backend.db_meta.enums import ClusterEntryType +from backend.db_meta.enums.cluster_type import ClusterType +from backend.db_meta.models import CLBEntryDetail, Cluster, ClusterEntry +from backend.flow.consts import DEFAULT_CONFIG_CONFIRM, DEFAULT_DB_MODULE_ID +from backend.flow.utils import dns_manage +from backend.flow.utils.mongodb.mongodb_password import MongoDBPassword + +logger = logging.getLogger("flow") + + +class MongoDBMigrateMeta(object): + """mongodb迁移元数据flow节点函数""" + + def __init__(self, info: dict): + self.info = info + + def action(self) -> bool: + function_name = self.info["meta_func_name"].lower() + if hasattr(self, function_name): + return getattr(self, function_name)() + + logger.error(_("找不到单据类型，请联系系统管理员")) + return False + + def check_dest_cluster(self): + """检查目标环境是否已经存在该cluster""" + + if Cluster.objects.filter(name=self.info["cluster_name"], bk_biz_id=self.info["bk_biz_id"]).count() > 0: + logger.error( + "error: cluster:{} has of bk_biz_id:{} been existed".format( + self.info["cluster_name"], str(self.info["bk_biz_id"]) + ) + ) + raise ValueError( + "error: cluster:{} has of bk_biz_id:{} been existed".format( + self.info["cluster_name"], str(self.info["bk_biz_id"]) + ) + ) + + def check_machine_spec(self): + """检查机器规格""" + + if self.info["cluster_type"] == ClusterType.MongoReplicaSet.value: + if not self.info["replicaset_spec"]["spec_id"]: + logger.error( + "error: machine spec of destination is not exist about {}".format( + ClusterType.MongoReplicaSet.value + ) + ) + raise ValueError( + "error: machine spec of destination is not exist about {}".format( + ClusterType.MongoReplicaSet.value + ) + ) + elif self.info["cluster_type"] == ClusterType.MongoShardedCluster.value: + for key, value in self.info["cluster_spec"].items(): + if not value["spec_id"]: + logger.error( + "error: machine spec of destination is not exist about {} {}".format( + ClusterType.MongoShardedCluster.value, key + ) + ) + raise ValueError( + "error: machine spec of destination is not exist about {} {}".format( + ClusterType.MongoShardedCluster.value, key + ) + ) + + def save_config(self): + """保存cluster的配置""" + + DBConfigApi.upsert_conf_item( + { + "conf_file_info": { + "conf_file": self.info["conf_file"], + "conf_type": self.info["conf_type"], + "namespace": self.info["namespace"], + }, + "conf_items": self.info["conf_items"], + "level_info": {"module": str(DEFAULT_DB_MODULE_ID)}, + "confirm": DEFAULT_CONFIG_CONFIRM, + "req_type": ReqType.SAVE_AND_PUBLISH, + "bk_biz_id": self.info["bk_biz_id"], + "level_name": LevelName.CLUSTER, + "level_value": self.info["cluster_name"], + } + ) + + def upsert_dba(self): + """更新dba""" + + DBAdministratorHandler.upsert_biz_admins(self.info["bk_biz_id"], self.info["db_admins"]) + + def save_password(self): + """保存密码到密码服务""" + + for username in self.info["usernames"]: + for password_info in self.info["password_infos"]: + result = MongoDBPassword().save_password_to_db( + instances=password_info["nodes"], + username=username, + password=password_info["password"][username], + operator=self.info["operator"], + ) + if result: + logger.error("save password fail, error: {}".format(result)) + + def change_domain_app(self): + """修改dns的app字段""" + + for domain in self.info["change_domain_app"]: + domain_name = domain if domain.endswith(".") else "{}.".format(domain) + try: + DnsApi.update_domain_belong_app( + { + "app": self.info["app"], + "new_app": self.info["new_app"], + "domain_name": domain_name, + "bk_cloud_id": self.info["bk_cloud_id"], + } + ) + except Exception as e: + logger.error( + "change domain:{} dns app fail, from old app:{} to new app:{}, error:{}".format( + domain_name, self.info["app"], self.info["new_app"], str(e) + ) + ) + raise ValueError( + "change domain:{} dns app fail, from old app:{} to new app:{}, error:{}".format( + domain_name, self.info["app"], self.info["new_app"], str(e) + ) + ) + + def shard_delete_domain(self): + """shard删除domain""" + + dns = dns_manage.DnsManage(bk_biz_id=self.info["bk_biz_id"], bk_cloud_id=self.info["bk_cloud_id"]) + for delete_domain in self.info["delete_domain"]: + dns.remove_domain_ip(del_instance_list=delete_domain["del_instance_list"], domain=delete_domain["domain"]) + + def add_clb_domain(self): + """添加clb到meta""" + + entry_type = ClusterEntryType.CLB + cluster = Cluster.objects.get( + bk_cloud_id=self.info["bk_cloud_id"], name=self.info["name"], bk_biz_id=self.info["bk_biz_id"] + ) + proxy_objs = cluster.proxyinstance_set.all() + clb_cluster_entry = ClusterEntry.objects.create( + cluster=cluster, + cluster_entry_type=entry_type, + entry=self.info["clb"]["clb_ip"], + creator=self.info["created_by"], + ) + clb_cluster_entry.save() + clb_cluster_entry.proxyinstance_set.add(*proxy_objs) + clb_domain = "" + if "clb_domain" in self.info["clb"]: + clb_domain = self.info["clb"]["clb_domain"] + if clb_domain != "": + entry_type = ClusterEntryType.CLBDNS + clbdns_cluster_entry = ClusterEntry.objects.create( + cluster=cluster, + cluster_entry_type=entry_type, + forward_to_id=clb_cluster_entry.id, + entry=clb_domain, + creator=self.info["created_by"], + ) + clbdns_cluster_entry.save() + clbdns_cluster_entry.proxyinstance_set.add(*proxy_objs) + clb_entry = CLBEntryDetail.objects.create( + clb_ip=self.info["clb"]["clb_ip"], + clb_id=self.info["clb"]["clb_id"], + listener_id=self.info["clb"]["clb_listener_id"], + clb_region=self.info["region"], + entry_id=clb_cluster_entry.id, + creator=self.info["created_by"], + ) + clb_entry.save() diff --git a/dbm-ui/backend/flow/utils/mongodb/mongodb_dataclass.py b/dbm-ui/backend/flow/utils/mongodb/mongodb_dataclass.py index ca4003d478..535f139457 100644 --- a/dbm-ui/backend/flow/utils/mongodb/mongodb_dataclass.py +++ b/dbm-ui/backend/flow/utils/mongodb/mongodb_dataclass.py @@ -124,6 +124,7 @@ def save_conf(self, namespace: str): key_file = "" cache_size = "" oplog_size = "" + conf_items = [] if namespace == ClusterType.MongoReplicaSet.value: cluster_name = self.replicaset_info["set_id"] key_file = self.replicaset_info["key_file"] @@ -134,11 +135,21 @@ def save_conf(self, namespace: str): key_file = self.payload["key_file"] cache_size = str(self.payload["shards"][0]["cacheSizeGB"]) oplog_size = str(self.payload["shards"][0]["oplogSizeMB"]) - conf_items = [ - {"conf_name": "key_file", "conf_value": key_file, "op_type": OpType.UPDATE}, - {"conf_name": "cacheSizeGB", "conf_value": cache_size, "op_type": OpType.UPDATE}, - {"conf_name": "oplogSizeMB", "conf_value": oplog_size, "op_type": OpType.UPDATE}, - ] + config_cache_size = str(self.payload["config"]["cacheSizeGB"]) + config_oplog_size = str(self.payload["config"]["oplogSizeMB"]) + conf_items.extend( + [ + {"conf_name": "config_cacheSizeGB", "conf_value": config_cache_size, "op_type": OpType.UPDATE}, + {"conf_name": "config_oplogSizeMB", "conf_value": config_oplog_size, "op_type": OpType.UPDATE}, + ] + ) + conf_items.extend( + [ + {"conf_name": "key_file", "conf_value": key_file, "op_type": OpType.UPDATE}, + {"conf_name": "cacheSizeGB", "conf_value": cache_size, "op_type": OpType.UPDATE}, + {"conf_name": "oplogSizeMB", "conf_value": oplog_size, "op_type": OpType.UPDATE}, + ] + ) DBConfigApi.upsert_conf_item( { "conf_file_info": { diff --git a/dbm-ui/backend/flow/utils/mongodb/mongodb_migrate_dataclass.py b/dbm-ui/backend/flow/utils/mongodb/mongodb_migrate_dataclass.py new file mode 100644 index 0000000000..268dfdcf92 --- /dev/null +++ b/dbm-ui/backend/flow/utils/mongodb/mongodb_migrate_dataclass.py @@ -0,0 +1,423 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. +Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +from dataclasses import dataclass + +from backend.components.dbconfig.constants import OpType +from backend.configuration.constants import DBType +from backend.db_meta.enums.cluster_type import ClusterType +from backend.db_meta.enums.instance_role import InstanceRole +from backend.db_meta.enums.machine_type import MachineType +from backend.db_meta.models import Machine +from backend.db_meta.models.spec import Spec +from backend.flow.consts import DEFAULT_DB_MODULE_ID, ConfigTypeEnum, MongoDBManagerUser +from backend.flow.utils.mongodb.migrate_meta import MongoDBMigrateMeta + + +@dataclass() +class MigrateActKwargs: + """节点私有变量数据类""" + + def __init__(self): + # 传入流程信息 + self.payload: dict = None + # 目标端的bk_biz_id + self.bk_biz_id: int = None + # 流程id + self.root_id: int = None + # 多个副本集的信息 + self.multi_replicaset_info: list = None + # 副本集的信息 + self.replicaset_info: str = None + # cluster的信息 + self.cluster_info: str = None + # 源端cluster信息 + self.source_cluster_info: dict = None + # 实例角色 + self.instance_role = [ + InstanceRole.MONGO_M1.value, + InstanceRole.MONGO_M2.value, + InstanceRole.MONGO_M3.value, + InstanceRole.MONGO_M4.value, + InstanceRole.MONGO_M5.value, + InstanceRole.MONGO_M6.value, + InstanceRole.MONGO_M7.value, + InstanceRole.MONGO_M8.value, + InstanceRole.MONGO_M9.value, + InstanceRole.MONGO_M10.value, + InstanceRole.MONGO_BACKUP.value, + ] + # 目标环境副本集mongodb机型规格 + self.dest_replicaset_machine_spec: dict = self.get_mongodb_spec_info( + cluster_type=ClusterType.MongoReplicaSet.value, machine_type=MachineType.MONGODB.value + ) + # 目标环境cluster mongos机型规格 + self.dest_mongos_machine_spec: dict = self.get_mongodb_spec_info( + cluster_type=ClusterType.MongoShardedCluster.value, machine_type=MachineType.MONGOS.value + ) + # 目标环境cluster config机型规格 + self.dest_config_machine_spec: dict = self.get_mongodb_spec_info( + cluster_type=ClusterType.MongoShardedCluster.value, machine_type=MachineType.MONOG_CONFIG.value + ) + # 目标环境cluster mongodb机型规格 + self.dest_mongodb_machine_spec: dict = self.get_mongodb_spec_info( + cluster_type=ClusterType.MongoShardedCluster.value, machine_type=MachineType.MONGODB.value + ) + # 副本集源环境机器映射目标环境机器规格 + self.replicaset_machine_spec: dict = None + # cluster源环境机器映射目标环境机器规格 + self.cluster_machine_spec: dict = None + # 管理员用户 + self.manager_users: list = [ + MongoDBManagerUser.DbaUser.value, + MongoDBManagerUser.AppDbaUser.value, + MongoDBManagerUser.MonitorUser.value, + MongoDBManagerUser.AppMonitorUser.value, + ] + + def skip_machine(self): + """副本集机器复用""" + + if self.source_cluster_info.get("cluster_type") == ClusterType.MongoReplicaSet.value: + for node in self.source_cluster_info.get("storages"): + if Machine.objects.filter(ip=node["ip"], bk_cloud_id=node["bk_cloud_id"]).count() > 0: + self.source_cluster_info["skip_machine"] = True + break + else: + self.source_cluster_info["skip_machine"] = False + + @staticmethod + def get_mongodb_spec_info(cluster_type: str, machine_type: str) -> dict: + """获取机器规格""" + + machine_specs = {} + for spec in Spec.objects.filter(spec_cluster_type=cluster_type, spec_machine_type=machine_type): + spec_info = {"spec_id": spec.spec_id, "spec_config": spec.get_spec_info()} + for device in spec.device_class: + machine_specs[device] = spec_info + return machine_specs + + def get_check_dest_cluster_info(self, cluster_name) -> dict: + """检查cluster是否已经在目标端存在信息""" + + return { + "bk_biz_id": self.bk_biz_id, + "cluster_name": cluster_name, + "meta_func_name": MongoDBMigrateMeta.check_dest_cluster.__name__, + } + + def get_check_spec_info(self) -> dict: + """检查目标端机器规格是否存在""" + + replicaset_spec = {} + cluster_spec = {} + if self.source_cluster_info.get("cluster_type") == ClusterType.MongoReplicaSet.value: + if self.source_cluster_info.get("machine_type") in self.dest_replicaset_machine_spec: + replicaset_spec = self.dest_replicaset_machine_spec.get(self.source_cluster_info.get("machine_type")) + else: + replicaset_spec = {"spec_id": 0, "spec_config": ""} + self.replicaset_machine_spec = replicaset_spec + elif self.source_cluster_info.get("cluster_type") == ClusterType.MongoShardedCluster.value: + # mnogos + if self.source_cluster_info.get("proxies_machine_type") in self.dest_mongos_machine_spec: + cluster_spec.update( + {"mongos": self.dest_mongos_machine_spec.get(self.source_cluster_info.get("proxies_machine_type"))} + ) + else: + cluster_spec.update({"mongos": {"spec_id": 0, "spec_config": ""}}) + # config + if self.source_cluster_info.get("configs_machine_type") in self.dest_config_machine_spec: + cluster_spec.update( + { + "mongo_config": self.dest_config_machine_spec.get( + self.source_cluster_info.get("configs_machine_type") + ) + } + ) + else: + cluster_spec.update({"mongo_config": {"spec_id": 0, "spec_config": ""}}) + # shard + if self.source_cluster_info.get("storages_machine_type") in self.dest_mongodb_machine_spec: + cluster_spec.update( + { + "mongodb": self.dest_mongodb_machine_spec.get( + self.source_cluster_info.get("storages_machine_type") + ) + } + ) + else: + cluster_spec.update({"mongodb": {"spec_id": 0, "spec_config": ""}}) + self.cluster_machine_spec = cluster_spec + return { + "cluster_type": self.source_cluster_info["cluster_type"], + "replicaset_spec": replicaset_spec, + "cluster_spec": cluster_spec, + "meta_func_name": MongoDBMigrateMeta.check_machine_spec.__name__, + } + + def get_save_conf_info(self, namespace: str) -> dict: + """获取保存到dbconfig的配置信息""" + + cluster_info = self.source_cluster_info + conf_type = ConfigTypeEnum.DBConf.value + conf_file = "{}-{}".format("Mongodb", cluster_info.get("major_version").split("-")[1].split(".")[0]) + cluster_name = "" + key_file = "" + cache_size = "" + oplog_size = "" + conf_items = [] + if namespace == ClusterType.MongoReplicaSet.value: + cluster_name = cluster_info["replsetname"] + key_file = cluster_info["key_file"] + cache_size = str(cluster_info["cache_size_gb"]) + oplog_size = str(cluster_info["oplog_size_mb"]) + elif namespace == ClusterType.MongoShardedCluster.value: + cluster_name = cluster_info["cluster_id"] + key_file = cluster_info["key_file"] + cache_size = str(cluster_info["storages_cache_size_gb"]) + oplog_size = str(cluster_info["storages_oplog_size_mb"]) + config_cache_size = str(cluster_info["configs_cache_size_gb"]) + config_oplog_size = str(cluster_info["configs_oplog_size_mb"]) + conf_items.extend( + [ + {"conf_name": "config_cacheSizeGB", "conf_value": config_cache_size, "op_type": OpType.UPDATE}, + {"conf_name": "config_oplogSizeMB", "conf_value": config_oplog_size, "op_type": OpType.UPDATE}, + ] + ) + conf_items.extend( + [ + {"conf_name": "key_file", "conf_value": key_file, "op_type": OpType.UPDATE}, + {"conf_name": "cacheSizeGB", "conf_value": cache_size, "op_type": OpType.UPDATE}, + {"conf_name": "oplogSizeMB", "conf_value": oplog_size, "op_type": OpType.UPDATE}, + ] + ) + return { + "conf_file": conf_file, + "conf_type": conf_type, + "namespace": namespace, + "conf_items": conf_items, + "bk_biz_id": str(self.bk_biz_id), + "cluster_name": cluster_name, + "meta_func_name": MongoDBMigrateMeta.save_config.__name__, + } + + def get_dba_info(self) -> dict: + """获取dba信息""" + + return { + "bk_biz_id": self.bk_biz_id, + "db_admins": [ + {"db_type": DBType.MongoDB.value, "users": self.source_cluster_info["mongodb_dbas"].split(",")} + ], + "meta_func_name": MongoDBMigrateMeta.upsert_dba.__name__, + } + + def instance_exchange_role(self, instances: dict) -> list: + """role转换""" + + nodes = [] + backup_node = {} + for instance in instances: + if instance.get("role") == "shardsvr-backup": + backup_node = instance + else: + nodes.append(instance) + nodes.append(backup_node) + nodes[-1]["role"] = self.instance_role[-1] + for index in range(len(nodes) - 1): + nodes[index]["role"] = self.instance_role[index] + return nodes + + def get_migrate_info(self) -> dict: + """获取迁移信息""" + + info = { + "bk_biz_id": self.bk_biz_id, + "major_version": self.source_cluster_info.get("major_version"), + "creator": self.source_cluster_info.get("mongodb_dbas").split(",")[0], + "region": self.source_cluster_info.get("region"), + "db_module_id": DEFAULT_DB_MODULE_ID, + "cluster_type": self.source_cluster_info.get("cluster_type"), + } + if self.source_cluster_info.get("cluster_type") == ClusterType.MongoReplicaSet.value: + # role转换 + # 找出backup,immute_domain + storages = [] + backup_node = {} + immute_domain = "" + for instance in self.source_cluster_info.get("storages"): + if instance.get("role") == "shardsvr-backup": + backup_node = instance + else: + if instance.get("role") == "shardsvr-primary": + immute_domain = instance.get("domain") + storages.append(instance) + storages.append(backup_node) + # 转换role + if len(storages) > 2: + storages[-1]["role"] = self.instance_role[-1] + for index in range(len(storages) - 1): + storages[index]["role"] = self.instance_role[index] + else: + for index in range(len(storages)): + storages[index]["role"] = self.instance_role[index] + info.update( + { + "skip_machine": self.source_cluster_info.get("skip_machine"), + "immute_domain": immute_domain, + "name": self.source_cluster_info.get("replsetname"), + "alias": self.source_cluster_info.get("replsetname"), + "spec_id": self.replicaset_machine_spec.get("spec_id"), + "spec_config": self.replicaset_machine_spec.get("spec_config"), + "bk_cloud_id": storages[0]["bk_cloud_id"], + "storages": storages, + } + ) + elif self.source_cluster_info.get("cluster_type") == ClusterType.MongoShardedCluster.value: + # proxies + proxies = [ + {"ip": mongos.get("ip"), "port": mongos.get("port")} + for mongos in self.source_cluster_info.get("proxies") + ] + + # configs + configs = [] + config_info = self.source_cluster_info.get("configsvr") + # role转换 config默认3node + nodes = self.instance_exchange_role(instances=config_info.get("nodes")) + configs.append({"shard": config_info.get("shard"), "nodes": nodes}) + # shard shard默认3个node + storages = [] + for shard in self.source_cluster_info.get("storages"): + # role转换 + nodes = self.instance_exchange_role(instances=shard.get("nodes")) + storages.append({"shard": shard.get("shard"), "nodes": nodes}) + + info.update( + { + "name": self.source_cluster_info.get("cluster_id"), + "alias": self.source_cluster_info.get("cluster_id"), + "bk_cloud_id": self.source_cluster_info.get("proxies")[0].get("bk_cloud_id"), + "machine_specs": self.cluster_machine_spec, + "immute_domain": self.source_cluster_info.get("immute_domain"), + "proxies": proxies, + "configs": configs, + "storages": storages, + } + ) + return info + + def get_save_password_info(self) -> dict: + """获取保存密码信息""" + + info = { + "usernames": self.manager_users, + "operator": self.source_cluster_info.get("mongodb_dbas").split(",")[0], + "password_infos": [], + "meta_func_name": MongoDBMigrateMeta.save_password.__name__, + } + if self.source_cluster_info.get("cluster_type") == ClusterType.MongoReplicaSet.value: + nodes = [ + {"ip": node.get("ip"), "port": node.get("port"), "bk_cloud_id": node.get("bk_cloud_id")} + for node in self.source_cluster_info.get("storages") + ] + info["password_infos"].append( + { + "nodes": nodes, + "password": self.source_cluster_info.get("password"), + } + ) + elif self.source_cluster_info.get("cluster_type") == ClusterType.MongoShardedCluster.value: + # mongos + proxie_nodes = [ + {"ip": node.get("ip"), "port": node.get("port"), "bk_cloud_id": node.get("bk_cloud_id")} + for node in self.source_cluster_info.get("proxies") + ] + info["password_infos"].append( + {"nodes": proxie_nodes, "password": self.source_cluster_info.get("proxies_password")} + ) + # config + config_nodes = [ + {"ip": node.get("ip"), "port": node.get("port"), "bk_cloud_id": node.get("bk_cloud_id")} + for node in self.source_cluster_info.get("configsvr")["nodes"] + ] + info["password_infos"].append( + {"nodes": config_nodes, "password": self.source_cluster_info.get("configsvr").get("password")} + ) + # shard + for shard in self.source_cluster_info.get("storages"): + nodes = [ + {"ip": node.get("ip"), "port": node.get("port"), "bk_cloud_id": node.get("bk_cloud_id")} + for node in shard.get("nodes") + ] + info["password_infos"].append({"nodes": nodes, "password": shard.get("password")}) + return info + + def get_change_dns_app_info(self) -> dict: + """获取修改dns的app字段信息""" + + change_domain_app = [] + bk_cloud_id = 0 + if self.source_cluster_info.get("cluster_type") == ClusterType.MongoReplicaSet.value: + bk_cloud_id = self.source_cluster_info.get("storages")[0].get("bk_cloud_id") + change_domain_app = [node.get("domain") for node in self.source_cluster_info.get("storages")] + elif self.source_cluster_info.get("cluster_type") == ClusterType.MongoShardedCluster.value: + bk_cloud_id = self.source_cluster_info.get("proxies")[0].get("bk_cloud_id") + change_domain_app = [self.source_cluster_info.get("immute_domain")] + # clb域名 + if self.source_cluster_info["clb"]: + change_domain_app.append(self.source_cluster_info["clb"]["clb_domain"]) + + return { + "app": self.source_cluster_info.get("app"), + "new_app": str(self.bk_biz_id), + "bk_cloud_id": bk_cloud_id, + "change_domain_app": change_domain_app, + "meta_func_name": MongoDBMigrateMeta.change_domain_app.__name__, + } + + def get_shard_delete_doamin_info(self): + """获取shard删除domain信息""" + + # config + delete_domain = [ + {"domain": node.get("domain"), "del_instance_list": ["{}#{}".format(node.get("ip"), node.get("port"))]} + for node in self.source_cluster_info.get("configsvr").get("nodes") + ] + # shard + for shard in self.source_cluster_info.get("storages"): + delete_domain.extend( + [ + { + "domain": node.get("domain"), + "del_instance_list": ["{}#{}".format(node.get("ip"), node.get("port"))], + } + for node in shard.get("nodes") + ] + ) + return { + "bk_biz_id": self.payload.get("app"), + "bk_cloud_id": self.source_cluster_info.get("proxies")[0].get("bk_cloud_id"), + "delete_domain": delete_domain, + "meta_func_name": MongoDBMigrateMeta.shard_delete_domain.__name__, + } + + def get_clb_info(self) -> dict: + """对于cluster，获取clb信息""" + + return { + "bk_biz_id": self.bk_biz_id, + "bk_cloud_id": self.source_cluster_info.get("proxies")[0].get("bk_cloud_id"), + "name": self.source_cluster_info.get("cluster_id"), + "clb": self.source_cluster_info.get("clb"), + "region": self.source_cluster_info.get("region"), + "created_by": self.source_cluster_info.get("mongodb_dbas").split(",")[0], + "meta_func_name": MongoDBMigrateMeta.add_clb_domain.__name__, + } diff --git a/dbm-ui/backend/flow/utils/mongodb/mongodb_password.py b/dbm-ui/backend/flow/utils/mongodb/mongodb_password.py index ec377acb06..7d2c9ee671 100644 --- a/dbm-ui/backend/flow/utils/mongodb/mongodb_password.py +++ b/dbm-ui/backend/flow/utils/mongodb/mongodb_password.py @@ -64,6 +64,8 @@ def save_password_to_db(self, instances: list, username: str, password: str, ope ) if result["code"] != RequestResultCode.Success.value: return result["message"] + " " + result["data"] + else: + return "" def delete_password_from_db(self, instances: list, usernames: list) -> str: """ diff --git a/dbm-ui/backend/flow/views/mongodb_scene.py b/dbm-ui/backend/flow/views/mongodb_scene.py index 4a1d154052..3dacd9c34a 100644 --- a/dbm-ui/backend/flow/views/mongodb_scene.py +++ b/dbm-ui/backend/flow/views/mongodb_scene.py @@ -236,3 +236,15 @@ def post(request): root_id = uuid.uuid1().hex MongoDBController(root_id=root_id, ticket_data=request.data).disable_cluster() return Response({"root_id": root_id}) + + +class MongoDBClusterMigrateView(FlowTestView): + """ + 迁移mongodb元数据 + """ + + @staticmethod + def post(request): + root_id = uuid.uuid1().hex + MongoDBController(root_id=root_id, ticket_data=request.data).migrate_meta() + return Response({"root_id": root_id})