From edf6c5f6910526b10ce6c176d7d196d78f1822f5 Mon Sep 17 00:00:00 2001 From: iSecloud <869820505@qq.com> Date: Thu, 21 Mar 2024 21:55:20 +0800 Subject: [PATCH] =?UTF-8?q?feat(backend):=20redis=E4=B8=BB=E4=BB=8E?= =?UTF-8?q?=E9=83=A8=E7=BD=B2=20&=20tendbcluster=E9=87=8D=E5=BB=BA=20?= =?UTF-8?q?=E8=81=94=E8=B0=83=20#4106?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../configuration/handlers/password.py | 9 + .../configuration/views/password_policy.py | 4 +- .../api/cluster/redisinstance/__init__.py | 10 ++ .../api/cluster/redisinstance/detail.py | 43 +++++ .../api/cluster/redisinstance/handler.py | 24 +++ .../db_services/dbbase/resources/query.py | 16 +- .../dbbase/resources/serializers.py | 6 +- .../db_services/mongodb/resources/query.py | 1 - .../mysql/resources/tendbcluster/query.py | 7 + .../redis/resources/redis_cluster/query.py | 2 + .../spider/remote_local_slave_recover.py | 1 + .../spider/remote_master_slave_migrate.py | 4 +- .../flow/utils/spider/spider_db_meta.py | 10 +- .../backend/tests/ticket/test_ticket_flow.py | 2 +- dbm-ui/backend/ticket/builders/common/base.py | 4 +- .../backend/ticket/builders/common/bigdata.py | 4 +- .../mysql/mysql_restore_local_slave.py | 5 +- dbm-ui/backend/ticket/builders/redis/base.py | 6 - .../ticket/builders/redis/redis_close.py | 20 +++ .../builders/redis/redis_cluster_apply.py | 84 +++++---- .../ticket/builders/redis/redis_destroy.py | 18 ++ .../builders/redis/redis_instance_apply.py | 161 ++++++++++++++++++ .../ticket/builders/redis/redis_open.py | 20 +++ .../tendbcluster/tendb_migrate_cluster.py | 80 +++++++++ .../tendbcluster/tendb_restore_local_slave.py | 84 +++++++++ .../tendbcluster/tendb_restore_slave.py | 79 +++++++++ dbm-ui/backend/ticket/constants.py | 8 +- .../backend/ticket/flow_manager/resource.py | 8 +- 28 files changed, 645 insertions(+), 75 deletions(-) create mode 100644 dbm-ui/backend/db_meta/api/cluster/redisinstance/__init__.py create mode 100644 dbm-ui/backend/db_meta/api/cluster/redisinstance/detail.py create mode 100644 dbm-ui/backend/db_meta/api/cluster/redisinstance/handler.py create mode 100644 dbm-ui/backend/ticket/builders/redis/redis_instance_apply.py create mode 100644 dbm-ui/backend/ticket/builders/tendbcluster/tendb_migrate_cluster.py create mode 100644 dbm-ui/backend/ticket/builders/tendbcluster/tendb_restore_local_slave.py create mode 100644 dbm-ui/backend/ticket/builders/tendbcluster/tendb_restore_slave.py diff --git a/dbm-ui/backend/configuration/handlers/password.py b/dbm-ui/backend/configuration/handlers/password.py index b9e0917a34..3d8f69876e 100644 --- a/dbm-ui/backend/configuration/handlers/password.py +++ b/dbm-ui/backend/configuration/handlers/password.py @@ -37,6 +37,15 @@ class DBPasswordHandler(object): """密码策略相关处理""" + @classmethod + def get_random_password(cls): + """ + 获取符合密码强度的字符串 + """ + random_password = DBPrivManagerApi.get_random_string({"security_rule_name": DBM_PASSWORD_SECURITY_NAME}) + random_password = base64_decode(random_password) + return random_password + @classmethod def verify_password_strength(cls, password: str, echo: bool = False): """ diff --git a/dbm-ui/backend/configuration/views/password_policy.py b/dbm-ui/backend/configuration/views/password_policy.py index df9251b34b..27b0796c40 100644 --- a/dbm-ui/backend/configuration/views/password_policy.py +++ b/dbm-ui/backend/configuration/views/password_policy.py @@ -8,7 +8,6 @@ an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ -import base64 import json from celery.schedules import crontab @@ -91,8 +90,7 @@ def verify_password_strength(self, request): ) @action(methods=["GET"], detail=False) def get_random_password(self, request, *args, **kwargs): - random_password = DBPrivManagerApi.get_random_string({"security_rule_name": DBM_PASSWORD_SECURITY_NAME}) - random_password = base64.b64decode(random_password).decode("utf-8") + random_password = DBPasswordHandler.get_random_password() return Response({"password": random_password}) @common_swagger_auto_schema( diff --git a/dbm-ui/backend/db_meta/api/cluster/redisinstance/__init__.py b/dbm-ui/backend/db_meta/api/cluster/redisinstance/__init__.py new file mode 100644 index 0000000000..aa5085c628 --- /dev/null +++ b/dbm-ui/backend/db_meta/api/cluster/redisinstance/__init__.py @@ -0,0 +1,10 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. +Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" diff --git a/dbm-ui/backend/db_meta/api/cluster/redisinstance/detail.py b/dbm-ui/backend/db_meta/api/cluster/redisinstance/detail.py new file mode 100644 index 0000000000..9df0665b23 --- /dev/null +++ b/dbm-ui/backend/db_meta/api/cluster/redisinstance/detail.py @@ -0,0 +1,43 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. +Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +from django.utils.translation import gettext as _ + +from backend.db_meta.api.cluster.base.graph import Graphic, LineLabel +from backend.db_meta.enums import InstanceRole +from backend.db_meta.models import Cluster + + +def scan_cluster(cluster: Cluster) -> Graphic: + """ + redis主从关系拓扑图 + """ + graph = Graphic(node_id=Graphic.generate_graphic_id(cluster)) + + # 获取master节点组 + master_insts, master_group = graph.add_instance_nodes( + cluster=cluster, roles=InstanceRole.REDIS_MASTER, group_name=_("Master 节点") + ) + + # 获取slave节点组 + slave_insts, slave_group = graph.add_instance_nodes( + cluster=cluster, roles=InstanceRole.REDIS_SLAVE, group_name=_("Slave 节点") + ) + + # 获得访问入口节点组 + entry = master_insts.first().bind_entry.first() + __, entry_group = graph.add_node(entry) + + # 访问入口 ---> Master/Slave节点，关系为：绑定 + graph.add_line(source=entry_group, target=master_group, label=LineLabel.Bind) + # Master ---> Slave, 关系为同步 + graph.add_line(source=master_group, target=slave_group, label=LineLabel.Rep) + + return graph diff --git a/dbm-ui/backend/db_meta/api/cluster/redisinstance/handler.py b/dbm-ui/backend/db_meta/api/cluster/redisinstance/handler.py new file mode 100644 index 0000000000..81b009867f --- /dev/null +++ b/dbm-ui/backend/db_meta/api/cluster/redisinstance/handler.py @@ -0,0 +1,24 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. +Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +from backend.db_meta.api.cluster.base.handler import ClusterHandler +from backend.db_meta.enums import ClusterType + +from .detail import scan_cluster + + +class RedisInstanceHandler(ClusterHandler): + + # 「必须」 集群类型 + cluster_type = ClusterType.RedisInstance + + def topo_graph(self): + """「必须」提供集群关系拓扑图""" + return scan_cluster(self.cluster).to_dict() diff --git a/dbm-ui/backend/db_services/dbbase/resources/query.py b/dbm-ui/backend/db_services/dbbase/resources/query.py index 0a806e417d..5799811aca 100644 --- a/dbm-ui/backend/db_services/dbbase/resources/query.py +++ b/dbm-ui/backend/db_services/dbbase/resources/query.py @@ -368,6 +368,8 @@ def _list_clusters( Q(name__in=query_params.get("name", "").split(",")) | Q(alias__in=query_params.get("name", "").split(",")) ), + # 集群类型 + "cluster_type": Q(cluster_type=query_params.get("cluster_type")), # 版本 "major_version": Q(major_version__in=query_params.get("major_version", "").split(",")), # 地域 @@ -568,6 +570,7 @@ def _list_instances( "port": Q(port__in=query_params.get("port", "").split(",")), "status": Q(status__in=query_params.get("status", "").split(",")), "cluster_id": Q(cluster__id=query_params.get("cluster_id")), + "cluster_type": Q(cluster__cluster_type=query_params.get("cluster_type")), "region": Q(region=query_params.get("region")), "role": Q(role__in=query_params.get("role", "").split(",")), "name": Q(cluster__name__in=query_params.get("name", "").split(",")), @@ -597,18 +600,20 @@ def _filter_instance_hook(cls, bk_biz_id, query_params, instances, **kwargs): instance_infos = [cls._to_instance_representation(inst, cluster_entry_map, **kwargs) for inst in instances] # 特例：如果有extra参数，则补充额外实例信息 if query_params.get("extra"): - cls._fill_instance_extra_info(bk_biz_id, instance_infos) + cls._fill_instance_extra_info(bk_biz_id, instance_infos, **kwargs) return instance_infos @classmethod - def _fill_instance_extra_info(cls, bk_biz_id: int, instance_infos: List[Dict]): + def _fill_instance_extra_info(cls, bk_biz_id: int, instance_infos: List[Dict], **kwargs): """ 补充实例的额外信息，这里的一个实现是补充主机和关联集群信息 @param bk_biz_id: 业务ID @param instance_infos: 实例字典信息 """ - instances_extra_info = InstanceHandler(bk_biz_id).check_instances(query_instances=instance_infos) + # db_type优先以指定的为准(比如mysql和tendbcluster是用的同一个handler)，然后以集群类型对应的组件为准 + db_type = kwargs.get("handler_db_type") or ClusterType.cluster_type_to_db_type(cls.cluster_types[0]) + instances_extra_info = InstanceHandler(bk_biz_id).check_instances(instance_infos, db_type=db_type) address__instance_extra_info = {inst["instance_address"]: inst for inst in instances_extra_info} for inst in instance_infos: extra_info = address__instance_extra_info[inst["instance_address"]] @@ -718,6 +723,7 @@ def _list_machines( "bk_host_id": Q(bk_host_id=query_params.get("bk_host_id")), "ip": Q(ip__in=query_params.get("ip", "").split(",")), "machine_type": Q(machine_type=query_params.get("machine_type")), + "bk_city_name": Q(bk_city__bk_idc_city_name__in=query_params.get("region", "").split(",")), "bk_os_name": Q(bk_os_name=query_params.get("bk_os_name")), "bk_cloud_id": Q(bk_cloud_id=query_params.get("bk_cloud_id")), "bk_agent_id": Q(bk_agent_id=query_params.get("bk_agent_id")), @@ -725,6 +731,10 @@ def _list_machines( Q(storageinstance__instance_role=query_params.get("instance_role")) | Q(proxyinstance__access_layer=query_params.get("instance_role")) ), + "cluster_ids": ( + Q(storageinstance__cluster__in=query_params.get("cluster_ids", "").split(",")) + | Q(proxyinstance__cluster__in=query_params.get("cluster_ids", "").split(",")) + ), "creator": Q(creator__icontains=query_params.get("creator")), } filter_params_map = {**inner_filter_params_map, **filter_params_map} diff --git a/dbm-ui/backend/db_services/dbbase/resources/serializers.py b/dbm-ui/backend/db_services/dbbase/resources/serializers.py index ecfc7ad721..d1e56964f7 100644 --- a/dbm-ui/backend/db_services/dbbase/resources/serializers.py +++ b/dbm-ui/backend/db_services/dbbase/resources/serializers.py @@ -31,6 +31,7 @@ class ListResourceSLZ(serializers.Serializer): status = serializers.CharField(required=False, help_text=_("状态")) db_module_id = serializers.CharField(required=False, help_text=_("所属DB模块")) bk_cloud_id = serializers.CharField(required=False, help_text=_("管控区域")) + cluster_type = serializers.ChoiceField(required=False, choices=ClusterType.get_choices()) class ListMySQLResourceSLZ(ListResourceSLZ): @@ -42,7 +43,6 @@ class ListSQLServerResourceSLZ(ListResourceSLZ): class ListMongoDBResourceSLZ(ListResourceSLZ): - cluster_type = serializers.ChoiceField(required=False, choices=ClusterType.get_choices()) domains = serializers.CharField(help_text=_("批量域名查询(逗号分割)"), required=False) @@ -96,11 +96,11 @@ class ListInstancesSerializer(InstanceAddressSerializer): status = serializers.CharField(help_text=_("状态"), required=False) role = serializers.CharField(help_text=_("角色"), required=False) cluster_id = serializers.CharField(help_text=_("集群ID"), required=False) + cluster_type = serializers.ChoiceField(help_text=_("集群类型"), required=False, choices=ClusterType.get_choices()) ip = serializers.CharField(required=False) class MongoDBListInstancesSerializer(ListInstancesSerializer): - cluster_type = serializers.ChoiceField(help_text=_("集群类型"), required=False, choices=ClusterType.get_choices()) exact_ip = serializers.CharField(help_text=_("精确IP查询"), required=False) @@ -119,6 +119,8 @@ class ListNodesSLZ(serializers.Serializer): class ListMachineSLZ(serializers.Serializer): bk_host_id = serializers.IntegerField(help_text=_("主机ID"), required=False) ip = serializers.CharField(help_text=_("IP(多个IP过滤以逗号分隔)"), required=False) + cluster_ids = serializers.CharField(help_text=_("集群ID(多个过滤以逗号分隔)"), required=False) + bk_city_name = serializers.CharField(help_text=_("城市名(多个过滤以逗号分隔)"), required=False) machine_type = serializers.ChoiceField(help_text=_("机器类型"), choices=MachineType.get_choices(), required=False) bk_os_name = serializers.CharField(help_text=_("os名字"), required=False) bk_cloud_id = serializers.IntegerField(help_text=_("云区域ID"), required=False) diff --git a/dbm-ui/backend/db_services/mongodb/resources/query.py b/dbm-ui/backend/db_services/mongodb/resources/query.py index 3b3cbd0140..e9dccb5296 100644 --- a/dbm-ui/backend/db_services/mongodb/resources/query.py +++ b/dbm-ui/backend/db_services/mongodb/resources/query.py @@ -49,7 +49,6 @@ def _list_clusters( ) -> ResourceList: """查询集群信息""" filter_params_map = { - "cluster_type": Q(cluster_type=query_params.get("cluster_type")), "domains": Q(immute_domain__in=query_params.get("domains", "").split(",")), } return super()._list_clusters( diff --git a/dbm-ui/backend/db_services/mysql/resources/tendbcluster/query.py b/dbm-ui/backend/db_services/mysql/resources/tendbcluster/query.py index f89baac385..38a8c4c2da 100644 --- a/dbm-ui/backend/db_services/mysql/resources/tendbcluster/query.py +++ b/dbm-ui/backend/db_services/mysql/resources/tendbcluster/query.py @@ -15,6 +15,7 @@ from django.forms import model_to_dict from django.utils.translation import ugettext_lazy as _ +from backend.configuration.constants import DBType from backend.db_meta.api.cluster.tendbcluster.detail import scan_cluster from backend.db_meta.enums import InstanceInnerRole, TenDBClusterSpiderRole from backend.db_meta.enums.cluster_type import ClusterType @@ -191,6 +192,12 @@ def _filter_instance_qs(cls, query_filters, query_params): return instances + @classmethod + def _filter_instance_hook(cls, bk_biz_id, query_params, instances, **kwargs): + # cluster handler + kwargs.update(handler_db_type=DBType.MySQL.value) + return super()._filter_instance_hook(bk_biz_id, query_params, instances, **kwargs) + @classmethod def _to_instance_representation(cls, instance: dict, cluster_entry_map: dict, **kwargs) -> Dict[str, Any]: """ diff --git a/dbm-ui/backend/db_services/redis/resources/redis_cluster/query.py b/dbm-ui/backend/db_services/redis/resources/redis_cluster/query.py index d0d77faace..c6dda1c2bd 100644 --- a/dbm-ui/backend/db_services/redis/resources/redis_cluster/query.py +++ b/dbm-ui/backend/db_services/redis/resources/redis_cluster/query.py @@ -15,6 +15,7 @@ from django.utils.translation import ugettext_lazy as _ from backend.db_meta.api.cluster.rediscluster.handler import RedisClusterHandler +from backend.db_meta.api.cluster.redisinstance.handler import RedisInstanceHandler from backend.db_meta.api.cluster.tendiscache.handler import TendisCacheClusterHandler from backend.db_meta.api.cluster.tendispluscluster.handler import TendisPlusClusterHandler from backend.db_meta.api.cluster.tendisssd.handler import TendisSSDClusterHandler @@ -45,6 +46,7 @@ class RedisListRetrieveResource(query.ListRetrieveResource): ClusterType.TendisTwemproxyRedisInstance: TendisCacheClusterHandler, ClusterType.TendisPredixyTendisplusCluster: TendisPlusClusterHandler, ClusterType.TendisPredixyRedisCluster: RedisClusterHandler, + ClusterType.RedisInstance: RedisInstanceHandler, } fields = [ diff --git a/dbm-ui/backend/flow/engine/bamboo/scene/spider/remote_local_slave_recover.py b/dbm-ui/backend/flow/engine/bamboo/scene/spider/remote_local_slave_recover.py index ed9fdfe14e..eb3e336cc1 100644 --- a/dbm-ui/backend/flow/engine/bamboo/scene/spider/remote_local_slave_recover.py +++ b/dbm-ui/backend/flow/engine/bamboo/scene/spider/remote_local_slave_recover.py @@ -69,6 +69,7 @@ def tendb_remote_slave_local_recover(self): self.data["root_id"] = self.root_id self.data["uid"] = self.ticket_data["uid"] self.data["ticket_type"] = self.ticket_data["ticket_type"] + self.data["created_by"] = self.ticket_data["created_by"] self.data["bk_biz_id"] = cluster_class.bk_biz_id self.data["db_module_id"] = cluster_class.db_module_id self.data["cluster_type"] = cluster_class.cluster_type diff --git a/dbm-ui/backend/flow/engine/bamboo/scene/spider/remote_master_slave_migrate.py b/dbm-ui/backend/flow/engine/bamboo/scene/spider/remote_master_slave_migrate.py index 0248726548..71663c0c1e 100644 --- a/dbm-ui/backend/flow/engine/bamboo/scene/spider/remote_master_slave_migrate.py +++ b/dbm-ui/backend/flow/engine/bamboo/scene/spider/remote_master_slave_migrate.py @@ -76,7 +76,7 @@ def migrate_master_slave_flow(self): # 构建流程 cluster_ids = [] for i in self.ticket_data["infos"]: - cluster_ids.extend(i["cluster_ids"]) + cluster_ids.append(i["cluster_id"]) tendb_migrate_pipeline_all = Builder( root_id=self.root_id, @@ -353,7 +353,7 @@ def migrate_master_slave_flow(self): ) ) uninstall_svr_sub_pipeline_list.append( - uninstall_svr_sub_pipeline.build_sub_process(sub_name=_("卸载remote节点{}".format(ip))) + uninstall_svr_sub_pipeline.build_sub_process(sub_name=_("卸载remote节点{}").format(ip)) ) # 安装实例 diff --git a/dbm-ui/backend/flow/utils/spider/spider_db_meta.py b/dbm-ui/backend/flow/utils/spider/spider_db_meta.py index a68132f30c..2a2136a4d3 100644 --- a/dbm-ui/backend/flow/utils/spider/spider_db_meta.py +++ b/dbm-ui/backend/flow/utils/spider/spider_db_meta.py @@ -161,6 +161,8 @@ def remotedb_migrate_add_install_nodes(self): """ remotedb 成对迁移添加初始化节点元数据 """ + cluster = Cluster.objects.get(id=self.cluster["cluster_id"]) + old_resource_spec = {MachineType.REMOTE.value: cluster.storageinstance_set.first().machine.spec_config} TenDBClusterMigrateRemoteDb.storage_create( cluster_id=self.cluster["cluster_id"], master_ip=self.cluster["new_master_ip"], @@ -168,7 +170,8 @@ def remotedb_migrate_add_install_nodes(self): ports=self.cluster["ports"], creator=self.global_data["created_by"], mysql_version=self.cluster["version"], - resource_spec=self.global_data["resource_spec"], + # 兼容资源池和手输机器两种情况 + resource_spec=self.global_data.get("resource_spec") or old_resource_spec, ) return True @@ -242,13 +245,16 @@ def tendb_slave_recover_add_nodes(self): """ remotedb 成对迁移添加初始化节点元数据 """ + cluster = Cluster.objects.get(id=self.cluster["cluster_id"]) + old_resource_spec = {MachineType.REMOTE.value: cluster.storageinstance_set.first().machine.spec_config} TenDBClusterMigrateRemoteDb.storage_create( cluster_id=self.cluster["cluster_id"], slave_ip=self.cluster["new_slave_ip"], ports=self.cluster["ports"], creator=self.global_data["created_by"], mysql_version=self.cluster["version"], - resource_spec=self.global_data["resource_spec"], + # 兼容资源池和手输机器两种情况 + resource_spec=self.global_data.get("resource_spec") or old_resource_spec, ) return True diff --git a/dbm-ui/backend/tests/ticket/test_ticket_flow.py b/dbm-ui/backend/tests/ticket/test_ticket_flow.py index 28dd728100..ea3b3ed0bb 100644 --- a/dbm-ui/backend/tests/ticket/test_ticket_flow.py +++ b/dbm-ui/backend/tests/ticket/test_ticket_flow.py @@ -180,7 +180,7 @@ def test_sql_import_flow(self, mocked_status, mocked__run, mocked_permission_cla lambda resource_request_id, node_infos: (1, APPLY_RESOURCE_RETURN_DATA), ) @patch( - "backend.ticket.flow_manager.resource.ResourceApplyFlow.patch_resource_params", lambda self, ticket_data: None + "backend.ticket.flow_manager.resource.ResourceApplyFlow.patch_resource_spec", lambda self, ticket_data: None ) @patch("backend.db_services.cmdb.biz.Permission", PermissionMock) @patch("backend.ticket.builders.mysql.mysql_ha_apply.DBConfigApi", DBConfigApiMock) diff --git a/dbm-ui/backend/ticket/builders/common/base.py b/dbm-ui/backend/ticket/builders/common/base.py index e1e60b2d76..9cfb621764 100644 --- a/dbm-ui/backend/ticket/builders/common/base.py +++ b/dbm-ui/backend/ticket/builders/common/base.py @@ -134,7 +134,9 @@ def validate_hosts_from_idle_pool(cls, bk_biz_id: int, host_list: List[int]) -> ) idle_host_list = [host["bk_host_id"] for host in idle_host_info_list] - return set(host_list) - set(idle_host_list) + host_not_in_idle = set(host_list) - set(idle_host_list) + if host_not_in_idle: + raise serializers.ValidationError(_("主机{}不在空闲机池，请保证所选的主机均来自空闲机").format(host_not_in_idle)) @classmethod def validate_hosts_not_in_db_meta(cls, host_infos: List[Dict]): diff --git a/dbm-ui/backend/ticket/builders/common/bigdata.py b/dbm-ui/backend/ticket/builders/common/bigdata.py index da29b5e8f3..4b6e8696da 100644 --- a/dbm-ui/backend/ticket/builders/common/bigdata.py +++ b/dbm-ui/backend/ticket/builders/common/bigdata.py @@ -46,9 +46,7 @@ def validate_hosts_from_idle_pool(cls, bk_biz_id, nodes: Dict): role_host_list = [node["bk_host_id"] for node in nodes[role] if node.get("bk_host_id")] hosts_set.update(role_host_list) - hosts_not_in_idle_pool = CommonValidate.validate_hosts_from_idle_pool(bk_biz_id, list(hosts_set)) - if hosts_not_in_idle_pool: - raise serializers.ValidationError(_("主机{}不在空闲机池，请保证所选的主机均来自空闲机").format(hosts_not_in_idle_pool)) + CommonValidate.validate_hosts_from_idle_pool(bk_biz_id, list(hosts_set)) @classmethod def validate_hosts_not_in_db_meta(cls, nodes: Dict): diff --git a/dbm-ui/backend/ticket/builders/mysql/mysql_restore_local_slave.py b/dbm-ui/backend/ticket/builders/mysql/mysql_restore_local_slave.py index ee93e822c3..c7cbecfa8c 100644 --- a/dbm-ui/backend/ticket/builders/mysql/mysql_restore_local_slave.py +++ b/dbm-ui/backend/ticket/builders/mysql/mysql_restore_local_slave.py @@ -18,10 +18,11 @@ from backend.ticket.builders.common.base import InstanceInfoSerializer from backend.ticket.builders.common.constants import MySQLBackupSource from backend.ticket.builders.mysql.base import BaseMySQLTicketFlowBuilder, MySQLBaseOperateDetailSerializer +from backend.ticket.builders.tendbcluster.base import TendbBaseOperateDetailSerializer from backend.ticket.constants import TicketType -class MysqlRestoreLocalSlaveDetailSerializer(MySQLBaseOperateDetailSerializer): +class MysqlRestoreLocalSlaveDetailSerializer(TendbBaseOperateDetailSerializer, MySQLBaseOperateDetailSerializer): class SlaveInfoSerializer(serializers.Serializer): slave = InstanceInfoSerializer(help_text=_("从库实例信息")) cluster_id = serializers.IntegerField(help_text=_("集群ID")) @@ -32,7 +33,7 @@ class SlaveInfoSerializer(serializers.Serializer): def validate(self, attrs): # 校验集群是否可用，集群类型为高可用 - super(MysqlRestoreLocalSlaveDetailSerializer, self).validate_cluster_can_access(attrs) + super(TendbBaseOperateDetailSerializer, self).validate_cluster_can_access(attrs) super(MysqlRestoreLocalSlaveDetailSerializer, self).validate_cluster_type(attrs, ClusterType.TenDBHA) # 校验实例的角色为slave diff --git a/dbm-ui/backend/ticket/builders/redis/base.py b/dbm-ui/backend/ticket/builders/redis/base.py index 8ce240d756..11ccf51570 100644 --- a/dbm-ui/backend/ticket/builders/redis/base.py +++ b/dbm-ui/backend/ticket/builders/redis/base.py @@ -149,9 +149,3 @@ def check_cluster_phase(cluster_id): def validate_cluster_id(self, cluster_id): return self.check_cluster_phase(cluster_id) - - def validate_src_cluster(self, cluster_id): - return self.check_cluster_phase(cluster_id) - - def validate_dst_cluster(self, cluster_id): - return self.check_cluster_phase(cluster_id) diff --git a/dbm-ui/backend/ticket/builders/redis/redis_close.py b/dbm-ui/backend/ticket/builders/redis/redis_close.py index 63e15500ee..d4897b7a51 100644 --- a/dbm-ui/backend/ticket/builders/redis/redis_close.py +++ b/dbm-ui/backend/ticket/builders/redis/redis_close.py @@ -9,11 +9,13 @@ specific language governing permissions and limitations under the License. """ from django.utils.translation import ugettext_lazy as _ +from rest_framework import serializers from backend.db_meta.enums import ClusterPhase from backend.flow.engine.controller.redis import RedisController from backend.iam_app.dataclass.actions import ActionEnum from backend.ticket import builders +from backend.ticket.builders.common.base import SkipToRepresentationMixin from backend.ticket.builders.redis.base import BaseRedisTicketFlowBuilder, RedisSingleOpsBaseDetailSerializer from backend.ticket.constants import TicketType @@ -47,3 +49,21 @@ class RedisCloseFlowBuilder(BaseRedisTicketFlowBuilder): serializer = RedisCloseDetailSerializer inner_flow_builder = RedisCloseFlowParamBuilder inner_flow_name = _("禁用集群") + + +class RedisInstanceCloseDetailSerializer(SkipToRepresentationMixin, serializers.Serializer): + cluster_ids = serializers.ListField(help_text=_("集群ID列表"), child=serializers.IntegerField()) + force = serializers.BooleanField(help_text=_("是否强制"), required=False, default=True) + + +class RedisInstanceCloseFlowParamBuilder(builders.FlowParamBuilder): + controller = RedisController.fake_scene + + +@builders.BuilderFactory.register( + TicketType.REDIS_INSTANCE_PROXY_CLOSE, phase=ClusterPhase.OFFLINE, iam=ActionEnum.REDIS_OPEN_CLOSE +) +class RedisInstanceCloseFlowBuilder(BaseRedisTicketFlowBuilder): + serializer = RedisInstanceCloseDetailSerializer + inner_flow_builder = RedisInstanceCloseFlowParamBuilder + inner_flow_name = _("禁用集群") diff --git a/dbm-ui/backend/ticket/builders/redis/redis_cluster_apply.py b/dbm-ui/backend/ticket/builders/redis/redis_cluster_apply.py index ea98f9f0e0..3fe2bc8521 100644 --- a/dbm-ui/backend/ticket/builders/redis/redis_cluster_apply.py +++ b/dbm-ui/backend/ticket/builders/redis/redis_cluster_apply.py @@ -8,13 +8,12 @@ an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ -from django.utils.crypto import get_random_string from django.utils.translation import ugettext as _ from rest_framework import serializers from backend.configuration.constants import AffinityEnum +from backend.configuration.handlers.password import DBPasswordHandler from backend.db_meta.enums import ClusterType -from backend.db_meta.models import Machine from backend.db_services.dbbase.constants import IpSource from backend.flow.engine.controller.redis import RedisController from backend.ticket import builders @@ -42,10 +41,10 @@ class RedisClusterApplyDetailSerializer(SkipToRepresentationMixin, serializers.S cluster_name = serializers.CharField(help_text=_("集群ID（英文数字及下划线）")) cluster_alias = serializers.CharField(help_text=_("集群别名（一般为中文别名）"), required=False, allow_blank=True) + proxy_pwd = serializers.CharField(help_text=_("proxy访问密码"), required=False) ip_source = serializers.ChoiceField(help_text=_("主机来源"), choices=IpSource.get_choices()) nodes = serializers.JSONField(help_text=_("部署节点"), required=False) - resource_spec = serializers.JSONField(help_text=_("proxy部署方案"), required=False) cluster_shard_num = serializers.IntegerField(help_text=_("集群分片数"), required=False) @@ -68,11 +67,22 @@ def validate(self, attrs): # 判断主机角色是否互斥 super().validate(attrs) + # 集群分片数至少>=3 + if attrs["cluster_shard_num"] < 3: + raise serializers.ValidationError(_("redis集群部署的集群分片数至少大于3")) + # 集群名校验 bk_biz_id, ticket_type = self.context["bk_biz_id"], self.context["ticket_type"] CommonValidate.validate_duplicate_cluster_name(bk_biz_id, ticket_type, attrs["cluster_name"]) - # 仅校验手工选择主机的情况 + # proxy密码校验，如果是用户输入，则必须满足密码强度 + if attrs.get("proxy_pwd"): + verify_result = DBPasswordHandler.verify_password_strength(attrs["proxy_pwd"], echo=True) + attrs["proxy_pwd"] = verify_result["password"] + if not verify_result["is_strength"]: + raise serializers.ValidationError(_("密码强度不符合要求，请重新输入密码。")) + + # 仅校验手工选择主机的情况 TODO: 目前redis已经不支持手动部署 if attrs["ip_source"] != IpSource.MANUAL_INPUT: return attrs @@ -83,35 +93,19 @@ def validate(self, attrs): all_nodes = [*master_nodes, *slave_nodes, *proxy_nodes] # 集群元数据检查 - exist_nodes = Machine.objects.filter(bk_host_id__in=all_nodes) - if exist_nodes.exists(): - exist_hosts = ",".join((exist_nodes.values_list("ip", flat=True))) - raise serializers.ValidationError(_("主机【{}】已经被注册到了集群元数据，请检查").format(exist_hosts)) + CommonValidate.validate_hosts_not_in_db_meta(host_infos=[{"bk_host_id": host_id} for host_id in all_nodes]) # 空闲机校验 - hosts_not_in_idle_pool = CommonValidate.validate_hosts_from_idle_pool(bk_biz_id, all_nodes) - if hosts_not_in_idle_pool: - host_id_to_ips = { - h["bk_host_id"]: h["ip"] for __, role_hosts in role__nodes_map.items() for h in role_hosts - } - hosts_not_in_idle_pool = {host_id_to_ips[h] for h in hosts_not_in_idle_pool} - raise serializers.ValidationError(_("主机{}不在空闲机池，请保证所选的主机均来自空闲机").format(hosts_not_in_idle_pool)) - - # TODO: master&slave 规格检查: cpu/mem/disk... - if master_nodes & slave_nodes: - raise serializers.ValidationError(_("master和slave中存在重复节点")) - - if master_nodes & proxy_nodes: - raise serializers.ValidationError(_("master和proxy中存在重复节点")) + CommonValidate.validate_hosts_from_idle_pool(bk_biz_id, all_nodes) - if slave_nodes & proxy_nodes: - raise serializers.ValidationError(_("slave和proxy中存在重复节点")) + # 校验不存在重复节点 + if (master_nodes & slave_nodes) or (master_nodes & proxy_nodes) or (slave_nodes & proxy_nodes): + raise serializers.ValidationError(_("master、slave、proxy中存在重复节点")) # 节点数检查 if not (len(master_nodes) and len(master_nodes) == len(slave_nodes)): raise serializers.ValidationError(_("至少提供1台master节点和1台slave节点，且master与slave节点数要保持一致")) - - if len(proxy_nodes) < REDIS_PROXY_MIN: + if len(proxy_nodes) < REDIS_PROXY_MIN and attrs["cluster_type"] != ClusterType.TendisRedisInstance: raise serializers.ValidationError(_("proxy至少提供2台机器")) return attrs @@ -119,9 +113,14 @@ def validate(self, attrs): class RedisClusterApplyFlowParamBuilder(builders.FlowParamBuilder): controllers = { + # tendis-cache部署flow ClusterType.TendisTwemproxyRedisInstance: RedisController.twemproxy_cluster_apply_scene, + # tendis-plus部署flow ClusterType.TendisPredixyTendisplusCluster: RedisController.predixy_cluster_apply_scene, + # tendis-ssd部署flow ClusterType.TwemproxyTendisSSDInstance: RedisController.twemproxy_cluster_apply_scene, + # redis-cluster部署flow + ClusterType.TendisPredixyRedisCluster: RedisController.predixy_cluster_apply_scene, } def build_controller_info(self) -> dict: @@ -174,40 +173,36 @@ def format_ticket_data(self): } ] }, - "redis_pwd": "JWe5UeSUAAvcpcY3", + "redis_pwd": "xxxxxxx", "ticket_type": "REDIS_CLUSTER_APPLY", - "proxy_pwd": "6Lenke4rWl6VU8lj", + "proxy_pwd": "xxxxxxx", "uid": 342 } """ - # 生成随机密码，长度16位，英文大小写+数字 - proxy_pwd = get_random_string(16) - proxy_admin_pwd = get_random_string(16) - redis_pwd = get_random_string(16) + # 生成随机密码，密码强度符合平台密码策略 + proxy_admin_pwd = DBPasswordHandler.get_random_password() + redis_pwd = DBPasswordHandler.get_random_password() + # proxy访问密码优先以用户为准 + proxy_pwd = self.ticket_data.get("proxy_pwd") or DBPasswordHandler.get_random_password() ticket_type = self.ticket_data["cluster_type"] # 默认db数量 DEFAULT_DATABASES = 2 # 域名映射 - if ticket_type in [ - ClusterType.TwemproxyTendisSSDInstance, - ]: + if ticket_type == ClusterType.TwemproxyTendisSSDInstance: domain_prefix = "ssd" - elif ticket_type in [ - ClusterType.TendisTwemproxyTendisplusIns, - ClusterType.TendisPredixyTendisplusCluster, - ]: - domain_prefix = "tendisplus" - else: + elif ticket_type == ClusterType.TendisPredixyRedisCluster: + domain_prefix = "rediscluster" + elif ticket_type == ClusterType.TendisTwemproxyRedisInstance: domain_prefix = "cache" - + elif ticket_type in [ClusterType.TendisTwemproxyTendisplusIns, ClusterType.TendisPredixyTendisplusCluster]: + domain_prefix = "tendisplus" domain_name = "{}.{}.{}.db".format( domain_prefix, self.ticket_data["cluster_name"], self.ticket_data["db_app_abbr"], ) - # 校验域名是否合法 CommonValidate._validate_domain_valid(domain_name) @@ -228,6 +223,7 @@ def format_ticket_data(self): } ) + # TODO: 目前redis已经不支持手动部署 if self.ticket_data["ip_source"] == IpSource.MANUAL_INPUT: # 如果是手动部署，根据前端传入的cap_key需充maxmemory, max_disk等参数 cap_key = self.ticket_data["cap_key"] @@ -279,6 +275,6 @@ def post_callback(self): class RedisClusterApplyFlowBuilder(BaseRedisTicketFlowBuilder): serializer = RedisClusterApplyDetailSerializer inner_flow_builder = RedisClusterApplyFlowParamBuilder - inner_flow_name = _("集群部署") + inner_flow_name = _("Redis 集群部署") resource_apply_builder = RedisApplyResourceParamBuilder pause_node_builder = RedisBasePauseParamBuilder diff --git a/dbm-ui/backend/ticket/builders/redis/redis_destroy.py b/dbm-ui/backend/ticket/builders/redis/redis_destroy.py index f624b149df..85691f0587 100644 --- a/dbm-ui/backend/ticket/builders/redis/redis_destroy.py +++ b/dbm-ui/backend/ticket/builders/redis/redis_destroy.py @@ -9,10 +9,12 @@ specific language governing permissions and limitations under the License. """ from django.utils.translation import ugettext_lazy as _ +from rest_framework import serializers from backend.db_meta.enums import ClusterPhase from backend.flow.engine.controller.redis import RedisController from backend.ticket import builders +from backend.ticket.builders.common.base import SkipToRepresentationMixin from backend.ticket.builders.redis.base import ( BaseRedisTicketFlowBuilder, RedisBasePauseParamBuilder, @@ -46,3 +48,19 @@ class RedisDestroyFlowBuilder(BaseRedisTicketFlowBuilder): inner_flow_builder = RedisDestroyFlowParamBuilder inner_flow_name = _("下架集群") pause_node_builder = RedisBasePauseParamBuilder + + +class RedisInstanceDestroyDetailSerializer(SkipToRepresentationMixin, serializers.Serializer): + cluster_ids = serializers.ListField(help_text=_("集群ID列表"), child=serializers.IntegerField()) + + +class RedisInstanceDestroyFlowParamBuilder(builders.FlowParamBuilder): + controller = RedisController.fake_scene + + +@builders.BuilderFactory.register(TicketType.REDIS_INSTANCE_DESTROY, phase=ClusterPhase.DESTROY) +class RedisInstanceCloseFlowBuilder(BaseRedisTicketFlowBuilder): + serializer = RedisInstanceDestroyDetailSerializer + inner_flow_builder = RedisInstanceDestroyFlowParamBuilder + inner_flow_name = _("下架集群") + pause_node_builder = RedisBasePauseParamBuilder diff --git a/dbm-ui/backend/ticket/builders/redis/redis_instance_apply.py b/dbm-ui/backend/ticket/builders/redis/redis_instance_apply.py new file mode 100644 index 0000000000..33cf360247 --- /dev/null +++ b/dbm-ui/backend/ticket/builders/redis/redis_instance_apply.py @@ -0,0 +1,161 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. +Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +from typing import Dict + +from django.utils.translation import ugettext as _ +from rest_framework import serializers + +from backend.configuration.constants import AffinityEnum +from backend.configuration.handlers.password import DBPasswordHandler +from backend.db_meta.models import Cluster, Machine, StorageInstance +from backend.db_services.dbbase.constants import IpSource +from backend.flow.engine.controller.redis import RedisController +from backend.iam_app.dataclass.actions import ActionEnum +from backend.ticket import builders +from backend.ticket.builders.common.base import CommonValidate, SkipToRepresentationMixin +from backend.ticket.builders.redis.base import BaseRedisTicketFlowBuilder +from backend.ticket.constants import TicketType + + +class RedisInstanceApplyDetailSerializer(SkipToRepresentationMixin, serializers.Serializer): + class InstanceInfoSerializer(serializers.Serializer): + cluster_name = serializers.CharField(help_text=_("集群ID（英文数字及下划线）")) + databases = serializers.IntegerField(help_text=_("db数量")) + backend_group = serializers.JSONField(help_text=_("追加部署的主机信息"), required=False) + + bk_cloud_id = serializers.IntegerField(help_text=_("云区域ID")) + db_app_abbr = serializers.CharField(help_text=_("业务英文缩写")) + city_code = serializers.CharField(help_text=_("城市代码")) + disaster_tolerance_level = serializers.ChoiceField( + help_text=_("容灾级别"), choices=AffinityEnum.get_choices(), required=False, default=AffinityEnum.NONE.value + ) + + port = serializers.IntegerField(help_text=_("集群端口"), required=False) + redis_pwd = serializers.CharField(help_text=_("访问密码"), required=False) + db_version = serializers.CharField(help_text=_("版本号"), required=False) + cluster_type = serializers.CharField(help_text=_("集群类型")) + infos = serializers.ListSerializer(help_text=_("集群信息"), child=InstanceInfoSerializer()) + + resource_spec = serializers.JSONField(help_text=_("proxy部署方案"), required=False) + ip_source = serializers.CharField(help_text=_("主机来源"), required=False, default=IpSource.RESOURCE_POOL) + append_apply = serializers.BooleanField(help_text=_("是否是追加部署")) + + city_name = serializers.SerializerMethodField(help_text=_("城市名")) + + def get_city_name(self, obj): + city_code = obj["city_code"] + return self.context["ticket_ctx"].city_map.get(city_code, city_code) + + def validate(self, attrs): + # 集群名校验 + bk_biz_id, ticket_type = self.context["bk_biz_id"], self.context["ticket_type"] + for info in attrs["infos"]: + CommonValidate.validate_duplicate_cluster_name(bk_biz_id, ticket_type, info["cluster_name"]) + + # 新部署机器组数要整除集群数 + if not attrs["append_apply"]: + machine_group = attrs["resource_spec"]["backend_group"]["count"] + cluster_num = len(attrs["infos"]) + if cluster_num % machine_group: + raise serializers.ValidationError(_("请保证机器组数{}能整除集群数{}").format(machine_group, cluster_num)) + + return attrs + + +class RedisInstanceApplyFlowParamBuilder(builders.FlowParamBuilder): + controller = RedisController.redis_instance_apply_scene + + def format_common_cluster_info(self): + """补充部署redis主从集群通用信息""" + for info in self.ticket_data["infos"]: + # 生成随机密码，密码强度符合平台密码策略 + redis_pwd = self.ticket_data.get("redis_pwd") or DBPasswordHandler.get_random_password() + # 域名生成规则：ins.{cluster_name}.{db_app_abbr}.db + domain_name = "ins.{}.{}.db".format(info["cluster_name"], self.ticket_data["db_app_abbr"]) + # 校验域名是否合法 + CommonValidate._validate_domain_valid(domain_name) + # 在info里，补充每个主从集群的部署信息 + info.update( + city=self.ticket_data["city_code"], + city_code=self.ticket_data["city_code"], + domain_name=domain_name, + cluster_alias=info["cluster_name"], + db_version=self.ticket_data["db_version"], + redis_pwd=redis_pwd, + ) + + def format_append_cluster_info(self): + """补充追加集群的信息""" + master_host_ids = [info["backend_group"]["master"]["bk_host_id"] for info in self.ticket_data["infos"]] + storages = StorageInstance.objects.prefetch_related("cluster").filter(machine__in=master_host_ids) + + master_host__cluster: Dict[int, Cluster] = {} + master_host__machine: Dict[int, Machine] = {} + master_host__max_port: Dict[int, int] = {} + # 获取主机IP与集群，主机和起始端口的映射 + for inst in storages: + cluster = inst.cluster.first() + master_host__machine[inst.machine.bk_host_id] = inst.machine + master_host__cluster[inst.machine.bk_host_id] = cluster + if inst.machine.bk_host_id not in master_host__max_port: + max_port = max(cluster.storageinstance_set.values_list("port", flat=True)) + master_host__max_port[inst.machine.bk_host_id] = max_port + + for info in self.ticket_data["infos"]: + master_host = info["backend_group"]["master"]["bk_host_id"] + # 更新起始端口+1 + master_host__max_port[master_host] += 1 + # 获取集群 + cluster = master_host__cluster[master_host] + # 更新追加集群的部署信息 + info.update( + disaster_tolerance_level=cluster.disaster_tolerance_level, + port=master_host__max_port[master_host], + db_version=cluster.major_version, + resource_spec=master_host__machine[master_host].spec_config, + # 追加部署的maxmemory为0，后续由周边程序分配 + maxmemory=0, + ) + + def format_ticket_data(self): + self.format_common_cluster_info() + if self.ticket_data["append_apply"]: + self.format_append_cluster_info() + + +class RedisInstanceApplyResourceParamBuilder(builders.ResourceApplyParamBuilder): + def format_apply_cluster_info(self, ticket_data): + """补充部署集群的信息""" + cluster_num, machine_group = len(ticket_data["infos"]), len(ticket_data["nodes"]["backend_group"]) + for index, info in enumerate(ticket_data["infos"]): + backend_group = ticket_data["nodes"]["backend_group"][index % machine_group] + info.update( + backend_group=backend_group, + # 在同一台机器部署的实例端口号要递增 + port=ticket_data["port"] + (index // machine_group), + db_version=ticket_data["db_version"], + resource_spec=ticket_data["resource_spec"]["master"], + # maxmemory = 机器内存 *0.9 / 实例数 + maxmemory=int(backend_group["master"]["bk_mem"] * 0.9 / (cluster_num / machine_group)), + ) + + def post_callback(self): + next_flow = self.ticket.next_flow() + self.format_apply_cluster_info(next_flow.details["ticket_data"]) + next_flow.save(update_fields=["details"]) + + +@builders.BuilderFactory.register(TicketType.REDIS_INS_APPLY, is_apply=True, iam=ActionEnum.REDIS_CLUSTER_APPLY) +class RedisClusterApplyFlowBuilder(BaseRedisTicketFlowBuilder): + serializer = RedisInstanceApplyDetailSerializer + inner_flow_builder = RedisInstanceApplyFlowParamBuilder + inner_flow_name = _("Redis 主从部署") + resource_apply_builder = RedisInstanceApplyResourceParamBuilder diff --git a/dbm-ui/backend/ticket/builders/redis/redis_open.py b/dbm-ui/backend/ticket/builders/redis/redis_open.py index 17570b2e04..0a58caeac9 100644 --- a/dbm-ui/backend/ticket/builders/redis/redis_open.py +++ b/dbm-ui/backend/ticket/builders/redis/redis_open.py @@ -9,11 +9,13 @@ specific language governing permissions and limitations under the License. """ from django.utils.translation import ugettext_lazy as _ +from rest_framework import serializers from backend.db_meta.enums import ClusterPhase from backend.flow.engine.controller.redis import RedisController from backend.iam_app.dataclass.actions import ActionEnum from backend.ticket import builders +from backend.ticket.builders.common.base import SkipToRepresentationMixin from backend.ticket.builders.redis.base import BaseRedisTicketFlowBuilder, RedisSingleOpsBaseDetailSerializer from backend.ticket.constants import TicketType @@ -44,3 +46,21 @@ class RedisOpenFlowBuilder(BaseRedisTicketFlowBuilder): serializer = RedisOpenDetailSerializer inner_flow_builder = RedisOpenFlowParamBuilder inner_flow_name = _("启用集群") + + +class RedisInstanceOpenDetailSerializer(SkipToRepresentationMixin, serializers.Serializer): + cluster_ids = serializers.ListField(help_text=_("集群ID列表"), child=serializers.IntegerField()) + force = serializers.BooleanField(help_text=_("是否强制"), required=False, default=True) + + +class RedisInstanceOpenFlowParamBuilder(builders.FlowParamBuilder): + controller = RedisController.fake_scene + + +@builders.BuilderFactory.register( + TicketType.REDIS_INSTANCE_PROXY_OPEN, phase=ClusterPhase.ONLINE, iam=ActionEnum.REDIS_OPEN_CLOSE +) +class RedisInstanceCloseFlowBuilder(BaseRedisTicketFlowBuilder): + serializer = RedisInstanceOpenDetailSerializer + inner_flow_builder = RedisInstanceOpenFlowParamBuilder + inner_flow_name = _("启用集群") diff --git a/dbm-ui/backend/ticket/builders/tendbcluster/tendb_migrate_cluster.py b/dbm-ui/backend/ticket/builders/tendbcluster/tendb_migrate_cluster.py new file mode 100644 index 0000000000..22ae00884a --- /dev/null +++ b/dbm-ui/backend/ticket/builders/tendbcluster/tendb_migrate_cluster.py @@ -0,0 +1,80 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. +Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + +from django.utils.translation import gettext_lazy as _ +from rest_framework import serializers + +from backend.db_services.dbbase.constants import IpSource +from backend.flow.engine.controller.spider import SpiderController +from backend.ticket import builders +from backend.ticket.builders.common.base import BaseOperateResourceParamBuilder, HostInfoSerializer +from backend.ticket.builders.common.constants import MySQLBackupSource +from backend.ticket.builders.mysql.base import BaseMySQLTicketFlowBuilder, MySQLBaseOperateDetailSerializer +from backend.ticket.constants import FlowRetryType, TicketType + + +class TendbClusterMigrateClusterDetailSerializer(MySQLBaseOperateDetailSerializer): + class MigrateClusterInfoSerializer(serializers.Serializer): + new_master = HostInfoSerializer(help_text=_("新主库主机"), required=False) + new_slave = HostInfoSerializer(help_text=_("新从库主机"), required=False) + old_master = HostInfoSerializer(help_text=_("旧主库主机"), required=False) + old_slave = HostInfoSerializer(help_text=_("旧从库主机"), required=False) + resource_spec = serializers.JSONField(help_text=_("资源规格"), required=False) + cluster_id = serializers.IntegerField(help_text=_("集群ID列表")) + + ip_source = serializers.ChoiceField( + help_text=_("机器来源"), choices=IpSource.get_choices(), required=False, default=IpSource.MANUAL_INPUT + ) + infos = serializers.ListSerializer(help_text=_("克隆主从信息"), child=MigrateClusterInfoSerializer()) + backup_source = serializers.ChoiceField( + help_text=_("备份源"), choices=MySQLBackupSource.get_choices(), default=MySQLBackupSource.REMOTE + ) + is_safe = serializers.BooleanField(help_text=_("安全模式"), default=True) + + def validate(self, attrs): + # 校验集群是否可用，集群类型为高可用 + super().validate_cluster_can_access(attrs) + # TODO: 校验old_master/old_slave是一对主从 + return attrs + + +class TendbClusterMigrateClusterParamBuilder(builders.FlowParamBuilder): + controller = SpiderController.tendb_cluster_remote_migrate + + def format_ticket_data(self): + if self.ticket_data["ip_source"] == IpSource.RESOURCE_POOL: + return + + for info in self.ticket_data["infos"]: + info["new_master_ip"], info["new_slave_ip"] = info["new_master"]["ip"], info["new_slave"]["ip"] + info["bk_new_master"], info["bk_new_slave"] = info.pop("new_master"), info.pop("new_slave") + info["old_master_ip"], info["old_slave_ip"] = info.pop("old_master")["ip"], info.pop("old_slave")["ip"] + + +class TendbClusterMigrateClusterResourceParamBuilder(BaseOperateResourceParamBuilder): + def post_callback(self): + next_flow = self.ticket.next_flow() + ticket_data = next_flow.details["ticket_data"] + for info in ticket_data["infos"]: + info["bk_new_master"], info["bk_new_slave"] = info.pop("new_master")[0], info.pop("new_slave")[0] + info["new_master_ip"], info["new_slave_ip"] = info["bk_new_master"]["ip"], info["bk_new_slave"]["ip"] + info["old_master_ip"], info["old_slave_ip"] = info.pop("old_master")["ip"], info.pop("old_slave")["ip"] + + next_flow.save(update_fields=["details"]) + + +@builders.BuilderFactory.register(TicketType.TENDBCLUSTER_MIGRATE_CLUSTER, is_apply=True) +class TendbClusterMigrateClusterFlowBuilder(BaseMySQLTicketFlowBuilder): + serializer = TendbClusterMigrateClusterDetailSerializer + inner_flow_builder = TendbClusterMigrateClusterParamBuilder + inner_flow_name = _("TenDB Cluster 主从迁移执行") + resource_batch_apply_builder = TendbClusterMigrateClusterResourceParamBuilder + retry_type = FlowRetryType.MANUAL_RETRY diff --git a/dbm-ui/backend/ticket/builders/tendbcluster/tendb_restore_local_slave.py b/dbm-ui/backend/ticket/builders/tendbcluster/tendb_restore_local_slave.py new file mode 100644 index 0000000000..7100f16b7a --- /dev/null +++ b/dbm-ui/backend/ticket/builders/tendbcluster/tendb_restore_local_slave.py @@ -0,0 +1,84 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. +Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + +import operator +from functools import reduce + +from django.db.models import Q +from django.utils.translation import gettext_lazy as _ +from rest_framework import serializers + +from backend.db_meta.models import StorageInstance +from backend.flow.engine.controller.spider import SpiderController +from backend.ticket import builders +from backend.ticket.builders.common.base import InstanceInfoSerializer +from backend.ticket.builders.common.constants import MySQLBackupSource +from backend.ticket.builders.mysql.base import BaseMySQLTicketFlowBuilder +from backend.ticket.builders.tendbcluster.base import TendbBaseOperateDetailSerializer +from backend.ticket.constants import TicketType + + +class TendbClusterRestoreLocalSlaveDetailSerializer(TendbBaseOperateDetailSerializer): + class SlaveInfoSerializer(serializers.Serializer): + slave = InstanceInfoSerializer(help_text=_("从库实例信息")) + cluster_id = serializers.IntegerField(help_text=_("集群ID")) + + infos = serializers.ListField(help_text=_("重建从库列表"), child=SlaveInfoSerializer()) + backup_source = serializers.ChoiceField(help_text=_("备份源"), choices=MySQLBackupSource.get_choices()) + force = serializers.BooleanField(help_text=_("是否强制执行"), required=False, default=False) + + def validate(self, attrs): + # 校验集群是否可用 + super(TendbBaseOperateDetailSerializer, self).validate_cluster_can_access(attrs) + # TODO 校验slave角色信息 + return attrs + + +class TendbClusterRestoreLocalSlaveParamBuilder(builders.FlowParamBuilder): + controller = SpiderController.tendb_cluster_remote_local_recover + + def format_ticket_data(self): + # 查询重建的slave实例 + slave_filters = reduce( + operator.or_, + [Q(machine__ip=info["slave"]["ip"], port=info["slave"]["port"]) for info in self.ticket_data["infos"]], + ) + # 查询实例对应的分片ID + slaves = StorageInstance.objects.prefetch_related( + "machine", "cluster", "as_receiver__tendbclusterstorageset" + ).filter(slave_filters) + # 按照ip聚合，补充slave重建信息 + ip__restore_infos: dict = {} + for slave in slaves: + shard_id = slave.as_receiver.first().tendbclusterstorageset.shard_id + machine = slave.machine + if slave.machine.ip not in ip__restore_infos: + ip__restore_infos[slave.machine.ip] = { + "cluster_id": slave.cluster.first().id, + "shard_ids": [shard_id], + "slave_ip": machine.ip, + "bk_slave": { + "bk_biz_id": slave.bk_biz_id, + "bk_host_id": machine.bk_host_id, + "bk_cloud_id": machine.bk_cloud_id, + "ip": machine.ip, + }, + } + else: + ip__restore_infos[slave.machine.ip]["shard_ids"].append(shard_id) + self.ticket_data["infos"] = list(ip__restore_infos.values()) + + +@builders.BuilderFactory.register(TicketType.TENDBCLUSTER_RESTORE_LOCAL_SLAVE) +class TendbClusterRestoreLocalSlaveFlowBuilder(BaseMySQLTicketFlowBuilder): + serializer = TendbClusterRestoreLocalSlaveDetailSerializer + inner_flow_builder = TendbClusterRestoreLocalSlaveParamBuilder + inner_flow_name = _("TenDB Cluster Slave原地重建执行") diff --git a/dbm-ui/backend/ticket/builders/tendbcluster/tendb_restore_slave.py b/dbm-ui/backend/ticket/builders/tendbcluster/tendb_restore_slave.py new file mode 100644 index 0000000000..6086b32c6e --- /dev/null +++ b/dbm-ui/backend/ticket/builders/tendbcluster/tendb_restore_slave.py @@ -0,0 +1,79 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. +Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + +from django.utils.translation import gettext_lazy as _ +from rest_framework import serializers + +from backend.db_meta.enums import ClusterType +from backend.db_services.dbbase.constants import IpSource +from backend.flow.engine.controller.spider import SpiderController +from backend.ticket import builders +from backend.ticket.builders.common.base import BaseOperateResourceParamBuilder, HostInfoSerializer +from backend.ticket.builders.common.constants import MySQLBackupSource +from backend.ticket.builders.mysql.base import BaseMySQLTicketFlowBuilder +from backend.ticket.builders.mysql.mysql_restore_slave import MysqlRestoreSlaveDetailSerializer +from backend.ticket.constants import TicketType + + +class TendbClusterRestoreSlaveDetailSerializer(MysqlRestoreSlaveDetailSerializer): + class RestoreInfoSerializer(serializers.Serializer): + old_slave = HostInfoSerializer(help_text=_("旧从库 IP")) + new_slave = HostInfoSerializer(help_text=_("新从库 IP"), required=False) + resource_spec = serializers.JSONField(help_text=_("新从库资源池参数"), required=False) + cluster_id = serializers.IntegerField(help_text=_("集群ID")) + + ip_source = serializers.ChoiceField( + help_text=_("机器来源"), choices=IpSource.get_choices(), required=False, default=IpSource.MANUAL_INPUT + ) + backup_source = serializers.ChoiceField(help_text=_("备份源"), choices=MySQLBackupSource.get_choices()) + infos = serializers.ListField(help_text=_("集群重建信息"), child=RestoreInfoSerializer()) + + def validate(self, attrs): + # 校验集群是否可用，集群类型为tendbcluster + super(MysqlRestoreSlaveDetailSerializer, self).validate_cluster_can_access(attrs) + super(MysqlRestoreSlaveDetailSerializer, self).validate_cluster_type(attrs, ClusterType.TenDBCluster) + # 校验新机器的云区域与集群一致 + if attrs["ip_source"] == IpSource.MANUAL_INPUT: + super(MysqlRestoreSlaveDetailSerializer, self).validate_hosts_clusters_in_same_cloud_area( + attrs, host_key=["new_slave"], cluster_key=["cluster_id"] + ) + return attrs + + +class TendbClusterRestoreSlaveParamBuilder(builders.FlowParamBuilder): + controller = SpiderController.tendb_cluster_remote_slave_recover + + def format_ticket_data(self): + if self.ticket_data["ip_source"] == IpSource.RESOURCE_POOL: + return + + for info in self.ticket_data["infos"]: + info["old_slave_ip"], info["new_slave_ip"] = info["old_slave"]["ip"], info["new_slave"]["ip"] + info["bk_old_slave"], info["bk_new_slave"] = info.pop("old_slave"), info.pop("new_slave") + + +class TendbClusterRestoreSlaveResourceParamBuilder(BaseOperateResourceParamBuilder): + def post_callback(self): + next_flow = self.ticket.next_flow() + ticket_data = next_flow.details["ticket_data"] + for info in ticket_data["infos"]: + info["bk_old_slave"], info["bk_new_slave"] = info.pop("old_slave"), info.pop("new_slave")[0] + info["old_slave_ip"], info["new_slave_ip"] = info["bk_old_slave"]["ip"], info["bk_new_slave"]["ip"] + + next_flow.save(update_fields=["details"]) + + +@builders.BuilderFactory.register(TicketType.TENDBCLUSTER_RESTORE_SLAVE, is_apply=True) +class TendbClusterRestoreSlaveFlowBuilder(BaseMySQLTicketFlowBuilder): + serializer = TendbClusterRestoreSlaveDetailSerializer + inner_flow_builder = TendbClusterRestoreSlaveParamBuilder + inner_flow_name = _("TenDB Cluster Slave重建") + resource_apply_builder = TendbClusterRestoreSlaveResourceParamBuilder diff --git a/dbm-ui/backend/ticket/constants.py b/dbm-ui/backend/ticket/constants.py index aa7d52788f..4aa83ffbd8 100644 --- a/dbm-ui/backend/ticket/constants.py +++ b/dbm-ui/backend/ticket/constants.py @@ -207,6 +207,9 @@ def get_db_type_by_ticket(cls, ticket_type): TENDBCLUSTER_SPIDER_MNT_DESTROY = TicketEnumField("TENDBCLUSTER_SPIDER_MNT_DESTROY", _("TenDB Cluster 下架运维节点"), _("运维 Spider 管理")) # noqa TENDBCLUSTER_SPIDER_SLAVE_APPLY = TicketEnumField("TENDBCLUSTER_SPIDER_SLAVE_APPLY", _("TenDB Cluster 部署只读接入层"), _("访问入口")) TENDBCLUSTER_SPIDER_SLAVE_DESTROY = TicketEnumField("TENDBCLUSTER_SPIDER_SLAVE_DESTROY", _("TenDB Cluster 只读接入层下架"), _("访问入口")) # noqa + TENDBCLUSTER_RESTORE_SLAVE = TicketEnumField("TENDBCLUSTER_RESTORE_SLAVE", _("TenDB Cluster Slave重建"), _("集群维护")) # noqa + TENDBCLUSTER_RESTORE_LOCAL_SLAVE = TicketEnumField("TENDBCLUSTER_RESTORE_LOCAL_SLAVE", _("TenDB Cluster Slave原地重建"), _("集群维护")) # noqa + TENDBCLUSTER_MIGRATE_CLUSTER = TicketEnumField("TENDBCLUSTER_MIGRATE_CLUSTER", _("TenDB Cluster 主从迁移"), _("集群维护")) # noqa TENDBCLUSTER_APPLY = TicketEnumField("TENDBCLUSTER_APPLY", _("TenDB Cluster 集群部署")) TENDBCLUSTER_ENABLE = TicketEnumField("TENDBCLUSTER_ENABLE", _("TenDB Cluster 集群启用"), register_iam=False) TENDBCLUSTER_DISABLE = TicketEnumField("TENDBCLUSTER_DISABLE", _("TenDB Cluster 集群禁用"), register_iam=False) @@ -263,13 +266,16 @@ def get_db_type_by_ticket(cls, ticket_type): REDIS_PLUGIN_DELETE_POLARIS = TicketEnumField("REDIS_PLUGIN_DELETE_POLARIS", _("Redis 删除Polaris"), _("集群管理")) REDIS_SINGLE_APPLY = TicketEnumField("REDIS_SINGLE_APPLY", _("Redis 单节点部署"), register_iam=False) REDIS_INS_APPLY = TicketEnumField("REDIS_INS_APPLY", _("Redis 主从节点部署"), register_iam=False) - REDIS_CLUSTER_APPLY = TicketEnumField("REDIS_CLUSTER_APPLY", _("Redis 集群部署"), _("集群管理"), register_iam=False) + REDIS_CLUSTER_APPLY = TicketEnumField("REDIS_CLUSTER_APPLY", _("Redis 集群部署"), _("集群管理")) REDIS_KEYS_EXTRACT = TicketEnumField("REDIS_KEYS_EXTRACT", _("Redis 提取 Key"), _("集群管理")) REDIS_KEYS_DELETE = TicketEnumField("REDIS_KEYS_DELETE", _("Redis 删除 key"), _("集群管理")) REDIS_BACKUP = TicketEnumField("REDIS_BACKUP", _("Redis 集群备份"), _("集群管理")) REDIS_PROXY_OPEN = TicketEnumField("REDIS_PROXY_OPEN", _("Redis 集群启用"), register_iam=False) REDIS_PROXY_CLOSE = TicketEnumField("REDIS_PROXY_CLOSE", _("Redis 集群禁用"), register_iam=False) REDIS_DESTROY = TicketEnumField("REDIS_DESTROY", _("Redis 集群删除"), _("集群管理")) + REDIS_INSTANCE_PROXY_OPEN = TicketEnumField("REDIS_INSTANCE_PROXY_OPEN", _("Redis 主从集群启用"), register_iam=False) + REDIS_INSTANCE_PROXY_CLOSE = TicketEnumField("REDIS_INSTANCE_PROXY_CLOSE", _("Redis 主从集群禁用"), register_iam=False) + REDIS_INSTANCE_DESTROY = TicketEnumField("REDIS_INSTANCE_DESTROY", _("Redis 主从集群删除"), _("集群管理")) REDIS_PURGE = TicketEnumField("REDIS_PURGE", _("Redis 集群清档"), _("集群管理")) REDIS_SCALE_UPDOWN = TicketEnumField("REDIS_SCALE_UPDOWN", _("Redis 集群容量变更"), _("集群维护")) diff --git a/dbm-ui/backend/ticket/flow_manager/resource.py b/dbm-ui/backend/ticket/flow_manager/resource.py index 433c4f5be0..40292dadc7 100644 --- a/dbm-ui/backend/ticket/flow_manager/resource.py +++ b/dbm-ui/backend/ticket/flow_manager/resource.py @@ -253,7 +253,7 @@ def fetch_apply_params(self, ticket_data): return details - def patch_resource_params(self, ticket_data, spec_map: Dict[int, Spec] = None): + def patch_resource_spec(self, ticket_data, spec_map: Dict[int, Spec] = None): """ 将资源池部署信息写入到ticket_data。 @param ticket_data: 待填充的字典 @@ -293,7 +293,7 @@ def _run(self) -> None: # 将机器信息写入ticket和inner flow self.write_node_infos(next_flow.details["ticket_data"], node_infos) - self.patch_resource_params(next_flow.details["ticket_data"]) + self.patch_resource_spec(next_flow.details["ticket_data"]) next_flow.save(update_fields=["details"]) # 相关信息回填到单据和resource flow中 self.ticket.update_details(resource_request_id=resource_request_id, nodes=node_infos) @@ -322,7 +322,7 @@ class ResourceBatchApplyFlow(ResourceApplyFlow): ] """ - def patch_resource_params(self, ticket_data): + def patch_resource_spec(self, ticket_data): spec_ids: List[int] = [] for info in ticket_data["infos"]: spec_ids.extend([data["spec_id"] for data in info["resource_spec"].values()]) @@ -330,7 +330,7 @@ def patch_resource_params(self, ticket_data): # 提前缓存数据库查询数据，避免多次IO spec_map = {spec.spec_id: spec for spec in Spec.objects.filter(spec_id__in=spec_ids)} for info in ticket_data["infos"]: - super().patch_resource_params(info, spec_map) + super().patch_resource_spec(info, spec_map) def write_node_infos(self, ticket_data, node_infos): """