feat(backend): redis主从部署 & tendbcluster重建 联调 TencentBlueKing#4106

dbm-ui/backend/configuration/handlers/password.py

@@ -37,6 +37,15 @@
 class DBPasswordHandler(object):
     """密码策略相关处理"""
 
+    @classmethod
+    def get_random_password(cls):
+        """
+        获取符合密码强度的字符串
+        """
+        random_password = DBPrivManagerApi.get_random_string({"security_rule_name": DBM_PASSWORD_SECURITY_NAME})
+        random_password = base64_decode(random_password)
+        return random_password
+
     @classmethod
     def verify_password_strength(cls, password: str, echo: bool = False):
         """

dbm-ui/backend/configuration/views/password_policy.py

@@ -8,7 +8,6 @@
 an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
 specific language governing permissions and limitations under the License.
 """
-import base64
 import json
 
 from celery.schedules import crontab
@@ -91,8 +90,7 @@ def verify_password_strength(self, request):
     )
     @action(methods=["GET"], detail=False)
     def get_random_password(self, request, *args, **kwargs):
-        random_password = DBPrivManagerApi.get_random_string({"security_rule_name": DBM_PASSWORD_SECURITY_NAME})
-        random_password = base64.b64decode(random_password).decode("utf-8")
+        random_password = DBPasswordHandler.get_random_password()
         return Response({"password": random_password})
 
     @common_swagger_auto_schema(

dbm-ui/backend/db_meta/api/cluster/redisinstance/__init__.py

@@ -0,0 +1,10 @@
+# -*- coding: utf-8 -*-
+"""
+TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available.
+Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved.
+Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at https://opensource.org/licenses/MIT
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+specific language governing permissions and limitations under the License.
+"""

dbm-ui/backend/db_meta/api/cluster/redisinstance/detail.py

@@ -0,0 +1,43 @@
+# -*- coding: utf-8 -*-
+"""
+TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available.
+Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved.
+Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at https://opensource.org/licenses/MIT
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+specific language governing permissions and limitations under the License.
+"""
+from django.utils.translation import gettext as _
+
+from backend.db_meta.api.cluster.base.graph import Graphic, LineLabel
+from backend.db_meta.enums import InstanceRole
+from backend.db_meta.models import Cluster
+
+
+def scan_cluster(cluster: Cluster) -> Graphic:
+    """
+    redis主从关系拓扑图
+    """
+    graph = Graphic(node_id=Graphic.generate_graphic_id(cluster))
+
+    # 获取master节点组
+    master_insts, master_group = graph.add_instance_nodes(
+        cluster=cluster, roles=InstanceRole.REDIS_MASTER, group_name=_("Master 节点")
+    )
+
+    # 获取slave节点组
+    slave_insts, slave_group = graph.add_instance_nodes(
+        cluster=cluster, roles=InstanceRole.REDIS_SLAVE, group_name=_("Slave 节点")
+    )
+
+    # 获得访问入口节点组
+    entry = master_insts.first().bind_entry.first()
+    __, entry_group = graph.add_node(entry)
+
+    # 访问入口 ---> Master/Slave节点，关系为：绑定
+    graph.add_line(source=entry_group, target=master_group, label=LineLabel.Bind)
+    # Master ---> Slave, 关系为同步
+    graph.add_line(source=master_group, target=slave_group, label=LineLabel.Rep)
+
+    return graph

dbm-ui/backend/db_meta/api/cluster/redisinstance/handler.py

@@ -0,0 +1,24 @@
+# -*- coding: utf-8 -*-
+"""
+TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available.
+Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved.
+Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at https://opensource.org/licenses/MIT
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+specific language governing permissions and limitations under the License.
+"""
+from backend.db_meta.api.cluster.base.handler import ClusterHandler
+from backend.db_meta.enums import ClusterType
+
+from .detail import scan_cluster
+
+
+class RedisInstanceHandler(ClusterHandler):
+
+    # 「必须」 集群类型
+    cluster_type = ClusterType.RedisInstance
+
+    def topo_graph(self):
+        """「必须」提供集群关系拓扑图"""
+        return scan_cluster(self.cluster).to_dict()

dbm-ui/backend/db_services/dbbase/resources/query.py

@@ -368,6 +368,8 @@ def _list_clusters(
                 Q(name__in=query_params.get("name", "").split(","))
                 | Q(alias__in=query_params.get("name", "").split(","))
             ),
+            # 集群类型
+            "cluster_type": Q(cluster_type=query_params.get("cluster_type")),
             # 版本
             "major_version": Q(major_version__in=query_params.get("major_version", "").split(",")),
             # 地域
@@ -568,6 +570,7 @@ def _list_instances(
             "port": Q(port__in=query_params.get("port", "").split(",")),
             "status": Q(status__in=query_params.get("status", "").split(",")),
             "cluster_id": Q(cluster__id=query_params.get("cluster_id")),
+            "cluster_type": Q(cluster__cluster_type=query_params.get("cluster_type")),
             "region": Q(region=query_params.get("region")),
             "role": Q(role__in=query_params.get("role", "").split(",")),
             "name": Q(cluster__name__in=query_params.get("name", "").split(",")),
@@ -597,18 +600,20 @@ def _filter_instance_hook(cls, bk_biz_id, query_params, instances, **kwargs):
         instance_infos = [cls._to_instance_representation(inst, cluster_entry_map, **kwargs) for inst in instances]
         # 特例：如果有extra参数，则补充额外实例信息
         if query_params.get("extra"):
-            cls._fill_instance_extra_info(bk_biz_id, instance_infos)
+            cls._fill_instance_extra_info(bk_biz_id, instance_infos, **kwargs)
 
         return instance_infos
 
     @classmethod
-    def _fill_instance_extra_info(cls, bk_biz_id: int, instance_infos: List[Dict]):
+    def _fill_instance_extra_info(cls, bk_biz_id: int, instance_infos: List[Dict], **kwargs):
         """
         补充实例的额外信息，这里的一个实现是补充主机和关联集群信息
         @param bk_biz_id: 业务ID
         @param instance_infos: 实例字典信息
         """
-        instances_extra_info = InstanceHandler(bk_biz_id).check_instances(query_instances=instance_infos)
+        # db_type优先以指定的为准(比如mysql和tendbcluster是用的同一个handler)，然后以集群类型对应的组件为准
+        db_type = kwargs.get("handler_db_type") or ClusterType.cluster_type_to_db_type(cls.cluster_types[0])
+        instances_extra_info = InstanceHandler(bk_biz_id).check_instances(instance_infos, db_type=db_type)
         address__instance_extra_info = {inst["instance_address"]: inst for inst in instances_extra_info}
         for inst in instance_infos:
             extra_info = address__instance_extra_info[inst["instance_address"]]
@@ -718,13 +723,18 @@ def _list_machines(
             "bk_host_id": Q(bk_host_id=query_params.get("bk_host_id")),
             "ip": Q(ip__in=query_params.get("ip", "").split(",")),
             "machine_type": Q(machine_type=query_params.get("machine_type")),
+            "bk_city_name": Q(bk_city__bk_idc_city_name__in=query_params.get("region", "").split(",")),
             "bk_os_name": Q(bk_os_name=query_params.get("bk_os_name")),
             "bk_cloud_id": Q(bk_cloud_id=query_params.get("bk_cloud_id")),
             "bk_agent_id": Q(bk_agent_id=query_params.get("bk_agent_id")),
             "instance_role": (
                 Q(storageinstance__instance_role=query_params.get("instance_role"))
                 | Q(proxyinstance__access_layer=query_params.get("instance_role"))
             ),
+            "cluster_ids": (
+                Q(storageinstance__cluster__in=query_params.get("cluster_ids", "").split(","))
+                | Q(proxyinstance__cluster__in=query_params.get("cluster_ids", "").split(","))
+            ),
             "creator": Q(creator__icontains=query_params.get("creator")),
         }
         filter_params_map = {**inner_filter_params_map, **filter_params_map}

dbm-ui/backend/db_services/dbbase/resources/serializers.py

@@ -31,6 +31,7 @@ class ListResourceSLZ(serializers.Serializer):
     status = serializers.CharField(required=False, help_text=_("状态"))
     db_module_id = serializers.CharField(required=False, help_text=_("所属DB模块"))
     bk_cloud_id = serializers.CharField(required=False, help_text=_("管控区域"))
+    cluster_type = serializers.ChoiceField(required=False, choices=ClusterType.get_choices())
 
 
 class ListMySQLResourceSLZ(ListResourceSLZ):
@@ -42,7 +43,6 @@ class ListSQLServerResourceSLZ(ListResourceSLZ):
 
 
 class ListMongoDBResourceSLZ(ListResourceSLZ):
-    cluster_type = serializers.ChoiceField(required=False, choices=ClusterType.get_choices())
     domains = serializers.CharField(help_text=_("批量域名查询(逗号分割)"), required=False)
 
 
@@ -96,11 +96,11 @@ class ListInstancesSerializer(InstanceAddressSerializer):
     status = serializers.CharField(help_text=_("状态"), required=False)
     role = serializers.CharField(help_text=_("角色"), required=False)
     cluster_id = serializers.CharField(help_text=_("集群ID"), required=False)
+    cluster_type = serializers.ChoiceField(help_text=_("集群类型"), required=False, choices=ClusterType.get_choices())
     ip = serializers.CharField(required=False)
 
 
 class MongoDBListInstancesSerializer(ListInstancesSerializer):
-    cluster_type = serializers.ChoiceField(help_text=_("集群类型"), required=False, choices=ClusterType.get_choices())
     exact_ip = serializers.CharField(help_text=_("精确IP查询"), required=False)
 
 
@@ -119,6 +119,8 @@ class ListNodesSLZ(serializers.Serializer):
 class ListMachineSLZ(serializers.Serializer):
     bk_host_id = serializers.IntegerField(help_text=_("主机ID"), required=False)
     ip = serializers.CharField(help_text=_("IP(多个IP过滤以逗号分隔)"), required=False)
+    cluster_ids = serializers.CharField(help_text=_("集群ID(多个过滤以逗号分隔)"), required=False)
+    bk_city_name = serializers.CharField(help_text=_("城市名(多个过滤以逗号分隔)"), required=False)
     machine_type = serializers.ChoiceField(help_text=_("机器类型"), choices=MachineType.get_choices(), required=False)
     bk_os_name = serializers.CharField(help_text=_("os名字"), required=False)
     bk_cloud_id = serializers.IntegerField(help_text=_("云区域ID"), required=False)

dbm-ui/backend/db_services/mongodb/resources/query.py

@@ -49,7 +49,6 @@ def _list_clusters(
     ) -> ResourceList:
         """查询集群信息"""
         filter_params_map = {
-            "cluster_type": Q(cluster_type=query_params.get("cluster_type")),
             "domains": Q(immute_domain__in=query_params.get("domains", "").split(",")),
         }
         return super()._list_clusters(

dbm-ui/backend/db_services/mysql/resources/tendbcluster/query.py

@@ -15,6 +15,7 @@
 from django.forms import model_to_dict
 from django.utils.translation import ugettext_lazy as _
 
+from backend.configuration.constants import DBType
 from backend.db_meta.api.cluster.tendbcluster.detail import scan_cluster
 from backend.db_meta.enums import InstanceInnerRole, TenDBClusterSpiderRole
 from backend.db_meta.enums.cluster_type import ClusterType
@@ -191,6 +192,12 @@ def _filter_instance_qs(cls, query_filters, query_params):
 
         return instances
 
+    @classmethod
+    def _filter_instance_hook(cls, bk_biz_id, query_params, instances, **kwargs):
+        # cluster handler
+        kwargs.update(handler_db_type=DBType.MySQL.value)
+        return super()._filter_instance_hook(bk_biz_id, query_params, instances, **kwargs)
+
     @classmethod
     def _to_instance_representation(cls, instance: dict, cluster_entry_map: dict, **kwargs) -> Dict[str, Any]:
         """

dbm-ui/backend/db_services/redis/resources/redis_cluster/query.py

@@ -15,6 +15,7 @@
 from django.utils.translation import ugettext_lazy as _
 
 from backend.db_meta.api.cluster.rediscluster.handler import RedisClusterHandler
+from backend.db_meta.api.cluster.redisinstance.handler import RedisInstanceHandler
 from backend.db_meta.api.cluster.tendiscache.handler import TendisCacheClusterHandler
 from backend.db_meta.api.cluster.tendispluscluster.handler import TendisPlusClusterHandler
 from backend.db_meta.api.cluster.tendisssd.handler import TendisSSDClusterHandler
@@ -45,6 +46,7 @@ class RedisListRetrieveResource(query.ListRetrieveResource):
         ClusterType.TendisTwemproxyRedisInstance: TendisCacheClusterHandler,
         ClusterType.TendisPredixyTendisplusCluster: TendisPlusClusterHandler,
         ClusterType.TendisPredixyRedisCluster: RedisClusterHandler,
+        ClusterType.RedisInstance: RedisInstanceHandler,
     }
 
     fields = [

dbm-ui/backend/flow/engine/bamboo/scene/spider/remote_local_slave_recover.py

@@ -69,6 +69,7 @@ def tendb_remote_slave_local_recover(self):
             self.data["root_id"] = self.root_id
             self.data["uid"] = self.ticket_data["uid"]
             self.data["ticket_type"] = self.ticket_data["ticket_type"]
+            self.data["created_by"] = self.ticket_data["created_by"]
             self.data["bk_biz_id"] = cluster_class.bk_biz_id
             self.data["db_module_id"] = cluster_class.db_module_id
             self.data["cluster_type"] = cluster_class.cluster_type

dbm-ui/backend/flow/engine/bamboo/scene/spider/remote_master_slave_migrate.py

@@ -76,7 +76,7 @@ def migrate_master_slave_flow(self):
         # 构建流程
         cluster_ids = []
         for i in self.ticket_data["infos"]:
-            cluster_ids.extend(i["cluster_ids"])
+            cluster_ids.append(i["cluster_id"])
 
         tendb_migrate_pipeline_all = Builder(
             root_id=self.root_id,
@@ -353,7 +353,7 @@ def migrate_master_slave_flow(self):
                     )
                 )
                 uninstall_svr_sub_pipeline_list.append(
-                    uninstall_svr_sub_pipeline.build_sub_process(sub_name=_("卸载remote节点{}".format(ip)))
+                    uninstall_svr_sub_pipeline.build_sub_process(sub_name=_("卸载remote节点{}").format(ip))
                 )
 
             # 安装实例

dbm-ui/backend/flow/utils/spider/spider_db_meta.py

@@ -161,14 +161,17 @@ def remotedb_migrate_add_install_nodes(self):
         """
         remotedb 成对迁移添加初始化节点元数据
         """
+        cluster = Cluster.objects.get(id=self.cluster["cluster_id"])
+        old_resource_spec = {MachineType.REMOTE.value: cluster.storageinstance_set.first().machine.spec_config}
         TenDBClusterMigrateRemoteDb.storage_create(
             cluster_id=self.cluster["cluster_id"],
             master_ip=self.cluster["new_master_ip"],
             slave_ip=self.cluster["new_slave_ip"],
             ports=self.cluster["ports"],
             creator=self.global_data["created_by"],
             mysql_version=self.cluster["version"],
-            resource_spec=self.global_data["resource_spec"],
+            # 兼容资源池和手输机器两种情况
+            resource_spec=self.global_data.get("resource_spec") or old_resource_spec,
         )
         return True
 
@@ -242,13 +245,16 @@ def tendb_slave_recover_add_nodes(self):
         """
         remotedb 成对迁移添加初始化节点元数据
         """
+        cluster = Cluster.objects.get(id=self.cluster["cluster_id"])
+        old_resource_spec = {MachineType.REMOTE.value: cluster.storageinstance_set.first().machine.spec_config}
         TenDBClusterMigrateRemoteDb.storage_create(
             cluster_id=self.cluster["cluster_id"],
             slave_ip=self.cluster["new_slave_ip"],
             ports=self.cluster["ports"],
             creator=self.global_data["created_by"],
             mysql_version=self.cluster["version"],
-            resource_spec=self.global_data["resource_spec"],
+            # 兼容资源池和手输机器两种情况
+            resource_spec=self.global_data.get("resource_spec") or old_resource_spec,
         )
         return True
 

dbm-ui/backend/tests/ticket/test_ticket_flow.py

@@ -180,7 +180,7 @@ def test_sql_import_flow(self, mocked_status, mocked__run, mocked_permission_cla
         lambda resource_request_id, node_infos: (1, APPLY_RESOURCE_RETURN_DATA),
     )
     @patch(
-        "backend.ticket.flow_manager.resource.ResourceApplyFlow.patch_resource_params", lambda self, ticket_data: None
+        "backend.ticket.flow_manager.resource.ResourceApplyFlow.patch_resource_spec", lambda self, ticket_data: None
     )
     @patch("backend.db_services.cmdb.biz.Permission", PermissionMock)
     @patch("backend.ticket.builders.mysql.mysql_ha_apply.DBConfigApi", DBConfigApiMock)

dbm-ui/backend/ticket/builders/common/base.py

@@ -134,7 +134,9 @@ def validate_hosts_from_idle_pool(cls, bk_biz_id: int, host_list: List[int]) ->
         )
         idle_host_list = [host["bk_host_id"] for host in idle_host_info_list]
 
-        return set(host_list) - set(idle_host_list)
+        host_not_in_idle = set(host_list) - set(idle_host_list)
+        if host_not_in_idle:
+            raise serializers.ValidationError(_("主机{}不在空闲机池，请保证所选的主机均来自空闲机").format(host_not_in_idle))
 
     @classmethod
     def validate_hosts_not_in_db_meta(cls, host_infos: List[Dict]):

dbm-ui/backend/ticket/builders/common/bigdata.py

@@ -46,9 +46,7 @@ def validate_hosts_from_idle_pool(cls, bk_biz_id, nodes: Dict):
             role_host_list = [node["bk_host_id"] for node in nodes[role] if node.get("bk_host_id")]
             hosts_set.update(role_host_list)
 
-        hosts_not_in_idle_pool = CommonValidate.validate_hosts_from_idle_pool(bk_biz_id, list(hosts_set))
-        if hosts_not_in_idle_pool:
-            raise serializers.ValidationError(_("主机{}不在空闲机池，请保证所选的主机均来自空闲机").format(hosts_not_in_idle_pool))
+        CommonValidate.validate_hosts_from_idle_pool(bk_biz_id, list(hosts_set))
 
     @classmethod
     def validate_hosts_not_in_db_meta(cls, nodes: Dict):

dbm-ui/backend/ticket/builders/mysql/mysql_restore_local_slave.py

@@ -18,10 +18,11 @@
 from backend.ticket.builders.common.base import InstanceInfoSerializer
 from backend.ticket.builders.common.constants import MySQLBackupSource
 from backend.ticket.builders.mysql.base import BaseMySQLTicketFlowBuilder, MySQLBaseOperateDetailSerializer
+from backend.ticket.builders.tendbcluster.base import TendbBaseOperateDetailSerializer
 from backend.ticket.constants import TicketType
 
 
-class MysqlRestoreLocalSlaveDetailSerializer(MySQLBaseOperateDetailSerializer):
+class MysqlRestoreLocalSlaveDetailSerializer(TendbBaseOperateDetailSerializer, MySQLBaseOperateDetailSerializer):
     class SlaveInfoSerializer(serializers.Serializer):
         slave = InstanceInfoSerializer(help_text=_("从库实例信息"))
         cluster_id = serializers.IntegerField(help_text=_("集群ID"))
@@ -32,7 +33,7 @@ class SlaveInfoSerializer(serializers.Serializer):
 
     def validate(self, attrs):
         # 校验集群是否可用，集群类型为高可用
-        super(MysqlRestoreLocalSlaveDetailSerializer, self).validate_cluster_can_access(attrs)
+        super(TendbBaseOperateDetailSerializer, self).validate_cluster_can_access(attrs)
         super(MysqlRestoreLocalSlaveDetailSerializer, self).validate_cluster_type(attrs, ClusterType.TenDBHA)
 
         # 校验实例的角色为slave

dbm-ui/backend/ticket/builders/redis/base.py

@@ -149,9 +149,3 @@ def check_cluster_phase(cluster_id):
 
     def validate_cluster_id(self, cluster_id):
         return self.check_cluster_phase(cluster_id)
-
-    def validate_src_cluster(self, cluster_id):
-        return self.check_cluster_phase(cluster_id)
-
-    def validate_dst_cluster(self, cluster_id):
-        return self.check_cluster_phase(cluster_id)