From 454b8429cc4ff4bd8516487a909f393dcc32055a Mon Sep 17 00:00:00 2001 From: yyhenryyy Date: Fri, 13 Sep 2024 17:29:16 +0800 Subject: [PATCH] =?UTF-8?q?fix(mongodb):=20clustername=E5=A2=9E=E5=8A=A0ap?= =?UTF-8?q?p=EF=BC=8C=E4=BF=AE=E6=94=B9=E5=AF=86=E7=A0=81=E8=A7=84?= =?UTF-8?q?=E5=88=99=E5=90=8D=E5=AD=97=EF=BC=8C=E4=BF=AE=E6=94=B9instance?= =?UTF-8?q?=E7=9A=84role=E5=90=8D=E5=AD=97=EF=BC=8C=E4=BF=AE=E6=94=B9?= =?UTF-8?q?=E6=89=A7=E8=A1=8C=E8=84=9A=E6=9C=AC=E5=B7=A5=E4=BD=9C=E7=9B=AE?= =?UTF-8?q?=E5=BD=95=20#6933?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../atommongodb/mongo_execute_script.go | 2 +- dbm-ui/backend/db_meta/enums/instance_role.py | 22 ++++++++--------- dbm-ui/backend/flow/consts.py | 7 +----- .../bamboo/scene/mongodb/mongodb_install.py | 2 +- .../mongodb/sub_task/cluster_increase_node.py | 1 - .../scene/mongodb/sub_task/mongos_install.py | 4 +--- .../mongodb/sub_task/replicaset_install.py | 4 +--- .../sub_task/replicaset_reduce_node.py | 1 - .../mongodb/sub_task/replicaset_replace.py | 3 --- .../sub_task/replicaset_set_increase_node.py | 1 - .../flow/utils/mongodb/calculate_cluster.py | 13 +++++----- .../flow/utils/mongodb/mongodb_dataclass.py | 24 ++++++++++--------- 12 files changed, 36 insertions(+), 48 deletions(-) diff --git a/dbm-services/mongodb/db-tools/dbactuator/pkg/atomjobs/atommongodb/mongo_execute_script.go b/dbm-services/mongodb/db-tools/dbactuator/pkg/atomjobs/atommongodb/mongo_execute_script.go index 069797deab..157a326f41 100644 --- a/dbm-services/mongodb/db-tools/dbactuator/pkg/atomjobs/atommongodb/mongo_execute_script.go +++ b/dbm-services/mongodb/db-tools/dbactuator/pkg/atomjobs/atommongodb/mongo_execute_script.go @@ -119,7 +119,7 @@ func (e *ExecScript) Init(runtime *jobruntime.JobGenericRuntime) error { // 获取各种目录 e.Mongo = filepath.Join(e.BinDir, "mongodb", "bin", "mongo") - e.ScriptDir = filepath.Join("/", "home", e.OsUser, e.runtime.UID) + e.ScriptDir = filepath.Join("/", "data", "dbbak", "mongoscript", e.runtime.UID) e.ScriptFilePath = filepath.Join(e.ScriptDir, strings.Join([]string{"script", "js"}, ".")) e.ResultFileName = strings.Join([]string{ e.ConfParams.ScriptName, strings.Join([]string{"result", "txt"}, ".")}, "_") diff --git a/dbm-ui/backend/db_meta/enums/instance_role.py b/dbm-ui/backend/db_meta/enums/instance_role.py index 3b2df8eebb..f6741a75ff 100644 --- a/dbm-ui/backend/db_meta/enums/instance_role.py +++ b/dbm-ui/backend/db_meta/enums/instance_role.py @@ -65,17 +65,17 @@ class InstanceRole(str, StructuredEnum): DORIS_OBSERVER = EnumField("doris_observer", _("Observer")) # MongoDB - MONGO_M1 = EnumField("mongo_m1", _("mongo_m1")) - MONGO_M2 = EnumField("mongo_m2", _("mongo_m2")) - MONGO_M3 = EnumField("mongo_m3", _("mongo_m3")) - MONGO_M4 = EnumField("mongo_m4", _("mongo_m4")) - MONGO_M5 = EnumField("mongo_m5", _("mongo_m5")) - MONGO_M6 = EnumField("mongo_m6", _("mongo_m6")) - MONGO_M7 = EnumField("mongo_m7", _("mongo_m7")) - MONGO_M8 = EnumField("mongo_m8", _("mongo_m8")) - MONGO_M9 = EnumField("mongo_m9", _("mongo_m9")) - MONGO_M10 = EnumField("mongo_m10", _("mongo_m10")) - MONGO_BACKUP = EnumField("mongo_backup", _("mongo_backup")) + MONGO_M1 = EnumField("m1", _("m1")) + MONGO_M2 = EnumField("m2", _("m2")) + MONGO_M3 = EnumField("m3", _("m3")) + MONGO_M4 = EnumField("m4", _("m4")) + MONGO_M5 = EnumField("m5", _("m5")) + MONGO_M6 = EnumField("m6", _("m6")) + MONGO_M7 = EnumField("m7", _("m7")) + MONGO_M8 = EnumField("m8", _("m8")) + MONGO_M9 = EnumField("m9", _("m9")) + MONGO_M10 = EnumField("m10", _("m10")) + MONGO_BACKUP = EnumField("backup", _("backup")) # Riak RIAK_NODE = EnumField("riak_node", _("Riak")) diff --git a/dbm-ui/backend/flow/consts.py b/dbm-ui/backend/flow/consts.py index f033ab8a25..396a5e8baf 100644 --- a/dbm-ui/backend/flow/consts.py +++ b/dbm-ui/backend/flow/consts.py @@ -431,11 +431,6 @@ class DBActuatorActionEnum(str, StructuredEnum): CreateToDBViaCtl = EnumField("create-to-db-via-ctl", _("重命名在中控建立目标库")) RenamePreDropToOnRemote = EnumField("rename-pre-drop-to-on-remote", _("TenDBCluster 重命名在remote预清理目标库")) RenameDropFromViaCtl = EnumField("rename-drop-from-via-ctl", _("TenDBCluster 重命名在中控删除源库")) - PushMySQLCrondConfig = EnumField("push-mysql-crond-config", _("推送mysql-crond配置")) - PushMySQLMonitorConfig = EnumField("push-mysql-monitor-config", _("推送mysql-monitor配置")) - PushChecksumConfig = EnumField("push-checksum-config", _("推送mysql-table-checksum配置")) - PushNewDbBackupConfig = EnumField("push-new-db-backup-config", _("推送备份配置")) - PushMySQLRotatebinlogConfig = EnumField("push-mysql-rotatebinlog-config", _("推送rotatebinlog配置")) class RedisActuatorActionEnum(str, StructuredEnum): @@ -1161,7 +1156,7 @@ class MongoDBPasswordRule(str, StructuredEnum): mongodb密码规则名 """ - RULE = EnumField("mongo_password", _("MongoDB密码规则")) + RULE = EnumField("mongodb_password", _("MongoDB密码规则")) class MongoDBClusterRole(str, StructuredEnum): diff --git a/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/mongodb_install.py b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/mongodb_install.py index cca22dba05..bbb88ba01e 100644 --- a/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/mongodb_install.py +++ b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/mongodb_install.py @@ -136,7 +136,7 @@ def multi_replicaset_install_flow(self): for replicaset_info in self.data["sets"]: kwargs = self.get_kwargs.get_add_relationship_to_meta_kwargs(replicaset_info=replicaset_info) pipeline.add_act( - act_name=_("MongoDB--添加复制集{}-{}关系到meta".format(self.data["app"], replicaset_info["set_id"])), + act_name=_("MongoDB--添加复制集{}关系到meta".format(replicaset_info["set_id"])), act_component_code=ExecAddRelationshipOperationComponent.code, kwargs=kwargs, ) diff --git a/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/cluster_increase_node.py b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/cluster_increase_node.py index a41cb3308d..9f3cc1b430 100644 --- a/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/cluster_increase_node.py +++ b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/cluster_increase_node.py @@ -37,7 +37,6 @@ def cluster_increase_node(root_id: str, ticket_data: Optional[Dict], sub_kwargs: sub_pipeline = SubBuilder(root_id=root_id, data=ticket_data) # 设置变量 - sub_get_kwargs.payload["app"] = sub_get_kwargs.payload["bk_app_abbr"] sub_get_kwargs.payload["cluster_type"] = ClusterType.MongoShardedCluster.value info["target"] = info["add_shard_nodes"][0] diff --git a/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/mongos_install.py b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/mongos_install.py index dc408a18d9..0217c6b58f 100644 --- a/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/mongos_install.py +++ b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/mongos_install.py @@ -71,7 +71,5 @@ def mongos_install( # 安装监控 return sub_pipeline.build_sub_process( - sub_name=_( - "MongoDB--安装mongos--{}-{}".format(sub_get_kwargs.payload["app"], sub_get_kwargs.payload["cluster_id"]) - ) + sub_name=_("MongoDB--安装mongos--{}".format(sub_get_kwargs.payload["cluster_id"])) ) diff --git a/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/replicaset_install.py b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/replicaset_install.py index b643f12b8f..fee6df5a4f 100644 --- a/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/replicaset_install.py +++ b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/replicaset_install.py @@ -126,7 +126,5 @@ def replicaset_install( # 安装监控 return sub_pipeline.build_sub_process( - sub_name=_( - "MongoDB--安装复制集--{}-{}".format(sub_get_kwargs.payload["app"], sub_get_kwargs.replicaset_info["set_id"]) - ) + sub_name=_("MongoDB--安装复制集-{}".format(sub_get_kwargs.replicaset_info["set_id"])) ) diff --git a/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/replicaset_reduce_node.py b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/replicaset_reduce_node.py index 05f5194ed6..9476244339 100644 --- a/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/replicaset_reduce_node.py +++ b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/replicaset_reduce_node.py @@ -39,7 +39,6 @@ def replicaset_reduce_node( if not cluster: # 设置变量 - sub_get_kwargs.payload["app"] = sub_get_kwargs.payload["bk_app_abbr"] sub_get_kwargs.payload["cluster_type"] = ClusterType.MongoReplicaSet.value # 获取集群信息并计算对应关系 diff --git a/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/replicaset_replace.py b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/replicaset_replace.py index 9327db4279..92c79146cf 100644 --- a/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/replicaset_replace.py +++ b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/replicaset_replace.py @@ -65,9 +65,6 @@ def replicaset_replace( act_name=_("MongoDB-机器初始化"), act_component_code=ExecuteDBActuatorJobComponent.code, kwargs=kwargs ) - # config计算参数 - # if cluster_role == MongoDBClusterRole.ConfigSvr.value: - # sub_get_kwargs.calc_param_replace(info=info, instance_num=0) # 进行替换——并行 以ip为维度 sub_sub_pipelines = [] for mongodb_instance in info["instances"]: diff --git a/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/replicaset_set_increase_node.py b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/replicaset_set_increase_node.py index 04e5b750fd..2966c85fe7 100644 --- a/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/replicaset_set_increase_node.py +++ b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/replicaset_set_increase_node.py @@ -38,7 +38,6 @@ def replicaset_set_increase_node( sub_pipeline = SubBuilder(root_id=root_id, data=ticket_data) # 设置变量 - sub_get_kwargs.payload["app"] = sub_get_kwargs.payload["bk_app_abbr"] info["target"] = info["add_shard_nodes"][0] # 计算cacheSize oplogSize diff --git a/dbm-ui/backend/flow/utils/mongodb/calculate_cluster.py b/dbm-ui/backend/flow/utils/mongodb/calculate_cluster.py index 2b0696bfeb..2337b895dc 100644 --- a/dbm-ui/backend/flow/utils/mongodb/calculate_cluster.py +++ b/dbm-ui/backend/flow/utils/mongodb/calculate_cluster.py @@ -101,12 +101,13 @@ def replicase_calc(payload: dict, payload_clusters: dict, app: str, domain_prefi else: domain = "{}.{}.{}.db".format(domain_prefix[machine_index], replica_set["set_id"], app) nodes.append({"ip": machine["ip"], "bk_cloud_id": machine["bk_cloud_id"], "domain": domain}) + set_id_key_file = "{}-{}".format(app, replica_set["set_id"]) sets.append( { - "set_id": replica_set["set_id"], + "set_id": set_id_key_file, "alias": replica_set["name"], "port": port, - "key_file": "{}-{}".format(app, replica_set["set_id"]), + "key_file": set_id_key_file, "cacheSizeGB": avg_mem_size_gb, "oplogSizeMB": oplog_size_mb, "skip_machine": skip_machine, @@ -122,7 +123,7 @@ def cluster_calc(payload: dict, payload_clusters: dict, app: str) -> dict: """cluster进行计算""" payload_clusters["alias"] = payload["cluster_alias"] - payload_clusters["cluster_id"] = payload["cluster_name"] + payload_clusters["cluster_id"] = "{}-{}".format(app, payload["cluster_name"]) payload_clusters["machine_specs"] = payload["machine_specs"] oplog_percent = payload["oplog_percent"] / 100 disaster_tolerance_level = payload["disaster_tolerance_level"] @@ -180,7 +181,7 @@ def cluster_calc(payload: dict, payload_clusters: dict, app: str) -> dict: # 分配机器 # mongo_config config = {} - config["set_id"] = "{}-{}".format(payload["cluster_name"], "conf") # 设置常量 + config["set_id"] = "{}-{}".format(payload_clusters["cluster_id"], "conf") # 设置常量 config["port"] = config_port # 设置常量 config["cacheSizeGB"] = config_mem_size_gb config["oplogSizeMB"] = config_oplog_size_mb @@ -198,7 +199,7 @@ def cluster_calc(payload: dict, payload_clusters: dict, app: str) -> dict: shard_port += 1 shard_info.append( { - "set_id": "{}-s{}".format(payload["cluster_name"], str(i + 1)), + "set_id": "{}-s{}".format(payload_clusters["cluster_id"], str(i + 1)), "port": shard_port, "cacheSizeGB": shard_avg_mem_size_gb, "oplogSizeMB": shard_oplog_size_mb, @@ -244,7 +245,7 @@ def calculate_cluster(payload: dict) -> dict: payload_clusters["ticket_type"] = payload["ticket_type"] payload_clusters["cluster_type"] = payload["cluster_type"] payload_clusters["city"] = payload["city_code"] - payload_clusters["app"] = payload["bk_app_abbr"] + payload_clusters["bk_app_abbr"] = payload["bk_app_abbr"] payload_clusters["disaster_tolerance_level"] = payload["disaster_tolerance_level"] app = payload["bk_app_abbr"] payload_clusters["db_version"] = payload["db_version"] diff --git a/dbm-ui/backend/flow/utils/mongodb/mongodb_dataclass.py b/dbm-ui/backend/flow/utils/mongodb/mongodb_dataclass.py index dc23784c9a..d436339106 100644 --- a/dbm-ui/backend/flow/utils/mongodb/mongodb_dataclass.py +++ b/dbm-ui/backend/flow/utils/mongodb/mongodb_dataclass.py @@ -391,7 +391,6 @@ def get_install_mongod_kwargs(self, node: dict, cluster_role: str) -> dict: "port": self.replicaset_info["port"], "dbVersion": self.payload["db_version"], "instanceType": MongoDBInstanceType.MongoD, - "app": self.payload["app"], "setId": self.replicaset_info["set_id"], "keyFile": key_file, "auth": True, @@ -439,7 +438,6 @@ def get_install_mongos_kwargs(self, node: dict, replace: bool) -> dict: "port": self.mongos_info["port"], "dbVersion": self.payload["db_version"], "instanceType": MongoDBInstanceType.MongoS, - "app": self.payload["app"], "setId": self.mongos_info["conf_set_id"], "keyFile": self.payload["key_file"], "auth": True, @@ -476,7 +474,6 @@ def get_replicaset_init_kwargs(self, config_svr: bool) -> dict: "payload": { "ip": self.replicaset_info["nodes"][0]["ip"], "port": self.replicaset_info["port"], - "app": self.payload["app"], "setId": self.replicaset_info["set_id"], "configSvr": config_svr, "ips": instances, @@ -638,7 +635,7 @@ def get_init_exec_script_kwargs(self, script_type: str) -> dict: create_extra_manager_user_status = False if self.payload["cluster_type"] == ClusterType.MongoReplicaSet.value: mongo_type = "replicaset" - set_name = "{}-{}".format(self.payload["app"], self.replicaset_info["set_id"]) + set_name = self.replicaset_info["set_id"] else: mongo_type = "cluster" set_name = "" @@ -685,7 +682,7 @@ def get_add_manager_user_kwargs(self) -> dict: """创建dba用户""" if self.payload["cluster_type"] == ClusterType.MongoReplicaSet.value: - set_name = "{}-{}".format(self.payload["app"], self.replicaset_info["set_id"]) + set_name = self.replicaset_info["set_id"] else: set_name = "" if float(".".join(self.payload["db_version"].split(".")[0:2])) < 2.6: @@ -731,7 +728,7 @@ def get_get_manager_password_kwargs(self) -> dict: """获取用户密码""" if self.payload["cluster_type"] == ClusterType.MongoReplicaSet.value: - set_name = "{}-{}".format(self.payload["app"], self.replicaset_info["set_id"]) + set_name = self.replicaset_info["set_id"] else: set_name = "" return {"set_trans_data_dataclass": CommonContext.__name__, "set_name": set_name, "users": self.manager_users} @@ -740,7 +737,7 @@ def get_add_password_to_db_kwargs(self, usernames: list, info: dict) -> dict: """添加密码到db""" if self.payload["cluster_type"] == ClusterType.MongoReplicaSet.value: - set_name = "{}-{}".format(self.payload["app"], self.replicaset_info["set_id"]) + set_name = self.replicaset_info["set_id"] else: set_name = "" @@ -1012,7 +1009,6 @@ def get_mongo_deinstall_kwargs( "payload": { "ip": node_info["ip"], "port": node_info["port"], - "app": self.payload["app"], "setId": self.payload["set_id"], "nodeInfo": nodes, "instanceType": instance_type, @@ -1123,7 +1119,7 @@ def get_instance_restart_kwargs( bk_cloud_id = host["bk_cloud_id"] port = instance["port"] username = MongoDBManagerUser.DbaUser.value - if instance["role"] == MongoDBInstanceType.MongoS: + if instance["role"] == MongoDBInstanceType.MongoS.value: instance_type = MongoDBInstanceType.MongoS.value else: instance_type = MongoDBInstanceType.MongoD.value @@ -1560,10 +1556,16 @@ def calc_increase_node(self, info: dict): self.get_cluster_info_deinstall(cluster_id=cluster_id) port = self.payload["nodes"][0]["port"] role = self.instance_role[current_shard_nodes_num - 1 + index] - node_name = role.split("_")[1] + node_name = role ip = add_shard_node["ip"] bk_cloud_id = add_shard_node["bk_cloud_id"] - domain = "{}.{}.{}.db".format(node_name, self.payload["cluster_name"], self.payload["app"]) + # self.payload["cluster_name"] 为 "app-clustername" + + domain = "{}.{}.{}.db".format( + node_name, + self.payload["cluster_name"].replace("{}-".format(self.payload["bk_app_abbr"]), ""), + self.payload["bk_app_abbr"], + ) db_instance = { "cluster_id": cluster_id, "role": role,