From 8364715f7e868450a717e5901da2b55287d53ad0 Mon Sep 17 00:00:00 2001 From: iSecloud <869820505@qq.com> Date: Thu, 11 Jan 2024 16:49:17 +0800 Subject: [PATCH 1/9] =?UTF-8?q?fix(backend):=20=E6=94=AF=E6=8C=81=E4=BA=91?= =?UTF-8?q?=E5=8C=BA=E5=9F=9F=E7=BB=84=E4=BB=B6=E9=83=A8=E7=BD=B2=E6=B5=81?= =?UTF-8?q?=E7=A8=8B=E4=B8=B2=E8=A1=8C=E5=8C=96=20&=20=E5=A2=9E=E5=8A=A0?= =?UTF-8?q?=E6=9F=A5=E8=AF=A2=E9=9B=86=E7=BE=A4=E9=80=9A=E7=94=A8=E4=BF=A1?= =?UTF-8?q?=E6=81=AF=E6=8E=A5=E5=8F=A3=20#3011?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../db_services/dbbase/resources/query.py | 55 +++++++++++++------ .../backend/db_services/dbbase/serializers.py | 16 ++++++ dbm-ui/backend/db_services/dbbase/views.py | 20 ++++++- .../bamboo/scene/cloud/base_service_flow.py | 28 ++++++++++ .../bamboo/scene/cloud/dns_service_flow.py | 16 ++++-- 5 files changed, 112 insertions(+), 23 deletions(-) diff --git a/dbm-ui/backend/db_services/dbbase/resources/query.py b/dbm-ui/backend/db_services/dbbase/resources/query.py index 1542ac7300..4ea4188782 100644 --- a/dbm-ui/backend/db_services/dbbase/resources/query.py +++ b/dbm-ui/backend/db_services/dbbase/resources/query.py @@ -9,7 +9,7 @@ specific language governing permissions and limitations under the License. """ import abc -from typing import Dict, List, Union +from typing import Dict, List, Tuple, Union import attr from django.db.models import Q @@ -114,15 +114,18 @@ def get_topo_graph(cls, bk_biz_id: int, cluster_id: int) -> dict: def get_fields(cls) -> List[Dict[str, str]]: return cls.fields - @classmethod - def export_cluster(cls, bk_biz_id: int, cluster_ids: list) -> HttpResponse: + @staticmethod + def common_query_cluster(bk_biz_id: int, cluster_types: list, cluster_ids: list) -> Tuple[List[Dict], List[Dict]]: + """集群的通用属性查询""" # 获取所有符合条件的集群对象 clusters = Cluster.objects.prefetch_related( "storageinstance_set", "proxyinstance_set", "storageinstance_set__machine", "proxyinstance_set__machine" - ).filter(bk_biz_id=bk_biz_id, cluster_type__in=cls.cluster_types) + ).filter(bk_biz_id=bk_biz_id, cluster_type__in=cluster_types) if cluster_ids: clusters = clusters.filter(id__in=cluster_ids) + cluster_entry_map = ClusterEntry.get_cluster_entry_map(cluster_ids=list(clusters.values_list("id", flat=True))) + # 初始化用于存储Excel数据的字典列表 headers = [ {"id": "cluster_id", "name": _("集群 ID")}, @@ -130,6 +133,7 @@ def export_cluster(cls, bk_biz_id: int, cluster_ids: list) -> HttpResponse: {"id": "cluster_alias", "name": _("集群别名")}, {"id": "cluster_type", "name": _("集群类型")}, {"id": "master_domain", "name": _("主域名")}, + {"id": "slave_domain", "name": _("从域名")}, {"id": "major_version", "name": _("主版本")}, {"id": "region", "name": _("地域")}, {"id": "disaster_tolerance_level", "name": _("容灾级别")}, @@ -149,8 +153,9 @@ def fill_instances_to_cluster_info( if role in cluster_info: cluster_info[role] += f"\n{ins.machine.ip}#{ins.port}" else: - if role not in headers: - headers.append({"id": role, "name": role}) + role_header = {"id": role, "name": role} + if role_header not in headers: + headers.append(role_header) cluster_info[role] = f"{ins.machine.ip}#{ins.port}" # 遍历所有的集群对象 @@ -163,6 +168,7 @@ def fill_instances_to_cluster_info( "cluster_alias": cluster.alias, "cluster_type": cluster.cluster_type, "master_domain": cluster.immute_domain, + "slave_domain": cluster_entry_map[cluster.id].get("slave_domain", ""), "major_version": cluster.major_version, "region": cluster.region, "disaster_tolerance_level": cluster.get_disaster_tolerance_level_display(), @@ -172,17 +178,13 @@ def fill_instances_to_cluster_info( # 将当前集群的信息追加到data_list列表中 data_list.append(cluster_info) - biz_name = AppCache.get_biz_name(bk_biz_id) - db_type = ClusterType.cluster_type_to_db_type(cls.cluster_types[0]) - wb = ExcelHandler.serialize(data_list, headers=headers, match_header=True) - return ExcelHandler.response( - wb, urlquote(_("{export_prefix}集群列表.xlsx").format(export_prefix=f"{biz_name}[{bk_biz_id}]{db_type}")) - ) - @classmethod - def export_instance(cls, bk_biz_id: int, bk_host_ids: list) -> HttpResponse: - # 查询实例 - query_condition = Q(bk_biz_id=bk_biz_id, cluster_type__in=cls.cluster_types) + return headers, data_list + + @staticmethod + def common_query_instance(bk_biz_id: int, cluster_types: list, bk_host_ids: list) -> Tuple[List[Dict], List[Dict]]: + """实例通用属性查询""" + query_condition = Q(bk_biz_id=bk_biz_id, cluster_type__in=cluster_types) if bk_host_ids: query_condition = query_condition & Q(machine__bk_host_id__in=bk_host_ids) storages = StorageInstance.objects.prefetch_related("machine", "machine__bk_city", "cluster").filter( @@ -229,9 +231,30 @@ def export_instance(cls, bk_biz_id: int, bk_host_ids: list) -> HttpResponse: } ) + return headers, data_list + + @classmethod + def export_cluster(cls, bk_biz_id: int, cluster_ids: list) -> HttpResponse: + """集群通用属性导出""" + headers, data_list = cls.common_query_cluster(bk_biz_id, cls.cluster_types, cluster_ids) + + biz_name = AppCache.get_biz_name(bk_biz_id) + db_type = ClusterType.cluster_type_to_db_type(cls.cluster_types[0]) + wb = ExcelHandler.serialize(data_list, headers=headers, match_header=True) + + return ExcelHandler.response( + wb, urlquote(_("{export_prefix}集群列表.xlsx").format(export_prefix=f"{biz_name}[{bk_biz_id}]{db_type}")) + ) + + @classmethod + def export_instance(cls, bk_biz_id: int, bk_host_ids: list) -> HttpResponse: + """实例通用属性导出""" + headers, data_list = cls.common_query_instance(bk_biz_id, cls.cluster_types, bk_host_ids) + biz_name = AppCache.get_biz_name(bk_biz_id) db_type = ClusterType.cluster_type_to_db_type(cls.cluster_types[0]) wb = ExcelHandler.serialize(data_list, headers=headers, match_header=True) + return ExcelHandler.response( wb, urlquote(_("{export_prefix}实例列表.xlsx").format(export_prefix=f"{biz_name}[{bk_biz_id}]{db_type}")) ) diff --git a/dbm-ui/backend/db_services/dbbase/serializers.py b/dbm-ui/backend/db_services/dbbase/serializers.py index 1f6882ab74..0b1e7a7aaf 100644 --- a/dbm-ui/backend/db_services/dbbase/serializers.py +++ b/dbm-ui/backend/db_services/dbbase/serializers.py @@ -47,3 +47,19 @@ def get_conditions(self, attr): class QueryAllTypeClusterResponseSerializer(serializers.Serializer): class Meta: swagger_schema_fields = {"example": [{"id": 47, "immute_domain": "mysql.dba.db.com"}]} + + +class CommonQueryClusterSerializer(serializers.Serializer): + bk_biz_id = serializers.IntegerField(help_text=_("业务ID")) + cluster_types = serializers.CharField(help_text=_("集群类型(逗号分隔)")) + cluster_ids = serializers.CharField(help_text=_("集群ID(逗号分割)"), required=False, default="") + + def validate(self, attrs): + attrs["cluster_types"] = attrs["cluster_types"].split(",") + attrs["cluster_ids"] = attrs["cluster_ids"].split(",") if attrs["cluster_ids"] else [] + return attrs + + +class CommonQueryClusterResponseSerializer(serializers.Serializer): + class Meta: + swagger_schema_fields = {"example": []} diff --git a/dbm-ui/backend/db_services/dbbase/views.py b/dbm-ui/backend/db_services/dbbase/views.py index a9255ad0c7..b9ac5cd41c 100644 --- a/dbm-ui/backend/db_services/dbbase/views.py +++ b/dbm-ui/backend/db_services/dbbase/views.py @@ -18,7 +18,10 @@ from backend.bk_web.pagination import AuditedLimitOffsetPagination from backend.bk_web.swagger import ResponseSwaggerAutoSchema, common_swagger_auto_schema from backend.db_meta.models import Cluster +from backend.db_services.dbbase.resources.query import ListRetrieveResource from backend.db_services.dbbase.serializers import ( + CommonQueryClusterResponseSerializer, + CommonQueryClusterSerializer, IsClusterDuplicatedResponseSerializer, IsClusterDuplicatedSerializer, QueryAllTypeClusterResponseSerializer, @@ -49,16 +52,29 @@ def verify_duplicated_cluster_name(self, request, *args, **kwargs): return Response(is_duplicated) @common_swagger_auto_schema( - operation_summary=_("查询全集群信息"), + operation_summary=_("查询全集群简略信息"), auto_schema=ResponseSwaggerAutoSchema, query_serializer=QueryAllTypeClusterSerializer(), responses={status.HTTP_200_OK: QueryAllTypeClusterResponseSerializer()}, tags=[SWAGGER_TAG], ) @action(methods=["GET"], detail=False, serializer_class=QueryAllTypeClusterSerializer) - def query_all_type_cluster(self, request, *args, **kwargs): + def simple_query_cluster(self, request, *args, **kwargs): data = self.params_validate(self.get_serializer_class()) conditions = self.get_serializer().get_conditions(data) cluster_queryset = Cluster.objects.filter(**conditions) cluster_infos = [cluster.simple_desc for cluster in cluster_queryset] return Response(cluster_infos) + + @common_swagger_auto_schema( + operation_summary=_("查询集群通用信息"), + auto_schema=ResponseSwaggerAutoSchema, + query_serializer=CommonQueryClusterSerializer(), + responses={status.HTTP_200_OK: CommonQueryClusterResponseSerializer()}, + tags=[SWAGGER_TAG], + ) + @action(methods=["GET"], detail=False, serializer_class=CommonQueryClusterSerializer) + def common_query_cluster(self, request, *args, **kwargs): + data = self.params_validate(self.get_serializer_class()) + __, cluster_infos = ListRetrieveResource.common_query_cluster(**data) + return Response(cluster_infos) diff --git a/dbm-ui/backend/flow/engine/bamboo/scene/cloud/base_service_flow.py b/dbm-ui/backend/flow/engine/bamboo/scene/cloud/base_service_flow.py index 12988f2412..f832817987 100644 --- a/dbm-ui/backend/flow/engine/bamboo/scene/cloud/base_service_flow.py +++ b/dbm-ui/backend/flow/engine/bamboo/scene/cloud/base_service_flow.py @@ -12,6 +12,7 @@ from dataclasses import asdict from typing import Any, Dict, List, Optional, Union +from bamboo_engine.builder import SubProcess from django.utils.crypto import get_random_string from django.utils.translation import ugettext as _ @@ -90,6 +91,33 @@ def _get_or_generate_usr_pwd(self, service: str, host_infos: Dict = None, force: def _get_access_hosts(self, host_infos): return [host["ip"] for host in host_infos] + def deploy_batch_service_flow( + self, + sub_pipeline_list: List[SubProcess], + pipeline: Union[Builder, SubBuilder], + name: str, + ratio: int = 1, + ): + """ + 分批串行化部署流程,每一批的子流程并行部署。一般用于灰度重装场景 + @param sub_pipeline_list: 子流程列表 + @param pipeline: 父流程 + @param name: 流程名称 + @param ratio: 分多少批次部署 + """ + step = len(sub_pipeline_list) // ratio + for batch in range(ratio): + st = batch * step + ed = (batch + 1) * step if batch != ratio - 1 else len(sub_pipeline_list) + + batch_sub_list = sub_pipeline_list[st:ed] + sub_pipeline = SubBuilder(self.root_id, data=self.data) + sub_pipeline.add_parallel_sub_pipeline(batch_sub_list) + + pipeline.add_sub_pipeline(sub_pipeline.build_sub_process(sub_name=_("[batch{}]{}").format(batch, name))) + + return pipeline + def deploy_service_flow( self, pipeline: Union[Builder, SubBuilder], diff --git a/dbm-ui/backend/flow/engine/bamboo/scene/cloud/dns_service_flow.py b/dbm-ui/backend/flow/engine/bamboo/scene/cloud/dns_service_flow.py index 89d1e12125..e3847cfddc 100644 --- a/dbm-ui/backend/flow/engine/bamboo/scene/cloud/dns_service_flow.py +++ b/dbm-ui/backend/flow/engine/bamboo/scene/cloud/dns_service_flow.py @@ -35,10 +35,12 @@ def _get_inventory_hosts(self): inventory_hosts.extend(self.data["drs"]["host_infos"]) return inventory_hosts - def build_dns_apply_flow(self, dns_pipeline: Union[Builder, SubBuilder]) -> Union[Builder, SubBuilder]: - sub_dns_pipeline = SubBuilder(self.root_id, data=self.data) + def build_dns_apply_flow( + self, dns_pipeline: Union[Builder, SubBuilder], grayscale: bool = False + ) -> Union[Builder, SubBuilder]: sub_dns_pipeline_list: List[SubProcess] = [] + # 构造dns部署子流程 for host_info in self.data["dns"]["host_infos"]: dns_deploy_pipeline = SubBuilder(self.root_id, data=self.data) dns_deploy_pipeline = self.deploy_dns_service_pipeline(host_info, dns_deploy_pipeline) @@ -46,8 +48,12 @@ def build_dns_apply_flow(self, dns_pipeline: Union[Builder, SubBuilder]) -> Unio dns_deploy_pipeline.build_sub_process(sub_name=_("主机{}部署dns服务").format(host_info["ip"])) ) - sub_dns_pipeline.add_parallel_sub_pipeline(sub_dns_pipeline_list) - dns_pipeline.add_sub_pipeline(sub_dns_pipeline.build_sub_process(sub_name=_("部署dns服务"))) + # 灰度部署的场景在重装会用到,每次按1/2的数量进行重启 + ratio = 2 if grayscale else 1 + dns_pipeline = self.deploy_batch_service_flow( + sub_pipeline_list=sub_dns_pipeline_list, pipeline=dns_pipeline, name=_("部署dns服务"), ratio=ratio + ) + return dns_pipeline def add_dns_flush_act(self, dns_pipeline: Union[Builder, SubBuilder], host_infos: Dict, flush_type: str): @@ -91,7 +97,7 @@ def service_reload_flow(self): """ # 重启/重装等同于重新部署,不影响其他组件 dns_pipeline = Builder(root_id=self.root_id, data=self.data) - dns_pipeline = self.build_dns_apply_flow(dns_pipeline) + dns_pipeline = self.build_dns_apply_flow(dns_pipeline, grayscale=True) dns_pipeline.run_pipeline() def service_add_flow(self): From 0947fd1af478f459cc1890c8f32293589ee1ddbc Mon Sep 17 00:00:00 2001 From: austinqli <1344583166@qq.com> Date: Thu, 11 Jan 2024 19:21:03 +0800 Subject: [PATCH 2/9] =?UTF-8?q?fix(frontend):=20dumper=E8=87=AA=E6=B5=8Bbu?= =?UTF-8?q?g=E4=BF=AE=E5=A4=8D=20#2791=20#=20Reviewed,=20transaction=20id:?= =?UTF-8?q?=201788?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../components/ModuleConfig.vue | 2 +- dbm-ui/frontend/src/services/dbbase.ts | 6 +- .../src/services/model/mysql/tendbha.ts | 4 + .../src/services/model/mysql/tendbsingle.ts | 4 + .../mysql/ha-cluster-list/components/List.vue | 94 +++++++++++-------- .../bigdata/expansion-capacity/Index.vue | 9 +- .../demand-factory/mysql/DumperInstall.vue | 2 +- 7 files changed, 75 insertions(+), 46 deletions(-) diff --git a/dbm-ui/frontend/src/layout/components/database-manage/components/ModuleConfig.vue b/dbm-ui/frontend/src/layout/components/database-manage/components/ModuleConfig.vue index e2f40dc3ae..70994fb692 100644 --- a/dbm-ui/frontend/src/layout/components/database-manage/components/ModuleConfig.vue +++ b/dbm-ui/frontend/src/layout/components/database-manage/components/ModuleConfig.vue @@ -253,7 +253,7 @@ margin-right: 10px; margin-left: 10px; font-size: 14px; - color: #63656E; + color: #939EB5; cursor: pointer; background: #2C323F; border-radius: 4px; diff --git a/dbm-ui/frontend/src/services/dbbase.ts b/dbm-ui/frontend/src/services/dbbase.ts index 56fc084cc3..2fa12ef0c4 100644 --- a/dbm-ui/frontend/src/services/dbbase.ts +++ b/dbm-ui/frontend/src/services/dbbase.ts @@ -1,7 +1,7 @@ import http from './http'; // 查询集群名字是否重复 -export function verifyDuplicatedClusterName(params: { +export function verifyDuplicatedClusterName(params: { cluster_type: string, name: string, bk_biz_id: number @@ -10,7 +10,7 @@ export function verifyDuplicatedClusterName(params: { } // 查询全集群信息 -export function queryAllTypeCluster(params: { +export function queryAllTypeCluster(params: { bk_biz_id: number cluster_types?: string, immute_domain?: string, @@ -24,5 +24,5 @@ export function queryAllTypeCluster(params: { major_version: string, name: string, region: string - }[]>('/apis/dbbase/query_all_type_cluster/', params); + }[]>('/apis/dbbase/simple_query_cluster/', params); } diff --git a/dbm-ui/frontend/src/services/model/mysql/tendbha.ts b/dbm-ui/frontend/src/services/model/mysql/tendbha.ts index a41d41418e..7ad0e975ea 100644 --- a/dbm-ui/frontend/src/services/model/mysql/tendbha.ts +++ b/dbm-ui/frontend/src/services/model/mysql/tendbha.ts @@ -25,6 +25,7 @@ export default class Tendbha { db_module_id: number; id: number; master_domain: string; + major_version: string; masters: { bk_biz_id: number, bk_cloud_id: number, @@ -51,6 +52,7 @@ export default class Tendbha { phase: string; phase_name: string; proxies: Tendbha['masters']; + region: string; slave_domain: string; slaves: Tendbha['masters']; status: string; @@ -69,12 +71,14 @@ export default class Tendbha { this.db_module_id = payload.db_module_id || 0; this.id = payload.id || 0; this.master_domain = payload.master_domain || ''; + this.major_version = payload.major_version || ''; this.masters = payload.masters || []; this.operations = payload.operations || []; this.permission = payload.permission || {}; this.phase = payload.phase || ''; this.phase_name = payload.phase_name || ''; this.proxies = payload.proxies || []; + this.region = payload.region || ''; this.slave_domain = payload.slave_domain || ''; this.slaves = payload.slaves || []; this.status = payload.status || ''; diff --git a/dbm-ui/frontend/src/services/model/mysql/tendbsingle.ts b/dbm-ui/frontend/src/services/model/mysql/tendbsingle.ts index 793788c7a3..3ce796b70f 100644 --- a/dbm-ui/frontend/src/services/model/mysql/tendbsingle.ts +++ b/dbm-ui/frontend/src/services/model/mysql/tendbsingle.ts @@ -24,6 +24,7 @@ export default class Tendbsingle { db_module_name: string; id: number; master_domain: string; + major_version: string; masters: { bk_biz_id: number, bk_cloud_id: number, @@ -50,6 +51,7 @@ export default class Tendbsingle { phase: string; phase_name: string; proxies: Tendbsingle['masters']; + region: string; slave_domain: string; slaves: Tendbsingle['masters']; status: string; @@ -67,12 +69,14 @@ export default class Tendbsingle { this.db_module_name = payload.db_module_name || ''; this.id = payload.id || 0; this.master_domain = payload.master_domain || ''; + this.major_version = payload.major_version || ''; this.masters = payload.masters || []; this.operations = payload.operations || []; this.permission = payload.permission || {}; this.phase = payload.phase || ''; this.phase_name = payload.phase_name || ''; this.proxies = payload.proxies || []; + this.region = payload.region || ''; this.slave_domain = payload.slave_domain || ''; this.slaves = payload.slaves || []; this.status = payload.status || ''; diff --git a/dbm-ui/frontend/src/views/mysql/ha-cluster-list/components/List.vue b/dbm-ui/frontend/src/views/mysql/ha-cluster-list/components/List.vue index 67defb580a..40811e466e 100644 --- a/dbm-ui/frontend/src/views/mysql/ha-cluster-list/components/List.vue +++ b/dbm-ui/frontend/src/views/mysql/ha-cluster-list/components/List.vue @@ -21,6 +21,7 @@ {{ t('实例申请') }} handleShowCreateSubscribeRuleSlider(data)}> - { t('数据订阅') } - + {isShowDumperEntry.value && ( + handleShowCreateSubscribeRuleSlider(data)}> + { t('数据订阅') } + + )} { data.isOnline ? ( item.field).map(item => ({ @@ -537,14 +551,14 @@ const getMenuList = async (item: SearchSelectItem | undefined, keyword: string) => { if (item?.id !== 'creator' && keyword) { - return getMenuListSearch(item, keyword, searchSelectData.value, state.filters); + return getMenuListSearch(item, keyword, searchSelectData, state.filters); } // 没有选中过滤标签 if (!item) { // 过滤掉已经选过的标签 const selected = (state.filters || []).map(value => value.id); - return searchSelectData.value.filter(item => !selected.includes(item.id)); + return searchSelectData.filter(item => !selected.includes(item.id)); } // 远程加载执行人 @@ -561,7 +575,7 @@ } // 不需要远层加载 - return searchSelectData.value.find(set => set.id === item.id)?.children || []; + return searchSelectData.find(set => set.id === item.id)?.children || []; }; const fetchData = (loading?:boolean) => { @@ -616,7 +630,7 @@ * 获取模块列表 */ const fetchModules = () => getModules({ - bk_biz_id: globalBizsStore.currentBizId, + bk_biz_id: currentBizId, cluster_type: ClusterTypes.TENDBHA, }).then((res) => { state.dbModuleList = res.map(item => ({ @@ -666,7 +680,7 @@ onConfirm: async () => { try { const params = { - bk_biz_id: globalBizsStore.currentBizId, + bk_biz_id: currentBizId, ticket_type: type, details: { cluster_ids: [data.id], @@ -705,7 +719,7 @@ onConfirm: async () => { try { const params = { - bk_biz_id: globalBizsStore.currentBizId, + bk_biz_id: currentBizId, ticket_type: TicketTypes.MYSQL_HA_DESTROY, details: { cluster_ids: [data.id], @@ -730,7 +744,7 @@ router.push({ name: 'SelfServiceApplyHa', query: { - bizId: globalBizsStore.currentBizId, + bizId: currentBizId, from: route.name as string, }, }); diff --git a/dbm-ui/frontend/src/views/tickets/common/components/demand-factory/bigdata/expansion-capacity/Index.vue b/dbm-ui/frontend/src/views/tickets/common/components/demand-factory/bigdata/expansion-capacity/Index.vue index eb90f5f298..9a866e1361 100644 --- a/dbm-ui/frontend/src/views/tickets/common/components/demand-factory/bigdata/expansion-capacity/Index.vue +++ b/dbm-ui/frontend/src/views/tickets/common/components/demand-factory/bigdata/expansion-capacity/Index.vue @@ -58,7 +58,9 @@
{{ t('已选IP') }}:
- +
+ +
@@ -164,4 +166,9 @@ diff --git a/dbm-ui/frontend/src/views/tickets/common/components/demand-factory/mysql/DumperInstall.vue b/dbm-ui/frontend/src/views/tickets/common/components/demand-factory/mysql/DumperInstall.vue index aee9549a7f..a270dca1dd 100644 --- a/dbm-ui/frontend/src/views/tickets/common/components/demand-factory/mysql/DumperInstall.vue +++ b/dbm-ui/frontend/src/views/tickets/common/components/demand-factory/mysql/DumperInstall.vue @@ -26,7 +26,7 @@
{{ t('数据源与接收端') }}: From d012c08e421185b04fa94778e92235687bb64772 Mon Sep 17 00:00:00 2001 From: hlinx <327159425@qq.com> Date: Thu, 11 Jan 2024 14:53:06 +0800 Subject: [PATCH 3/9] =?UTF-8?q?feat(frontend):=20=E6=96=B0=E5=A2=9E?= =?UTF-8?q?=E5=81=A5=E5=BA=B7=E6=8A=A5=E5=91=8A=E6=95=B0=E6=8D=AE=20#3001?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dbm-ui/frontend/src/services/report.ts | 27 ++++++- .../views/inspection-manage/report/Index.vue | 5 ++ .../components/backup-inspection/Index.vue | 8 ++ .../components/data-validation/Index.vue | 4 +- .../components/InstacenDetail.vue | 76 ------------------- .../components/metadata-check/Index.vue | 13 +++- .../report/components/more/Index.vue | 18 +++++ 7 files changed, 71 insertions(+), 80 deletions(-) delete mode 100644 dbm-ui/frontend/src/views/inspection-manage/report/components/data-validation/components/InstacenDetail.vue create mode 100644 dbm-ui/frontend/src/views/inspection-manage/report/components/more/Index.vue diff --git a/dbm-ui/frontend/src/services/report.ts b/dbm-ui/frontend/src/services/report.ts index f1063d55f5..44d334bbaa 100644 --- a/dbm-ui/frontend/src/services/report.ts +++ b/dbm-ui/frontend/src/services/report.ts @@ -10,6 +10,7 @@ interface IResult { format: 'text'|'status'|'fail_slave_instance' }[] } + // 数据校验 export const getChecksumReport = function (params: Record) { return http.get('/db_report/checksum_check/report', params); @@ -27,7 +28,6 @@ export const getChecksumInstance = function (params: Record) { }[]>>('/db_report/checksum_check/instance', params); }; - // 元数据检查报告列表 export const getMetaCheckInsganceBelong = function (params: Record) { return http.get('/db_report/meta_check/instance_belong', params); @@ -42,3 +42,28 @@ export const getmysqlCheckBinlogBackup = function (params: Record) export const getmysqlCheckFullBackup = function (params: Record) { return http.get('/db_report/mysql_check/full_backup', params); }; + +// dbmon心跳超时检查报告 +export const getDbmonHeartbeat = function (params: Record) { + return http.get('/db_report/dbmon/heartbeat', params); +}; + +// redis binlog检查报告 +export const getRedisCheckBinlogBackup = function (params: Record) { + return http.get('/db_report/redis_check/binlog_backup', params); +}; + +// redis 全备检查报告 +export const getRedisCheckFullBackup = function (params: Record) { + return http.get('/db_report/redis_check/full_backup', params); +}; + +// redis 孤立节点检查报告 +export const getRedisMetaCheckAloneInstance = function (params: Record) { + return http.get('/db_report/redis_meta_check/alone_instance', params); +}; + +// 实例状态异常检查 +export const getRedisMetaCheckStatusAbnormal = function (params: Record) { + return http.get('/db_report/redis_meta_check/status_abnormal', params); +}; diff --git a/dbm-ui/frontend/src/views/inspection-manage/report/Index.vue b/dbm-ui/frontend/src/views/inspection-manage/report/Index.vue index ae4f82e3b3..f2c66ac4c5 100644 --- a/dbm-ui/frontend/src/views/inspection-manage/report/Index.vue +++ b/dbm-ui/frontend/src/views/inspection-manage/report/Index.vue @@ -13,6 +13,9 @@ +
; return comMap[tabType.value]; diff --git a/dbm-ui/frontend/src/views/inspection-manage/report/components/backup-inspection/Index.vue b/dbm-ui/frontend/src/views/inspection-manage/report/components/backup-inspection/Index.vue index 2adf97f9ae..5e35d61f31 100644 --- a/dbm-ui/frontend/src/views/inspection-manage/report/components/backup-inspection/Index.vue +++ b/dbm-ui/frontend/src/views/inspection-manage/report/components/backup-inspection/Index.vue @@ -5,11 +5,19 @@ + + - diff --git a/dbm-ui/frontend/src/views/inspection-manage/report/components/metadata-check/Index.vue b/dbm-ui/frontend/src/views/inspection-manage/report/components/metadata-check/Index.vue index c3d967a908..d1852fa2e9 100644 --- a/dbm-ui/frontend/src/views/inspection-manage/report/components/metadata-check/Index.vue +++ b/dbm-ui/frontend/src/views/inspection-manage/report/components/metadata-check/Index.vue @@ -2,10 +2,19 @@ + + From c0c7d348c3f2cea4a020b5ca9890b205c13fd12b Mon Sep 17 00:00:00 2001 From: ymakedaq <996156275@qq.com> Date: Thu, 11 Jan 2024 16:36:59 +0800 Subject: [PATCH 4/9] =?UTF-8?q?fix(backend):=20=E5=85=BC=E5=AE=B9=E8=BF=81?= =?UTF-8?q?=E7=A7=BB=E6=97=A7spider=E9=9B=86=E7=BE=A4=E8=87=AA=E5=A2=9E?= =?UTF-8?q?=E6=AD=A5=E9=95=BF=E4=B8=BA17=E6=89=A9=E5=AE=B9spider=E5=8D=95?= =?UTF-8?q?=E6=8D=AE=20close=20#2991?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../db-resource/internal/controller/manage/list.go | 5 ++++- dbm-ui/backend/flow/utils/spider/get_spider_incr.py | 10 ++++++++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/dbm-services/common/db-resource/internal/controller/manage/list.go b/dbm-services/common/db-resource/internal/controller/manage/list.go index 3f907c151e..d183ec707c 100644 --- a/dbm-services/common/db-resource/internal/controller/manage/list.go +++ b/dbm-services/common/db-resource/internal/controller/manage/list.go @@ -43,6 +43,7 @@ type MachineResourceGetterInputParam struct { Mem apply.MeasureRange `json:"mem"` Disk apply.MeasureRange `json:"disk"` DiskType string `json:"disk_type"` + OsType string `json:"os_type"` StorageSpecs []apply.DiskSpec `json:"storage_spec"` // true,false,"" GseAgentAlive string `json:"gse_agent_alive"` @@ -198,7 +199,9 @@ func (c *MachineResourceGetterInputParam) queryBs(db *gorm.DB) (err error) { db.Where("(?)", model.JSONQuery("dedicated_bizs").JointOrContains(cmutil.IntSliceToStrSlice(c.ForBizs))) } } - + if cmutil.IsNotEmpty(c.OsType) { + db.Where("os_type = ?", c.OsType) + } db.Order("create_time desc") return nil } diff --git a/dbm-ui/backend/flow/utils/spider/get_spider_incr.py b/dbm-ui/backend/flow/utils/spider/get_spider_incr.py index 1ab7ddba49..4dea960291 100644 --- a/dbm-ui/backend/flow/utils/spider/get_spider_incr.py +++ b/dbm-ui/backend/flow/utils/spider/get_spider_incr.py @@ -14,7 +14,6 @@ from backend.components import DRSApi from backend.db_meta.models import Cluster -from backend.flow.consts import MAX_SPIDER_MASTER_COUNT from backend.flow.engine.bamboo.scene.spider.common.exceptions import FailedToAssignIncrException logger = logging.getLogger("root") @@ -49,10 +48,17 @@ def get_spider_master_incr(cluster: Cluster, add_spiders: list) -> list: # 生成对比list tmp_list = [int(info["SPIDER_AUTO_INCREMENT_MODE_VALUE"]) for info in res[0]["cmd_results"][1]["table_data"]] + increment_step_list = list( + set([int(info["SPIDER_AUTO_INCREMENT_STEP"]) for info in res[0]["cmd_results"][1]["table_data"]]) + ) + if len(increment_step_list) > 1: + raise FailedToAssignIncrException(message=_("there are several different self incrementing steps")) + max_spider_master_count = increment_step_list[0] + logger.info("get the spider_auto_increment val: {}".format(max_spider_master_count)) # incr_number 从1开始寻找,如果已使用则跳过,直至到未使用则赋值给对应的待加入的spider-master节点,且跳出 start = 0 for spider in new_add_spiders: - for incr_number in range(start + 1, MAX_SPIDER_MASTER_COUNT + 1): + for incr_number in range(start + 1, max_spider_master_count + 1): if incr_number not in tmp_list: spider["incr_number"] = incr_number break From 94efec0605d206b47c4c8ea45c46cd314d5d7655 Mon Sep 17 00:00:00 2001 From: iSecloud <869820505@qq.com> Date: Thu, 11 Jan 2024 19:58:09 +0800 Subject: [PATCH 5/9] =?UTF-8?q?fix(backend):=20=E8=B5=84=E6=BA=90=E6=B1=A0?= =?UTF-8?q?=E8=BF=87=E6=BB=A4=E5=A2=9E=E5=8A=A0=E6=93=8D=E4=BD=9C=E7=B3=BB?= =?UTF-8?q?=E7=BB=9F=E7=B1=BB=E5=9E=8B=20#3016?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dbm-ui/backend/db_services/dbresource/serializers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/dbm-ui/backend/db_services/dbresource/serializers.py b/dbm-ui/backend/db_services/dbresource/serializers.py index 18146365f2..5b92fdfbb8 100644 --- a/dbm-ui/backend/db_services/dbresource/serializers.py +++ b/dbm-ui/backend/db_services/dbresource/serializers.py @@ -80,6 +80,7 @@ class ResourceLimitSerializer(serializers.Serializer): city = serializers.CharField(help_text=_("城市"), required=False) subzones = serializers.CharField(help_text=_("园区"), required=False) + os_type = serializers.CharField(help_text=_("操作系统类型"), required=False) cpu = serializers.CharField(help_text=_("cpu资源限制"), required=False) mem = serializers.CharField(help_text=_("内存资源限制"), required=False) disk = serializers.CharField(help_text=_("磁盘资源限制"), required=False) From 6839eb3472f1b5bc375984baff30f89ebe0eb376 Mon Sep 17 00:00:00 2001 From: austinqli <1344583166@qq.com> Date: Thu, 11 Jan 2024 21:18:26 +0800 Subject: [PATCH 6/9] =?UTF-8?q?feat(frontend):=20=E8=B5=84=E6=BA=90?= =?UTF-8?q?=E6=B1=A0=E7=AE=A1=E7=90=86=E9=A1=B5=E9=9D=A2=E5=A2=9E=E5=8A=A0?= =?UTF-8?q?=E6=93=8D=E4=BD=9C=E7=B3=BB=E7=BB=9F=E7=B1=BB=E5=9E=8B=20#3023?= =?UTF-8?q?=20#=20Reviewed,=20transaction=20id:=201794?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dbm-ui/frontend/src/locales/zh-cn.json | 2 + .../services/model/db-resource/DbResource.ts | 8 ++- .../src/services/source/dbresourceResource.ts | 12 +++- .../src/views/resource-manage/pool/Index.vue | 5 ++ .../components/com-factory/Index.vue | 2 + .../com-factory/components/OSType.vue | 70 +++++++++++++++++++ .../search-box/components/field-config.ts | 4 ++ .../components/field-input/Index.vue | 6 +- .../pool/hooks/useTableSetting.ts | 5 ++ 9 files changed, 111 insertions(+), 3 deletions(-) create mode 100644 dbm-ui/frontend/src/views/resource-manage/pool/components/search-box/components/com-factory/components/OSType.vue diff --git a/dbm-ui/frontend/src/locales/zh-cn.json b/dbm-ui/frontend/src/locales/zh-cn.json index de7ba4969d..1e6437e8bd 100644 --- a/dbm-ui/frontend/src/locales/zh-cn.json +++ b/dbm-ui/frontend/src/locales/zh-cn.json @@ -2553,5 +2553,7 @@ "规则下存在实例_不可删除": "规则下存在实例,不可删除", "确认删除该规则?": "确认删除该规则?", "IP_PORT_或_域名_端口": "IP:PORT 或 域名:端口", + "请选择操作系统类型": "请选择操作系统类型", + "操作系统类型": "操作系统类型", "这行勿动!新增翻译请在上一行添加!": "" } \ No newline at end of file diff --git a/dbm-ui/frontend/src/services/model/db-resource/DbResource.ts b/dbm-ui/frontend/src/services/model/db-resource/DbResource.ts index 7583f313eb..21ed820d7c 100644 --- a/dbm-ui/frontend/src/services/model/db-resource/DbResource.ts +++ b/dbm-ui/frontend/src/services/model/db-resource/DbResource.ts @@ -33,6 +33,9 @@ export default class DbResource { ip: string; label: string; net_device_id: string; + os_bit: string; + os_type: string; + os_version: string; rack_id: string; raid: string; resource_types: string[]; @@ -67,9 +70,12 @@ export default class DbResource { this.create_time = payload.create_time; this.device_class = payload.device_class; this.for_bizs = payload.for_bizs || []; - this.ip = payload.ip; + this.ip = payload.ip; this.label = payload.label; this.net_device_id = payload.net_device_id; + this.os_bit = payload.os_bit; + this.os_type = payload.os_type; + this.os_version = payload.os_version; this.rack_id = payload.rack_id; this.raid = payload.raid; this.resource_types = payload.resource_types || []; diff --git a/dbm-ui/frontend/src/services/source/dbresourceResource.ts b/dbm-ui/frontend/src/services/source/dbresourceResource.ts index 7cc4bcf3a2..2fd9696431 100644 --- a/dbm-ui/frontend/src/services/source/dbresourceResource.ts +++ b/dbm-ui/frontend/src/services/source/dbresourceResource.ts @@ -171,7 +171,17 @@ export function updateResource(params: { resource_types: string[], set_empty_biz: boolean, set_empty_resource_type: boolean, - storage_device: Record + storage_device: Record }) { return http.post(`${path}/update/`, params); } + +/** + * 获取操作系统类型 + */ +export function getOsTypeList(params: { + offset?: number, + limit?: number, +}) { + return http.get(`${path}/get_os_types/`, params); +} diff --git a/dbm-ui/frontend/src/views/resource-manage/pool/Index.vue b/dbm-ui/frontend/src/views/resource-manage/pool/Index.vue index ddcb787560..2bca031063 100644 --- a/dbm-ui/frontend/src/views/resource-manage/pool/Index.vue +++ b/dbm-ui/frontend/src/views/resource-manage/pool/Index.vue @@ -172,6 +172,11 @@ field: 'device_class', render: ({ data }: {data: DbResourceModel}) => data.device_class || '--', }, + { + label: t('操作系统类型'), + field: 'os_type', + render: ({ data }: {data: DbResourceModel}) => data.os_type || '--', + }, { label: t('地域'), field: 'city', diff --git a/dbm-ui/frontend/src/views/resource-manage/pool/components/search-box/components/com-factory/Index.vue b/dbm-ui/frontend/src/views/resource-manage/pool/components/search-box/components/com-factory/Index.vue index 1a469f5dad..d5c51b4853 100644 --- a/dbm-ui/frontend/src/views/resource-manage/pool/components/search-box/components/com-factory/Index.vue +++ b/dbm-ui/frontend/src/views/resource-manage/pool/components/search-box/components/com-factory/Index.vue @@ -57,6 +57,7 @@ import Hosts from './components/Hosts.vue'; import Mem from './components/Mem.vue'; import MountPoint from './components/MountPoint.vue'; + import OSType from './components/OSType.vue'; import ResourceTypes from './components/ResourceTypes.vue'; import SpecId from './components/SpecId.vue'; import Subzones from './components/Subzones.vue'; @@ -89,6 +90,7 @@ agent_status: AgentStatus, city: City, device_class: DeviceClass, + os_type: OSType, mount_point: MountPoint, cpu: Cpu, mem: Mem, diff --git a/dbm-ui/frontend/src/views/resource-manage/pool/components/search-box/components/com-factory/components/OSType.vue b/dbm-ui/frontend/src/views/resource-manage/pool/components/search-box/components/com-factory/components/OSType.vue new file mode 100644 index 0000000000..65658449d0 --- /dev/null +++ b/dbm-ui/frontend/src/views/resource-manage/pool/components/search-box/components/com-factory/components/OSType.vue @@ -0,0 +1,70 @@ + + + + + diff --git a/dbm-ui/frontend/src/views/resource-manage/pool/components/search-box/components/field-config.ts b/dbm-ui/frontend/src/views/resource-manage/pool/components/search-box/components/field-config.ts index 3911f27ca3..140880670a 100644 --- a/dbm-ui/frontend/src/views/resource-manage/pool/components/search-box/components/field-config.ts +++ b/dbm-ui/frontend/src/views/resource-manage/pool/components/search-box/components/field-config.ts @@ -105,6 +105,10 @@ export default { label: t('机型'), component: 'device_class', }, + os_type: { + label: t('操作系统类型'), + component: 'os_type', + }, mount_point: { label: t('磁盘挂载点'), component: 'mount_point', diff --git a/dbm-ui/frontend/src/views/resource-manage/pool/components/search-box/components/field-input/Index.vue b/dbm-ui/frontend/src/views/resource-manage/pool/components/search-box/components/field-input/Index.vue index 77866972ce..ab1a1e603c 100644 --- a/dbm-ui/frontend/src/views/resource-manage/pool/components/search-box/components/field-input/Index.vue +++ b/dbm-ui/frontend/src/views/resource-manage/pool/components/search-box/components/field-input/Index.vue @@ -87,7 +87,11 @@ :model="localValueMemo" name="subzones" @change="handleChange" /> -
+
diff --git a/dbm-ui/frontend/src/views/resource-manage/pool/hooks/useTableSetting.ts b/dbm-ui/frontend/src/views/resource-manage/pool/hooks/useTableSetting.ts index b39baa329e..ecab9df8f3 100644 --- a/dbm-ui/frontend/src/views/resource-manage/pool/hooks/useTableSetting.ts +++ b/dbm-ui/frontend/src/views/resource-manage/pool/hooks/useTableSetting.ts @@ -50,6 +50,10 @@ export default function () { label: t('机型'), field: 'device_class', }, + { + label: t('操作系统类型'), + field: 'os_type', + }, { label: t('地域'), field: 'city', @@ -83,6 +87,7 @@ export default function () { 'bk_cpu', 'bk_mem', 'bk_disk', + 'os_type', ], size: cache.size || 'small', }); From 8c0aeb002b49c0934d9fe9c7fc4cd624b3c07f13 Mon Sep 17 00:00:00 2001 From: iSecloud <869820505@qq.com> Date: Thu, 11 Jan 2024 20:33:38 +0800 Subject: [PATCH 7/9] =?UTF-8?q?fix(backend):=20=E8=A1=A5=E5=85=85=E8=B5=84?= =?UTF-8?q?=E6=BA=90=E6=B1=A0=E6=9F=A5=E8=AF=A2=E6=93=8D=E4=BD=9C=E7=B3=BB?= =?UTF-8?q?=E7=BB=9F=E7=B1=BB=E5=9E=8B=E6=8E=A5=E5=8F=A3=20#3019?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../backend/db_services/dbresource/views/resource.py | 10 +++++++++- dbm-ui/backend/db_services/ipchooser/constants.py | 12 ++++++++++++ dbm-ui/backend/ticket/builders/es/es_scale_up.py | 4 ++-- 3 files changed, 23 insertions(+), 3 deletions(-) diff --git a/dbm-ui/backend/db_services/dbresource/views/resource.py b/dbm-ui/backend/db_services/dbresource/views/resource.py index 03779baac6..b1587a60bd 100644 --- a/dbm-ui/backend/db_services/dbresource/views/resource.py +++ b/dbm-ui/backend/db_services/dbresource/views/resource.py @@ -53,7 +53,7 @@ SpecCountResourceResponseSerializer, SpecCountResourceSerializer, ) -from backend.db_services.ipchooser.constants import ModeType +from backend.db_services.ipchooser.constants import BkOsType, ModeType from backend.db_services.ipchooser.handlers.host_handler import HostHandler from backend.db_services.ipchooser.handlers.topo_handler import TopoHandler from backend.db_services.ipchooser.query.resource import ResourceQueryHelper @@ -261,6 +261,14 @@ def get_mountpoints(self, request): def get_disktypes(self, request): return Response(DBResourceApi.get_disktypes()) + @common_swagger_auto_schema( + operation_summary=_("获取操作系统类型"), + tags=[SWAGGER_TAG], + ) + @action(detail=False, methods=["GET"]) + def get_os_types(self, request): + return Response(BkOsType.get_values()) + @common_swagger_auto_schema( operation_summary=_("根据逻辑城市查询园区"), query_serializer=ListSubzonesSerializer(), diff --git a/dbm-ui/backend/db_services/ipchooser/constants.py b/dbm-ui/backend/db_services/ipchooser/constants.py index ea8ab93686..a49c46fb0f 100644 --- a/dbm-ui/backend/db_services/ipchooser/constants.py +++ b/dbm-ui/backend/db_services/ipchooser/constants.py @@ -16,6 +16,7 @@ from backend.utils import env from backend.utils.basic import choices_to_namedtuple, tuple_choices from backend.utils.enum import EnhanceEnum +from blue_krill.data_types.enum import EnumField, StructuredEnum class CommonEnum(EnhanceEnum): @@ -49,6 +50,17 @@ def _get_member__alias_map(cls) -> Dict[Enum, str]: } +class BkOsType(str, StructuredEnum): + """蓝鲸操作系统类型""" + + LINUX = EnumField("Linux", "Linux") + WINDOWS = EnumField("Windows", "Windows") + AIX = EnumField("Aix", "Aix") + UNIX = EnumField("Unix", "Unix") + SOLARIS = EnumField("Solaris", "Solaris") + FREEBSD = EnumField("FreeBSD", "FreeBSD") + + class ScopeType(EnhanceEnum): """作用域类型""" diff --git a/dbm-ui/backend/ticket/builders/es/es_scale_up.py b/dbm-ui/backend/ticket/builders/es/es_scale_up.py index e7bce4b43f..6aaf99f310 100644 --- a/dbm-ui/backend/ticket/builders/es/es_scale_up.py +++ b/dbm-ui/backend/ticket/builders/es/es_scale_up.py @@ -40,8 +40,8 @@ def validate(self, attrs): for role_nodes in role_nodes_list: node_list.extend(role_nodes) - min_instance_num = min([node["instance_num"] for node in node_list if "instance_num" in node.keys()]) - if min_instance_num <= 0: + instance_num_list = [node["instance_num"] for node in node_list if "instance_num" in node.keys()] + if instance_num_list and min(instance_num_list) <= 0: raise serializers.ValidationError(_("实例数必须为正数,请确保实例的合法性")) return attrs From 99717d4ff9022bf3a2267dc8b985c81d0bb19a09 Mon Sep 17 00:00:00 2001 From: xiepaup Date: Fri, 12 Jan 2024 10:48:41 +0800 Subject: [PATCH 8/9] =?UTF-8?q?feat(redis):=20=E6=8F=90=E4=BA=A4=E4=B8=8B?= =?UTF-8?q?=E6=9E=B6=E5=AE=9E=E4=BE=8B=E5=8D=95=E6=8D=AE=20#3005?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../builders/redis/redis_toolbox_instance_shutdown.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/dbm-ui/backend/ticket/builders/redis/redis_toolbox_instance_shutdown.py b/dbm-ui/backend/ticket/builders/redis/redis_toolbox_instance_shutdown.py index 91588dd777..56f0ff0687 100644 --- a/dbm-ui/backend/ticket/builders/redis/redis_toolbox_instance_shutdown.py +++ b/dbm-ui/backend/ticket/builders/redis/redis_toolbox_instance_shutdown.py @@ -27,8 +27,12 @@ class HostInfoSerializer(serializers.Serializer): cluster_id = serializers.IntegerField(help_text=_("集群ID")) # bk_cloud_id = serializers.IntegerField(help_text=_("云区域ID")) - proxy = serializers.ListField(help_text=_("proxy列表"), child=HostInfoSerializer(), required=False) - redis_slave = serializers.ListField(help_text=_("slave列表"), child=HostInfoSerializer(), required=False) + proxy = serializers.ListField( + help_text=_("proxy列表"), allow_empty=True, child=serializers.IPAddressField(), required=False + ) + redis_slave = serializers.ListField( + help_text=_("slave列表"), allow_empty=True, child=serializers.IPAddressField(), required=False + ) infos = serializers.ListField(help_text=_("批量操作参数列表"), child=InfoSerializer()) From 40dda8f6368ab742f5d921880fcdc52a3ecdea1c Mon Sep 17 00:00:00 2001 From: yyhenryyy Date: Thu, 4 Jan 2024 15:07:06 +0800 Subject: [PATCH 9/9] =?UTF-8?q?feat(mongodb):=20mongodb=E5=AE=89=E8=A3=85?= =?UTF-8?q?=EF=BC=8C=E5=8D=B8=E8=BD=BD=EF=BC=8C=E6=B7=BB=E5=8A=A0=E7=94=A8?= =?UTF-8?q?=E6=88=B7=EF=BC=8C=E5=88=A0=E9=99=A4=E7=94=A8=E6=88=B7=EF=BC=8C?= =?UTF-8?q?=E6=89=A7=E8=A1=8C=E8=84=9A=E6=9C=ACflow=20#2321?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dbm-ui/backend/flow/consts.py | 93 ++ .../bamboo/scene/common/get_file_list.py | 17 + .../engine/bamboo/scene/mongodb/__init__.py | 0 .../engine/bamboo/scene/mongodb/base_flow.py | 48 + .../bamboo/scene/mongodb/mongodb_backup.py | 98 ++ .../bamboo/scene/mongodb/mongodb_deinstall.py | 75 ++ .../scene/mongodb/mongodb_exec_script.py | 55 + .../scene/mongodb/mongodb_fake_install.py | 155 +++ .../bamboo/scene/mongodb/mongodb_install.py | 174 +++ .../bamboo/scene/mongodb/mongodb_remove_ns.py | 99 ++ .../bamboo/scene/mongodb/mongodb_replace.py | 62 + .../bamboo/scene/mongodb/mongodb_user.py | 60 + .../bamboo/scene/mongodb/sub_task/__init__.py | 16 + .../bamboo/scene/mongodb/sub_task/backup.py | 91 ++ .../scene/mongodb/sub_task/deinstall.py | 116 ++ .../scene/mongodb/sub_task/exec_script.py | 54 + .../scene/mongodb/sub_task/mongos_install.py | 66 ++ .../scene/mongodb/sub_task/remove_ns.py | 88 ++ .../mongodb/sub_task/replicaset_install.py | 123 ++ .../scene/mongodb/sub_task/send_media.py | 32 + .../bamboo/scene/mongodb/sub_task/user.py | 60 + .../backend/flow/engine/controller/mongodb.py | 50 + .../collections/mongodb/__init__.py | 0 .../collections/mongodb/add_domain_to_dns.py | 66 ++ .../collections/mongodb/add_password_to_db.py | 75 ++ .../mongodb/add_relationship_to_meta.py | 95 ++ .../mongodb/delete_domain_from_dns.py | 66 ++ .../mongodb/delete_password_from_db.py | 63 + .../mongodb/delete_relationship_from_meta.py | 62 + .../collections/mongodb/exec_actuator_job.py | 186 +++ .../collections/mongodb/exec_actuator_job2.py | 127 ++ .../mongodb/get_manager_user_password.py | 82 ++ .../collections/mongodb/send_media.py | 92 ++ dbm-ui/backend/flow/urls.py | 12 + .../flow/utils/mongodb/calculate_cluster.py | 122 ++ .../flow/utils/mongodb/mongodb_dataclass.py | 1026 +++++++++++++++++ .../flow/utils/mongodb/mongodb_password.py | 100 ++ .../utils/mongodb/mongodb_script_template.py | 118 ++ dbm-ui/backend/flow/views/mongodb_scene.py | 75 ++ 39 files changed, 3999 insertions(+) create mode 100644 dbm-ui/backend/flow/engine/bamboo/scene/mongodb/__init__.py create mode 100644 dbm-ui/backend/flow/engine/bamboo/scene/mongodb/base_flow.py create mode 100644 dbm-ui/backend/flow/engine/bamboo/scene/mongodb/mongodb_backup.py create mode 100644 dbm-ui/backend/flow/engine/bamboo/scene/mongodb/mongodb_deinstall.py create mode 100644 dbm-ui/backend/flow/engine/bamboo/scene/mongodb/mongodb_exec_script.py create mode 100644 dbm-ui/backend/flow/engine/bamboo/scene/mongodb/mongodb_fake_install.py create mode 100644 dbm-ui/backend/flow/engine/bamboo/scene/mongodb/mongodb_install.py create mode 100644 dbm-ui/backend/flow/engine/bamboo/scene/mongodb/mongodb_remove_ns.py create mode 100644 dbm-ui/backend/flow/engine/bamboo/scene/mongodb/mongodb_replace.py create mode 100644 dbm-ui/backend/flow/engine/bamboo/scene/mongodb/mongodb_user.py create mode 100644 dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/__init__.py create mode 100644 dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/backup.py create mode 100644 dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/deinstall.py create mode 100644 dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/exec_script.py create mode 100644 dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/mongos_install.py create mode 100644 dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/remove_ns.py create mode 100644 dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/replicaset_install.py create mode 100644 dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/send_media.py create mode 100644 dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/user.py create mode 100644 dbm-ui/backend/flow/engine/controller/mongodb.py create mode 100644 dbm-ui/backend/flow/plugins/components/collections/mongodb/__init__.py create mode 100644 dbm-ui/backend/flow/plugins/components/collections/mongodb/add_domain_to_dns.py create mode 100644 dbm-ui/backend/flow/plugins/components/collections/mongodb/add_password_to_db.py create mode 100644 dbm-ui/backend/flow/plugins/components/collections/mongodb/add_relationship_to_meta.py create mode 100644 dbm-ui/backend/flow/plugins/components/collections/mongodb/delete_domain_from_dns.py create mode 100644 dbm-ui/backend/flow/plugins/components/collections/mongodb/delete_password_from_db.py create mode 100644 dbm-ui/backend/flow/plugins/components/collections/mongodb/delete_relationship_from_meta.py create mode 100644 dbm-ui/backend/flow/plugins/components/collections/mongodb/exec_actuator_job.py create mode 100644 dbm-ui/backend/flow/plugins/components/collections/mongodb/exec_actuator_job2.py create mode 100644 dbm-ui/backend/flow/plugins/components/collections/mongodb/get_manager_user_password.py create mode 100644 dbm-ui/backend/flow/plugins/components/collections/mongodb/send_media.py create mode 100644 dbm-ui/backend/flow/utils/mongodb/calculate_cluster.py create mode 100644 dbm-ui/backend/flow/utils/mongodb/mongodb_dataclass.py create mode 100644 dbm-ui/backend/flow/utils/mongodb/mongodb_password.py create mode 100644 dbm-ui/backend/flow/utils/mongodb/mongodb_script_template.py create mode 100644 dbm-ui/backend/flow/views/mongodb_scene.py diff --git a/dbm-ui/backend/flow/consts.py b/dbm-ui/backend/flow/consts.py index 34929dbb94..98c7d52689 100644 --- a/dbm-ui/backend/flow/consts.py +++ b/dbm-ui/backend/flow/consts.py @@ -167,6 +167,7 @@ class NameSpaceEnum(str, StructuredEnum): Influxdb = EnumField("influxdb", _("Influxdb")) TenDBCluster = EnumField("tendbcluster", _("tendbcluster")) Riak = EnumField("riak", _("Riak")) + MongoDB = EnumField("mongodb", _("mongodb")) class ConfigTypeEnum(str, StructuredEnum): @@ -184,6 +185,8 @@ class ConfigTypeEnum(str, StructuredEnum): HdfsSite = EnumField("hdfs-site", _("HDFS实例hdfs-site配置")) CoreSite = EnumField("core-site", _("HDFS实例core-site配置")) HdfsInstall = EnumField("install", _("HDFS实例安装配置")) + MongoD = EnumField("mongod", _("mongod配置")) + MongoS = EnumField("mongos", _("mongos配置")) class ConfigFileEnum(str, StructuredEnum): @@ -198,6 +201,8 @@ class ConfigFileEnum(str, StructuredEnum): Base = EnumField("base", _("基本配置")) HotKey = EnumField("hotkey", _("热key配置")) BigKey = EnumField("bigkey", _("大key配置")) + DefaultConf = EnumField("defaultconf", _("默认配置")) + OsConf = EnumField("osconf", _("os配置")) class DbBackupRoleEnum(str, StructuredEnum): @@ -240,6 +245,28 @@ class MediumEnum(str, StructuredEnum): RiakMonitor = EnumField("riak-monitor", _("riak-monitor")) RedisDts = EnumField("redis-dts", _("redis-dts")) TBinlogDumper = EnumField("tbinlogdumper", _("tbinlogdumper实例")) + MongoDB = EnumField("mongodb", _("mongodb")) + MongoD = EnumField("mongod", _("mongod")) + MongoS = EnumField("mongos", _("mongos")) + MongoShardSvr = EnumField("shardsvr", _("shardsvr")) + MongoConfigSvr = EnumField("configsvr", _("configsvr")) + AuthDB = EnumField("admin", _("admin")) + DbaUser = EnumField("dba", _("dba")) + AppDbaUser = EnumField("appdba", _("appdba")) + MonitorUser = EnumField("monitor", _("monitor")) + AppMonitorUser = EnumField("appmonitor", _("appmonitor")) + RootRole = EnumField("root", _("root")) + BackupRole = EnumField("backup", _("backup")) + ClusterMonitorRole = EnumField("clusterMonitor", _("clusterMonitor")) + ReadAnyDatabaseRole = EnumField("readAnyDatabase", _("readAnyDatabase")) + HostManagerRole = EnumField("hostManager", _("hostManager")) + ReadWriteRole = EnumField("readWrite", _("readWrite")) + UserAdminAnyDatabaseRole = EnumField("userAdminAnyDatabase", _("userAdminAnyDatabase")) + DbAdminAnyDatabaseRole = EnumField("dbAdminAnyDatabase", _("dbAdminAnyDatabase")) + ReadWriteAnyDatabaseRole = EnumField("readWriteAnyDatabase", _("readWriteAnyDatabase")) + ClusterAdminRole = EnumField("clusterAdmin", _("clusterAdmin")) + MongoDBInitSet = EnumField("mongodb_init_set", _("mongodb_init_set")) + MongoDBExtraUserCreate = EnumField("mongodb_extra_user_create", _("mongodb_extra_user_create")) class CloudServiceName(str, StructuredEnum): @@ -401,6 +428,20 @@ class RedisActuatorActionEnum(str, StructuredEnum): REUPLOAD_OLD_BACKUP_RECORDS = EnumField("reupload_old_backup_records", _("reupload_old_backup_records")) +class MongoDBActuatorActionEnum(str, StructuredEnum): + OsInit = EnumField("os_mongo_init", _("os_mongo_init")) + mongoDInstall = EnumField("mongod_install", _("mongod_install")) + mongoSInstall = EnumField("mongos_install", _("mongos_install")) + InitReplicaset = EnumField("init_replicaset", _("init_replicaset")) + AddUser = EnumField("add_user", _("add_user")) + DeleteUser = EnumField("delete_user", _("delete_user")) + MongoExecuteScript = EnumField("mongo_execute_script", _("mongo_execute_script")) + Backup = EnumField("mongodb_backup", _("mongodb_backup")) + RemoveNs = EnumField("mongodb_remove_ns", _("mongodb_remove_ns")) + Restore = EnumField("mongodb_restore", _("mongodb_restore")) + PitRestore = EnumField("mongodb_pit_restore", _("mongodb_pit_restore")) + + class EsActuatorActionEnum(str, StructuredEnum): Init = EnumField("init", _("init")) DecompressPkg = EnumField("decompress_pkg", _("decompress_pkg")) @@ -946,6 +987,58 @@ class MySQLPrivComponent(str, StructuredEnum): PULSAR_FAKE_USER = EnumField("pulsar_user", _("pulsar_user")) +class RequestResultCode(int, StructuredEnum): + """ + 请求结果code状态 + """ + + Success = EnumField(0, _("success")) + + +class MongoDBPasswordRule(str, StructuredEnum): + """ + mongodb密码规则 + """ + + RULE = EnumField("password", _("密码规则")) + + +class MongoDBClusterRole(str, StructuredEnum): + """ + mongodb cluster role + """ + + ConfigSvr = EnumField("configsvr", _("configsvr")) + ShardSvr = EnumField("shardsvr", _("shardsvr")) + + +class MongoDBTotalCache(float, StructuredEnum): + """ + cache占机器内存的百分比 + """ + + Cache_Percent = EnumField(0.65, _("cache_percent")) + + +class MongoDBDomainPrefix(str, StructuredEnum): + """ + mongodb domain Prefix + """ + + MONGOS = EnumField("mongos", _("mongos")) + M1 = EnumField("m1", _("m1")) + M2 = EnumField("m2", _("m2")) + M3 = EnumField("m3", _("m3")) + M4 = EnumField("m4", _("m4")) + M5 = EnumField("m5", _("m5")) + M6 = EnumField("m6", _("m6")) + M7 = EnumField("m7", _("m7")) + M8 = EnumField("m8", _("m8")) + M9 = EnumField("m9", _("m9")) + M10 = EnumField("m10", _("m10")) + BACKUP = EnumField("backup", _("backup")) + + class TBinlogDumperProtocolType(str, StructuredEnum): """ 定义tbinlogdumper的对端协议 diff --git a/dbm-ui/backend/flow/engine/bamboo/scene/common/get_file_list.py b/dbm-ui/backend/flow/engine/bamboo/scene/common/get_file_list.py index 9e86c6e2e5..91282ec92b 100644 --- a/dbm-ui/backend/flow/engine/bamboo/scene/common/get_file_list.py +++ b/dbm-ui/backend/flow/engine/bamboo/scene/common/get_file_list.py @@ -527,3 +527,20 @@ def get_tbinlogdumper_package(self): f"{env.BKREPO_PROJECT}/{env.BKREPO_BUCKET}/{tbinlogdumper_pkg.path}", f"{env.BKREPO_PROJECT}/{env.BKREPO_BUCKET}/{self.actuator_pkg.path}", ] + + def mongodb_pkg(self, db_version: str) -> list: + """ + 部署mongodb,需要的pkg包 + """ + + mongodb_pkg = Package.get_latest_package( + version=db_version, pkg_type=MediumEnum.MongoDB, db_type=DBType.MongoDB + ) + # bkdbmon_pkg = Package.get_latest_package( + # version=MediumEnum.Latest, pkg_type=MediumEnum.DbMon, db_type=DBType.MongoDB + # ) + return [ + f"{env.BKREPO_PROJECT}/{env.BKREPO_BUCKET}{self.actuator_pkg.path}", + f"{env.BKREPO_PROJECT}/{env.BKREPO_BUCKET}{mongodb_pkg.path}", + # f"{env.BKREPO_PROJECT}/{env.BKREPO_BUCKET}/{bkdbmon_pkg.path}", + ] diff --git a/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/__init__.py b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/base_flow.py b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/base_flow.py new file mode 100644 index 0000000000..fd2e5af227 --- /dev/null +++ b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/base_flow.py @@ -0,0 +1,48 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. +Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + +from typing import Dict, Optional + +from backend.flow.utils.mongodb.mongodb_dataclass import MongoDBCluster + + +def start(): + pass + """ + fix me + """ + + +class MongoBaseFlow(object): + """MongoRemoveNsFlowflow + 分析 payload,检查输入,生成Flow""" + + def __init__(self, root_id: str, data: Optional[Dict]): + """ + 传入参数 + @param root_id : 任务流程定义的root_id + @param data : 单据传递过来的参数列表,是dict格式 + """ + + self.root_id = root_id + self.payload = data + + # 检查 cluster 和 输入中的bk_biz_id字段是否相同. + @classmethod + def check_cluster(cls, cluster: MongoDBCluster, payload): + if cluster is None: + raise Exception("row.cluster_domain is not exists.") + if str(cluster.bk_biz_id) != payload["bk_biz_id"]: + raise Exception( + "bad bk_biz_id {} vs {} {} {}".format( + cluster.bk_biz_id, payload["bk_biz_id"], type(cluster.bk_biz_id), type(payload["bk_biz_id"]) + ) + ) diff --git a/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/mongodb_backup.py b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/mongodb_backup.py new file mode 100644 index 0000000000..d59445fd3c --- /dev/null +++ b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/mongodb_backup.py @@ -0,0 +1,98 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. +Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +import logging.config +from typing import Dict, Optional + +from django.utils.translation import ugettext as _ + +from backend.configuration.constants import DBType +from backend.flow.engine.bamboo.scene.common.builder import Builder +from backend.flow.engine.bamboo.scene.common.get_file_list import GetFileList +from backend.flow.engine.bamboo.scene.mongodb.base_flow import MongoBaseFlow +from backend.flow.engine.bamboo.scene.mongodb.mongodb_fake_install import FlowActKwargs +from backend.flow.engine.bamboo.scene.mongodb.sub_task.backup import BackupSubTask +from backend.flow.plugins.components.collections.mongodb.send_media import ExecSendMediaOperationComponent +from backend.flow.utils.mongodb.mongodb_dataclass import MongoRepository + +logger = logging.getLogger("flow") + + +class MongoBackupFlow(MongoBaseFlow): + """MongoDB备份flow + 分析 payload,检查输入,生成Flow""" + + def __init__(self, root_id: str, data: Optional[Dict]): + """ + 传入参数 + @param root_id : 任务流程定义的root_id + @param data : 单据传递过来的参数列表,是dict格式 + """ + + self.root_id = root_id + self.payload = data + + def start(self): + """ + mongo_backup install流程 + """ + + # backup_dir 提前创建好的,在部署的时候就创建好了. + backup_dir = FlowActKwargs(self.payload).get_backup_dir() + file_list = GetFileList(db_type=DBType.MongoDB).get_db_actuator_package() + + # 创建流程实例 + pipeline = Builder(root_id=self.root_id, data=self.payload) + + # 解析输入 + # 1. 解析每个集群Id的节点列表 + # 2. 备份一般在某个Secondary且非Backup节点上执行. 但由于无法连接mongod,这里怎么搞? + # 3. 获得密码列表 + # 4. 生成并发子任务. + # 介质下发——job的api可以多个IP并行执行 + + sub_pipelines = [] + # bk_host {ip:"x.x.x.x", bk_cloud_id: "0"} + bk_host_list = [] + + # 域名忽略大小写. + cluster_domain_list = [row["cluster_domain"].lower() for row in self.payload["data_for_backup"]] + clusters = MongoRepository.fetch_many_cluster_dict(immute_domain__in=cluster_domain_list) + + for row in self.payload["data_for_backup"]: + domain_lower = row["cluster_domain"].lower() + cluster = clusters[domain_lower] + self.check_cluster(cluster, self.payload) + sub_pl, sub_bk_host_list = BackupSubTask.process_cluster( + root_id=self.root_id, + ticket_data=self.payload, + sub_ticket_data=row, + cluster=cluster, + backup_dir=backup_dir, + ) + bk_host_list.extend(sub_bk_host_list) + sub_pipelines.append(sub_pl.build_sub_process(_("MongoDB-备份-{}".format(cluster.name)))) + + send_media_kwargs = { + "file_list": file_list, + "ip_list": bk_host_list, + "exec_ips": [item["ip"] for item in bk_host_list], + "file_target_path": backup_dir + "/install", + } + pipeline.add_act( + act_name=_("MongoDB-介质下发"), + act_component_code=ExecSendMediaOperationComponent.code, + kwargs=send_media_kwargs, + ) + + pipeline.add_parallel_sub_pipeline(sub_flow_list=sub_pipelines) + + # 运行流程 + pipeline.run_pipeline() diff --git a/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/mongodb_deinstall.py b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/mongodb_deinstall.py new file mode 100644 index 0000000000..87f3960c83 --- /dev/null +++ b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/mongodb_deinstall.py @@ -0,0 +1,75 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. +Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +import logging.config +from typing import Dict, Optional + +from django.utils.translation import ugettext as _ + +from backend.flow.engine.bamboo.scene.common.builder import Builder +from backend.flow.engine.bamboo.scene.mongodb.sub_task import deinstall +from backend.flow.plugins.components.collections.mongodb.send_media import ExecSendMediaOperationComponent +from backend.flow.utils.mongodb.mongodb_dataclass import ActKwargs + +logger = logging.getLogger("flow") + + +class MongoDBDeInstallFlow(object): + """MongoDB卸载flow""" + + def __init__(self, root_id: str, data: Optional[Dict]): + """ + 传入参数 + @param root_id : 任务流程定义的root_id + @param data : 单据传递过来的参数列表,是dict格式 + """ + + self.root_id = root_id + self.data = data + self.get_kwargs = ActKwargs() + self.get_kwargs.payload = data + self.get_kwargs.get_file_path() + + def prepare_job(self, pipeline: Builder): + """ + 准备工作 + """ + + # 介质下发——job的api可以多个IP并行执行 + kwargs = self.get_kwargs.get_send_media_kwargs() + pipeline.add_act( + act_name=_("MongoDB-介质下发"), act_component_code=ExecSendMediaOperationComponent.code, kwargs=kwargs + ) + + def multi_cluster_deinstall_flow(self): + """ + multi cluster deinstall流程 + """ + + # 创建流程实例 + pipeline = Builder(root_id=self.root_id, data=self.data) + + # 获取所有的cluster主机信息 + self.get_kwargs.get_hosts_deinstall() + + # 下发介质 + self.prepare_job(pipeline=pipeline) + + # 卸载——子流程并行 + sub_pipelines = [] + for cluster_id in self.data["cluster_ids"]: + sub_pipline = deinstall( + root_id=self.root_id, ticket_data=self.data, sub_kwargs=self.get_kwargs, cluster_id=cluster_id + ) + sub_pipelines.append(sub_pipline) + pipeline.add_parallel_sub_pipeline(sub_flow_list=sub_pipelines) + + # 运行流程 + pipeline.run_pipeline() diff --git a/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/mongodb_exec_script.py b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/mongodb_exec_script.py new file mode 100644 index 0000000000..f440b7b460 --- /dev/null +++ b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/mongodb_exec_script.py @@ -0,0 +1,55 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. +Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +import logging.config +from typing import Dict, Optional + +from backend.flow.engine.bamboo.scene.common.builder import Builder +from backend.flow.engine.bamboo.scene.mongodb.sub_task import exec_script +from backend.flow.utils.mongodb.mongodb_dataclass import ActKwargs + +logger = logging.getLogger("flow") + + +class MongoExecScriptFlow(object): + """MongoDB执行脚本flow""" + + def __init__(self, root_id: str, data: Optional[Dict]): + """ + 传入参数 + @param root_id : 任务流程定义的root_id + @param data : 单据传递过来的参数列表,是dict格式 + """ + + self.root_id = root_id + self.data = data + self.get_kwargs = ActKwargs() + self.get_kwargs.payload = data + self.get_kwargs.get_file_path() + + def multi_cluster_exec_script_flow(self): + """ + multi cluster execute script流程 + """ + + # 创建流程实例 + pipeline = Builder(root_id=self.root_id, data=self.data) + + # 执行脚本——并行 + sub_pipelines = [] + for cluster_id in self.data["cluster_ids"]: + sub_pipline = exec_script( + root_id=self.root_id, ticket_data=self.data, sub_kwargs=self.get_kwargs, cluster_id=cluster_id + ) + sub_pipelines.append(sub_pipline) + pipeline.add_parallel_sub_pipeline(sub_flow_list=sub_pipelines) + + # 运行流程 + pipeline.run_pipeline() diff --git a/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/mongodb_fake_install.py b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/mongodb_fake_install.py new file mode 100644 index 0000000000..a65d431cc8 --- /dev/null +++ b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/mongodb_fake_install.py @@ -0,0 +1,155 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. +Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +import logging.config +from dataclasses import dataclass +from typing import Dict, Optional + +from django.utils.translation import ugettext as _ + +from backend.configuration.constants import DBType +from backend.db_meta.enums import ClusterType, InstanceRole +from backend.db_meta.models import Cluster +from backend.flow.engine.bamboo.scene.common.builder import Builder +from backend.flow.engine.bamboo.scene.common.get_file_list import GetFileList +from backend.flow.plugins.components.collections.mongodb.add_relationship_to_meta import ( + ExecAddRelationshipOperationComponent, +) +from backend.flow.utils.mongodb.mongodb_dataclass import ActKwargs + +logger = logging.getLogger("flow") + + +# FlowActKwargs 备份、回档、清档专用. +@dataclass() +class FlowActKwargs(ActKwargs): + def __init__(self, payload: dict): + self.payload = payload + + @staticmethod + def get_file_list_for_backup(cls, db_version) -> dict: + """介质下发的kwargs""" + + file_list = GetFileList(db_type=DBType.MongoDB).mongodb_pkg(db_version=db_version) + return {"file_list": file_list} + + +class MongoFakeInstallFlow(object): + """MongoFakeInstallFlow 用作调试 不在生产环境使用""" + + def __init__(self, root_id: str, data: Optional[Dict]): + """ + 传入参数 + @param root_id : 任务流程定义的root_id + @param data : 单据传递过来的参数列表,是dict格式 + """ + + self.root_id = root_id + self.data = data + self.get_kwargs = FlowActKwargs() + self.get_kwargs.payload = data + self.get_kwargs.get_backup_dir() + self.payload = data + + def start(self): + """ + mongo_backup install流程 + """ + + # 创建流程实例 + pipeline = Builder(root_id=self.root_id, data=self.data) + + # 解析输入 + # 1. 解析每个集群Id的节点列表 + # 2. 备份一般在某个Secondary且非Backup节点上执行. 但由于无法连接mongod,这里怎么搞? + # 3. 获得密码列表 + # 4. 生成并发子任务. + # 介质下发——job的api可以多个IP并行执行 + + # backup_dir = self.get_kwargs.get_backup_dir() + # 复制集关系写入meta + kwargs = self.get_add_info_to_meta_kwargs() + logger.info("get_add_info_to_meta_kwargs return", kwargs) + # 检查是否已存在. + + try: + v = Cluster.objects.get(bk_biz_id=kwargs["bk_biz_id"], name=kwargs["name"]) + except Cluster.DoesNotExist: + logging.getLogger("flow").info("pass") + pass + else: + raise Exception( + "cluster is already exists {}:{}:{}".format(kwargs["bk_biz_id"], kwargs["name"], v.cluster_type) + ) + + pipeline.add_act( + act_name=_("MongoDB--添加关系到meta"), + act_component_code=ExecAddRelationshipOperationComponent.code, + kwargs=kwargs, + ) + + # 运行流程 + pipeline.run_pipeline() + + def get_add_info_to_meta_kwargs(self) -> dict: + """添加关系到meta的kwargs""" + + info = { + "bk_biz_id": int(self.payload["bk_biz_id"]), + "app": self.payload["app"], + "name": "{}-{}-{}".format(self.payload["app"], self.payload["areaId"], self.payload["setId"]), + "alias": self.payload["alias"], + "major_version": self.payload["db_version"], + "creator": self.payload["created_by"], + "bk_cloud_id": 0, + "region": self.payload["city"], + } + + instance_role = [ + InstanceRole.MONGO_M1, + InstanceRole.MONGO_M2, + InstanceRole.MONGO_M3, + InstanceRole.MONGO_M4, + InstanceRole.MONGO_M5, + InstanceRole.MONGO_M6, + InstanceRole.MONGO_M7, + InstanceRole.MONGO_M8, + InstanceRole.MONGO_M9, + InstanceRole.MONGO_M10, + InstanceRole.MONGO_BACKUP, + ] + if self.payload["cluster_type"] == ClusterType.MongoReplicaSet.value: + info["cluster_type"] = ClusterType.MongoReplicaSet.value + info["db_module_id"] = 1 + info["immute_domain"] = self.payload["nodes"][0]["domain"] + info["storages"] = [] + if len(self.payload["nodes"]) <= 11: + for index, node in enumerate(self.payload["nodes"]): + if index == len(self.payload["nodes"]) - 1: + info["storages"].append( + { + "role": InstanceRole.MONGO_BACKUP, + "ip": node["ip"], + "port": node["port"], + "domain": node["domain"], + } + ) + else: + info["storages"].append( + { + "role": instance_role[index], + "ip": node["ip"], + "port": node["port"], + "domain": node["domain"], + } + ) + else: + raise Exception("bad cluster_type {}".format(self.payload["cluster_type"])) + return info diff --git a/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/mongodb_install.py b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/mongodb_install.py new file mode 100644 index 0000000000..27f8c094ba --- /dev/null +++ b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/mongodb_install.py @@ -0,0 +1,174 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. +Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +import logging.config +from typing import Dict, Optional + +from django.utils.translation import ugettext as _ + +from backend.flow.consts import MongoDBClusterRole +from backend.flow.engine.bamboo.scene.common.builder import Builder +from backend.flow.engine.bamboo.scene.mongodb.sub_task import mongos_install, replicaset_install +from backend.flow.plugins.components.collections.mongodb.add_domain_to_dns import ExecAddDomainToDnsOperationComponent +from backend.flow.plugins.components.collections.mongodb.add_relationship_to_meta import ( + ExecAddRelationshipOperationComponent, +) +from backend.flow.plugins.components.collections.mongodb.exec_actuator_job import ExecuteDBActuatorJobComponent +from backend.flow.plugins.components.collections.mongodb.get_manager_user_password import ( + ExecGetPasswordOperationComponent, +) +from backend.flow.plugins.components.collections.mongodb.send_media import ExecSendMediaOperationComponent +from backend.flow.utils.mongodb.calculate_cluster import calculate_cluster +from backend.flow.utils.mongodb.mongodb_dataclass import ActKwargs + +logger = logging.getLogger("flow") + + +class MongoDBInstallFlow(object): + """MongoDB安装flow""" + + def __init__(self, root_id: str, data: Optional[Dict]): + """ + 传入参数 + @param root_id : 任务流程定义的root_id + @param data : 单据传递过来的参数列表,是dict格式 + """ + + self.root_id = root_id + # 计算cluster分布 + payload_clusters = calculate_cluster(data) + self.data = payload_clusters + self.get_kwargs = ActKwargs() + self.get_kwargs.payload = payload_clusters + self.get_kwargs.get_inti_info() + self.get_kwargs.get_file_path() + + def prepare_job(self, pipeline: Builder): + """ + 准备工作 + """ + + # 介质下发——job的api可以多个IP并行执行 + kwargs = self.get_kwargs.get_send_media_kwargs() + pipeline.add_act( + act_name=_("MongoDB-介质下发"), act_component_code=ExecSendMediaOperationComponent.code, kwargs=kwargs + ) + + # 机器初始化——job的api可以多个IP并行执行 + kwargs = self.get_kwargs.get_os_init_kwargs() + pipeline.add_act( + act_name=_("MongoDB-机器初始化"), act_component_code=ExecuteDBActuatorJobComponent.code, kwargs=kwargs + ) + + def multi_replicaset_install_flow(self): + """ + multi replicaset install流程 + """ + + # 创建流程实例 + pipeline = Builder(root_id=self.root_id, data=self.data) + + # 下发介质和os初始化 + self.prepare_job(pipeline=pipeline) + + # 复制集安装——子流程并行 + sub_pipelines = [] + for replicaset_info in self.data["sets"]: + self.get_kwargs.replicaset_info = replicaset_info + sub_pipline = replicaset_install( + root_id=self.root_id, + ticket_data=self.data, + sub_kwargs=self.get_kwargs, + cluster=False, + cluster_role="", + config_svr=False, + ) + sub_pipelines.append(sub_pipline) + pipeline.add_parallel_sub_pipeline(sub_flow_list=sub_pipelines) + + # 复制集关系写入meta + for replicaset_info in self.data["sets"]: + kwargs = self.get_kwargs.get_add_relationship_to_meta_kwargs(replicaset_info=replicaset_info) + pipeline.add_act( + act_name=_("MongoDB--添加复制集{}-{}关系到meta".format(self.data["app"], replicaset_info["set_id"])), + act_component_code=ExecAddRelationshipOperationComponent.code, + kwargs=kwargs, + ) + + # 运行流程 + pipeline.run_pipeline() + + def cluster_install_flow(self): + """ + cluster install流程 + """ + + # 创建流程实例 + pipeline = Builder(root_id=self.root_id, data=self.data) + + # 下发介质和os初始化 + self.prepare_job(pipeline=pipeline) + + # 密码服务获取管理用户密码 shard,config的密码保持一致 + kwargs = self.get_kwargs.get_get_manager_password_kwargs() + pipeline.add_act( + act_name=_("MongoDB--获取管理员用户密码"), act_component_code=ExecGetPasswordOperationComponent.code, kwargs=kwargs + ) + + # cluster安装 + # config和shard安装——子流程并行 + sub_pipelines = [] + # 安装shard + for replicaset_info in self.data["shards"]: + self.get_kwargs.replicaset_info = replicaset_info + sub_pipline = replicaset_install( + root_id=self.root_id, + ticket_data=self.data, + sub_kwargs=self.get_kwargs, + cluster=True, + cluster_role=MongoDBClusterRole.ShardSvr, + config_svr=False, + ) + sub_pipelines.append(sub_pipline) + # 安装config + self.get_kwargs.replicaset_info = self.data["config"] + sub_pipline = replicaset_install( + root_id=self.root_id, + ticket_data=self.data, + sub_kwargs=self.get_kwargs, + cluster=True, + cluster_role=MongoDBClusterRole.ConfigSvr, + config_svr=True, + ) + sub_pipelines.append(sub_pipline) + pipeline.add_parallel_sub_pipeline(sub_flow_list=sub_pipelines) + + # mongos安装——子流程并行 + self.get_kwargs.mongos_info = self.data["mongos"] + sub_pipline = mongos_install(root_id=self.root_id, ticket_data=self.data, sub_kwargs=self.get_kwargs) + pipeline.add_sub_pipeline(sub_flow=sub_pipline) + + # cluster关系写入meta + kwargs = self.get_kwargs.get_add_relationship_to_meta_kwargs(replicaset_info={}) + pipeline.add_act( + act_name=_("MongoDB--添加关系到meta"), + act_component_code=ExecAddRelationshipOperationComponent.code, + kwargs=kwargs, + ) + # 域名写入dns + kwargs = self.get_kwargs.get_add_domain_to_dns_kwargs(cluster=True) + pipeline.add_act( + act_name=_("MongoDB--添加domain到dns"), + act_component_code=ExecAddDomainToDnsOperationComponent.code, + kwargs=kwargs, + ) + + # 运行流程 + pipeline.run_pipeline() diff --git a/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/mongodb_remove_ns.py b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/mongodb_remove_ns.py new file mode 100644 index 0000000000..e6da9d329e --- /dev/null +++ b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/mongodb_remove_ns.py @@ -0,0 +1,99 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. +Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +import logging.config +from typing import Dict, Optional + +from django.utils.translation import ugettext as _ + +from backend.configuration.constants import DBType +from backend.flow.engine.bamboo.scene.common.builder import Builder +from backend.flow.engine.bamboo.scene.common.get_file_list import GetFileList +from backend.flow.engine.bamboo.scene.mongodb.sub_task.remove_ns import RemoveNsSubTask +from backend.flow.plugins.components.collections.mongodb.send_media import ExecSendMediaOperationComponent +from backend.flow.utils.mongodb.mongodb_dataclass import ActKwargs, MongoRepository + +logger = logging.getLogger("flow") + + +class MongoRemoveNsFlow(object): + """MongoRemoveNsFlowflow + 分析 payload,检查输入,生成Flow""" + + def __init__(self, root_id: str, data: Optional[Dict]): + """ + 传入参数 + @param root_id : 任务流程定义的root_id + @param data : 单据传递过来的参数列表,是dict格式 + """ + + self.root_id = root_id + self.payload = data + + def start(self): + """ + fix me + """ + helper = ActKwargs() + # 创建流程实例 + pipeline = Builder(root_id=self.root_id, data=self.payload) + backup_dir = helper.get_backup_dir() + file_list = GetFileList(db_type=DBType.MongoDB).get_db_actuator_package() + + print("data_for_backup", self.payload["data_for_backup"]) + + # todo 改为批量查询. + sub_pipelines = [] + # bk_host {ip:"1.1.1.1", bk_cloud_id: "0"} + bk_host_list = [] + # todo 同一机器的多个集群一起备份时,执行备份的机器要尽量错开. + + for row in self.payload["data_for_remove_ns"]: + cluster = MongoRepository.fetch_one_cluster(immute_domain=row["cluster_domain"]) + check_cluster(cluster, self.payload) + print("sub_pipline start", row) + sub_pl, sub_bk_host_list = RemoveNsSubTask.process_cluster( + root_id=self.root_id, + ticket_data=self.payload, + sub_ticket_data=row, + cluster=cluster, + backup_dir=backup_dir, + ) + bk_host_list.extend(sub_bk_host_list) + sub_pipelines.append(sub_pl.build_sub_process(_("MongoDB-备份-{}".format(cluster.name)))) + + send_media_kwargs = { + "file_list": file_list, + "ip_list": bk_host_list, + "exec_ips": [item["ip"] for item in bk_host_list], + "file_target_path": backup_dir + "/install", + } + print("send_media_kwargs", send_media_kwargs) + pipeline.add_act( + act_name=_("MongoDB-介质下发"), + act_component_code=ExecSendMediaOperationComponent.code, + kwargs=send_media_kwargs, + ) + + pipeline.add_parallel_sub_pipeline(sub_flow_list=sub_pipelines) + + # 运行流程 + pipeline.run_pipeline() + + +def check_cluster(cluster, payload): + if cluster is None: + raise Exception("row.cluster_domain is not exists.") + if str(cluster.bk_biz_id) != payload["bk_biz_id"]: + raise Exception( + "bad bk_biz_id {} vs {} {} {}".format( + cluster.bk_biz_id, payload["bk_biz_id"], type(cluster.bk_biz_id), type(payload["bk_biz_id"]) + ) + ) diff --git a/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/mongodb_replace.py b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/mongodb_replace.py new file mode 100644 index 0000000000..c639d4a540 --- /dev/null +++ b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/mongodb_replace.py @@ -0,0 +1,62 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. +Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +import logging.config +from typing import Dict, Optional + +from django.utils.translation import ugettext as _ + +from backend.flow.engine.bamboo.scene.common.builder import Builder +from backend.flow.plugins.components.collections.mongodb.exec_actuator_job import ExecuteDBActuatorJobComponent +from backend.flow.utils.mongodb.mongodb_dataclass import ActKwargs + +logger = logging.getLogger("flow") + + +class MongoReplaceFlow(object): + """MongoDB整机替换flow""" + + def __init__(self, root_id: str, data: Optional[Dict]): + """ + 传入参数 + @param root_id : 任务流程定义的root_id + @param data : 单据传递过来的参数列表,是dict格式 + """ + + self.root_id = root_id + self.data = data + self.get_kwargs = ActKwargs() + self.get_kwargs.payload = data + self.get_kwargs.get_file_path() + + def multi_replace_flow(self): + """ + multi replicaset execute script流程 + """ + + # 创建流程实例 + pipeline = Builder(root_id=self.root_id, data=self.data) + + # 获取所有的根据主机ip获取cluster信息 + self.get_kwargs.get_hosts_deinstall() + + # 创建整机替换——并行 + acts_list = [] + for cluster_id in self.data["cluster_ids"]: + kwargs = self.get_kwargs.get_exec_script_kwargs(cluster_id=cluster_id) + acts_list.append( + { + "act_name": _("MongoDB-{}-整机替换".format(str(cluster_id))), + "act_component_code": ExecuteDBActuatorJobComponent.code, + "kwargs": kwargs, + } + ) + # 运行流程 + pipeline.run_pipeline() diff --git a/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/mongodb_user.py b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/mongodb_user.py new file mode 100644 index 0000000000..16733e26ab --- /dev/null +++ b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/mongodb_user.py @@ -0,0 +1,60 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. +Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +import logging.config +from typing import Dict, Optional + +from backend.flow.engine.bamboo.scene.common.builder import Builder +from backend.flow.engine.bamboo.scene.mongodb.sub_task import user +from backend.flow.utils.mongodb.mongodb_dataclass import ActKwargs + +logger = logging.getLogger("flow") + + +class MongoUserFlow(object): + """MongoDB创建业务用户flow""" + + def __init__(self, root_id: str, data: Optional[Dict]): + """ + 传入参数 + @param root_id : 任务流程定义的root_id + @param data : 单据传递过来的参数列表,是dict格式 + """ + + self.root_id = root_id + self.data = data + self.get_kwargs = ActKwargs() + self.get_kwargs.payload = data + self.get_kwargs.get_file_path() + + def multi_cluster_create_user_flow(self, create: bool): + """ + multi replicaset create/delete user流程 + create True:创建 + create False:删除 + """ + + # 创建流程实例 + pipeline = Builder(root_id=self.root_id, data=self.data) + + # 创建/删除用户子流程并行 + sub_pipelines = [] + for cluster_id in self.data["cluster_ids"]: + sub_pipline = user( + root_id=self.root_id, + ticket_data=self.data, + sub_kwargs=self.get_kwargs, + cluster_id=cluster_id, + create=create, + ) + sub_pipelines.append(sub_pipline) + pipeline.add_parallel_sub_pipeline(sub_flow_list=sub_pipelines) + # 运行流程 + pipeline.run_pipeline() diff --git a/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/__init__.py b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/__init__.py new file mode 100644 index 0000000000..b19d645c72 --- /dev/null +++ b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/__init__.py @@ -0,0 +1,16 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. +Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + +from .deinstall import deinstall +from .exec_script import exec_script +from .mongos_install import mongos_install +from .replicaset_install import replicaset_install +from .user import user diff --git a/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/backup.py b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/backup.py new file mode 100644 index 0000000000..0d55fd6100 --- /dev/null +++ b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/backup.py @@ -0,0 +1,91 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. +Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + +from typing import Dict, List, Optional, Tuple + +from django.utils.translation import ugettext as _ + +from backend.flow.consts import MongoDBActuatorActionEnum +from backend.flow.engine.bamboo.scene.common.builder import SubBuilder +from backend.flow.plugins.components.collections.mongodb.exec_actuator_job2 import ExecuteDBActuatorJobComponent +from backend.flow.utils.mongodb.mongodb_dataclass import CommonContext, MongoDBCluster, ReplicaSet + + +# BackupSubTask 处理某个Cluster的备份任务. +class BackupSubTask: + """ + payload: 整体的ticket_data + sub_payload: 这个子任务的ticket_data + rs: + backup_dir: + """ + + def __init__(self): + pass + + @classmethod + def make_kwargs(cls, payload: Dict, sub_payload: Dict, rs: ReplicaSet, backup_dir: str) -> dict: + """备份 kwargs""" + print("get_backup_node", sub_payload) + node = rs.get_backup_node() + if node is None: + raise Exception("no backup node. rs:{}".format(rs.set_name)) + + return { + "set_trans_data_dataclass": CommonContext.__name__, + "get_trans_data_ip_var": None, + "bk_cloud_id": node.bk_cloud_id, + "exec_ip": node.ip, + "db_act_template": { + "action": MongoDBActuatorActionEnum.Backup, + "backup_dir": backup_dir, + "payload": { + "ip": node.ip, + "port": node.port, + "user": "root", + "pass": "root", + "authDb": "admin", + "ns_filter": sub_payload["ns_filter"], + }, + }, + } + + @classmethod + def process_cluster( + cls, + root_id: str, + ticket_data: Optional[Dict], + sub_ticket_data: Optional[Dict], + cluster: MongoDBCluster, + backup_dir: str, + ) -> Tuple[SubBuilder, List]: + """ + backup a ReplicaSet or backup a ShardedCluster + """ + + # 创建子流程 + sb = SubBuilder(root_id=root_id, data=ticket_data) + acts_list = [] + for rs in cluster.get_shards(): + acts_list.append( + { + "act_name": _("MongoDB备份[{}]".format(rs.set_name)), + "act_component_code": ExecuteDBActuatorJobComponent.code, + "kwargs": cls.make_kwargs(ticket_data, sub_ticket_data, rs, backup_dir), + } + ) + + sb.add_parallel_acts(acts_list=acts_list) + sub_bk_host_list = [] + for v in acts_list: + sub_bk_host_list.append({"ip": v["kwargs"]["exec_ip"], "bk_cloud_id": v["kwargs"]["bk_cloud_id"]}) + + return sb, sub_bk_host_list diff --git a/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/deinstall.py b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/deinstall.py new file mode 100644 index 0000000000..e8b500bfbb --- /dev/null +++ b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/deinstall.py @@ -0,0 +1,116 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. +Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + +from copy import deepcopy +from typing import Dict, Optional + +from django.utils.translation import ugettext as _ + +from backend.db_meta.enums.cluster_type import ClusterType +from backend.flow.consts import MediumEnum +from backend.flow.engine.bamboo.scene.common.builder import SubBuilder +from backend.flow.plugins.components.collections.mongodb.delete_domain_from_dns import ( + ExecDeleteDomainFromDnsOperationComponent, +) +from backend.flow.plugins.components.collections.mongodb.delete_password_from_db import ( + ExecDeletePasswordFromDBOperationComponent, +) +from backend.flow.plugins.components.collections.mongodb.exec_actuator_job import ExecuteDBActuatorJobComponent +from backend.flow.utils.mongodb.mongodb_dataclass import ActKwargs + + +def mongo_deinstall_parallel(sub_get_kwargs: ActKwargs, nodes: list, instance_type: str) -> list: + acts_list = [] + for node in nodes: + kwargs = sub_get_kwargs.get_mongo_deinstall_kwargs( + node_info=node, nodes_info=nodes, instance_type=instance_type + ) + acts_list.append( + { + "act_name": _("MongoDB-{}-{}卸载".format(node["ip"], instance_type)), + "act_component_code": ExecuteDBActuatorJobComponent.code, + "kwargs": kwargs, + } + ) + return acts_list + + +def deinstall(root_id: str, ticket_data: Optional[Dict], sub_kwargs: ActKwargs, cluster_id: int) -> SubBuilder: + """ + 单个cluster卸载流程 + """ + + # 获取变量 + sub_get_kwargs = deepcopy(sub_kwargs) + # 创建子流程 + sub_pipeline = SubBuilder(root_id=root_id, data=ticket_data) + + # 获取单个cluster信息 + sub_get_kwargs.get_cluster_info_deinstall(cluster_id=cluster_id) + + # mongo卸载 + # 复制集卸载 + if sub_get_kwargs.payload["cluster_type"] == ClusterType.MongoReplicaSet.value: + # mongo卸载——并行 + acts_list = mongo_deinstall_parallel( + sub_get_kwargs=sub_get_kwargs, nodes=sub_get_kwargs.payload["nodes"], instance_type=MediumEnum.MongoD + ) + sub_pipeline.add_parallel_acts(acts_list=acts_list) + # cluster卸载 + elif sub_get_kwargs.payload["cluster_type"] == ClusterType.MongoShardedCluster.value: + # mongos卸载——并行 + acts_list = mongo_deinstall_parallel( + sub_get_kwargs=sub_get_kwargs, + nodes=sub_get_kwargs.payload["mongos_nodes"], + instance_type=MediumEnum.MongoS, + ) + sub_pipeline.add_parallel_acts(acts_list=acts_list) + # shard卸载——并行 + many_acts_list = [] + for shard_nodes in sub_get_kwargs.payload["shards_nodes"]: + acts_list = mongo_deinstall_parallel( + sub_get_kwargs=sub_get_kwargs, nodes=shard_nodes["nodes"], instance_type=MediumEnum.MongoD + ) + many_acts_list.extend(acts_list) + sub_pipeline.add_parallel_acts(acts_list=many_acts_list) + # config卸载——并行 + acts_list = mongo_deinstall_parallel( + sub_get_kwargs=sub_get_kwargs, + nodes=sub_get_kwargs.payload["config_nodes"], + instance_type=MediumEnum.MongoD, + ) + sub_pipeline.add_parallel_acts(acts_list=acts_list) + + # 删除dns + kwargs = sub_get_kwargs.get_delete_domain_kwargs() + sub_pipeline.add_act( + act_name=_("MongoDB-删除域名"), + act_component_code=ExecDeleteDomainFromDnsOperationComponent.code, + kwargs=kwargs, + ) + + # 删除保存在密码服务的密码 + kwargs = sub_get_kwargs.get_delete_pwd_kwargs() + sub_pipeline.add_act( + act_name=_("MongoDB-删除密码"), + act_component_code=ExecDeletePasswordFromDBOperationComponent.code, + kwargs=kwargs, + ) + + # 删除db_meta关系 + kwargs = {"cluster_id": cluster_id} + sub_pipeline.add_act( + act_name=_("MongoDB-删除db_meta关系"), + act_component_code=ExecDeletePasswordFromDBOperationComponent.code, + kwargs=kwargs, + ) + + return sub_pipeline.build_sub_process(sub_name=_("MongoDB--卸载--cluster_id:{}".format(str(cluster_id)))) diff --git a/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/exec_script.py b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/exec_script.py new file mode 100644 index 0000000000..4afe94ea7a --- /dev/null +++ b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/exec_script.py @@ -0,0 +1,54 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. +Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + +from copy import deepcopy +from typing import Dict, Optional + +from django.utils.translation import ugettext as _ + +from backend.flow.consts import MediumEnum +from backend.flow.engine.bamboo.scene.common.builder import SubBuilder +from backend.flow.plugins.components.collections.mongodb.exec_actuator_job import ExecuteDBActuatorJobComponent +from backend.flow.plugins.components.collections.mongodb.send_media import ExecSendMediaOperationComponent +from backend.flow.utils.mongodb.mongodb_dataclass import ActKwargs + + +def exec_script(root_id: str, ticket_data: Optional[Dict], sub_kwargs: ActKwargs, cluster_id: int) -> SubBuilder: + """ + 单个cluster执行脚本流程 + """ + + # 获取变量 + sub_get_kwargs = deepcopy(sub_kwargs) + + # 创建子流程 + sub_pipeline = SubBuilder(root_id=root_id, data=ticket_data) + + # 获取信息 + sub_get_kwargs.get_cluster_info_create_user(cluster_id=cluster_id, admin_user=MediumEnum.DbaUser) + + # 介质下发 + kwargs = sub_get_kwargs.get_send_media_kwargs() + sub_pipeline.add_act( + act_name=_("MongoDB-介质下发"), act_component_code=ExecSendMediaOperationComponent.code, kwargs=kwargs + ) + + # 执行脚本 + kwargs = sub_get_kwargs.get_exec_script_kwargs() + sub_pipeline.add_act( + act_name=_("MongoDB-cluster_id:{}-执行脚本".format(str(cluster_id))), + act_component_code=ExecuteDBActuatorJobComponent.code, + kwargs=kwargs, + ) + + return sub_pipeline.build_sub_process( + sub_name=_("MongoDB--创建用户--cluster_id:{}-{}".format(str(cluster_id), sub_get_kwargs.payload["hosts"][0]["ip"])) + ) diff --git a/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/mongos_install.py b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/mongos_install.py new file mode 100644 index 0000000000..b4dc2fe48c --- /dev/null +++ b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/mongos_install.py @@ -0,0 +1,66 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. +Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + +from copy import deepcopy +from typing import Dict, Optional + +from django.utils.translation import ugettext as _ + +from backend.flow.consts import MediumEnum +from backend.flow.engine.bamboo.scene.common.builder import SubBuilder +from backend.flow.plugins.components.collections.mongodb.add_password_to_db import ( + ExecAddPasswordToDBOperationComponent, +) +from backend.flow.plugins.components.collections.mongodb.exec_actuator_job import ExecuteDBActuatorJobComponent +from backend.flow.utils.mongodb.mongodb_dataclass import ActKwargs + + +def mongos_install(root_id: str, ticket_data: Optional[Dict], sub_kwargs: ActKwargs) -> SubBuilder: + """ + 多个mongos安装流程 + """ + + # 获取变量 + sub_get_kwargs = deepcopy(sub_kwargs) + # 创建子流程 + sub_pipeline = SubBuilder(root_id=root_id, data=ticket_data) + + # mongod安装——并行 + acts_list = [] + for node in sub_get_kwargs.mongos_info["nodes"]: + kwargs = sub_get_kwargs.get_install_mongos_kwargs(node=node) + acts_list.append( + { + "act_name": _("MongoDB-{}-mongos安装".format(node["ip"])), + "act_component_code": ExecuteDBActuatorJobComponent.code, + "kwargs": kwargs, + } + ) + sub_pipeline.add_parallel_acts(acts_list=acts_list) + + # dba,appdba,monitor,monitor用户用户密码写入密码服务 + kwargs = sub_get_kwargs.get_add_password_to_db_kwargs( + usernames=[MediumEnum.DbaUser, MediumEnum.AppDbaUser, MediumEnum.MonitorUser, MediumEnum.AppMonitorUser], + info=sub_get_kwargs.mongos_info, + ) + sub_pipeline.add_act( + act_name=_("MongoDB--保存dba用户及额外管理用户密码"), + act_component_code=ExecAddPasswordToDBOperationComponent.code, + kwargs=kwargs, + ) + + # 安装监控 + + return sub_pipeline.build_sub_process( + sub_name=_( + "MongoDB--安装mongos--{}-{}".format(sub_get_kwargs.payload["app"], sub_get_kwargs.replicaset_info["areaId"]) + ) + ) diff --git a/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/remove_ns.py b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/remove_ns.py new file mode 100644 index 0000000000..0a9bb19d6a --- /dev/null +++ b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/remove_ns.py @@ -0,0 +1,88 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. +Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + +from typing import Dict, List, Optional, Tuple + +from django.utils.translation import ugettext as _ + +from backend.flow.consts import MongoDBActuatorActionEnum +from backend.flow.engine.bamboo.scene.common.builder import SubBuilder +from backend.flow.plugins.components.collections.mongodb.exec_actuator_job import ExecuteDBActuatorJobComponent +from backend.flow.utils.mongodb.mongodb_dataclass import CommonContext, MongoDBCluster, ReplicaSet + + +# BackupSubTask 处理某个Cluster的备份任务. +class RemoveNsSubTask: + """ + payload: 整体的ticket_data + sub_payload: 这个子任务的ticket_data + rs: + backup_dir: + """ + + @classmethod + def make_dbactuator_kwargs(cls, payload: Dict, sub_payload: Dict, rs: ReplicaSet, backup_dir: str) -> dict: + """备份 kwargs""" + nodes = rs.get_not_backup_nodes() + if len(nodes) == 0: + raise Exception("no backup node. rs:{}".format(rs.set_name)) + + node = nodes[0] + return { + "set_trans_data_dataclass": CommonContext.__name__, + "get_trans_data_ip_var": None, + "bk_cloud_id": node.bk_cloud_id, + "exec_ip": node.ip, + "db_act_template": { + "action": MongoDBActuatorActionEnum.RemoveNs, + "backup_dir": backup_dir, + "payload": { + "ip": node.ip, + "port": node.port, + "user": "root", + "pass": "root", + "authDb": "admin", + "ns_filter": sub_payload["ns_filter"], + }, + }, + } + + @classmethod + def process_cluster( + cls, + root_id: str, + ticket_data: Optional[Dict], + sub_ticket_data: Optional[Dict], + cluster: MongoDBCluster, + backup_dir: str, + ) -> Tuple[SubBuilder, List]: + """ + cluster can be a ReplicaSet or a ShardedCluster + """ + + # 创建子流程 + sub_pipeline = SubBuilder(root_id=root_id, data=ticket_data) + acts_list = [] + for rs in cluster.get_shards(): + acts_list.append( + { + "act_name": _("MongoDB-RemoveNs[{}]".format(rs.set_name)), + "act_component_code": ExecuteDBActuatorJobComponent.code, + "kwargs": cls.make_dbactuator_kwargs(ticket_data, sub_ticket_data, rs, backup_dir), + } + ) + + sub_pipeline.add_parallel_acts(acts_list=acts_list) + sub_bk_host_list = [] + for v in acts_list: + sub_bk_host_list.append({"ip": v["kwargs"]["exec_ip"], "bk_cloud_id": v["kwargs"]["bk_cloud_id"]}) + + return sub_pipeline, sub_bk_host_list diff --git a/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/replicaset_install.py b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/replicaset_install.py new file mode 100644 index 0000000000..a5e3e0e9d0 --- /dev/null +++ b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/replicaset_install.py @@ -0,0 +1,123 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. +Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + +from copy import deepcopy +from typing import Dict, Optional + +from django.utils.translation import ugettext as _ + +from backend.flow.consts import MediumEnum +from backend.flow.engine.bamboo.scene.common.builder import SubBuilder +from backend.flow.plugins.components.collections.mongodb.add_domain_to_dns import ExecAddDomainToDnsOperationComponent +from backend.flow.plugins.components.collections.mongodb.add_password_to_db import ( + ExecAddPasswordToDBOperationComponent, +) +from backend.flow.plugins.components.collections.mongodb.exec_actuator_job import ExecuteDBActuatorJobComponent +from backend.flow.plugins.components.collections.mongodb.get_manager_user_password import ( + ExecGetPasswordOperationComponent, +) +from backend.flow.utils.mongodb.mongodb_dataclass import ActKwargs + + +def replicaset_install( + root_id: str, + ticket_data: Optional[Dict], + sub_kwargs: ActKwargs, + cluster: bool, + cluster_role: str, + config_svr: bool, +) -> SubBuilder: + """ + 单个replicaset安装流程 + """ + + # 获取变量 + sub_get_kwargs = deepcopy(sub_kwargs) + # 创建子流程 + sub_pipeline = SubBuilder(root_id=root_id, data=ticket_data) + + # mongod安装——并行 + acts_list = [] + for node in sub_get_kwargs.replicaset_info["nodes"]: + kwargs = sub_get_kwargs.get_install_mongod_kwargs(node=node, cluster_role=cluster_role) + acts_list.append( + { + "act_name": _("MongoDB-{}-mongod安装".format(node["ip"])), + "act_component_code": ExecuteDBActuatorJobComponent.code, + "kwargs": kwargs, + } + ) + sub_pipeline.add_parallel_acts(acts_list=acts_list) + + # 创建复制集 + kwargs = sub_get_kwargs.get_replicaset_init_kwargs(config_svr=config_svr) + sub_pipeline.add_act( + act_name=_("MongoDB-{}-replicaset初始化".format(sub_get_kwargs.replicaset_info["nodes"][0]["ip"])), + act_component_code=ExecuteDBActuatorJobComponent.code, + kwargs=kwargs, + ) + + if not cluster: + # 密码服务获取管理用户密码 + kwargs = sub_get_kwargs.get_get_manager_password_kwargs() + sub_pipeline.add_act( + act_name=_("MongoDB--获取管理员用户密码"), act_component_code=ExecGetPasswordOperationComponent.code, kwargs=kwargs + ) + + # 创建dba用户 + kwargs = sub_get_kwargs.get_add_manager_user_kwargs() + sub_pipeline.add_act( + act_name=_("MongoDB--创建dba用户"), act_component_code=ExecuteDBActuatorJobComponent.code, kwargs=kwargs + ) + + # 创建appdba,monitor,monitor用户 + kwargs = sub_get_kwargs.get_init_exec_script_kwargs(script_type=MediumEnum.MongoDBExtraUserCreate) + sub_pipeline.add_act( + act_name=_("MongoDB--创建额外管理用户"), act_component_code=ExecuteDBActuatorJobComponent.code, kwargs=kwargs + ) + + # dba, appdba,monitor,monitor用户密码写入密码服务 + kwargs = sub_get_kwargs.get_add_password_to_db_kwargs( + usernames=[MediumEnum.DbaUser, MediumEnum.AppDbaUser, MediumEnum.MonitorUser, MediumEnum.AppMonitorUser], + info=sub_get_kwargs.replicaset_info, + ) + sub_pipeline.add_act( + act_name=_("MongoDB--保存dba用户及额外管理用户密码"), + act_component_code=ExecAddPasswordToDBOperationComponent.code, + kwargs=kwargs, + ) + + # 进行初始配置 + # 创建oplog重放权限的role,把role授权给dba,appdba 把admin库的gcs_heartbeat授予给monitor用户 + # 3.x版本修改验证方式 + kwargs = sub_get_kwargs.get_init_exec_script_kwargs(script_type=MediumEnum.MongoDBInitSet) + sub_pipeline.add_act( + act_name=_("MongoDB-{}-db初始设置".format(sub_get_kwargs.replicaset_info["nodes"][0]["ip"])), + act_component_code=ExecuteDBActuatorJobComponent.code, + kwargs=kwargs, + ) + + if not cluster: + # 域名写入dns + kwargs = sub_get_kwargs.get_add_domain_to_dns_kwargs(cluster=False) + sub_pipeline.add_act( + act_name=_("MongoDB--添加domain到dns"), + act_component_code=ExecAddDomainToDnsOperationComponent.code, + kwargs=kwargs, + ) + + # 安装监控 + + return sub_pipeline.build_sub_process( + sub_name=_( + "MongoDB--安装复制集--{}-{}".format(sub_get_kwargs.payload["app"], sub_get_kwargs.replicaset_info["set_id"]) + ) + ) diff --git a/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/send_media.py b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/send_media.py new file mode 100644 index 0000000000..a30767035b --- /dev/null +++ b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/send_media.py @@ -0,0 +1,32 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. +Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + + +# BackupSubTask 处理某个Cluster的备份任务. +class SendMediaSubTask: + """ + payload: 整体的ticket_data + sub_payload: 这个子任务的ticket_data + rs: + backup_dir: + """ + + def __init__(self): + pass + + @classmethod + def make_act_kwargs(cls, file_list, bk_host_list, file_target_path: str) -> dict: + return { + "file_list": file_list, + "ip_list": bk_host_list, + "exec_ips": [item["ip"] for item in bk_host_list], + "file_target_path": file_target_path, + } diff --git a/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/user.py b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/user.py new file mode 100644 index 0000000000..57757078a0 --- /dev/null +++ b/dbm-ui/backend/flow/engine/bamboo/scene/mongodb/sub_task/user.py @@ -0,0 +1,60 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. +Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + +from copy import deepcopy +from typing import Dict, Optional + +from django.utils.translation import ugettext as _ + +from backend.flow.consts import MediumEnum +from backend.flow.engine.bamboo.scene.common.builder import SubBuilder +from backend.flow.plugins.components.collections.mongodb.exec_actuator_job import ExecuteDBActuatorJobComponent +from backend.flow.plugins.components.collections.mongodb.send_media import ExecSendMediaOperationComponent +from backend.flow.utils.mongodb.mongodb_dataclass import ActKwargs + + +def user( + root_id: str, ticket_data: Optional[Dict], sub_kwargs: ActKwargs, cluster_id: int, create: bool +) -> SubBuilder: + """ + 单个cluster 创建/删除用户流程 + """ + + # 获取变量 + sub_get_kwargs = deepcopy(sub_kwargs) + + # 创建子流程 + sub_pipeline = SubBuilder(root_id=root_id, data=ticket_data) + + # 获取信息 + sub_get_kwargs.get_cluster_info_user(cluster_id=cluster_id, admin_user=MediumEnum.DbaUser) + + # 介质下发 + kwargs = sub_get_kwargs.get_send_media_kwargs() + sub_pipeline.add_act( + act_name=_("MongoDB-介质下发"), act_component_code=ExecSendMediaOperationComponent.code, kwargs=kwargs + ) + + # 创建用户 + kwargs = sub_get_kwargs.get_user_kwargs(create=create, admin_user=MediumEnum.DbaUser) + if create: + act_name = _("MongoDB-cluster_id:{}-创建用户".format(str(cluster_id))) + sub_name = _( + "MongoDB--创建用户--cluster_id:{}-{}".format(str(cluster_id), sub_get_kwargs.payload["hosts"][0]["ip"]) + ) + else: + act_name = _("MongoDB-cluster_id:{}-删除用户".format(str(cluster_id))) + sub_name = _( + "MongoDB--删除用户--cluster_id:{}-{}".format(str(cluster_id), sub_get_kwargs.payload["hosts"][0]["ip"]) + ) + sub_pipeline.add_act(act_name=act_name, act_component_code=ExecuteDBActuatorJobComponent.code, kwargs=kwargs) + + return sub_pipeline.build_sub_process(sub_name=sub_name) diff --git a/dbm-ui/backend/flow/engine/controller/mongodb.py b/dbm-ui/backend/flow/engine/controller/mongodb.py new file mode 100644 index 0000000000..585c42601b --- /dev/null +++ b/dbm-ui/backend/flow/engine/controller/mongodb.py @@ -0,0 +1,50 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. +Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +from backend.flow.engine.bamboo.scene.mongodb.mongodb_backup import MongoBackupFlow +from backend.flow.engine.bamboo.scene.mongodb.mongodb_fake_install import MongoFakeInstallFlow +from backend.flow.engine.bamboo.scene.mongodb.mongodb_install import MongoDBInstallFlow +from backend.flow.engine.controller.base import BaseController + + +class MongoDBController(BaseController): + """ + 名字服务相关控制器 + """ + + def multi_replicaset_create(self): + """ + 安装复制集 + """ + + flow = MongoDBInstallFlow(root_id=self.root_id, data=self.ticket_data) + flow.multi_replicaset_install_flow() + + def cluster_create(self): + """ + cluster安装 + """ + + flow = MongoDBInstallFlow(root_id=self.root_id, data=self.ticket_data) + flow.cluster_install_flow() + + def mongo_backup(self): + """ + 发起备份任务 + """ + flow = MongoBackupFlow(root_id=self.root_id, data=self.ticket_data) + flow.start() + + def fake_install(self): + """ + 在Meta中生成一个ReplicaSet,用于测试 + """ + flow = MongoFakeInstallFlow(root_id=self.root_id, data=self.ticket_data) + flow.start() diff --git a/dbm-ui/backend/flow/plugins/components/collections/mongodb/__init__.py b/dbm-ui/backend/flow/plugins/components/collections/mongodb/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/dbm-ui/backend/flow/plugins/components/collections/mongodb/add_domain_to_dns.py b/dbm-ui/backend/flow/plugins/components/collections/mongodb/add_domain_to_dns.py new file mode 100644 index 0000000000..f03f81b290 --- /dev/null +++ b/dbm-ui/backend/flow/plugins/components/collections/mongodb/add_domain_to_dns.py @@ -0,0 +1,66 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. +Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +import logging +from typing import List + +from pipeline.component_framework.component import Component +from pipeline.core.flow.activity import Service + +from backend.flow.plugins.components.collections.common.base_service import BaseService +from backend.flow.utils import dns_manage + +logger = logging.getLogger("json") + + +class ExecAddDomainToDnsOperation(BaseService): + """ + NameServiceCreate服务 + """ + + def _execute(self, data, parent_data) -> bool: + """ + 执行创建名字服务功能的函数 + global_data 单据全局变量,格式字典 + kwargs 私有变量 + """ + + # 从流程节点中获取变量 + kwargs = data.get_one_of_inputs("kwargs") + + # 写入dns + for domain in kwargs["domains"]: + result = dns_manage.DnsManage( + bk_biz_id=kwargs["bk_biz_id"], bk_cloud_id=kwargs["bk_cloud_id"] + ).create_domain(instance_list=domain["instance_list"], add_domain_name=domain["domain"]) + if not result: + self.log_error( + "add domain:{} with instance:{} to dns fail".format(domain["domain"], domain["instance_list"]) + ) + return False + self.log_info("add domain to dns successfully") + return True + + # 流程节点输入参数 + def inputs_format(self) -> List: + return [ + Service.InputItem(name="kwargs", key="kwargs", type="dict", required=True), + Service.InputItem(name="global_data", key="global_data", type="dict", required=True), + ] + + +class ExecAddDomainToDnsOperationComponent(Component): + """ + ExecAddDomainToDnsOperation组件 + """ + + name = __name__ + code = "add_domain_to_dns_operation" + bound_service = ExecAddDomainToDnsOperation diff --git a/dbm-ui/backend/flow/plugins/components/collections/mongodb/add_password_to_db.py b/dbm-ui/backend/flow/plugins/components/collections/mongodb/add_password_to_db.py new file mode 100644 index 0000000000..b685faa680 --- /dev/null +++ b/dbm-ui/backend/flow/plugins/components/collections/mongodb/add_password_to_db.py @@ -0,0 +1,75 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. +Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +import logging +from typing import List + +from pipeline.component_framework.component import Component +from pipeline.core.flow.activity import Service + +import backend.flow.utils.mongodb.mongodb_dataclass as flow_context +from backend.flow.plugins.components.collections.common.base_service import BaseService +from backend.flow.utils.mongodb.mongodb_password import MongoDBPassword + +logger = logging.getLogger("json") + + +class ExecAddPasswordToDBOperation(BaseService): + """ + NameServiceCreate服务 + """ + + def _execute(self, data, parent_data) -> bool: + """ + 执行创建名字服务功能的函数 + global_data 单据全局变量,格式字典 + kwargs 私有变量 + """ + + # 从流程节点中获取变量 + kwargs = data.get_one_of_inputs("kwargs") + trans_data = data.get_one_of_inputs("trans_data") + usernames = kwargs["usernames"] + + if trans_data is None or trans_data == "${trans_data}": + # 表示没有加载上下文内容,则在此添加 + trans_data = getattr(flow_context, kwargs["set_trans_data_dataclass"])() + + # 把密码写入db + for username in usernames: + if kwargs["set_name"]: + password = trans_data[kwargs["set_name"]][username] + else: + password = trans_data[username] + result = MongoDBPassword().save_password_to_db( + instances=kwargs["nodes"], username=username, password=password, operator=kwargs["operator"] + ) + if result is not None: + self.log_error("add password of user:{} to db fail, error:{}".format(username, result)) + return False + self.log_info("add password of users:{} to db successfully".format(",".join(usernames))) + return True + + # 流程节点输入参数 + def inputs_format(self) -> List: + return [ + Service.InputItem(name="kwargs", key="kwargs", type="dict", required=True), + Service.InputItem(name="global_data", key="global_data", type="dict", required=True), + ] + + +class ExecAddPasswordToDBOperationComponent(Component): + """ + ExecAddPasswordToDBOperation组件 + """ + + name = __name__ + code = "add_password_to_db" + bound_service = ExecAddPasswordToDBOperation diff --git a/dbm-ui/backend/flow/plugins/components/collections/mongodb/add_relationship_to_meta.py b/dbm-ui/backend/flow/plugins/components/collections/mongodb/add_relationship_to_meta.py new file mode 100644 index 0000000000..f0e6b235e9 --- /dev/null +++ b/dbm-ui/backend/flow/plugins/components/collections/mongodb/add_relationship_to_meta.py @@ -0,0 +1,95 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. +Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +import logging +from typing import List + +from pipeline.component_framework.component import Component +from pipeline.core.flow.activity import Service + +from backend.db_meta.api.cluster.mongocluster import pkg_create_mongo_cluster +from backend.db_meta.api.cluster.mongorepset import pkg_create_mongoset +from backend.db_meta.enums.cluster_type import ClusterType +from backend.flow.plugins.components.collections.common.base_service import BaseService + +logger = logging.getLogger("json") + + +class ExecAddRelationshipOperation(BaseService): + """ + NameServiceCreate服务 + """ + + def _execute(self, data, parent_data) -> bool: + """ + 执行创建名字服务功能的函数 + global_data 单据全局变量,格式字典 + kwargs 私有变量 + """ + + # 从流程节点中获取变量 + kwargs = data.get_one_of_inputs("kwargs") + + # 写入meta + try: + if kwargs["cluster_type"] == ClusterType.MongoReplicaSet.value: + + pkg_create_mongoset( + bk_biz_id=kwargs["bk_biz_id"], + name=kwargs["name"], + immute_domain=kwargs["immute_domain"], + alias=kwargs["alias"], + major_version=kwargs["major_version"], + storages=kwargs["storages"], + creator=kwargs["creator"], + bk_cloud_id=kwargs["bk_cloud_id"], + db_module_id=kwargs["db_module_id"], + region=kwargs["region"], + skip_machine=kwargs["skip_machine"], + spec_id=kwargs["spec_id"], + spec_config=kwargs["spec_config"], + ) + elif kwargs["cluster_type"] == ClusterType.MongoShardedCluster.value: + pkg_create_mongo_cluster( + bk_biz_id=kwargs["bk_biz_id"], + name=kwargs["name"], + immute_domain=kwargs["immute_domain"], + alias=kwargs["alias"], + major_version=kwargs["major_version"], + proxies=kwargs["proxies"], + configs=kwargs["configs"], + storages=kwargs["storages"], + creator=kwargs["creator"], + bk_cloud_id=kwargs["bk_cloud_id"], + region=kwargs["region"], + machine_specs=kwargs["machine_specs"], + ) + except Exception as e: + self.log_error("add relationship to meta fail, error:{}".format(str(e))) + return False + self.log_info("add mongodb relationship to meta successfully") + return True + + # 流程节点输入参数 + def inputs_format(self) -> List: + return [ + Service.InputItem(name="kwargs", key="kwargs", type="dict", required=True), + Service.InputItem(name="global_data", key="global_data", type="dict", required=True), + ] + + +class ExecAddRelationshipOperationComponent(Component): + """ + ExecAddRelationshipOperation组件 + """ + + name = __name__ + code = "add_relationship_to_meta_operation" + bound_service = ExecAddRelationshipOperation diff --git a/dbm-ui/backend/flow/plugins/components/collections/mongodb/delete_domain_from_dns.py b/dbm-ui/backend/flow/plugins/components/collections/mongodb/delete_domain_from_dns.py new file mode 100644 index 0000000000..52a699efc1 --- /dev/null +++ b/dbm-ui/backend/flow/plugins/components/collections/mongodb/delete_domain_from_dns.py @@ -0,0 +1,66 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. +Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +import logging +from typing import List + +from pipeline.component_framework.component import Component +from pipeline.core.flow.activity import Service + +from backend.flow.plugins.components.collections.common.base_service import BaseService +from backend.flow.utils import dns_manage + +logger = logging.getLogger("json") + + +class ExecDeleteDomainFromDnsOperation(BaseService): + """ + DeleteDomainFromDns服务 + """ + + def _execute(self, data, parent_data) -> bool: + """ + 执行删除domain功能的函数 + global_data 单据全局变量,格式字典 + kwargs 私有变量 + """ + + # 从流程节点中获取变量 + kwargs = data.get_one_of_inputs("kwargs") + + # 从dns删除domain + for del_domain in kwargs["del_domains"]: + result = dns_manage.DnsManage( + bk_biz_id=kwargs["bk_biz_id"], bk_cloud_id=kwargs["bk_cloud_id"] + ).remove_domain_ip(del_instance_list=del_domain["del_instance_list"], domain=del_domain["domain"]) + if not result: + self.log_error( + "delete domain:{} with instance:{} to dns fail".format(kwargs["domain"], kwargs["instance_list"]) + ) + return False + self.log_info("delete domain from dns successfully") + return True + + # 流程节点输入参数 + def inputs_format(self) -> List: + return [ + Service.InputItem(name="kwargs", key="kwargs", type="dict", required=True), + Service.InputItem(name="global_data", key="global_data", type="dict", required=True), + ] + + +class ExecDeleteDomainFromDnsOperationComponent(Component): + """ + ExecDeleteDomainFromDnsOperation组件 + """ + + name = __name__ + code = "delete_domain_from_dns_operation" + bound_service = ExecDeleteDomainFromDnsOperation diff --git a/dbm-ui/backend/flow/plugins/components/collections/mongodb/delete_password_from_db.py b/dbm-ui/backend/flow/plugins/components/collections/mongodb/delete_password_from_db.py new file mode 100644 index 0000000000..afb9226008 --- /dev/null +++ b/dbm-ui/backend/flow/plugins/components/collections/mongodb/delete_password_from_db.py @@ -0,0 +1,63 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. +Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +import logging +from typing import List + +from pipeline.component_framework.component import Component +from pipeline.core.flow.activity import Service + +from backend.flow.plugins.components.collections.common.base_service import BaseService +from backend.flow.utils.mongodb.mongodb_password import MongoDBPassword + +logger = logging.getLogger("json") + + +class ExecDeletePasswordFromDBOperation(BaseService): + """ + DeletePasswordFromDB服务 + """ + + def _execute(self, data, parent_data) -> bool: + """ + 删除密码功能的函数 + global_data 单据全局变量,格式字典 + kwargs 私有变量 + """ + + # 从流程节点中获取变量 + kwargs = data.get_one_of_inputs("kwargs") + + # 从db中删除管理员密码 + result = MongoDBPassword().delete_password_from_db( + instances=kwargs["instances"], usernames=kwargs["usernames"] + ) + if result is not None: + self.log_error("delete password of admin user from db fail, error:{}".format(result)) + return False + self.log_info("delete password of admin user to db successfully") + return True + + # 流程节点输入参数 + def inputs_format(self) -> List: + return [ + Service.InputItem(name="kwargs", key="kwargs", type="dict", required=True), + Service.InputItem(name="global_data", key="global_data", type="dict", required=True), + ] + + +class ExecDeletePasswordFromDBOperationComponent(Component): + """ + ExecAddPasswordToDBOperation组件 + """ + + name = __name__ + code = "delete_password_from_db" + bound_service = ExecDeletePasswordFromDBOperation diff --git a/dbm-ui/backend/flow/plugins/components/collections/mongodb/delete_relationship_from_meta.py b/dbm-ui/backend/flow/plugins/components/collections/mongodb/delete_relationship_from_meta.py new file mode 100644 index 0000000000..c8d35aa4f9 --- /dev/null +++ b/dbm-ui/backend/flow/plugins/components/collections/mongodb/delete_relationship_from_meta.py @@ -0,0 +1,62 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. +Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +import logging +from typing import List + +from pipeline.component_framework.component import Component +from pipeline.core.flow.activity import Service + +from backend.db_meta.models import Cluster +from backend.flow.plugins.components.collections.common.base_service import BaseService + +logger = logging.getLogger("json") + + +class ExecDeleteRelationshipOperation(BaseService): + """ + DeleteRelationship服务 + """ + + def _execute(self, data, parent_data) -> bool: + """ + 执行删除关系功能的函数 + global_data 单据全局变量,格式字典 + kwargs 私有变量 + """ + + # 从流程节点中获取变量 + kwargs = data.get_one_of_inputs("kwargs") + + # 从meta删除关系 + try: + Cluster.objects.filter(id=kwargs["cluster_id"]).delete() + except Exception as e: + self.log_error("delete relationship from meta fail, error:{}".format(str(e))) + return False + self.log_info("delete mongodb relationship from meta successfully") + return True + + # 流程节点输入参数 + def inputs_format(self) -> List: + return [ + Service.InputItem(name="kwargs", key="kwargs", type="dict", required=True), + Service.InputItem(name="global_data", key="global_data", type="dict", required=True), + ] + + +class ExecDeleteRelationshipOperationComponent(Component): + """ + ExecDeleteRelationshipOperation组件 + """ + + name = __name__ + code = "delete_relationship_from_meta_operation" + bound_service = ExecDeleteRelationshipOperation diff --git a/dbm-ui/backend/flow/plugins/components/collections/mongodb/exec_actuator_job.py b/dbm-ui/backend/flow/plugins/components/collections/mongodb/exec_actuator_job.py new file mode 100644 index 0000000000..3912c8fe68 --- /dev/null +++ b/dbm-ui/backend/flow/plugins/components/collections/mongodb/exec_actuator_job.py @@ -0,0 +1,186 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. +Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +import base64 +import json +import logging +import re +from typing import List + +from django.utils.translation import ugettext as _ +from jinja2 import Environment +from pipeline.component_framework.component import Component +from pipeline.core.flow.activity import Service + +import backend.flow.utils.redis.redis_context_dataclass as flow_context +from backend import env +from backend.components import JobApi +from backend.flow.consts import MediumEnum +from backend.flow.models import FlowNode +from backend.flow.plugins.components.collections.common.base_service import BkJobService +from backend.flow.utils.mongodb.mongodb_script_template import ( + mongodb_actuator_template, + mongodb_fast_execute_script_common_kwargs, + mongodb_os_init_actuator_template, +) + +logger = logging.getLogger("json") +cpl = re.compile("(?P.+?)") # 非贪婪模式,只匹配第一次出现的自定义tag + + +class ExecuteDBActuatorJobService(BkJobService): + """ + 根据db-actuator组件,绑定fast_execute_script api接口访问。 + """ + + def _execute(self, data, parent_data) -> bool: + """ + 执行fast_execute_script脚本 + global_data 单据全局变量,格式字典 + trans_data 单据上下文 + kwargs 字典传入格式: + { + root_id: db-actuator任务必须参数,做录入日志平台的条件 + node_id: db-actuator任务必须参数,做录入日志平台的条件 + node_name: db-actuator任务必须参数,做录入日志平台的条件 + get_redis_payload_func : 表示获取执行 redis的db-actuator 参数方法名称,对应RedisActPayload类 + exec_ip: 表示执行的ip节点 + get_trans_data_ip_name: 表示从上下文获取到执行ip的变量名,对应单据的获取到上下文dataclass类 + cluster: 操作的集群名称 + extend_attr: 额外的字段数据,需要从RedisApplyContext中获取 + } + """ + kwargs = data.get_one_of_inputs("kwargs") + global_data = data.get_one_of_inputs("global_data") + trans_data = data.get_one_of_inputs("trans_data") + + if trans_data is None or trans_data == "${trans_data}": + # 表示没有加载上下文内容,则在此添加 + trans_data = getattr(flow_context, kwargs["set_trans_data_dataclass"])() + + root_id = kwargs["root_id"] + node_name = kwargs["node_name"] + node_id = kwargs["node_id"] + + # 创建db管理员账号,从上游流程节点获取密码 + if kwargs.get("create_manager_user", False): + if kwargs["set_name"]: + kwargs["db_act_template"]["payload"]["password"] = trans_data[kwargs["set_name"]][ + kwargs["db_act_template"]["payload"]["username"] + ] + else: + kwargs["db_act_template"]["payload"]["password"] = trans_data[ + kwargs["db_act_template"]["payload"]["username"] + ] + + # 创建额外管理员账号 + if kwargs.get("create_extra_manager_user", False): + if kwargs["set_name"]: + kwargs["db_act_template"]["payload"]["adminPassword"] = trans_data[kwargs["set_name"]][ + kwargs["db_act_template"]["payload"]["adminUsername"] + ] + kwargs["db_act_template"]["payload"]["script"].replace( + "{{appdba_pwd}}", trans_data[kwargs["set_name"]][MediumEnum.AppDbaUser] + ) + kwargs["db_act_template"]["payload"]["script"].replace( + "{{monitor_pwd}}", trans_data[kwargs["set_name"]][MediumEnum.MonitorUser] + ) + kwargs["db_act_template"]["payload"]["script"].replace( + "{{appmonitor_pwd}}", trans_data[kwargs["set_name"]][MediumEnum.AppMonitorUser] + ) + else: + kwargs["db_act_template"]["payload"]["adminPassword"] = trans_data[ + kwargs["db_act_template"]["payload"]["adminUsername"] + ] + kwargs["db_act_template"]["payload"]["script"].replace( + "{{appdba_pwd}}", trans_data[MediumEnum.AppDbaUser] + ) + kwargs["db_act_template"]["payload"]["script"].replace( + "{{monitor_pwd}}", trans_data[MediumEnum.MonitorUser] + ) + kwargs["db_act_template"]["payload"]["script"].replace( + "{{appmonitor_pwd}}", trans_data[MediumEnum.AppMonitorUser] + ) + + # 进行db初始设置获,从上游流程节点获取密码 + if kwargs.get("db_init_set", False): + if kwargs["set_name"]: + kwargs["db_act_template"]["payload"]["adminPassword"] = trans_data[kwargs["set_name"]][ + kwargs["db_act_template"]["payload"]["adminUsername"] + ] + else: + kwargs["db_act_template"]["payload"]["adminPassword"] = trans_data[ + kwargs["db_act_template"]["payload"]["adminUsername"] + ] + + # 拼接节点执行ip所需要的信息,ip信息统一用list处理拼接 + if kwargs["get_trans_data_ip_var"]: + exec_ips = self.splice_exec_ips_list( + ticket_ips=kwargs["exec_ip"], pool_ips=getattr(trans_data, kwargs["get_trans_data_ip_var"]) + ) + else: + exec_ips = self.splice_exec_ips_list(ticket_ips=kwargs["exec_ip"]) + + if not exec_ips: + self.log_error(_("该节点获取到执行ip信息为空,请联系系统管理员{}").format(exec_ips)) + return False + + target_ip_info = [{"bk_cloud_id": kwargs["bk_cloud_id"], "ip": ip} for ip in exec_ips] + self.log_info("{} exec {}".format(target_ip_info, kwargs["node_name"])) + + # 获取 actuator 组件所需要执行的参数, + db_act_template = kwargs["db_act_template"] + db_act_template["root_id"] = root_id + db_act_template["node_id"] = node_id + db_act_template["version_id"] = self._runtime_attrs["version"] + db_act_template["uid"] = global_data["uid"] + db_act_template["payload"] = str( + base64.b64encode(json.dumps(db_act_template["payload"]).encode("utf-8")), "utf-8" + ) + + FlowNode.objects.filter(root_id=kwargs["root_id"], node_id=node_id).update(hosts=exec_ips) + + # 脚本内容 + jinja_env = Environment() + template = jinja_env.from_string(mongodb_actuator_template) + if kwargs.get("init_flag", False): + template = jinja_env.from_string(mongodb_os_init_actuator_template) + body = { + "bk_biz_id": env.JOB_BLUEKING_BIZ_ID, + "task_name": f"DBM_{node_name}_{node_id}", + "script_content": str(base64.b64encode(template.render(db_act_template).encode("utf-8")), "utf-8"), + "script_language": 1, + "target_server": {"ip_list": target_ip_info}, + } + + self.log_info( + "[{}] ready start task with body {} {}".format(node_name, mongodb_fast_execute_script_common_kwargs, body) + ) + resp = JobApi.fast_execute_script({**mongodb_fast_execute_script_common_kwargs, **body}, raw=True) + + # 传入调用结果,并单调监听任务状态 + data.outputs.ext_result = resp + data.outputs.exec_ips = exec_ips + return True + + def inputs_format(self) -> List: + return [ + Service.InputItem(name="kwargs", key="kwargs", type="dict", required=True), + Service.InputItem(name="global_data", key="global_data", type="dict", required=True), + ] + + def outputs_format(self) -> List: + return [Service.OutputItem(name="exec_ips", key="exec_ips", type="list")] + + +class ExecuteDBActuatorJobComponent(Component): + name = __name__ + code = "mongodb_db_actuator_execute" + bound_service = ExecuteDBActuatorJobService diff --git a/dbm-ui/backend/flow/plugins/components/collections/mongodb/exec_actuator_job2.py b/dbm-ui/backend/flow/plugins/components/collections/mongodb/exec_actuator_job2.py new file mode 100644 index 0000000000..b054ea44d2 --- /dev/null +++ b/dbm-ui/backend/flow/plugins/components/collections/mongodb/exec_actuator_job2.py @@ -0,0 +1,127 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. +Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +import base64 +import json +import logging +import re +from typing import List + +from django.utils.translation import ugettext as _ +from jinja2 import Environment +from pipeline.component_framework.component import Component +from pipeline.core.flow.activity import Service + +import backend.flow.utils.redis.redis_context_dataclass as flow_context +from backend import env +from backend.components import JobApi +from backend.flow.models import FlowNode +from backend.flow.plugins.components.collections.common.base_service import BkJobService +from backend.flow.utils.mongodb.mongodb_script_template import make_script_common_kwargs, mongodb_actuator_template2 + +logger = logging.getLogger("json") +cpl = re.compile("(?P.+?)") # 非贪婪模式,只匹配第一次出现的自定义tag + + +class _ExecBkJobService(BkJobService): + """ + 根据db-actuator组件,绑定fast_execute_script api接口访问。 + """ + + def _execute(self, data, parent_data) -> bool: + """ + 执行fast_execute_script脚本 + global_data 单据全局变量,格式字典 + trans_data 单据上下文 + kwargs 字典传入格式: + { + root_id: db-actuator任务必须参数,做录入日志平台的条件 + node_id: db-actuator任务必须参数,做录入日志平台的条件 + node_name: db-actuator任务必须参数,做录入日志平台的条件 + get_redis_payload_func : 表示获取执行 redis的db-actuator 参数方法名称,对应RedisActPayload类 + exec_ip: 表示执行的ip节点 + get_trans_data_ip_name: 表示从上下文获取到执行ip的变量名,对应单据的获取到上下文dataclass类 + cluster: 操作的集群名称 + extend_attr: 额外的字段数据,需要从RedisApplyContext中获取 + } + """ + kwargs = data.get_one_of_inputs("kwargs") + global_data = data.get_one_of_inputs("global_data") + trans_data = data.get_one_of_inputs("trans_data") + + if trans_data is None or trans_data == "${trans_data}": + # 表示没有加载上下文内容,则在此添加 + trans_data = getattr(flow_context, kwargs["set_trans_data_dataclass"])() + + root_id = kwargs["root_id"] + node_name = kwargs["node_name"] + node_id = kwargs["node_id"] + + exec_ips = self.splice_exec_ips_list(ticket_ips=kwargs["exec_ip"]) + + self.log_info("exec_ips {}".format(exec_ips)) + self.log_info("trans_data {}".format(trans_data)) + + if not exec_ips: + self.log_error(_("该节点获取到执行ip信息为空,请联系系统管理员{}").format(exec_ips)) + return False + + target_ip_info = [{"bk_cloud_id": kwargs["bk_cloud_id"], "ip": ip} for ip in exec_ips] + self.log_info("{} exec {}".format(target_ip_info, kwargs["node_name"])) + + # 获取 actuator 组件所需要执行的参数, + db_act_template = kwargs["db_act_template"] + db_act_template["root_id"] = root_id + db_act_template["node_id"] = node_id + db_act_template["version_id"] = self._runtime_attrs["version"] + db_act_template["uid"] = global_data["uid"] + db_act_template["payload"] = str( + base64.b64encode(json.dumps(db_act_template["payload"]).encode("utf-8")), "utf-8" + ) + + FlowNode.objects.filter(root_id=kwargs["root_id"], node_id=node_id).update(hosts=exec_ips) + + # 脚本内容 + jinja_env = Environment() + template = jinja_env.from_string(mongodb_actuator_template2) + body = { + "bk_biz_id": env.JOB_BLUEKING_BIZ_ID, + "task_name": f"DBM_{node_name}_{node_id}", + "script_content": str(base64.b64encode(template.render(db_act_template).encode("utf-8")), "utf-8"), + "script_language": 1, + "target_server": {"ip_list": target_ip_info}, + } + + self.log_info("[{}] ready start task with body {} {}".format(node_name, "", body)) + resp = JobApi.fast_execute_script( + {**(make_script_common_kwargs(timeout=3600, exec_account="root")), **body}, raw=True + ) + + # 传入调用结果,并单调监听任务状态 + data.outputs.ext_result = resp + data.outputs.exec_ips = exec_ips + return True + + @staticmethod + def inputs_format() -> List: + return [ + Service.InputItem(name="kwargs", key="kwargs", type="dict", required=True), + Service.InputItem(name="global_data", key="global_data", type="dict", required=True), + ] + + @staticmethod + def outputs_format() -> List: + return [Service.OutputItem(name="exec_ips", key="exec_ips", type="list")] + + +class ExecuteDBActuatorJobComponent(Component): + name = __name__ + code = "mongodb_db_actuator_execute2" + bound_service = _ExecBkJobService diff --git a/dbm-ui/backend/flow/plugins/components/collections/mongodb/get_manager_user_password.py b/dbm-ui/backend/flow/plugins/components/collections/mongodb/get_manager_user_password.py new file mode 100644 index 0000000000..facb9dd53c --- /dev/null +++ b/dbm-ui/backend/flow/plugins/components/collections/mongodb/get_manager_user_password.py @@ -0,0 +1,82 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. +Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +import logging +from typing import List + +from pipeline.component_framework.component import Component +from pipeline.core.flow.activity import Service + +from backend.flow.plugins.components.collections.common.base_service import BaseService +from backend.flow.utils.mongodb.mongodb_password import MongoDBPassword + +logger = logging.getLogger("json") + + +class ExecGetPasswordOperation(BaseService): + """ + NameServiceCreate服务 + """ + + def _execute(self, data, parent_data) -> bool: + """ + 执行创建名字服务功能的函数 + global_data 单据全局变量,格式字典 + kwargs 私有变量 + """ + + # 从流程节点中获取变量 + kwargs = data.get_one_of_inputs("kwargs") + trans_data = data.get_one_of_inputs("trans_data") + # + # if trans_data is None or trans_data == "${trans_data}": + # # 表示没有加载上下文内容,则在此添加 + # trans_data = getattr(flow_context, kwargs["set_trans_data_dataclass"])() + + # 从密码服务获取密码 + user_password = {} + for user in kwargs["users"]: + result = MongoDBPassword().create_user_password() + if result["password"] is None: + self.log_error("user:{} get password fail, error:{}".format(user, result["info"])) + return False + user_password[user] = result["password"] + self.log_info("manager users get password successfully") + if trans_data is None or trans_data == "${trans_data}": + data.outputs["trans_data"] = {} + if kwargs["set_name"]: + data.outputs["trans_data"][kwargs["set_name"]] = user_password + else: + data.outputs["trans_data"] = user_password + elif isinstance(trans_data, dict): + if kwargs["set_name"]: + # users_pwd_info[kwargs["set_name"]] = user_password + data.outputs["trans_data"][kwargs["set_name"]] = user_password + else: + data.outputs["trans_data"] = user_password + + return True + + # 流程节点输入参数 + def inputs_format(self) -> List: + return [ + Service.InputItem(name="kwargs", key="kwargs", type="dict", required=True), + Service.InputItem(name="global_data", key="global_data", type="dict", required=True), + ] + + +class ExecGetPasswordOperationComponent(Component): + """ + ExecGetPasswordOperation组件 + """ + + name = __name__ + code = "get_password_operation" + bound_service = ExecGetPasswordOperation diff --git a/dbm-ui/backend/flow/plugins/components/collections/mongodb/send_media.py b/dbm-ui/backend/flow/plugins/components/collections/mongodb/send_media.py new file mode 100644 index 0000000000..4db1ea55ae --- /dev/null +++ b/dbm-ui/backend/flow/plugins/components/collections/mongodb/send_media.py @@ -0,0 +1,92 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. +Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +import copy +import logging +from typing import List + +from django.utils.translation import ugettext as _ +from pipeline.component_framework.component import Component +from pipeline.core.flow.activity import Service + +from backend import env +from backend.components import JobApi +from backend.core import consts +from backend.flow.consts import MediumFileTypeEnum +from backend.flow.models import FlowNode +from backend.flow.plugins.components.collections.common.base_service import BkJobService + +logger = logging.getLogger("json") + + +class ExecSendMediaOperation(BkJobService): + """ + NameServiceCreate服务 + """ + + def _execute(self, data, parent_data) -> bool: + """ + 执行创建名字服务功能的函数 + global_data 单据全局变量,格式字典 + kwargs 私有变量 + """ + + # 从流程节点中获取变量 + kwargs = data.get_one_of_inputs("kwargs") + root_id = kwargs["root_id"] + node_name = kwargs["node_name"] + node_id = kwargs["node_id"] + exec_ips = kwargs["exec_ips"] + + # 介质下发 + # 拼接fast_trans_file 接口请求参数 + payload = copy.deepcopy(consts.BK_TRANSFER_REPO_PAYLOAD) + payload["bk_biz_id"] = env.JOB_BLUEKING_BIZ_ID + payload["file_source_list"].append( + { + "file_list": kwargs["file_list"], + "file_type": MediumFileTypeEnum.Repo.value, + "file_source_code": env.APP_CODE, + } + ) + payload["file_target_path"] = kwargs["file_target_path"] + payload["target_server"]["ip_list"] = kwargs["ip_list"] + self.log_info(_("[{}] 下发介质包参数:{}").format(node_name, payload)) + FlowNode.objects.filter(root_id=root_id, node_id=node_id).update(hosts=exec_ips) + + # 请求传输 + resp = JobApi.fast_transfer_file(payload, raw=True) + if resp["code"] != 0: + self.log_error(_("下发介质包失败,resp:{}").format(resp)) + return False + + # 传入调用结果,并单调监听任务状态 + data.outputs.ext_result = resp + return True + + # 流程节点输入参数 + def inputs_format(self) -> List: + return [ + Service.InputItem(name="kwargs", key="kwargs", type="dict", required=True), + Service.InputItem(name="global_data", key="global_data", type="dict", required=True), + ] + + def outputs_format(self) -> List: + return [Service.OutputItem(name="exec_ips", key="exec_ips", type="list")] + + +class ExecSendMediaOperationComponent(Component): + """ + ExecSendMediaOperation组件 + """ + + name = __name__ + code = "send_media" + bound_service = ExecSendMediaOperation diff --git a/dbm-ui/backend/flow/urls.py b/dbm-ui/backend/flow/urls.py index 299838733b..2000752b54 100644 --- a/dbm-ui/backend/flow/urls.py +++ b/dbm-ui/backend/flow/urls.py @@ -62,6 +62,12 @@ RedisClusterMigrateLoad, RedisClusterMigratePrecheck, ) +from backend.flow.views.mongodb_scene import ( + ClusterInstallApiView, + MongoBackupApiView, + MongoFakeInstallApiView, + MultiReplicasetInstallApiView, +) from backend.flow.views.mysql_add_slave import AddMysqlSlaveSceneApiView from backend.flow.views.mysql_add_slave_remote import AddMysqlSlaveRemoteSceneApiView from backend.flow.views.mysql_checksum import MysqlChecksumSceneApiView @@ -238,6 +244,12 @@ url(r"^scene/nameservice_polaris_create$", PolarisCreateSceneApiView.as_view()), url(r"^scene/nameservice_polaris_delete$", PolarisDeleteSceneApiView.as_view()), # name_service end + # mongodb start + url(r"^scene/multi_replicaset_create$", MultiReplicasetInstallApiView.as_view()), + url(r"^scene/cluster_create$", ClusterInstallApiView.as_view()), + url(r"^scene/mongo_backup$", MongoBackupApiView.as_view()), + url(r"^scene/install_rs_fake$", MongoFakeInstallApiView.as_view()), + # mongodb end url(r"^scene/install_mysql_apply$", InstallMySQLSingleSceneApiView.as_view()), url(r"^scene/install_mysql_ha_apply$", InstallMySQLHASceneApiView.as_view()), url(r"^scene/destroy_mysql_ha$", DestroyMySQLHASceneApiView.as_view()), diff --git a/dbm-ui/backend/flow/utils/mongodb/calculate_cluster.py b/dbm-ui/backend/flow/utils/mongodb/calculate_cluster.py new file mode 100644 index 0000000000..4c69a808f8 --- /dev/null +++ b/dbm-ui/backend/flow/utils/mongodb/calculate_cluster.py @@ -0,0 +1,122 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. +Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +from copy import deepcopy + +from backend.configuration.constants import AffinityEnum +from backend.db_meta.enums.cluster_type import ClusterType +from backend.flow.consts import MongoDBDomainPrefix, MongoDBTotalCache + + +def calculate_cluster(payload: dict) -> dict: + """ 计算cluster""" + + payload_clusters = {} + payload_clusters["uid"] = payload["uid"] + payload_clusters["created_by"] = payload["created_by"] + payload_clusters["bk_biz_id"] = payload["bk_biz_id"] + payload_clusters["ticket_type"] = payload["ticket_type"] + payload_clusters["cluster_type"] = payload["cluster_type"] + payload_clusters["city"] = payload["city_code"] + payload_clusters["app"] = payload["bk_app_abbr"] + app = payload["bk_app_abbr"] + payload_clusters["db_version"] = payload["db_version"] + # 目前只支持11个节点 + domain_prefix = [ + MongoDBDomainPrefix.M1, + MongoDBDomainPrefix.M2, + MongoDBDomainPrefix.M3, + MongoDBDomainPrefix.M4, + MongoDBDomainPrefix.M5, + MongoDBDomainPrefix.M6, + MongoDBDomainPrefix.M7, + MongoDBDomainPrefix.M8, + MongoDBDomainPrefix.M9, + MongoDBDomainPrefix.M10, + MongoDBDomainPrefix.BACKUP, + ] + + if payload["cluster_type"] == ClusterType.MongoReplicaSet.value: + payload_clusters["spec_id"] = payload["spec_id"] + payload_clusters["spec_config"] = payload["infos"][0]["resource_spec"]["spec_config"] + # 获取全部主机 + hosts = [] + for info in payload["infos"]: + for machine in info["mongo_machine_set"]: + hosts.append({"ip": machine["ip"], "bk_cloud_id": machine["bk_cloud_id"]}) + payload_clusters["hosts"] = hosts + # 获取复制集实例 + sets = [] + node_replica_count = payload["node_replica_count"] + print("node_replica_count") + print(node_replica_count) + port = payload["start_port"] + oplog_percent = payload["oplog_percent"] / 100 + data_disk = "/data1" + # 计算cacheSizeGB和oplogSizeMB bk_mem:MB ["/data1"]["size"]:GB + avg_mem_size_gb = int( + payload["infos"][0]["mongo_machine_set"][0]["bk_mem"] + * MongoDBTotalCache.Cache_Percent + / node_replica_count + / 1024 + ) + if payload["infos"][0]["mongo_machine_set"][0]["storage"].get("/data1"): + data_disk = "/data1" + elif payload["infos"][0]["mongo_machine_set"][0]["storage"].get("/data"): + data_disk = "/data" + oplog_size_mb = int( + payload["infos"][0]["mongo_machine_set"][0]["storage"].get(data_disk)["size"] * 1024 * oplog_percent + ) + # 分配机器 + for index, info in enumerate(payload["infos"]): + machines = [] + # 主从节点分布在不同的机房 + if payload["disaster_tolerance_level"] == AffinityEnum.CROS_SUBZONE: + mongo_machine_set = deepcopy(info["mongo_machine_set"]) + if machines: + machines.clear() + machines.append(mongo_machine_set[0]) + mongo_machine_set.remove(mongo_machine_set[0]) + for machine in mongo_machine_set: + if machine["sub_zone_id"] != machines[0]["sub_zone_id"]: + machines.append(machine) + break + mongo_machine_set.remove(machines[1]) + machines.extend(mongo_machine_set) + elif payload["disaster_tolerance_level"] == AffinityEnum.SAME_SUBZONE: + machines = info["mongo_machine_set"] + replica_sets = payload["replica_sets"][index * node_replica_count : node_replica_count * (index + 1)] + for replica_set_index, replica_set in enumerate(replica_sets): + skip_machine = True + if replica_set_index == 0: + skip_machine = False + nodes = [] + for machine_index, machine in enumerate(machines): + if machine_index == len(machines) - 1: + domain = "{}.{}.{}.db".format(domain_prefix[-1], replica_set["set_id"], app) + else: + domain = "{}.{}.{}.db".format(domain_prefix[machine_index], replica_set["set_id"], app) + nodes.append({"ip": machine["ip"], "bk_cloud_id": machine["bk_cloud_id"], "domain": domain}) + sets.append( + { + "set_id": replica_set["set_id"], + "alias": replica_set["name"], + "port": port, + "cacheSizeGB": avg_mem_size_gb, + "oplogSizeMB": oplog_size_mb, + "skip_machine": skip_machine, + "nodes": nodes, + } + ) + port += 1 + payload_clusters["sets"] = sets + return payload_clusters + elif "cluster_type" == ClusterType.MongoShardedCluster.value: + pass diff --git a/dbm-ui/backend/flow/utils/mongodb/mongodb_dataclass.py b/dbm-ui/backend/flow/utils/mongodb/mongodb_dataclass.py new file mode 100644 index 0000000000..e9ef52b067 --- /dev/null +++ b/dbm-ui/backend/flow/utils/mongodb/mongodb_dataclass.py @@ -0,0 +1,1026 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. +Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +import os +from dataclasses import dataclass +from typing import Any, List, Optional + +from backend.components import DBConfigApi +from backend.components.dbconfig.constants import FormatType, LevelName +from backend.configuration.constants import DBType +from backend.db_meta.enums import machine_type +from backend.db_meta.enums.cluster_entry_type import ClusterEntryType +from backend.db_meta.enums.cluster_type import ClusterType +from backend.db_meta.enums.instance_role import InstanceRole +from backend.db_meta.models import Cluster +from backend.db_meta.models.machine import Machine +from backend.db_package.models import Package +from backend.flow.consts import ConfigTypeEnum, MediumEnum, MongoDBActuatorActionEnum +from backend.flow.engine.bamboo.scene.common.get_file_list import GetFileList +from backend.flow.utils.mongodb import mongodb_password, mongodb_script_template +from backend.flow.utils.mongodb.mongodb_password import MongoDBPassword + + +@dataclass() +class ActKwargs: + """节点私有变量数据类""" + + def __init__(self): + # 传入流程信息 + self.payload: dict = None + # 流程id + self.root_id: int = None + # 创建单个复制集信息 + self.replicaset_info: dict = None + # 创建mongos + self.mongos_info: dict = None + # 集群类型 + self.cluster_type: str = None + # db大版本 + self.db_main_version: str = None + # 备份所在目录 + self.file_path: str = None + # 管理员用户 + self.manager_users: list = [ + MediumEnum.DbaUser.value, + MediumEnum.AppDbaUser.value, + MediumEnum.MonitorUser.value, + MediumEnum.AppMonitorUser.value, + ] + + def __get_define_config(self, namespace: str, conf_file: str, conf_type: str) -> Any: + """获取一些全局的参数配置""" + + data = DBConfigApi.query_conf_item( + params={ + "bk_biz_id": self.payload["bk_biz_id"], + "level_name": LevelName.APP, + "level_value": self.payload["bk_biz_id"], + "conf_file": conf_file, + "conf_type": conf_type, + "namespace": namespace, + "format": FormatType.MAP, + } + ) + return data["content"] + + def get_inti_info(self): + """获取初始信息一些信息""" + + # 集群类型 + self.cluster_type = self.payload["cluster_type"] + # db大版本 + self.db_main_version = str(self.payload["db_version"].split(".")[0]) + + def get_file_path(self): + """安装文件存放路径""" + + self.file_path = self.__get_define_config( + namespace=self.cluster_type, + conf_type=ConfigTypeEnum.DBConf, + conf_file="{}-{}".format("Mongodb", self.db_main_version), + )["file_path"] + + def get_backup_dir(self): + """安装文件存放路径""" + + resp = self.__get_define_config( + namespace=self.cluster_type, + conf_type=ConfigTypeEnum.DBConf, + conf_file="{}-{}".format("Mongodb", self.db_main_version), + ) + return resp["backup_dir"] + + def get_send_media_kwargs(self) -> dict: + """介质下发的kwargs""" + + file_list = GetFileList(db_type=DBType.MongoDB).mongodb_pkg(db_version=self.payload["db_version"]) + ip_list = self.payload["hosts"] + exec_ips = [host["ip"] for host in ip_list] + return { + "file_list": file_list, + "ip_list": ip_list, + "exec_ips": exec_ips, + "file_target_path": self.file_path + "/install", + } + + def get_os_init_kwargs(self) -> dict: + """os初始化的kwargs""" + + # 获取os配置 + result = self.__get_define_config( + namespace=self.cluster_type, + conf_type=ConfigTypeEnum.DBConf, + conf_file="{}-{}".format("Mongodb", self.db_main_version), + ) + # 获取os密码 + password_info = mongodb_password.MongoDBPassword().get_password_from_db( + "0.0.0.0", 0, self.payload["hosts"][0]["bk_cloud_id"], result["user"] + ) + if password_info["password"] is None: + raise ValueError( + "get password of os user:{} from password service fail, error:{}".format( + result["user"], password_info["info"] + ) + ) + return { + "init_flag": True, + "set_trans_data_dataclass": CommonContext.__name__, + "get_trans_data_ip_var": None, + "bk_cloud_id": self.payload["hosts"][0]["bk_cloud_id"], + "exec_ip": self.payload["hosts"], + "file_target_path": self.file_path + "/install", + "db_act_template": { + "action": MongoDBActuatorActionEnum.OsInit, + "file_path": self.file_path, + "user": result["user"], + "group": result["group"], + "data_dir": result["install_dir"], + "backup_dir": result["backup_dir"], + "payload": { + "user": result["user"], + "password": password_info["password"], + }, + }, + } + + def get_install_mongod_kwargs(self, node: dict, cluster_role: str) -> dict: + """复制集mongod安装的kwargs""" + + # 获取安装包名以及MD5值 + pkg = Package.get_latest_package( + version=self.payload["db_version"], pkg_type=MediumEnum.MongoDB, db_type=DBType.MongoDB + ) + # 获取配置 + db_config = self.__get_define_config( + namespace=self.cluster_type, + conf_type=ConfigTypeEnum.DBConf, + conf_file="{}-{}".format("Mongodb", self.db_main_version), + ) + db_config["cacheSizeGB"] = self.replicaset_info["cacheSizeGB"] + db_config["oplogSizeMB"] = self.replicaset_info["oplogSizeMB"] + return { + "set_trans_data_dataclass": CommonContext.__name__, + "get_trans_data_ip_var": None, + "bk_cloud_id": node["bk_cloud_id"], + "exec_ip": node["ip"], + "db_act_template": { + "action": MongoDBActuatorActionEnum.mongoDInstall, + "file_path": self.file_path, + "payload": { + "mediapkg": { + "pkg": os.path.basename(pkg.path), + "pkg_md5": pkg.md5, + }, + "ip": node["ip"], + "port": self.replicaset_info["port"], + "dbVersion": self.payload["db_version"], + "instanceType": MediumEnum.MongoD, + "app": self.payload["app"], + "setId": self.replicaset_info["set_id"], + "auth": True, + "clusterRole": cluster_role, + "dbConfig": db_config, + }, + }, + } + + def get_install_mongos_kwargs(self, node: dict) -> dict: + """mongos安装""" + + pkg = Package.get_latest_package( + version=self.payload["db_version"], pkg_type=MediumEnum.MongoDB, db_type=DBType.MongoDB + ) + # 获取配置 + db_config = self.__get_define_config( + namespace=self.cluster_type, + conf_type=ConfigTypeEnum.DBConf, + conf_file="{}-{}".format("Mongodb", self.db_main_version), + ) + # 获取configDB配置 + config_db = [ + "{}:{}".format(node["ip"], str(self.payload["config"]["port"])) for node in self.payload["config"]["nodes"] + ] + return { + "set_trans_data_dataclass": CommonContext.__name__, + "get_trans_data_ip_var": None, + "bk_cloud_id": node["bk_cloud_id"], + "exec_ip": node["ip"], + "db_act_template": { + "action": MongoDBActuatorActionEnum.mongoSInstall, + "file_path": self.file_path, + "payload": { + "mediapkg": { + "pkg": os.path.basename(pkg.path), + "pkg_md5": pkg.md5, + }, + "ip": node["ip"], + "port": self.mongos_info["port"], + "dbVersion": self.payload["db_version"], + "instanceType": MediumEnum.MongoS, + "app": self.payload["app"], + "setId": self.mongos_info["set_id"], + "auth": True, + "configDB": config_db, + "dbConfig": db_config, + }, + }, + } + + def get_replicaset_init_kwargs(self, config_svr: bool) -> dict: + """复制集初始化""" + + priority = {} + hidden = {} + instances = [ + "{}:{}".format(node["ip"], str(self.replicaset_info["port"])) for node in self.replicaset_info["nodes"] + ] + for index, instance in enumerate(instances): + if index == len(instances) - 1: + priority[instance] = 0 + hidden[instance] = True + else: + priority[instance] = 1 + hidden[instance] = False + + return { + "set_trans_data_dataclass": CommonContext.__name__, + "get_trans_data_ip_var": None, + "bk_cloud_id": self.replicaset_info["nodes"][0]["bk_cloud_id"], + "exec_ip": self.replicaset_info["nodes"][0]["ip"], + "db_act_template": { + "action": MongoDBActuatorActionEnum.InitReplicaset, + "file_path": self.file_path, + "payload": { + "ip": self.replicaset_info["nodes"][0]["ip"], + "port": self.replicaset_info["port"], + "app": self.payload["app"], + "setId": self.replicaset_info["set_id"], + "configSvr": config_svr, + "ips": instances, + "priority": priority, + "hidden": hidden, + }, + }, + } + + def get_add_relationship_to_meta_kwargs(self, replicaset_info: dict) -> dict: + """添加replicaset关系到meta的kwargs""" + + info = { + "bk_biz_id": int(self.payload["bk_biz_id"]), + "major_version": self.payload["db_version"], + "creator": self.payload["created_by"], + "region": self.payload["city"], + } + instance_role = [ + InstanceRole.MONGO_M1, + InstanceRole.MONGO_M2, + InstanceRole.MONGO_M3, + InstanceRole.MONGO_M4, + InstanceRole.MONGO_M5, + InstanceRole.MONGO_M6, + InstanceRole.MONGO_M7, + InstanceRole.MONGO_M8, + InstanceRole.MONGO_M9, + InstanceRole.MONGO_M10, + InstanceRole.MONGO_BACKUP, + ] + if self.payload["cluster_type"] == ClusterType.MongoReplicaSet.value: + info["cluster_type"] = ClusterType.MongoReplicaSet.value + info["skip_machine"] = replicaset_info["skip_machine"] + info["immute_domain"] = replicaset_info["nodes"][0]["domain"] + info["name"] = "{}-{}".format(self.payload["app"], replicaset_info["set_id"]) + info["alias"] = replicaset_info["alias"] + info["spec_id"] = self.payload["spec_id"] + info["spec_config"] = self.payload["spec_config"] + info["bk_cloud_id"] = replicaset_info["nodes"][0]["bk_cloud_id"] + info["db_module_id"] = 0 + # 复制集节点 + info["storages"] = [] + if len(replicaset_info["nodes"]) <= 11: + for index, node in enumerate(replicaset_info["nodes"]): + if index == len(replicaset_info["nodes"]) - 1: + info["storages"].append( + { + "role": instance_role[-1], + "ip": node["ip"], + "port": replicaset_info["port"], + "domain": node["domain"], + } + ) + else: + info["storages"].append( + { + "role": instance_role[index], + "ip": node["ip"], + "port": replicaset_info["port"], + "domain": node["domain"], + } + ) + elif self.payload["cluster_type"] == ClusterType.MongoShardedCluster.value: + info["cluster_type"] = ClusterType.MongoShardedCluster.value + info["name"] = "{}-{}".format(self.payload["app"], self.payload["config"]["set_id"]) + info["alias"] = self.payload["alias"] + info["bk_cloud_id"] = self.payload["config"]["nodes"][0]["bk_cloud_id"] + info["machine_specs"] = self.payload["machine_specs"] + info["immute_domain"] = self.payload["mongos"]["domain"] + # mongos + info["proxies"] = [ + {"ip": node["ip"], "port": self.payload["mongos"]["port"]} for node in self.payload["mongos"]["nodes"] + ] + # config + info["configs"] = [] + # TODO config name + for index, node in enumerate(self.payload["config"]["nodes"]): + if index == len(self.payload["config"]["nodes"]) - 1: + info["configs"].append( + {"ip": node["ip"], "port": self.payload["config"]["port"], "role": instance_role[-1]} + ) + else: + info["configs"].append( + {"ip": node["ip"], "port": self.payload["config"]["port"], "role": instance_role[index]} + ) + + # shard + info["storages"] = [] + for shard in self.payload["shards"]: + storage = { + "shard": "{}-{}".format(self.payload["app"], shard["set_id"]), + "nodes": [], + } + if len(shard["nodes"]) <= 11: + for index, node in enumerate(shard["nodes"]): + if index == len(shard["nodes"]) - 1: + storage["nodes"].append( + {"role": instance_role[-1], "ip": node["ip"], "port": shard["port"]} + ) + else: + storage["nodes"].append( + {"role": instance_role[index], "ip": node["ip"], "port": shard["port"]} + ) + info["storages"].append(storage) + return info + + def get_add_domain_to_dns_kwargs(self, cluster: bool) -> dict: + """添加域名到dns的kwargs""" + if not cluster: + domains = [ + { + "domain": node["domain"], + "instance_list": ["{}#{}".format(node["ip"], str(self.replicaset_info["port"]))], + } + for node in self.replicaset_info["nodes"] + ] + return { + "bk_biz_id": self.payload["bk_biz_id"], + "bk_cloud_id": self.replicaset_info["nodes"][0]["bk_cloud_id"], + "domains": domains, + } + else: + domains = [ + { + "domain": self.payload["mongos"]["domain"], + "instance_list": ["{}#{}".format(node["ip"], str(self.payload["mongos"]["port"]))], + } + for node in self.payload["mongos"]["nodes"] + ] + return { + "bk_biz_id": self.payload["bk_biz_id"], + "bk_cloud_id": self.payload["mongos"]["nodes"][0]["bk_cloud_id"], + "domains": domains, + } + + def get_init_exec_script_kwargs(self, script_type: str) -> dict: + """通过执行脚本""" + + db_init_set_status = False + create_extra_manager_user_status = False + if self.payload["cluster_type"] == ClusterType.MongoReplicaSet.value: + mongo_type = "replicaset" + set_name = "{}-{}".format(self.payload["app"], self.replicaset_info["set_id"]) + else: + mongo_type = "cluster" + set_name = "" + + if script_type == MediumEnum.MongoDBExtraUserCreate: + create_extra_manager_user_status = True + db_init_set_status = False + script = mongodb_script_template.mongo_extra_manager_user_create_js_script + elif script_type == MediumEnum.MongoDBInitSet: + create_extra_manager_user_status = False + db_init_set_status = True + script = mongodb_script_template.mongo_init_set_js_script + + return { + "set_trans_data_dataclass": CommonContext.__name__, + "get_trans_data_ip_var": None, + "db_init_set": db_init_set_status, + "create_extra_manager_user": create_extra_manager_user_status, + "set_name": set_name, + "bk_cloud_id": self.replicaset_info["nodes"][0]["bk_cloud_id"], + "exec_ip": self.replicaset_info["nodes"][0]["ip"], + "db_act_template": { + "action": MongoDBActuatorActionEnum.MongoExecuteScript, + "file_path": self.file_path, + "payload": { + "ip": self.replicaset_info["nodes"][0]["ip"], + "port": self.replicaset_info["port"], + "script": script, + "type": mongo_type, + "secondary": False, + "adminUsername": MediumEnum.DbaUser, + "adminPassword": "", + "repoUrl": "", + "repoUsername": "", + "repoToken": "", + "repoProject": "", + "repoRepo": "", + "repoPath": "", + }, + }, + } + + def get_add_manager_user_kwargs(self) -> dict: + """创建dba用户""" + + if self.payload["cluster_type"] == ClusterType.MongoReplicaSet.value: + set_name = "{}-{}".format(self.payload["app"], self.replicaset_info["set_id"]) + else: + set_name = "" + return { + "create_manager_user": True, + "set_trans_data_dataclass": CommonContext.__name__, + "get_trans_data_ip_var": None, + "set_name": set_name, + "bk_cloud_id": self.replicaset_info["nodes"][0]["bk_cloud_id"], + "exec_ip": self.replicaset_info["nodes"][0]["ip"], + "db_act_template": { + "action": MongoDBActuatorActionEnum.AddUser, + "file_path": self.file_path, + "payload": { + "ip": self.replicaset_info["nodes"][0]["ip"], + "port": self.replicaset_info["port"], + "instanceType": MediumEnum.MongoD, + "username": MediumEnum.DbaUser, + "password": "", + "adminUsername": "", + "adminPassword": "", + "authDb": MediumEnum.AuthDB, + "dbs": [], + "privileges": [MediumEnum.RootRole], + }, + }, + } + + def get_get_manager_password_kwargs(self) -> dict: + """获取用户密码""" + + if self.payload["cluster_type"] == ClusterType.MongoReplicaSet.value: + set_name = "{}-{}".format(self.payload["app"], self.replicaset_info["set_id"]) + else: + set_name = "" + return {"set_trans_data_dataclass": CommonContext.__name__, "set_name": set_name, "users": self.manager_users} + + def get_add_password_to_db_kwargs(self, usernames: list, info: dict) -> dict: + """添加密码到db""" + + if self.payload["cluster_type"] == ClusterType.MongoReplicaSet.value: + set_name = "{}-{}".format(self.payload["app"], self.replicaset_info["set_id"]) + else: + set_name = "" + + nodes = [ + {"ip": node["ip"], "port": info["port"], "bk_cloud_id": node["bk_cloud_id"]} for node in info["nodes"] + ] + return {"nodes": nodes, "usernames": usernames, "set_name": set_name, "operator": self.payload["created_by"]} + + def get_cluster_info_user(self, cluster_id: int, admin_user: str): + """创建/删除用户获取cluster信息""" + + # 获取集群信息 + cluster_info = MongoRepository().fetch_one_cluster(id=cluster_id) + bk_cloud_id = cluster_info.bk_cloud_id + exec_ip: str = None + port: int = None + instance_type: str = None + if cluster_info.cluster_type == ClusterType.MongoReplicaSet.value: + exec_ip = cluster_info.get_shards()[0].members[0].ip + port = int(cluster_info.get_shards()[0].members[0].port) + instance_type = MediumEnum.MongoD + elif cluster_info.cluster_type == ClusterType.MongoShardedCluster.value: + exec_ip = cluster_info.get_mongos()[0].ip + port = int(cluster_info.get_mongos()[0].port) + instance_type = MediumEnum.MongoS + + # 获取用户密码 + result = MongoDBPassword().get_password_from_db( + ip=exec_ip, port=port, bk_cloud_id=bk_cloud_id, username=admin_user + ) + if result["info"] != "": + raise ValueError("get password of dba fail, error:{}".format(result["info"])) + self.payload["db_version"] = cluster_info.major_version + self.payload["hosts"] = [{"ip": exec_ip}] + self.payload["bk_cloud_id"] = bk_cloud_id + self.payload["port"] = port + self.payload["instance_type"] = instance_type + self.payload["admin_password"] = result["password"] + + def get_user_kwargs(self, create: bool, admin_user: str) -> dict: + """用户""" + + if create: + # 创建 + return { + "set_trans_data_dataclass": CommonContext.__name__, + "get_trans_data_ip_var": None, + "bk_cloud_id": self.payload["bk_cloud_id"], + "exec_ip": self.payload["hosts"][0]["ip"], + "db_act_template": { + "action": MongoDBActuatorActionEnum.AddUser, + "file_path": self.file_path, + "payload": { + "ip": self.payload["hosts"][0]["ip"], + "port": self.payload["port"], + "instanceType": self.payload["instance_type"], + "username": self.payload["username"], + "password": self.payload["password"], + "adminUsername": admin_user, + "adminPassword": self.payload["admin_password"], + "authDb": self.payload["authDb"], + "dbs": self.payload["dbs"], + "privileges": self.payload["privileges"], + }, + }, + } + else: + # 删除 + return { + "set_trans_data_dataclass": CommonContext.__name__, + "get_trans_data_ip_var": None, + "bk_cloud_id": self.payload["bk_cloud_id"], + "exec_ip": self.payload["hosts"][0]["ip"], + "db_act_template": { + "action": MongoDBActuatorActionEnum.DeleteUser, + "file_path": self.file_path, + "payload": { + "ip": self.payload["hosts"][0]["ip"], + "port": self.payload["port"], + "instanceType": self.payload["instance_type"], + "username": self.payload["username"], + "adminUsername": admin_user, + "adminPassword": self.payload["admin_password"], + "authDb": self.payload["authDb"], + }, + }, + } + + def get_exec_script_kwargs(self, cluster_id: int, admin_user: str) -> dict: + """执行脚本""" + + return { + "set_trans_data_dataclass": CommonContext.__name__, + "get_trans_data_ip_var": None, + "bk_cloud_id": self.payload["bk_cloud_id"], + "exec_ip": self.payload["hosts"][0]["ip"], + "db_act_template": { + "action": MongoDBActuatorActionEnum.MongoExecuteScript, + "file_path": self.file_path, + "payload": { + "ip": self.payload["hosts"][0]["ip"], + "port": self.payload["port"], + "script": self.payload["script"], + "Type": self.payload["instance_type"], + "secondary": False, + "adminUsername": admin_user, + "adminPassword": self.payload["admin_password"], + "repoUrl": self.payload["fileserver"]["url"], + "repoUsername": self.payload["fileserver"]["username"], + "repoToken": self.payload["fileserver"]["password"], + "repoProject": self.payload["fileserver"]["project"], + "repoRepo": self.payload["fileserver"]["bucket"], + "repoPath": self.payload["rules"][self.payload["cluster_ids"].index(cluster_id)]["path"], + }, + }, + } + + def get_hosts_deinstall(self): + """获取所有需要卸载的cluster的hosts""" + + hosts = set() + for cluster_id in self.payload["cluster_ids"]: + cluster_info = MongoRepository().fetch_one_cluster(id=cluster_id) + if cluster_info.cluster_type == ClusterType.MongoReplicaSet.value: + for member in cluster_info.get_shards()[0].members: + hosts.add(member.ip) + elif cluster_info.cluster_type == ClusterType.MongoShardedCluster.value: + mongos = cluster_info.get_mongos() + shards = cluster_info.get_shards() + config = cluster_info.get_config() + for mongo in mongos: + hosts.add(mongo.ip) + for member in config.members: + hosts.add(member.ip) + for shard in shards: + for member in shard.members: + hosts.add(member.ip) + list_hosts = [] + for host in hosts: + list_hosts.append({"ip": host}) + self.payload["hosts"] = list_hosts + + def get_cluster_info_deinstall(self, cluster_id: str): + """卸载流程获取cluster信息""" + + cluster_info = MongoRepository().fetch_one_cluster(id=cluster_id) + self.payload["cluster_type"] = cluster_info.cluster_type + self.payload["app"] = cluster_info.name.split("-")[0] + self.payload["set_id"] = cluster_info.name.split("-")[1] + self.payload["bk_cloud_id"] = cluster_info.bk_cloud_id + nodes = [] + if cluster_info.cluster_type == ClusterType.MongoReplicaSet.value: + for member in cluster_info.get_shards()[0].members: + nodes.append( + { + "ip": member.ip, + "port": int(member.port), + "bk_cloud_id": member.bk_cloud_id, + "domain": member.domain, + } + ) + self.payload["nodes"] = nodes + elif cluster_info.cluster_type == ClusterType.MongoShardedCluster.value: + mongos = cluster_info.get_mongos() + shards = cluster_info.get_shards() + config = cluster_info.get_config() + mongos_nodes = [] + shards_nodes = [] + config_nodes = [] + for mongo in mongos: + mongos_nodes.append( + {"ip": mongo.ip, "port": int(mongo.port), "bk_cloud_id": mongo.bk_cloud_id, "domain": mongo.domain} + ) + for shard in shards: + shard_info = {"shard": shard.set_name} + nodes = [] + for member in shard.members: + nodes.append({"ip": member.ip, "port": int(member.port), "bk_cloud_id": member.bk_cloud_id}) + shard_info["nodes"] = nodes + shards_nodes.append(shard_info) + for member in config.members: + config_nodes.append({"ip": member.ip, "port": int(member.port), "bk_cloud_id": member.bk_cloud_id}) + self.payload["mongos_nodes"] = mongos_nodes + self.payload["shards_nodes"] = shards_nodes + self.payload["config_nodes"] = config_nodes + + def get_mongo_deinstall_kwargs(self, node_info: dict, instance_type: str, nodes_info) -> dict: + """卸载mongo""" + nodes = [] + for node in nodes_info: + nodes.append(node["ip"]) + return { + "set_trans_data_dataclass": CommonContext.__name__, + "get_trans_data_ip_var": None, + "bk_cloud_id": self.payload["bk_cloud_id"], + "exec_ip": node_info["ip"], + "db_act_template": { + "action": MongoDBActuatorActionEnum.MongoExecuteScript, + "file_path": self.file_path, + "payload": { + "ip": node_info["ip"], + "port": node_info["port"], + "app": self.payload["app"], + "setId": self.payload["set_id"], + "nodeInfo": nodes, + "instanceType": instance_type, + }, + }, + } + + def get_delete_domain_kwargs(self): + """删除dns""" + + del_domains = [] + if self.payload["cluster_type"] == ClusterType.MongoReplicaSet.value: + for node in self.payload["nodes"]: + del_domains.append( + {"domain": node["domain"], "del_instance_list": ["{}#{}".format(node["ip"], str(node["port"]))]} + ) + elif self.payload["cluster_type"] == ClusterType.MongoShardedCluster.value: + del_instance_list = [ + "{}#{}".format(node["ip"], str(node["port"])) for node in self.payload["mongos_nodes"] + ] + del_domains.append( + {"domain": self.payload["mongos_nodes"][0]["domain"], "del_instance_list": del_instance_list} + ) + + return { + "bk_cloud_id": self.payload["bk_cloud_id"], + "bk_biz_id": self.payload["bk_biz_id"], + "del_domains": del_domains, + } + + def get_delete_pwd_kwargs(self): + """删除密码""" + + instances = [] + if self.payload["cluster_type"] == ClusterType.MongoReplicaSet.value: + for node in self.payload["nodes"]: + instances.append({"ip": node["ip"], "port": node["port"], "bk_cloud_id": node["bk_cloud_id"]}) + elif self.payload["cluster_type"] == ClusterType.MongoShardedCluster.value: + for node in self.payload["mongos_nodes"]: + instances.append({"ip": node["ip"], "port": node["port"], "bk_cloud_id": node["bk_cloud_id"]}) + for node in self.payload["config_nodes"]: + instances.append({"ip": node["ip"], "port": node["port"], "bk_cloud_id": node["bk_cloud_id"]}) + for shard in self.payload["shards_nodes"]: + for node in shard["nodes"]: + instances.append({"ip": node["ip"], "port": node["port"], "bk_cloud_id": node["bk_cloud_id"]}) + return { + "instances": self.payload["bk_cloud_id"], + "usernames": [ + MediumEnum.DbaUser, + MediumEnum.AppDbaUser, + MediumEnum.MonitorUser, + MediumEnum.AppMonitorUser, + ], + } + + def get_cluster_by_ip_replace(self): + """通过ip获取cluster信息""" + + replace_replicaset = [] + for ip_info in self.payload["replicaset"]: + # 获取machine + machine = Machine.objects.filter(ip=ip_info["ip"], spec_id=ip_info["spec_id"])[0] + # 获取存储实例 + store_instances = machine.storageinstance_set.all() + for store_instance in store_instances: + name = store_instance.cluster.first().name + set_id = name.split("-")[1] + port = store_instance.port + cluster_info = { + "cluster_id": store_instance.cluster.first().id, + "cluster_info": { + "setId": set_id, + "port": port, + "nodes": [ + { + "ip": ip_info["target"]["ip"], + "bk_cloud_id": ip_info["target"]["bk_cloud_id"], + "domain": store_instance.bind_entry.first( + cluster_entry_type=ClusterEntryType.DNS + ).entry, + "bk_host_id": ip_info["target"]["bk_host_id"], + "status": ip_info["target"]["status"], + } + ], + "source": { + "ip": machine.ip, + "bk_cloud_id": machine.bk_cloud_id, + "spec_id": ip_info["spec_id"], + }, + }, + } + replace_replicaset.append(cluster_info) + # for ip_info in self.payload["cluster"]: + + +@dataclass() +class CommonContext: + """通用可读写上下文""" + + # 调用第三方接口返回的数据 + def __init__(self): + pass + + output: Optional[Any] = None + + +# entities +# Node -> ReplicaSet -> Cluster[Rs,ShardedCluster] + + +class MongoNode: + def __init__(self, ip, port, role, bk_cloud_id, domain): + self.ip: str = ip + self.port: str = port + self.role: str = role + self.bk_cloud_id: int = bk_cloud_id + self.domain: str = domain + + +class ReplicaSet: + def __init__(self): + self.set_name: str = None + self.members: List[MongoNode] = None + + # get_backup_node 返回MONGO_BACKUP member + def get_backup_node(self): + i = len(self.members) - 1 + while i >= 0: + if self.members[i].role == InstanceRole.MONGO_BACKUP: + return self.members[i] + i = i - 1 + + return None + + # get_not_backup_nodes 返回非MONGO_BACKUP的member + def get_not_backup_nodes(self): + members = [] + # i = len(self.members) - 1 + for m in self.members: + if m.role == InstanceRole.MONGO_BACKUP: + members.append(m) + + return members + + def get_bk_cloud_id(self): + for i in self.members: + return i.bk_cloud_id + return None + + +# MongoDBCluster 有cluster_id cluster_name cluster_type +class MongoDBCluster: + bk_cloud_id: int + bk_biz_id: int + creator: str + name: str + app: str + immute_domain: str + alias: str + major_version: str + region: str + cluster_type: str + id: str + + def __init__(self): + self.id: str = None + self.name: str = None + self.cluster_type: str = None + + # get_shards interface + def get_shards(self) -> List[ReplicaSet]: + pass + + def get_mongos(self) -> List[MongoNode]: + return None + + def get_config(self) -> ReplicaSet: + return None + + def get_bk_cloud_id(self): + pass + + +class ReplicaSetCluster(MongoDBCluster): + shard: ReplicaSet # storages + + def __init__(self): + self.cluster_type: str = ClusterType.MongoReplicaSet.value + + def get_shards(self): + return [self.shard] + + def get_mongos(self) -> List[MongoNode]: + return None + + +class ShardedCluster(MongoDBCluster): + shards: List[ReplicaSet] # storages + mongos: List[MongoNode] # proxies + config: ReplicaSet # configs + + def __init__(self): + self.cluster_type: str = ClusterType.MongoShardedCluster.value + + def get_shards(self) -> List[ReplicaSet]: + return self.shards + + def get_config(self) -> ReplicaSet: + return self.config + + def get_mongos(self) -> List[MongoNode]: + return self.mongos + + +# MongoRepository +# +class MongoRepository: + def __init__(self): + pass + + @classmethod + def fetch_many_cluster(cls, **kwargs): + rows: List[MongoDBCluster] = [] + v = Cluster.objects.filter(**kwargs) + for i in v: + if i.cluster_type == ClusterType.MongoReplicaSet.value: + row = ReplicaSetCluster() + row.id = i.id + row.name = i.name + row.cluster_type = i.cluster_type + row.major_version = i.major_version + row.bk_biz_id = i.bk_biz_id + row.immute_domain = i.immute_domain + row.bk_cloud_id = i.bk_cloud_id + + # MongoReplicaSet 只有一个Set + row.shard = ReplicaSet() + row.shard.set_name = row.name + row.shard.members = [] + + for m in i.storageinstance_set.all(): + row.shard.members.append( + MongoNode( + m.ip_port.split(":")[0], + str(m.port), + m.instance_role, + m.machine.bk_cloud_id, + m.bind_entry.first().entry, + ) + ) + + rows.append(row) + elif i.cluster_type == ClusterType.MongoShardedCluster.value: + row = ShardedCluster() + row.id = i.id + row.name = i.name + row.cluster_type = i.cluster_type + row.major_version = i.major_version + row.bk_biz_id = i.bk_biz_id + row.immute_domain = i.immute_domain + row.mongos = [] + row.shards = [] + + for m in i.proxyinstance_set.all(): + row.mongos.append( + MongoNode( + m.ip_port.split(":")[0], + str(m.port), + m.instance_role, + m.machine.bk_cloud_id, + m.bind_entry.first().entry, + ) + ) + + for m in i.nosqlstoragesetdtl_set.all(): + # seg_range + shard = ReplicaSet() + shard.set_name = m.seg_range + shard.members = [] + # add primary + node = m.instance + shard.members.append( + MongoNode( + node.ip_port.split(":")[0], + str(node.port), + node.instance_role, + node.machine.bk_cloud_id, + "", + ) + ) + + # add secondary + for e in m.instance.as_ejector.all(): + node = e.receiver + shard.members.append( + MongoNode( + node.ip_port.split(":")[0], + str(node.port), + node.instance_role, + node.machine.bk_cloud_id, + "", + ) + ) + + if m.instance.machine_type == machine_type.MachineType.MONOG_CONFIG.value: + row.config = shard + else: + row.shards.append(shard) + + rows.append(row) + + return rows + + @classmethod + def fetch_one_cluster(cls, **kwargs): + rows = cls.fetch_many_cluster(**kwargs) + if len(rows) > 0: + return rows[0] + return None + + @classmethod + def fetch_many_cluster_dict(cls, **kwargs): + clusters = cls.fetch_many_cluster(**kwargs) + clusters_map = {} + for cluster in clusters: + clusters_map[cluster.immute_domain.lower()] = cluster + return clusters_map diff --git a/dbm-ui/backend/flow/utils/mongodb/mongodb_password.py b/dbm-ui/backend/flow/utils/mongodb/mongodb_password.py new file mode 100644 index 0000000000..70c0240a85 --- /dev/null +++ b/dbm-ui/backend/flow/utils/mongodb/mongodb_password.py @@ -0,0 +1,100 @@ +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. +Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + +import base64 + +from backend.components import MySQLPrivManagerApi +from backend.flow.consts import MediumEnum, MongoDBPasswordRule, RequestResultCode + + +class MongoDBPassword(object): + """ + mongodb用户密码 + """ + + def __init__(self): + self.security_rule_name = MongoDBPasswordRule.RULE + self.component = MediumEnum.MongoDB.value + + @staticmethod + def base64_encode(password: str) -> str: + """进行base64编码""" + + return str(base64.b64encode(password.encode("utf-8")), "utf-8") + + @staticmethod + def base64_decode(password: str) -> str: + """进行base64解码""" + + return str(base64.b64decode(password), "utf-8") + + def create_user_password(self) -> dict: + """创建密码""" + + result = MySQLPrivManagerApi.get_random_string( + {"security_rule_name": self.security_rule_name}, + raw=True, + ) + if result["code"] != RequestResultCode.Success: + return {"password": None, "info": result["message"]} + if self.base64_decode(result["data"])[0] == "-" or self.base64_decode(result["data"])[0:2] == "--": + return {"password": self.base64_decode(result["data"]).replace("-", "!"), "info": None} + return {"password": self.base64_decode(result["data"]), "info": None} + + def save_password_to_db(self, instances: list, username: str, password: str, operator: str) -> str: + """把密码保存到db中""" + + result = MySQLPrivManagerApi.modify_password( + { + "instances": instances, + "username": username, + "component": self.component, + "password": self.base64_encode(password), + "operator": operator, + "security_rule_name": self.security_rule_name, + }, + raw=True, + ) + if result["code"] != RequestResultCode.Success: + return result["message"] + " " + result["data"] + + def delete_password_from_db(self, instances: list, usernames: list) -> str: + """ + 从db中删除密码 + instances [{"ip":"x.x.x.x","port":1234,"bk_cloud_id":0}] + usernames ["user1", "user2"] + """ + + users = [] + for user in usernames: + users.append({"username": user, "component": self.component}) + result = MySQLPrivManagerApi.delete_password( + { + "instances": instances, + "users": users, + }, + raw=True, + ) + if result["code"] != RequestResultCode.Success: + return result["message"] + + def get_password_from_db(self, ip: str, port: int, bk_cloud_id: int, username: str) -> dict: + """从db获取密码""" + + result = MySQLPrivManagerApi.get_password( + { + "instances": [{"ip": ip, "port": port, "bk_cloud_id": bk_cloud_id}], + "users": [{"username": username, "component": self.component}], + }, + raw=True, + ) + if result["code"] != RequestResultCode.Success: + return {"password": None, "info": result["message"]} + return {"password": self.base64_decode(result["data"]["items"][0]["password"]), "info": None} diff --git a/dbm-ui/backend/flow/utils/mongodb/mongodb_script_template.py b/dbm-ui/backend/flow/utils/mongodb/mongodb_script_template.py new file mode 100644 index 0000000000..544b75dd82 --- /dev/null +++ b/dbm-ui/backend/flow/utils/mongodb/mongodb_script_template.py @@ -0,0 +1,118 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. +Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +# fast_execute_script接口固定参数 +# 这里独立出来,遇到过全局变量被其他db修改,导致用户错乱的问题 +mongodb_fast_execute_script_common_kwargs = { + "timeout": 3600, + "account_alias": "root", + "is_param_sensitive": 0, +} + +mongodb_actuator_template = """ +mkdir -p {{file_path}}/install/dbactuator-{{uid}}/logs +cp {{file_path}}/install/mongo-dbactuator {{file_path}}/install/dbactuator-{{uid}} +cd {{file_path}}/install/dbactuator-{{uid}} +chmod +x mongo-dbactuator +./mongo-dbactuator --uid {{uid}} --root_id {{root_id}} --node_id {{node_id}} \ +--version_id {{version_id}} --payload {{payload}} --atom-job-list {{action}} +""" + +mongodb_script_template = {"mongodb_actuator_template": mongodb_actuator_template} + +mongodb_os_init_actuator_template = """ +mkdir -p {{file_path}}/install/dbactuator-{{uid}}/logs +cp {{file_path}}/install/mongo-dbactuator {{file_path}}/install/dbactuator-{{uid}} +cd {{file_path}}/install/dbactuator-{{uid}} +chmod +x mongo-dbactuator +./mongo-dbactuator --uid {{uid}} --root_id {{root_id}} --node_id {{node_id}} \ +--version_id {{version_id}} --payload {{payload}} --atom-job-list {{action}} \ +--data_dir={{data_dir}} --backup_dir={{backup_dir}} --user={{user}} --group={{group}} +""" + +mongo_init_set_js_script = """ +db = db.getSiblingDB('admin'); +var num = db.system.roles.count({'_id':'admin.applyOps'}); +if (num == 0) { + db.createRole({role:'applyOps',privileges:[{resource:{anyResource:true},actions:["anyAction"]}],roles:[]}); + db.grantRolesToUser('dba',[{role:'applyOps',db:'admin'}]); + db.grantRolesToUser('appdba',[{role:'applyOps',db:'admin'}]); +} +var num = db.system.roles.count({'_id':'admin.heartbeatOps'}); +if (num == 0) { + db.createRole({role:'heartbeatOps',privileges:[{resource:{db:'admin',collection:'gcs_heartbeat'}, +actions:['find','insert','update','remove']}],roles:[]}); + db.grantRolesToUser('monitor',[{role:'heartbeatOps',db:'admin'}]); +} +var v = db.version(); +if (v.match(/^3\\./)) { + db.system.version.insert({ '_id' : 'authSchema', 'currentVersion' : 3 }); +} +""" + +mongo_extra_manager_user_create_js_script = """ +db = db.getSiblingDB('admin'); +var v = db.version(); +var main = v.slice(0,1); +var int_main = Number(main); +var num = db.system.users.count({'_id' : 'admin.appdba'}); +if (num == 0) { + if (int_main >= 3) { + db.createUser({user:'appdba',pwd:'{{appdba_pwd}}', + roles:[{role:'userAdminAnyDatabase',db:'admin'},{role:'dbAdminAnyDatabase',db:'admin'}, + {role:'readWriteAnyDatabase',db:'admin'},{role:'clusterAdmin',db:'admin'}]}); + } else { + db.addUser({user:'appdba',pwd:'{{appdba_pwd}}', + roles:[{role:'userAdminAnyDatabase',db:'admin'},{role:'dbAdminAnyDatabase',db:'admin'}, + {role:'readWriteAnyDatabase',db:'admin'},{role:'clusterAdmin',db:'admin'}]}); + } +} +var num = db.system.users.count({'_id' : 'admin.monitor'}); +if (num == 0) { + if (int_main >= 3) { + db.createUser({user:'monitor',pwd:'{{monitor_pwd}}', + roles:[{role:'backup',db:'admin'},{role:'clusterMonitor',db:'admin'}, + {role:'readAnyDatabase',db:'admin'},{role:'hostManager',db:'admin'}]}); + } else { + db.addUser({user:'monitor',pwd:'{{monitor_pwd}}', + roles:[{role:'backup',db:'admin'},{role:'clusterMonitor',db:'admin'}, + {role:'readAnyDatabase',db:'admin'},{role:'hostManager',db:'admin'}]}); + } +} +var num = db.system.users.count({'_id' : 'admin.appmonitor'}); +if (num == 0) { + if (int_main >= 3) { + db.createUser({user:'appmonitor',pwd:'{{appmonitor_pwd}}', + roles:[{role:'backup',db:'admin'},{role:'clusterMonitor',db:'admin'}, + {role:'readAnyDatabase',db:'admin'},{role:'hostManager',db:'admin'}]}); + } else { + db.addUser({user:'appmonitor',pwd:'{{appmonitor_pwd}}', + roles:[{role:'backup',db:'admin'},{role:'clusterMonitor',db:'admin'}, + {role:'readAnyDatabase',db:'admin'},{role:'hostManager',db:'admin'}]}); + } +} +""" + +mongodb_actuator_template2 = """ +mkdir -p {{file_path}}/install/dbactuator-{{uid}}/logs +cp {{file_path}}/install/mongo-dbactuator {{file_path}}/install/dbactuator-{{uid}} +cd {{file_path}}/install/dbactuator-{{uid}} +chmod +x mongo-dbactuator +./mongo-dbactuator --uid {{uid}} --root_id {{root_id}} --node_id {{node_id}} \ +--version_id {{version_id}} --payload {{payload}} --atom-job-list {{action}} +""" + + +def make_script_common_kwargs(timeout=3600, exec_account="root", is_param_sensitive=0): + return { + "timeout": timeout, + "account_alias": exec_account, + "is_param_sensitive": is_param_sensitive, + } diff --git a/dbm-ui/backend/flow/views/mongodb_scene.py b/dbm-ui/backend/flow/views/mongodb_scene.py new file mode 100644 index 0000000000..b1ecd6f435 --- /dev/null +++ b/dbm-ui/backend/flow/views/mongodb_scene.py @@ -0,0 +1,75 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. +Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +import logging +import uuid + +from rest_framework.response import Response + +from backend.flow.engine.controller.mongodb import MongoDBController +from backend.flow.views.base import FlowTestView + +logger = logging.getLogger("root") + + +class MultiReplicasetInstallApiView(FlowTestView): + """复制集安装""" + + @staticmethod + def post(request): + root_id = uuid.uuid1().hex + MongoDBController(root_id=root_id, ticket_data=request.data).multi_replicaset_create() + return Response({"root_id": root_id}) + + +class ClusterInstallApiView(FlowTestView): + """cluster安装""" + + @staticmethod + def post(request): + root_id = uuid.uuid1().hex + MongoDBController(root_id=root_id, ticket_data=request.data).cluster_create() + return Response({"root_id": root_id}) + + +class MongoBackupApiView(FlowTestView): + """ + Mongo Backup Api + """ + + @staticmethod + def post(request): + """ + mongo_backup + """ + root_id = uuid.uuid1().hex + # root_id 32位字串 320ba2d87a1411eebbed525400066689 + # request 是一个request对象 + # request.data 输入Json + MongoDBController(root_id=root_id, ticket_data=request.data).mongo_backup() + return Response({"root_id": root_id}) + + +class MongoFakeInstallApiView(FlowTestView): + """ + Mongo Backup Api + """ + + @staticmethod + def post(request): + """ + mongo_backup + """ + root_id = uuid.uuid1().hex + # root_id 32位字串 320ba2d87a1411eebbed525400066689 + # request 是一个request对象 + # request.data 输入Json + MongoDBController(root_id=root_id, ticket_data=request.data).fake_install() + return Response({"root_id": root_id})