diff --git a/apps/backend/subscription/serializers.py b/apps/backend/subscription/serializers.py index d8ab5d605..70c41e10c 100644 --- a/apps/backend/subscription/serializers.py +++ b/apps/backend/subscription/serializers.py @@ -19,6 +19,7 @@ from apps.node_man import constants, models, tools from apps.node_man.models import ProcessStatus from apps.node_man.serializers import policy +from apps.node_man.serializers.base import SubScopeInstSelectorSerializer from apps.utils import basic @@ -27,7 +28,7 @@ class GatewaySerializer(serializers.Serializer): bk_app_code = serializers.CharField() -class ScopeSerializer(serializers.Serializer): +class ScopeSerializer(SubScopeInstSelectorSerializer): bk_biz_id = serializers.IntegerField(required=False, default=None) # TODO: 是否取消掉这个范围内的scope bk_biz_scope = serializers.ListField(required=False) @@ -124,7 +125,7 @@ class GetSubscriptionSerializer(GatewaySerializer): class UpdateSubscriptionSerializer(GatewaySerializer): - class UpdateScopeSerializer(serializers.Serializer): + class UpdateScopeSerializer(SubScopeInstSelectorSerializer): node_type = serializers.ChoiceField(choices=models.Subscription.NODE_TYPE_CHOICES) nodes = serializers.ListField() bk_biz_id = serializers.IntegerField(required=False, default=None) @@ -157,7 +158,7 @@ class SwitchSubscriptionSerializer(GatewaySerializer): class RunSubscriptionSerializer(GatewaySerializer): - class RunScopeSerializer(serializers.Serializer): + class RunScopeSerializer(SubScopeInstSelectorSerializer): node_type = serializers.ChoiceField(choices=models.Subscription.NODE_TYPE_CHOICES, label="节点类型") nodes = serializers.ListField(child=serializers.DictField(), label="拓扑节点列表") diff --git a/apps/backend/subscription/tools.py b/apps/backend/subscription/tools.py index 4310d9957..745c65baa 100644 --- a/apps/backend/subscription/tools.py +++ b/apps/backend/subscription/tools.py @@ -41,6 +41,7 @@ from apps.utils.batch_request import batch_request, request_multi_thread from apps.utils.cache import func_cache_decorator from apps.utils.time_handler import strftime_local +from apps.core.ipchooser.tools.base import HostQuerySqlHelper logger = logging.getLogger("app") @@ -680,6 +681,7 @@ def wrapper(scope: Dict[str, Union[Dict, Any]], *args, **kwargs) -> Dict[str, Di "object_type": scope["object_type"], "node_type": scope["node_type"], "nodes": list(nodes), + "instance_selector": scope.get("instance_selector") }, **kwargs, } @@ -727,6 +729,11 @@ def get_instances_by_scope(scope: Dict[str, Union[Dict, int, Any]]) -> Dict[str, "host|instance|host|yyyy": {...}, } """ + instance_selector = scope.get("instance_selector") + # 不进行主机筛选时传入 None，传入空列表则识别为全部过滤 + if instance_selector == []: + return {} + instances = [] bk_biz_id = scope["bk_biz_id"] if bk_biz_id: @@ -808,13 +815,35 @@ def get_instances_by_scope(scope: Dict[str, Union[Dict, int, Any]]) -> Dict[str, "object_type": scope["object_type"], "node_type": models.Subscription.NodeType.INSTANCE, } + + bk_host_ids = [] + for instance in instances: - if data["object_type"] == models.Subscription.ObjectType.HOST: - data.update(instance["host"]) - else: - data.update(instance["service"]) + is_host = data["object_type"] == models.Subscription.ObjectType.HOST + instance_data = instance["host"] if is_host else instance["service"] + + data.update(instance_data) + bk_host_ids.append(instance_data.get("bk_host_id")) instances_dict[create_node_id(data)] = instance + # 对 instances 进行二次过滤 + if instance_selector and bk_host_ids: + instance_selector_host_ids = HostQuerySqlHelper.multiple_cond_sql( + params={"bk_host_id": bk_host_ids, "conditions": instance_selector}, + biz_scope=[bk_biz_id], + return_all_node_type=True + ).values_list("bk_host_id", flat=True) + + selector_instances_dict = {} + for node_id, instance in instances_dict.items(): + is_host = data["object_type"] == models.Subscription.ObjectType.HOST + instance_data = instance["host"] if is_host else instance["service"] + + if instance_data["bk_host_id"] in instance_selector_host_ids: + selector_instances_dict[node_id] = instance if is_host else instance["service"] + + return selector_instances_dict + return instances_dict diff --git a/apps/backend/subscription/views.py b/apps/backend/subscription/views.py index a689f5e7b..1f490176c 100644 --- a/apps/backend/subscription/views.py +++ b/apps/backend/subscription/views.py @@ -84,6 +84,7 @@ def create_subscription(self, request): object_type=scope["object_type"], node_type=scope["node_type"], nodes=scope["nodes"], + instance_selector=scope.get("instance_selector"), target_hosts=params.get("target_hosts"), from_system=params["bk_app_code"] or "blueking", enable=enable, @@ -202,6 +203,9 @@ def update_subscription(self, request): subscription.node_type = scope["node_type"] subscription.nodes = scope["nodes"] subscription.bk_biz_id = scope.get("bk_biz_id") + # 避免空列表误判 + if scope.get("instance_selector") is not None: + subscription.instance_selector = scope["instance_selector"] # 策略部署新增 subscription.plugin_name = params.get("plugin_name") subscription.bk_biz_scope = params.get("bk_biz_scope") diff --git a/apps/backend/tests/subscription/test_tools.py b/apps/backend/tests/subscription/test_tools.py index e93ccddc6..e3042aff9 100644 --- a/apps/backend/tests/subscription/test_tools.py +++ b/apps/backend/tests/subscription/test_tools.py @@ -22,6 +22,7 @@ CmdbClient, list_biz_hosts_without_info_client, ) +from apps.node_man import models, constants # 全局使用的mock run_task = mock.patch("apps.backend.subscription.tasks.run_subscription_task").start() @@ -57,6 +58,25 @@ def setUp(self): self.get_process_by_biz_id_client.start() self.batch_request_client.start() + models.Host.objects.create( + bk_host_id=1, + bk_biz_id=2, + bk_cloud_id=0, + inner_ip="127.0.0.1", + outer_ip=None, + login_ip="127.0.0.1", + data_ip="127.0.0.1", + os_type="LINUX", + node_type="AGENT", + ap_id=1, + ) + models.ProcessStatus.objects.create( + bk_host_id=1, + name=models.ProcessStatus.GSE_AGENT_PROCESS_NAME, + proc_type=constants.ProcType.AGENT, + source_type=models.ProcessStatus.SourceType.DEFAULT, + ) + def tearDown(self): self.tools_client.stop() self.commons_client.stop() @@ -155,3 +175,31 @@ def test_get_service_instance_scope(self): instance = instances[instance_id] self.assertEqual(instance["service"]["id"], 10) self.assertSetEqual({"process", "scope", "host", "service"}, set(instance.keys())) + + def test_get_instance_selector_scope(self): + instances = get_instances_by_scope( + { + "bk_biz_id": 2, + "object_type": "HOST", + "node_type": "INSTANCE", + "instance_selector": [{"key": "os_type", "value": ["WINDOWS"]}], + "nodes": [ + {"ip": "127.0.0.1", "bk_cloud_id": 0, "bk_supplier_id": 0}, + ], + } + ) + self.assertEqual(len(list(instances.keys())), 0) + + def test_get_empty_list_instance_selector_scope(self): + instances = get_instances_by_scope( + { + "bk_biz_id": 2, + "object_type": "HOST", + "node_type": "INSTANCE", + "instance_selector": [], + "nodes": [ + {"ip": "127.0.0.1", "bk_cloud_id": 0, "bk_supplier_id": 0}, + ], + } + ) + self.assertEqual(len(list(instances.keys())), 0) diff --git a/apps/node_man/handlers/policy.py b/apps/node_man/handlers/policy.py index 983fa87a9..e0f7ae5e9 100644 --- a/apps/node_man/handlers/policy.py +++ b/apps/node_man/handlers/policy.py @@ -395,6 +395,7 @@ def migrate_preview(cls, query_params: Dict[str, Any]) -> List[Dict[str, Any]]: object_type=scope["object_type"], node_type=scope["node_type"], nodes=scope["nodes"], + instance_selector=scope.get("instance_selector"), target_hosts=query_params.get("target_hosts"), # SaaS侧均为主程序部署 is_main=True, diff --git a/apps/node_man/migrations/0072_subscription_instance_selector.py b/apps/node_man/migrations/0072_subscription_instance_selector.py new file mode 100644 index 000000000..c66939f3a --- /dev/null +++ b/apps/node_man/migrations/0072_subscription_instance_selector.py @@ -0,0 +1,19 @@ +# Generated by Django 3.2.4 on 2023-08-04 02:19 + +from django.db import migrations +import django_mysql.models + + +class Migration(migrations.Migration): + + dependencies = [ + ('node_man', '0071_update_ap_gse_version_to_v2'), + ] + + operations = [ + migrations.AddField( + model_name='subscription', + name='instance_selector', + field=django_mysql.models.JSONField(blank=True, default=dict, null=True, verbose_name='订阅任务范围主机属性筛选'), + ), + ] diff --git a/apps/node_man/models.py b/apps/node_man/models.py index be451fd26..6b6affa6d 100644 --- a/apps/node_man/models.py +++ b/apps/node_man/models.py @@ -1765,6 +1765,7 @@ class CategoryType(object): object_type = models.CharField(_("对象类型"), max_length=20, choices=OBJECT_TYPE_CHOICES, db_index=True) node_type = models.CharField(_("节点类型"), max_length=20, choices=NODE_TYPE_CHOICES, db_index=True) nodes = JSONField(_("节点"), default=list) + instance_selector = JSONField(_("订阅任务范围主机属性筛选"), null=True, blank=True) target_hosts = JSONField(_("下发的目标机器"), default=None, null=True) from_system = models.CharField(_("所属系统"), max_length=30) update_time = models.DateTimeField(_("更新时间"), auto_now=True, db_index=True) @@ -1808,6 +1809,7 @@ def scope(self): "node_type": self.node_type, "nodes": self.nodes, "need_register": need_register, + "instance_selector": self.instance_selector, } @classmethod diff --git a/apps/node_man/serializers/base.py b/apps/node_man/serializers/base.py index a019beab2..6faff445d 100644 --- a/apps/node_man/serializers/base.py +++ b/apps/node_man/serializers/base.py @@ -15,6 +15,15 @@ from apps.node_man import constants, exceptions, models +# 放在后台会导致循坏导入 +class SubScopeInstSelectorSerializer(serializers.Serializer): + instance_selector = serializers.ListField( + child=serializers.DictField(), + required=False, + label="实例筛选器" + ) + + # 安装插件配置 class StepSerializer(serializers.Serializer): class SettingSerializer(serializers.Serializer): @@ -46,7 +55,7 @@ def validate(self, data): # 策略范围 -class ScopeSerializer(serializers.Serializer): +class ScopeSerializer(SubScopeInstSelectorSerializer): class NodeSerializer(serializers.Serializer): bk_biz_id = serializers.IntegerField(label="业务ID") bk_inst_id = serializers.IntegerField(required=False, label="实例ID") diff --git a/docs/apidoc/create_subscription.md b/docs/apidoc/create_subscription.md index f02e54f43..cd1bd5557 100644 --- a/docs/apidoc/create_subscription.md +++ b/docs/apidoc/create_subscription.md @@ -34,12 +34,13 @@ | 字段 | 类型 | 必选 | 描述 | | ------------- | --------- | --- | ----------------------------------------------------------------------------------- | -| bk_biz_id | int | 否 | 蓝鲸业务ID | -| bk_biz_scope | int array | 否 | 蓝鲸业务ID列表 | -| node_type | string | 是 | 节点类别，1: TOPO，动态实例（拓扑）2: INSTANCE，静态实例 3: SERVICE_TEMPLATE，服务模板 4: SET_TEMPLATE，集群模板 | -| object_type | string | 是 | 对象类型，1：HOST，主机类型  2：SERVICE，服务类型 | -| need_register | bool | 否 | 是否需要注册到CMDB，false是不注册，true是注册。默认为不注册 | -| nodes | objects | 是 | 节点列表，见nodes定义 | +| bk_biz_id | int | 否 | 蓝鲸业务ID | +| bk_biz_scope | int array | 否 | 蓝鲸业务ID列表 | +| node_type | string | 是 | 节点类别，1: TOPO，动态实例（拓扑）2: INSTANCE，静态实例 3: SERVICE_TEMPLATE，服务模板 4: SET_TEMPLATE，集群模板 | +| object_type | string | 是 | 对象类型，1：HOST，主机类型  2：SERVICE，服务类型 | +| need_register | bool | 否 | 是否需要注册到CMDB，false是不注册，true是注册。默认为不注册 | +| nodes | objects | 是 | 节点列表，见nodes定义 +| instance_selector | objects | 否 | 主机属性筛选列表 | ##### config @@ -119,6 +120,13 @@ instance_info | enable_compression | bool | 否 | 数据压缩开关，默认是关闭 | | data_path | string | 否 | 数据文件路径 | +###### instance_selector + +| 字段 | 类型 | 必选 | 描述 | +| ------------------- | ------ | --- | ----------------------- | +| key | string | 否 | 主机属性 | +| value | string | 否 | 主机属性值列表 | + ###### job_type Agent @@ -184,6 +192,7 @@ Plugin "bk_token": "xxx", "run_immediately": true, "scope": { + "instance_selector": [{"key": "os_type", "value": ["LINUX"]}], "bk_biz_id": 2, "object_type": "SERVICE", "node_type": "TOPO", diff --git a/docs/apidoc/update_subscription.md b/docs/apidoc/update_subscription.md index a70749a9f..e921771cc 100644 --- a/docs/apidoc/update_subscription.md +++ b/docs/apidoc/update_subscription.md @@ -31,9 +31,10 @@ | 字段 | 类型 | 必选 | 描述 | | --------- | ------------- | --- | ----------------------------------------------------------------------------------- | -| bk_biz_id | int | 否 | 蓝鲸业务ID | -| node_type | string | 是 | 节点类别，1: TOPO，动态实例（拓扑）2: INSTANCE，静态实例 3: SERVICE_TEMPLATE，服务模板 4: SET_TEMPLATE，集群模板 | -| nodes | objects | 是 | 节点列表，见nodes定义 | +| bk_biz_id | int | 否 | 蓝鲸业务ID | +| node_type | string | 是 | 节点类别，1: TOPO，动态实例（拓扑）2: INSTANCE，静态实例 3: SERVICE_TEMPLATE，服务模板 4: SET_TEMPLATE，集群模板 | +| nodes | objects | 是 | 节点列表，见nodes定义 | +| instance_selector | objects | 否 | 主机属性筛选列表 | ##### config @@ -113,6 +114,13 @@ instance_info | data_path | string | 否 | 数据文件路径 | | enable_compression | bool | 否 | 数据压缩开关 | +###### instance_selector + +| 字段 | 类型 | 必选 | 描述 | +| ------------------- | ------ | --- | ----------------------- | +| key | string | 否 | 主机属性 | +| value | string | 否 | 主机属性值 | + ###### job_type Agent @@ -180,6 +188,7 @@ Plugin "run_immediately": true, "subscription_id": 1, "scope": { + "instance_selector": [{"key": "os_type", "value": ["LINUX"]}], "bk_biz_id": 2, "object_type": "SERVICE", "node_type": "TOPO",