From 900cf1f6176027125e856c105770bb439e532ca5 Mon Sep 17 00:00:00 2001 From: pycook Date: Mon, 25 Nov 2024 20:19:01 +0800 Subject: [PATCH] feat(api): update ipam --- cmdb-api/api/lib/cmdb/cache.py | 2 +- cmdb-api/api/lib/cmdb/ci.py | 12 +++++- cmdb-api/api/lib/cmdb/ci_type.py | 7 +++- cmdb-api/api/lib/cmdb/ipam/address.py | 40 ++++++++----------- cmdb-api/api/lib/cmdb/ipam/history.py | 4 ++ cmdb-api/api/lib/cmdb/ipam/stats.py | 16 ++++---- cmdb-api/api/lib/cmdb/ipam/subnet.py | 25 ++++++++---- cmdb-api/api/models/cmdb.py | 1 + .../ipam/{histories.py => ipam_history.py} | 0 9 files changed, 65 insertions(+), 42 deletions(-) rename cmdb-api/api/views/cmdb/ipam/{histories.py => ipam_history.py} (100%) diff --git a/cmdb-api/api/lib/cmdb/cache.py b/cmdb-api/api/lib/cmdb/cache.py index 6ce0aa67..b51a310e 100644 --- a/cmdb-api/api/lib/cmdb/cache.py +++ b/cmdb-api/api/lib/cmdb/cache.py @@ -512,7 +512,7 @@ def flush_adc_counter(cls): result[i.type_id]['rule_count'] = len(adts) + AutoDiscoveryCITypeRelation.get_by( ad_type_id=i.type_id, only_query=True).count() result[i.type_id]['exec_target_count'] = len( - set([i.oneagent_id for adt in adts for i in db.session.query( + set([j.oneagent_id for adt in adts for j in db.session.query( AutoDiscoveryRuleSyncHistory.oneagent_id).filter( AutoDiscoveryRuleSyncHistory.adt_id == adt.id)])) diff --git a/cmdb-api/api/lib/cmdb/ci.py b/cmdb-api/api/lib/cmdb/ci.py index 30a6b616..b048ed3d 100644 --- a/cmdb-api/api/lib/cmdb/ci.py +++ b/cmdb-api/api/lib/cmdb/ci.py @@ -357,6 +357,7 @@ def add(cls, ci_type_name, is_auto_discovery=False, _is_admin=False, ticket_id=None, + _sync=False, **ci_dict): """ add ci @@ -366,6 +367,7 @@ def add(cls, ci_type_name, :param is_auto_discovery: default is False :param _is_admin: default is False :param ticket_id: + :param _sync: :param ci_dict: :return: """ @@ -496,10 +498,16 @@ def add(cls, ci_type_name, record_id = cls.save_password(ci.id, attr_id, password_dict[attr_id], record_id, ci_type.id) if record_id or has_dynamic: # has changed - ci_cache.apply_async(args=(ci.id, operate_type, record_id), queue=CMDB_QUEUE) + if not _sync: + ci_cache.apply_async(args=(ci.id, operate_type, record_id), queue=CMDB_QUEUE) + else: + ci_cache(ci.id, operate_type, record_id) if ref_ci_dict: # add relations - ci_relation_add.apply_async(args=(ref_ci_dict, ci.id, current_user.uid), queue=CMDB_QUEUE) + if not _sync: + ci_relation_add.apply_async(args=(ref_ci_dict, ci.id, current_user.uid), queue=CMDB_QUEUE) + else: + ci_relation_add(ref_ci_dict, ci.id, current_user.uid) return ci.id diff --git a/cmdb-api/api/lib/cmdb/ci_type.py b/cmdb-api/api/lib/cmdb/ci_type.py index f0eae839..36f421f0 100644 --- a/cmdb-api/api/lib/cmdb/ci_type.py +++ b/cmdb-api/api/lib/cmdb/ci_type.py @@ -879,6 +879,8 @@ def get_children(_id): def _wrap_relation_type_dict(type_id, relation_inst): ci_type_dict = CITypeCache.get(type_id).to_dict() ci_type_dict["ctr_id"] = relation_inst.id + show_key = AttributeCache.get(ci_type_dict.get('show_id') or ci_type_dict['unique_id']) + ci_type_dict["show_key"] = show_key and show_key.name ci_type_dict["attributes"] = CITypeAttributeManager.get_attributes_by_type_id(ci_type_dict["id"]) attr_filter = CIFilterPermsCRUD.get_attr_filter(type_id) if attr_filter: @@ -1551,7 +1553,10 @@ def _import_attribute_group(type2attribute_group, type_id_map, attr_id_map): if existed is None: _group['type_id'] = type_id_map.get(_group['type_id'], _group['type_id']) - existed = CITypeAttributeGroup.create(flush=True, **_group) + try: + existed = CITypeAttributeGroup.create(flush=True, **_group) + except: + continue for order, attr in enumerate(group['attributes'] or []): item_existed = CITypeAttributeGroupItem.get_by(group_id=existed.id, diff --git a/cmdb-api/api/lib/cmdb/ipam/address.py b/cmdb-api/api/lib/cmdb/ipam/address.py index 1ebb82be..b9bcfe5f 100644 --- a/cmdb-api/api/lib/cmdb/ipam/address.py +++ b/cmdb-api/api/lib/cmdb/ipam/address.py @@ -3,7 +3,6 @@ import redis_lock from flask import abort -from api.extensions import db from api.extensions import rd from api.lib.cmdb.cache import CITypeCache from api.lib.cmdb.ci import CIManager @@ -21,9 +20,8 @@ class IpAddressManager(object): def __init__(self): - self.ci_type = CITypeCache.get(BuiltinModelEnum.IPAM_ADDRESS) - not self.ci_type and abort(400, ErrFormat.ipam_address_model_not_found.format( - BuiltinModelEnum.IPAM_ADDRESS)) + self.ci_type = CITypeCache.get(BuiltinModelEnum.IPAM_ADDRESS) or abort( + 404, ErrFormat.ipam_address_model_not_found.format(BuiltinModelEnum.IPAM_ADDRESS)) self.type_id = self.ci_type.id @@ -48,25 +46,28 @@ def _add_relation(parent_id, child_id): CIRelationManager().add(parent_id, child_id, valid=False, apply_async=False) @staticmethod - def calc_free_count(subnet_id): - db.session.commit() + def calc_used_count(subnet_id): q = "{}:(0;2),-{}:true".format(IPAddressBuiltinAttributes.ASSIGN_STATUS, IPAddressBuiltinAttributes.IS_USED) - return len(set(RelationSearch([subnet_id], level=[1], query=q).search(only_ids=True) or [])) + return len(set(RelationSearch([subnet_id], level=[1], query=q, count=1000000).search(only_ids=True) or [])) - def _update_subnet_count(self, subnet_id, assign_count, used_count=None): + @staticmethod + def _calc_assign_count(subnet_id): + q = "{}:(0;2)".format(IPAddressBuiltinAttributes.ASSIGN_STATUS) + + return len(set(RelationSearch([subnet_id], level=[1], query=q, count=1000000).search(only_ids=True) or [])) + + def _update_subnet_count(self, subnet_id, assign_count_computed, used_count=None): payload = {} cur = CIManager.get_ci_by_id(subnet_id, need_children=False) - if assign_count is not None: - payload[SubnetBuiltinAttributes.ASSIGN_COUNT] = (cur.get( - SubnetBuiltinAttributes.ASSIGN_COUNT) or 0) + assign_count - + if assign_count_computed: + payload[SubnetBuiltinAttributes.ASSIGN_COUNT] = self._calc_assign_count(subnet_id) if used_count is not None: payload[SubnetBuiltinAttributes.USED_COUNT] = used_count payload[SubnetBuiltinAttributes.FREE_COUNT] = (cur[SubnetBuiltinAttributes.HOSTS_COUNT] - - self.calc_free_count(subnet_id)) + self.calc_used_count(subnet_id)) CIManager().update(subnet_id, **payload) def assign_ips(self, ips, subnet_id, cidr, **kwargs): @@ -95,35 +96,28 @@ def assign_ips(self, ips, subnet_id, cidr, **kwargs): ip2ci = {ci[IPAddressBuiltinAttributes.IP]: ci for ci in cis} ci_ids = [] - status_change_num = 0 for ip in ips: kwargs['name'] = ip kwargs[IPAddressBuiltinAttributes.IP] = ip if ip not in ip2ci: ci_id = CIManager.add(self.type_id, _sync=True, **kwargs) - status_change_num += 1 else: ci_id = ip2ci[ip]['_id'] CIManager().update(ci_id, _sync=True, **kwargs) - if IPAddressBuiltinAttributes.ASSIGN_STATUS in kwargs and ( - (kwargs[IPAddressBuiltinAttributes.ASSIGN_STATUS] or 2) != - (ip2ci[ip].get(IPAddressBuiltinAttributes.ASSIGN_STATUS) or 2)): - status_change_num += 1 ci_ids.append(ci_id) self._add_relation(subnet_id, ci_id) if ips and IPAddressBuiltinAttributes.ASSIGN_STATUS in kwargs: - self._update_subnet_count(subnet_id, -status_change_num if kwargs.get( - IPAddressBuiltinAttributes.ASSIGN_STATUS) == IPAddressAssignStatus.UNASSIGNED else status_change_num) + self._update_subnet_count(subnet_id, True) if ips and IPAddressBuiltinAttributes.IS_USED in kwargs: q = "{}:true".format(IPAddressBuiltinAttributes.IS_USED) cur_used_ids = RelationSearch([subnet_id], level=[1], query=q).search(only_ids=True) for _id in set(cur_used_ids) - set(ci_ids): - CIManager().update(_id, _sync=True, **{IPAddressBuiltinAttributes.IS_USED: False}) + CIManager().update(_id, **{IPAddressBuiltinAttributes.IS_USED: False}) - self._update_subnet_count(subnet_id, None, used_count=len(ips)) + self._update_subnet_count(subnet_id, False, used_count=len(ips)) if kwargs.get(IPAddressBuiltinAttributes.ASSIGN_STATUS) in ( IPAddressAssignStatus.ASSIGNED, IPAddressAssignStatus.RESERVED): diff --git a/cmdb-api/api/lib/cmdb/ipam/history.py b/cmdb-api/api/lib/cmdb/ipam/history.py index 86a356f7..67b61284 100644 --- a/cmdb-api/api/lib/cmdb/ipam/history.py +++ b/cmdb-api/api/lib/cmdb/ipam/history.py @@ -50,6 +50,10 @@ def add(self, **kwargs): if scan_rule is not None: scan_rule.update(last_scan_time=kwargs.get('start_at')) + for i in self.cls.get_by(subnet_scan_id=kwargs.get('subnet_scan_id'), only_query=True).order_by( + self.cls.id.desc()).offset(100): + i.delete() + def _can_update(self, **kwargs): pass diff --git a/cmdb-api/api/lib/cmdb/ipam/stats.py b/cmdb-api/api/lib/cmdb/ipam/stats.py index cf412360..21a175a6 100644 --- a/cmdb-api/api/lib/cmdb/ipam/stats.py +++ b/cmdb-api/api/lib/cmdb/ipam/stats.py @@ -18,15 +18,13 @@ class Stats(object): def __init__(self): - self.address_type = CITypeCache.get(BuiltinModelEnum.IPAM_ADDRESS) - not self.address_type and abort(400, ErrFormat.ipam_address_model_not_found.format( - BuiltinModelEnum.IPAM_ADDRESS)) + self.address_type = CITypeCache.get(BuiltinModelEnum.IPAM_ADDRESS) or abort( + 404, ErrFormat.ipam_address_model_not_found.format(BuiltinModelEnum.IPAM_ADDRESS)) self.address_type_id = self.address_type.id - self.subnet_type = CITypeCache.get(BuiltinModelEnum.IPAM_SUBNET) - not self.subnet_type and abort(400, ErrFormat.ipam_address_model_not_found.format( - BuiltinModelEnum.IPAM_ADDRESS)) + self.subnet_type = CITypeCache.get(BuiltinModelEnum.IPAM_SUBNET) or abort( + 404, ErrFormat.ipam_address_model_not_found.format(BuiltinModelEnum.IPAM_ADDRESS)) self.subnet_type_id = self.subnet_type.id @@ -40,8 +38,10 @@ def leaf_nodes(self, parent_id): return list(set(ci_ids) - set(has_children_ci_ids)) else: - type_id = CIManager().get_by_id(parent_id).type_id - key = [(str(parent_id), type_id)] + _type = CIManager().get_by_id(parent_id) + if not _type: + return abort(404, ErrFormat.ipam_subnet_not_found) + key = [(str(parent_id), _type.type_id)] result = [] while True: res = [json.loads(x).items() for x in [i or '{}' for i in rd.get( diff --git a/cmdb-api/api/lib/cmdb/ipam/subnet.py b/cmdb-api/api/lib/cmdb/ipam/subnet.py index cd407502..c718cb95 100644 --- a/cmdb-api/api/lib/cmdb/ipam/subnet.py +++ b/cmdb-api/api/lib/cmdb/ipam/subnet.py @@ -2,6 +2,7 @@ from collections import defaultdict +import datetime import ipaddress from flask import abort @@ -9,7 +10,7 @@ from api.lib.cmdb.cache import CITypeCache from api.lib.cmdb.ci import CIManager from api.lib.cmdb.ci import CIRelationManager -from api.lib.cmdb.const import BuiltinModelEnum, BUILTIN_ATTRIBUTES +from api.lib.cmdb.const import BuiltinModelEnum from api.lib.cmdb.ipam.const import OperateTypeEnum from api.lib.cmdb.ipam.const import SubnetBuiltinAttributes from api.lib.cmdb.ipam.history import OperateHistoryManager @@ -22,9 +23,8 @@ class SubnetManager(object): def __init__(self): - self.ci_type = CITypeCache.get(BuiltinModelEnum.IPAM_SUBNET) - not self.ci_type and abort(400, ErrFormat.ipam_subnet_model_not_found.format( - BuiltinModelEnum.IPAM_SUBNET)) + self.ci_type = CITypeCache.get(BuiltinModelEnum.IPAM_SUBNET) or abort( + 404, ErrFormat.ipam_subnet_model_not_found.format(BuiltinModelEnum.IPAM_SUBNET)) self.type_id = self.ci_type.id @@ -47,7 +47,7 @@ def scan_rules(self, oneagent_id, last_update_at=None): new_last_update_at = "" for i in result: - __last_update_at = max([i['updated_at'] or "", i['created_at'] or ""]) + __last_update_at = max([i['rule_updated_at'] or "", i['created_at'] or ""]) if new_last_update_at < __last_update_at: new_last_update_at = __last_update_at @@ -131,7 +131,11 @@ def _build_tree(_tree, parent_id=None): @staticmethod def _is_valid_cidr(cidr): try: - return str(ipaddress.ip_network(cidr)) + cidr = ipaddress.ip_network(cidr) + if not (8 <= cidr.prefixlen <= 31): + raise ValueError + + return str(cidr) except ValueError: return abort(400, ErrFormat.ipam_cidr_invalid_notation.format(cidr)) @@ -143,6 +147,7 @@ def _check_root_node_is_overlapping(self, cidr, _id=None): root_nodes = set(all_nodes) - set(none_root_nodes) - set(_id and [_id] or []) response, _, _, _, _, _ = SearchFromDB("_type:{}".format(self.type_id), ci_ids=list(root_nodes), + count=1000000, parent_node_perm_passed=True).search() cur_subnet = ipaddress.ip_network(cidr) @@ -163,6 +168,7 @@ def _check_child_node_is_overlapping(self, parent_id, cidr, _id=None): response, _, _, _, _, _ = SearchFromDB("_type:{}".format(self.type_id), ci_ids=list(child_nodes), + count=1000000, parent_node_perm_passed=True).search() cur_subnet = ipaddress.ip_network(cidr) @@ -240,7 +246,8 @@ def _update_subnet(_id, **kwargs): def _update_scan_rule(ci_id, agent_id, cron, scan_enabled=True): existed = IPAMSubnetScan.get_by(ci_id=ci_id, first=True, to_dict=False) if existed is not None: - existed.update(ci_id=ci_id, agent_id=agent_id, cron=cron, scan_enabled=scan_enabled) + existed.update(ci_id=ci_id, agent_id=agent_id, cron=cron, scan_enabled=scan_enabled, + rule_updated_at=datetime.datetime.now()) else: IPAMSubnetScan.create(ci_id=ci_id, agent_id=agent_id, cron=cron, scan_enabled=scan_enabled) @@ -273,7 +280,9 @@ def delete(cls, _id): existed = IPAMSubnetScan.get_by(ci_id=_id, first=True, to_dict=False) existed and existed.delete() + delete_ci_ids = [] for i in CIRelation.get_by(first_ci_id=_id, to_dict=False): + delete_ci_ids.append(i.second_ci_id) i.delete() cur = CIManager.get_ci_by_id(_id, need_children=False) @@ -284,6 +293,8 @@ def delete(cls, _id): cidr=cur.get(SubnetBuiltinAttributes.CIDR), description=cur.get(SubnetBuiltinAttributes.CIDR)) + # batch_delete_ci.apply_async(args=(delete_ci_ids,)) + return _id diff --git a/cmdb-api/api/models/cmdb.py b/cmdb-api/api/models/cmdb.py index 2a9cc636..3936f757 100644 --- a/cmdb-api/api/models/cmdb.py +++ b/cmdb-api/api/models/cmdb.py @@ -676,6 +676,7 @@ class IPAMSubnetScan(Model): ci_id = db.Column(db.Integer, index=True, nullable=False) scan_enabled = db.Column(db.Boolean, default=True) + rule_updated_at = db.Column(db.DateTime) last_scan_time = db.Column(db.DateTime) # scan rules diff --git a/cmdb-api/api/views/cmdb/ipam/histories.py b/cmdb-api/api/views/cmdb/ipam/ipam_history.py similarity index 100% rename from cmdb-api/api/views/cmdb/ipam/histories.py rename to cmdb-api/api/views/cmdb/ipam/ipam_history.py