feat(api): update ipam

cmdb-api/api/lib/cmdb/cache.py

@@ -512,7 +512,7 @@ def flush_adc_counter(cls):
                 result[i.type_id]['rule_count'] = len(adts) + AutoDiscoveryCITypeRelation.get_by(
                     ad_type_id=i.type_id, only_query=True).count()
                 result[i.type_id]['exec_target_count'] = len(
-                    set([i.oneagent_id for adt in adts for i in db.session.query(
+                    set([j.oneagent_id for adt in adts for j in db.session.query(
                         AutoDiscoveryRuleSyncHistory.oneagent_id).filter(
                         AutoDiscoveryRuleSyncHistory.adt_id == adt.id)]))
 

cmdb-api/api/lib/cmdb/ci.py

@@ -357,6 +357,7 @@ def add(cls, ci_type_name,
             is_auto_discovery=False,
             _is_admin=False,
             ticket_id=None,
+            _sync=False,
             **ci_dict):
         """
         add ci
@@ -366,6 +367,7 @@ def add(cls, ci_type_name,
         :param is_auto_discovery: default is False
         :param _is_admin: default is False
         :param ticket_id:
+        :param _sync:
         :param ci_dict:
         :return:
         """
@@ -496,10 +498,16 @@ def add(cls, ci_type_name,
                 record_id = cls.save_password(ci.id, attr_id, password_dict[attr_id], record_id, ci_type.id)
 
         if record_id or has_dynamic:  # has changed
-            ci_cache.apply_async(args=(ci.id, operate_type, record_id), queue=CMDB_QUEUE)
+            if not _sync:
+                ci_cache.apply_async(args=(ci.id, operate_type, record_id), queue=CMDB_QUEUE)
+            else:
+                ci_cache(ci.id, operate_type, record_id)
 
         if ref_ci_dict:  # add relations
-            ci_relation_add.apply_async(args=(ref_ci_dict, ci.id, current_user.uid), queue=CMDB_QUEUE)
+            if not _sync:
+                ci_relation_add.apply_async(args=(ref_ci_dict, ci.id, current_user.uid), queue=CMDB_QUEUE)
+            else:
+                ci_relation_add(ref_ci_dict, ci.id, current_user.uid)
 
         return ci.id
 

cmdb-api/api/lib/cmdb/ci_type.py

@@ -879,6 +879,8 @@ def get_children(_id):
     def _wrap_relation_type_dict(type_id, relation_inst):
         ci_type_dict = CITypeCache.get(type_id).to_dict()
         ci_type_dict["ctr_id"] = relation_inst.id
+        show_key = AttributeCache.get(ci_type_dict.get('show_id') or ci_type_dict['unique_id'])
+        ci_type_dict["show_key"] = show_key and show_key.name
         ci_type_dict["attributes"] = CITypeAttributeManager.get_attributes_by_type_id(ci_type_dict["id"])
         attr_filter = CIFilterPermsCRUD.get_attr_filter(type_id)
         if attr_filter:
@@ -1551,7 +1553,10 @@ def _import_attribute_group(type2attribute_group, type_id_map, attr_id_map):
                 if existed is None:
                     _group['type_id'] = type_id_map.get(_group['type_id'], _group['type_id'])
 
-                    existed = CITypeAttributeGroup.create(flush=True, **_group)
+                    try:
+                        existed = CITypeAttributeGroup.create(flush=True, **_group)
+                    except:
+                        continue
 
                 for order, attr in enumerate(group['attributes'] or []):
                     item_existed = CITypeAttributeGroupItem.get_by(group_id=existed.id,

cmdb-api/api/lib/cmdb/ipam/address.py

@@ -3,7 +3,6 @@
 import redis_lock
 from flask import abort
 
-from api.extensions import db
 from api.extensions import rd
 from api.lib.cmdb.cache import CITypeCache
 from api.lib.cmdb.ci import CIManager
@@ -21,9 +20,8 @@
 
 class IpAddressManager(object):
     def __init__(self):
-        self.ci_type = CITypeCache.get(BuiltinModelEnum.IPAM_ADDRESS)
-        not self.ci_type and abort(400, ErrFormat.ipam_address_model_not_found.format(
-            BuiltinModelEnum.IPAM_ADDRESS))
+        self.ci_type = CITypeCache.get(BuiltinModelEnum.IPAM_ADDRESS) or abort(
+            404, ErrFormat.ipam_address_model_not_found.format(BuiltinModelEnum.IPAM_ADDRESS))
 
         self.type_id = self.ci_type.id
 
@@ -48,25 +46,28 @@ def _add_relation(parent_id, child_id):
         CIRelationManager().add(parent_id, child_id, valid=False, apply_async=False)
 
     @staticmethod
-    def calc_free_count(subnet_id):
-        db.session.commit()
+    def calc_used_count(subnet_id):
         q = "{}:(0;2),-{}:true".format(IPAddressBuiltinAttributes.ASSIGN_STATUS, IPAddressBuiltinAttributes.IS_USED)
 
-        return len(set(RelationSearch([subnet_id], level=[1], query=q).search(only_ids=True) or []))
+        return len(set(RelationSearch([subnet_id], level=[1], query=q, count=1000000).search(only_ids=True) or []))
 
-    def _update_subnet_count(self, subnet_id, assign_count, used_count=None):
+    @staticmethod
+    def _calc_assign_count(subnet_id):
+        q = "{}:(0;2)".format(IPAddressBuiltinAttributes.ASSIGN_STATUS)
+
+        return len(set(RelationSearch([subnet_id], level=[1], query=q, count=1000000).search(only_ids=True) or []))
+
+    def _update_subnet_count(self, subnet_id, assign_count_computed, used_count=None):
         payload = {}
 
         cur = CIManager.get_ci_by_id(subnet_id, need_children=False)
-        if assign_count is not None:
-            payload[SubnetBuiltinAttributes.ASSIGN_COUNT] = (cur.get(
-                SubnetBuiltinAttributes.ASSIGN_COUNT) or 0) + assign_count
-
+        if assign_count_computed:
+            payload[SubnetBuiltinAttributes.ASSIGN_COUNT] = self._calc_assign_count(subnet_id)
         if used_count is not None:
             payload[SubnetBuiltinAttributes.USED_COUNT] = used_count
 
         payload[SubnetBuiltinAttributes.FREE_COUNT] = (cur[SubnetBuiltinAttributes.HOSTS_COUNT] -
-                                                       self.calc_free_count(subnet_id))
+                                                       self.calc_used_count(subnet_id))
         CIManager().update(subnet_id, **payload)
 
     def assign_ips(self, ips, subnet_id, cidr, **kwargs):
@@ -95,35 +96,28 @@ def assign_ips(self, ips, subnet_id, cidr, **kwargs):
             ip2ci = {ci[IPAddressBuiltinAttributes.IP]: ci for ci in cis}
 
             ci_ids = []
-            status_change_num = 0
             for ip in ips:
                 kwargs['name'] = ip
                 kwargs[IPAddressBuiltinAttributes.IP] = ip
                 if ip not in ip2ci:
                     ci_id = CIManager.add(self.type_id, _sync=True, **kwargs)
-                    status_change_num += 1
                 else:
                     ci_id = ip2ci[ip]['_id']
                     CIManager().update(ci_id, _sync=True, **kwargs)
-                    if IPAddressBuiltinAttributes.ASSIGN_STATUS in kwargs and (
-                            (kwargs[IPAddressBuiltinAttributes.ASSIGN_STATUS] or 2) !=
-                            (ip2ci[ip].get(IPAddressBuiltinAttributes.ASSIGN_STATUS) or 2)):
-                        status_change_num += 1
                 ci_ids.append(ci_id)
 
                 self._add_relation(subnet_id, ci_id)
 
             if ips and IPAddressBuiltinAttributes.ASSIGN_STATUS in kwargs:
-                self._update_subnet_count(subnet_id, -status_change_num if kwargs.get(
-                    IPAddressBuiltinAttributes.ASSIGN_STATUS) == IPAddressAssignStatus.UNASSIGNED else status_change_num)
+                self._update_subnet_count(subnet_id, True)
 
             if ips and IPAddressBuiltinAttributes.IS_USED in kwargs:
                 q = "{}:true".format(IPAddressBuiltinAttributes.IS_USED)
                 cur_used_ids = RelationSearch([subnet_id], level=[1], query=q).search(only_ids=True)
                 for _id in set(cur_used_ids) - set(ci_ids):
-                    CIManager().update(_id, _sync=True, **{IPAddressBuiltinAttributes.IS_USED: False})
+                    CIManager().update(_id, **{IPAddressBuiltinAttributes.IS_USED: False})
 
-                self._update_subnet_count(subnet_id, None, used_count=len(ips))
+                self._update_subnet_count(subnet_id, False, used_count=len(ips))
 
         if kwargs.get(IPAddressBuiltinAttributes.ASSIGN_STATUS) in (
                 IPAddressAssignStatus.ASSIGNED, IPAddressAssignStatus.RESERVED):

cmdb-api/api/lib/cmdb/ipam/history.py

@@ -50,6 +50,10 @@ def add(self, **kwargs):
             if scan_rule is not None:
                 scan_rule.update(last_scan_time=kwargs.get('start_at'))
 
+        for i in self.cls.get_by(subnet_scan_id=kwargs.get('subnet_scan_id'), only_query=True).order_by(
+                self.cls.id.desc()).offset(100):
+            i.delete()
+
     def _can_update(self, **kwargs):
         pass
 

cmdb-api/api/lib/cmdb/ipam/stats.py

@@ -18,15 +18,13 @@
 
 class Stats(object):
     def __init__(self):
-        self.address_type = CITypeCache.get(BuiltinModelEnum.IPAM_ADDRESS)
-        not self.address_type and abort(400, ErrFormat.ipam_address_model_not_found.format(
-            BuiltinModelEnum.IPAM_ADDRESS))
+        self.address_type = CITypeCache.get(BuiltinModelEnum.IPAM_ADDRESS) or abort(
+            404, ErrFormat.ipam_address_model_not_found.format(BuiltinModelEnum.IPAM_ADDRESS))
 
         self.address_type_id = self.address_type.id
 
-        self.subnet_type = CITypeCache.get(BuiltinModelEnum.IPAM_SUBNET)
-        not self.subnet_type and abort(400, ErrFormat.ipam_address_model_not_found.format(
-            BuiltinModelEnum.IPAM_ADDRESS))
+        self.subnet_type = CITypeCache.get(BuiltinModelEnum.IPAM_SUBNET) or abort(
+            404, ErrFormat.ipam_address_model_not_found.format(BuiltinModelEnum.IPAM_ADDRESS))
 
         self.subnet_type_id = self.subnet_type.id
 
@@ -40,8 +38,10 @@ def leaf_nodes(self, parent_id):
             return list(set(ci_ids) - set(has_children_ci_ids))
 
         else:
-            type_id = CIManager().get_by_id(parent_id).type_id
-            key = [(str(parent_id), type_id)]
+            _type = CIManager().get_by_id(parent_id)
+            if not _type:
+                return abort(404, ErrFormat.ipam_subnet_not_found)
+            key = [(str(parent_id), _type.type_id)]
             result = []
             while True:
                 res = [json.loads(x).items() for x in [i or '{}' for i in rd.get(

cmdb-api/api/lib/cmdb/ipam/subnet.py

@@ -2,14 +2,15 @@
 
 from collections import defaultdict
 
+import datetime
 import ipaddress
 from flask import abort
 
 from api.lib.cmdb.cache import AttributeCache
 from api.lib.cmdb.cache import CITypeCache
 from api.lib.cmdb.ci import CIManager
 from api.lib.cmdb.ci import CIRelationManager
-from api.lib.cmdb.const import BuiltinModelEnum, BUILTIN_ATTRIBUTES
+from api.lib.cmdb.const import BuiltinModelEnum
 from api.lib.cmdb.ipam.const import OperateTypeEnum
 from api.lib.cmdb.ipam.const import SubnetBuiltinAttributes
 from api.lib.cmdb.ipam.history import OperateHistoryManager
@@ -22,9 +23,8 @@
 
 class SubnetManager(object):
     def __init__(self):
-        self.ci_type = CITypeCache.get(BuiltinModelEnum.IPAM_SUBNET)
-        not self.ci_type and abort(400, ErrFormat.ipam_subnet_model_not_found.format(
-            BuiltinModelEnum.IPAM_SUBNET))
+        self.ci_type = CITypeCache.get(BuiltinModelEnum.IPAM_SUBNET) or abort(
+            404, ErrFormat.ipam_subnet_model_not_found.format(BuiltinModelEnum.IPAM_SUBNET))
 
         self.type_id = self.ci_type.id
 
@@ -47,7 +47,7 @@ def scan_rules(self, oneagent_id, last_update_at=None):
 
         new_last_update_at = ""
         for i in result:
-            __last_update_at = max([i['updated_at'] or "", i['created_at'] or ""])
+            __last_update_at = max([i['rule_updated_at'] or "", i['created_at'] or ""])
             if new_last_update_at < __last_update_at:
                 new_last_update_at = __last_update_at
 
@@ -131,7 +131,11 @@ def _build_tree(_tree, parent_id=None):
     @staticmethod
     def _is_valid_cidr(cidr):
         try:
-            return str(ipaddress.ip_network(cidr))
+            cidr = ipaddress.ip_network(cidr)
+            if not (8 <= cidr.prefixlen <= 31):
+                raise ValueError
+
+            return str(cidr)
         except ValueError:
             return abort(400, ErrFormat.ipam_cidr_invalid_notation.format(cidr))
 
@@ -143,6 +147,7 @@ def _check_root_node_is_overlapping(self, cidr, _id=None):
         root_nodes = set(all_nodes) - set(none_root_nodes) - set(_id and [_id] or [])
         response, _, _, _, _, _ = SearchFromDB("_type:{}".format(self.type_id),
                                                ci_ids=list(root_nodes),
+                                               count=1000000,
                                                parent_node_perm_passed=True).search()
 
         cur_subnet = ipaddress.ip_network(cidr)
@@ -163,6 +168,7 @@ def _check_child_node_is_overlapping(self, parent_id, cidr, _id=None):
 
         response, _, _, _, _, _ = SearchFromDB("_type:{}".format(self.type_id),
                                                ci_ids=list(child_nodes),
+                                               count=1000000,
                                                parent_node_perm_passed=True).search()
 
         cur_subnet = ipaddress.ip_network(cidr)
@@ -240,7 +246,8 @@ def _update_subnet(_id, **kwargs):
     def _update_scan_rule(ci_id, agent_id, cron, scan_enabled=True):
         existed = IPAMSubnetScan.get_by(ci_id=ci_id, first=True, to_dict=False)
         if existed is not None:
-            existed.update(ci_id=ci_id, agent_id=agent_id, cron=cron, scan_enabled=scan_enabled)
+            existed.update(ci_id=ci_id, agent_id=agent_id, cron=cron, scan_enabled=scan_enabled,
+                           rule_updated_at=datetime.datetime.now())
         else:
             IPAMSubnetScan.create(ci_id=ci_id, agent_id=agent_id, cron=cron, scan_enabled=scan_enabled)
 
@@ -273,7 +280,9 @@ def delete(cls, _id):
         existed = IPAMSubnetScan.get_by(ci_id=_id, first=True, to_dict=False)
         existed and existed.delete()
 
+        delete_ci_ids = []
         for i in CIRelation.get_by(first_ci_id=_id, to_dict=False):
+            delete_ci_ids.append(i.second_ci_id)
             i.delete()
 
         cur = CIManager.get_ci_by_id(_id, need_children=False)
@@ -284,6 +293,8 @@ def delete(cls, _id):
                                     cidr=cur.get(SubnetBuiltinAttributes.CIDR),
                                     description=cur.get(SubnetBuiltinAttributes.CIDR))
 
+        # batch_delete_ci.apply_async(args=(delete_ci_ids,))
+
         return _id
 
 

cmdb-api/api/models/cmdb.py

@@ -676,6 +676,7 @@ class IPAMSubnetScan(Model):
 
     ci_id = db.Column(db.Integer, index=True, nullable=False)
     scan_enabled = db.Column(db.Boolean, default=True)
+    rule_updated_at = db.Column(db.DateTime)
     last_scan_time = db.Column(db.DateTime)
 
     # scan rules