From 3f640e8018292919f6a7483398934d4e7bbdf379 Mon Sep 17 00:00:00 2001 From: tyderian1978 Date: Tue, 17 Nov 2026 23:06:11 +1100 Subject: [PATCH] inventory: index lookups instead of scanning every object Syncing N objects compared each one with every object of its type in several places, so a sync cost O(N^2): finding a VM/device by name and cluster or a VM interface/disk by name and VM, listing the interfaces, IPs, MACs and disks of an object, the longest matching prefix, matching VMs by MAC, serial or primary IP and resolving NetBox IDs. The inventory now keeps per object type indexes from an attribute value to the objects carrying it (NetBox ID, primary key, interface parent, IP address, assigned object, MAC address, prefix network, serial, asset tag, primary IPs). An index is built on the first lookup of its type. Invariants: * An index only selects candidates. Every lookup still applies its original comparison to them in inventory order, so it returns exactly what the full scan returned (first match wins, case-insensitive names). * Keys depend only on the object's own data. The primary key index files an object under its lower-cased primary key value, which every display name starts with; a lookup tries the whole name and every part in front of a " (". Values without a stable string (i.e. objects) are filed as volatile and returned for every lookup. * NetBoxObject.data is an NBDataDict which reports every write, and assigning data or nb_id is reported too, so an object is re-keyed right away whatever changes it (update(), resolve_relations(), direct writes). * An index is rebuilt if its object list was replaced or changed outside add_object(), as tests do. SourceBase.ip_address_objects_for() (4f5f686) keeps its interface and callers but now answers from the inventory's address index. That index follows address changes of existing objects and a replaced inventory list, which the append-only per-source index did not, and it is the same index the primary IP matching uses, so there is one address index instead of two. --- module/netbox/inventory.py | 474 ++++++++++++++++++++++++++- module/netbox/object_classes.py | 94 +++++- module/sources/common/source_base.py | 22 +- module/sources/vmware/connection.py | 4 +- tests/test_inventory_indexes.py | 379 +++++++++++++++++++++ tests/test_ip_address_index.py | 12 + 6 files changed, 951 insertions(+), 34 deletions(-) create mode 100644 tests/test_inventory_indexes.py diff --git a/module/netbox/inventory.py b/module/netbox/inventory.py index 5900bd02..ded81700 100644 --- a/module/netbox/inventory.py +++ b/module/netbox/inventory.py @@ -9,6 +9,7 @@ import json import asyncio +from ipaddress import IPv4Address, IPv6Address, IPv4Network, IPv6Network, IPv4Interface, IPv6Interface from module.netbox import * from module.common.misc import grab @@ -17,6 +18,245 @@ log = get_logger() +""" +Indexes of the inventory + +Most lookups in the inventory used to compare every object of a type with the value to find, which makes +syncing N objects cost N^2. The indexes below map an attribute value to the objects carrying it, so a +lookup only needs to look at a handful of candidates. + +An index is only ever used to pick candidates. Every lookup still applies its original comparison to the +candidates, in the order the objects were added to the inventory, so the result is exactly the one the +full scan returned. For that to hold an index has to list every object which could match, which is +guaranteed by these invariants: + + * an index key is derived only from the data of the object itself (never from objects it references), + * every write to NetBoxObject.data and NetBoxObject.nb_id is reported to the inventory (see NBDataDict and + NetBoxObject.__setattr__) and the object is re-keyed straight away, + * a value which can't be turned into a reliable key is filed as "volatile" and returned for every lookup, + * a value which can never match a lookup (i.e. an interface parent which isn't an object yet) isn't indexed. + +The index of an object type is built on its first lookup. It is rebuilt whenever its list in +'base_structure' was replaced or changed behind the inventory's back (tests do that). +""" + +# key of the bucket holding objects which have to be considered for every lookup +_VOLATILE = object() + +# key for values which can never match a lookup, objects with this key are not indexed +_SKIP = object() + +# primary key values whose string representation is stable and can be used as key +_STABLE_PRIMARY_KEY_TYPES = (str, int, float, bool, IPv4Address, IPv6Address, IPv4Network, IPv6Network, + IPv4Interface, IPv6Interface) + + +def _display_name_key(value): + """ + Key of the primary key index. The display name of every object type starts with the string of its + primary key value, followed by nothing or " (" and its secondary key, so the lower-cased primary key + string is a prefix of every lower-cased display name this object can produce. + """ + if value is None or isinstance(value, _STABLE_PRIMARY_KEY_TYPES): + return f"{value}".lower() + # i.e. a NetBoxObject as primary key value, its string changes with the referenced object + return _VOLATILE + + +def _value_key(value): + """key for plain equality lookups""" + try: + hash(value) + except TypeError: + return _VOLATILE + return value + + +def _reference_key(value): + """key for lookups by referenced object, only objects can be equal to an object""" + if isinstance(value, NetBoxObject): + return value + return _SKIP + + +def _ip_address_key(value): + """key for IP address strings, the address without prefix length""" + if isinstance(value, str): + return value.split("/")[0] + return _SKIP + + +def _ip_network_key(value): + """key for IP prefixes, an IP can be matched against every possible network of all prefix lengths""" + if isinstance(value, (IPv4Network, IPv6Network)): + return value.version, value.prefixlen, int(value.network_address) + if value is None: + return _SKIP + return _VOLATILE + + +def _primary_ip_key(value): + """key for primary IPs of devices and VMs, they can be an object, a NetBox dict or an ID""" + if isinstance(value, NBIPAddress): + return value + if isinstance(value, (dict, int)): + return _VOLATILE + return _SKIP + + +class _IndexSpec: + """ + describes one index of an object type + + name: name of the index + watch: name of the data key which changes the key of this index ('__nb_id__' for the NetBox ID) + key_function: turns the watched value into an index key + """ + + __slots__ = ("name", "watch", "key_function") + + nb_id_key = "__nb_id__" + + def __init__(self, name, watch, key_function): + self.name = name + self.watch = watch + self.key_function = key_function + + def key_of(self, this_object): + + if self.watch == self.nb_id_key: + return self.key_function(this_object.nb_id) + + data = this_object.data + if not isinstance(data, dict): + return _SKIP + + return self.key_function(data.get(self.watch)) + + +# additional indexes on top of NetBox ID and primary key, data key as index name +_type_index_definitions = { + NBVM: {"serial": _value_key, "primary_ip4": _primary_ip_key, "primary_ip6": _primary_ip_key}, + NBDevice: {"serial": _value_key, "asset_tag": _value_key, + "primary_ip4": _primary_ip_key, "primary_ip6": _primary_ip_key}, + NBVMInterface: {"virtual_machine": _reference_key, "mac_address": _value_key}, + NBInterface: {"device": _reference_key, "mac_address": _value_key}, + NBIPAddress: {"address": _ip_address_key, "assigned_object_id": _reference_key}, + NBMACAddress: {"mac_address": _value_key, "assigned_object_id": _reference_key}, + NBPrefix: {"prefix": _ip_network_key}, + NBVirtualDisk: {"virtual_machine": _reference_key} +} + +_index_spec_cache = dict() + + +def _get_index_specs(object_type): + + specs = _index_spec_cache.get(object_type) + if specs is None: + specs = [ + _IndexSpec("__nb_id__", _IndexSpec.nb_id_key, _value_key), + _IndexSpec("__primary_key__", object_type.primary_key, _display_name_key) + ] + for data_key, key_function in _type_index_definitions.get(object_type, dict()).items(): + specs.append(_IndexSpec(data_key, data_key, key_function)) + + _index_spec_cache[object_type] = specs + + return specs + + +class _TypeIndex: + """ + All indexes of one object type. + + position: inventory position of every indexed object, used to return candidates in inventory order + keys: index name -> object -> key of this object + buckets: index name -> key -> objects with this key (dict used as ordered set) + """ + + def __init__(self, object_type, items): + + self.items = items + self.specs = _get_index_specs(object_type) + self.watch = dict() + for spec in self.specs: + self.watch.setdefault(spec.watch, list()).append(spec) + + self.position = dict() + self.keys = {spec.name: dict() for spec in self.specs} + self.buckets = {spec.name: dict() for spec in self.specs} + + for position, this_object in enumerate(items): + self.add(this_object, position) + + def add(self, this_object, position): + + # first occurrence defines the position, like a scan would + if this_object in self.position: + return + + self.position[this_object] = position + for spec in self.specs: + self._file(this_object, spec, spec.key_of(this_object)) + + def _file(self, this_object, spec, key): + + self.keys[spec.name][this_object] = key + if key is not _SKIP: + self.buckets[spec.name].setdefault(key, dict())[this_object] = None + + def update(self, this_object, watch_key=None): + + if this_object not in self.position: + return + + specs = self.specs if watch_key is None else self.watch.get(watch_key) + if not specs: + return + + for spec in specs: + old_key = self.keys[spec.name].get(this_object, _SKIP) + new_key = spec.key_of(this_object) + + if old_key is new_key or (type(old_key) is type(new_key) and old_key == new_key): + continue + + if old_key is not _SKIP: + bucket = self.buckets[spec.name].get(old_key) + if bucket is not None: + bucket.pop(this_object, None) + if len(bucket) == 0: + del self.buckets[spec.name][old_key] + + self._file(this_object, spec, new_key) + + def candidates(self, index_name, keys): + """ + return all objects filed under one of the keys plus all volatile ones in inventory order. + Returns None if a key can't be looked up, the caller needs to scan all objects then. + """ + + buckets = self.buckets[index_name] + + result = dict() + for key in keys: + try: + bucket = buckets.get(key) + except TypeError: + return None + if bucket: + result.update(bucket) + + volatile = buckets.get(_VOLATILE) + if volatile: + result.update(volatile) + + if len(result) <= 1: + return list(result) + + return sorted(result, key=self.position.__getitem__) + class NetBoxInventory: """ @@ -43,10 +283,167 @@ def init(self): # object types read from NetBox in this run self.loaded_types = set() + # indexes per object type name, built on first lookup + self._indexes = dict() + for object_type in NetBoxObject.__subclasses__(): self.base_structure[object_type.name] = list() + def _get_index(self, object_type): + """ + return the index of an object type, (re)build it if it doesn't match the current list of objects + + Parameters + ---------- + object_type: NetBoxObject subclass + object type to return the index for + + Returns + ------- + _TypeIndex: index of this object type + """ + + items = self.base_structure.get(object_type.name) + if items is None: + return _TypeIndex(object_type, list()) + + indexes = self.__dict__.get("_indexes") + if indexes is None: + indexes = self._indexes = dict() + + type_index = indexes.get(object_type.name) + if type_index is None or type_index.items is not items or len(type_index.position) != len(items): + type_index = indexes[object_type.name] = _TypeIndex(object_type, items) + + return type_index + + def object_data_changed(self, this_object, data_key=None): + """ + Called by a NetBoxObject whenever its data or its NetBox ID changes, keeps the indexes up to date. + + Parameters + ---------- + this_object: NetBoxObject + the changed object + data_key: str + the changed data key, None if the whole data set changed + """ + + indexes = self.__dict__.get("_indexes") + if not indexes: + return + + type_index = indexes.get(this_object.name) + if type_index is not None: + type_index.update(this_object, data_key) + + def get_index_candidates(self, object_type, index_name, values): + """ + Return all objects of $object_type which possibly carry one of $values in the indexed + attribute $index_name, in the order of the inventory. The result can contain objects which don't + match, callers have to apply their own comparison to each object. + + Parameters + ---------- + object_type: NetBoxObject subclass + object type to find + index_name: str + name of the index, the data key of one of the indexes in '_type_index_definitions' + values: list, tuple + values to look for + + Returns + ------- + list: of candidate objects + """ + + type_index = self._get_index(object_type) + + key_function = None + for spec in type_index.specs: + if spec.name == index_name: + key_function = spec.key_function + break + + if key_function is None: + raise ValueError(f"No index '{index_name}' defined for {object_type.name} objects") + + keys = list() + for value in values: + key = key_function(value) + if key is _VOLATILE: + return list(type_index.items) + if key is not _SKIP: + keys.append(key) + + candidates = type_index.candidates(index_name, keys) + if candidates is None: + return list(type_index.items) + + return candidates + + def get_prefix_candidates(self, ip_to_match): + """ + Return all prefixes which possibly contain $ip_to_match, in the order of the inventory. + Callers have to apply their own comparison to each prefix. + + Parameters + ---------- + ip_to_match: IPv4Address, IPv6Address + IP address to find prefixes for + + Returns + ------- + list: of candidate NBPrefix objects + """ + + if not isinstance(ip_to_match, (IPv4Address, IPv6Address)): + return list(self.get_all_items(NBPrefix)) + + max_prefix_length = ip_to_match.max_prefixlen + all_ones = (1 << max_prefix_length) - 1 + ip_int = int(ip_to_match) + + keys = list() + for prefix_length in range(max_prefix_length + 1): + netmask = all_ones ^ (all_ones >> prefix_length) + keys.append((ip_to_match.version, prefix_length, ip_int & netmask)) + + candidates = self._get_index(NBPrefix).candidates("prefix", keys) + if candidates is None: + return list(self.get_all_items(NBPrefix)) + + return candidates + + def get_primary_ip_candidates(self, object_type, ip_addresses): + """ + Return all objects of $object_type which possibly use one of $ip_addresses (without prefix length) + as primary IPv4 or IPv6 address, in the order of the inventory. Callers have to apply their own + comparison to each object. + + Parameters + ---------- + object_type: NBDevice, NBVM + object type to find + ip_addresses: list + IP address strings without prefix length + + Returns + ------- + list: of candidate objects + """ + + ip_objects = self.get_index_candidates(NBIPAddress, "address", [x for x in ip_addresses if x is not None]) + + result = dict() + for index_name in ["primary_ip4", "primary_ip6"]: + for this_object in self.get_index_candidates(object_type, index_name, ip_objects): + result[this_object] = None + + position = self._get_index(object_type).position + return sorted(result, key=position.__getitem__) + def add_source(self, source_handler=None): """ adds $source_tag to list of disabled sources @@ -83,7 +480,11 @@ def get_by_id(self, object_type, nb_id=None): if nb_id is None or self.base_structure[object_type.name] is None: return None - for this_object in self.base_structure[object_type.name]: + candidates = self._get_index(object_type).candidates("__nb_id__", [_value_key(nb_id)]) + if candidates is None: + candidates = self.base_structure[object_type.name] + + for this_object in candidates: if this_object.nb_id == nb_id: return this_object @@ -130,28 +531,34 @@ def get_by_data(self, object_type, data=None): elif data.get(object_type.primary_key) is not None: if object_type in (NBVMInterface, NBVirtualDisk) and isinstance(data.get("virtual_machine"), NBVM): # VM names are only unique within a cluster; use the resolved parent object. - for this_object in self.get_all_items(object_type): + for this_object in self.get_index_candidates(object_type, "virtual_machine", + [data["virtual_machine"]]): if this_object.data.get("virtual_machine") is data["virtual_machine"] and \ str(this_object.data.get(object_type.primary_key)).lower() == \ str(data[object_type.primary_key]).lower(): return this_object return None - object_name_to_find = None - for this_object in self.get_all_items(object_type): + object_name_to_find = self.get_all_items(object_type)[0].get_display_name(data, including_second_key=True) + name_to_find = f"{object_name_to_find}".lower() - if object_name_to_find is None: - object_name_to_find = this_object.get_display_name(data, including_second_key=True) + for this_object in self._get_display_name_candidates(object_type, name_to_find): # compare lower key - if f"{object_name_to_find}".lower() == \ - f"{this_object.get_display_name(including_second_key=True)}".lower(): + if name_to_find == f"{this_object.get_display_name(including_second_key=True)}".lower(): return this_object # try to match all data attributes else: - for this_object in self.get_all_items(object_type): + candidates = None + for attr_name, attr_value in data.items(): + if attr_name in ("serial", "asset_tag") and \ + attr_name in _type_index_definitions.get(object_type, dict()).keys(): + candidates = self.get_index_candidates(object_type, attr_name, [attr_value]) + break + + for this_object in candidates if candidates is not None else self.get_all_items(object_type): all_items_match = True for attr_name, attr_value in data.items(): @@ -164,6 +571,39 @@ def get_by_data(self, object_type, data=None): return None + def _get_display_name_candidates(self, object_type, name_to_find): + """ + Return all objects of $object_type whose lower-cased display name (including second key) can be + $name_to_find, in the order of the inventory. + + A display name is the primary key value, optionally followed by " (" and the secondary key. So the + primary key of a matching object is either the whole name or the part in front of one of the " (" + in it. + + Parameters + ---------- + object_type: NetBoxObject subclass + object type to find + name_to_find: str + lower-cased display name to find + + Returns + ------- + list: of candidate objects + """ + + keys = [name_to_find] + separator_position = name_to_find.find(" (") + while separator_position != -1: + keys.append(name_to_find[:separator_position]) + separator_position = name_to_find.find(" (", separator_position + 1) + + candidates = self._get_index(object_type).candidates("__primary_key__", keys) + if candidates is None: + return list(self.get_all_items(object_type)) + + return candidates + def slug_used(self, object_type: NetBoxObject, slug: str) -> bool: """ Determine if a slug for an object_tpe is already used @@ -215,7 +655,17 @@ def add_object(self, object_type, data=None, read_from_netbox=False, source=None new_object = object_type(data, read_from_netbox=read_from_netbox, inventory=self, source=source) # add to inventory - self.base_structure[object_type.name].append(new_object) + items = self.base_structure[object_type.name] + items.append(new_object) + + # add to index if one exists, otherwise it gets built with the next lookup + indexes = self.__dict__.get("_indexes") + type_index = indexes.get(object_type.name) if indexes else None + if type_index is not None: + if type_index.items is items and len(type_index.position) == len(items) - 1: + type_index.add(new_object, len(items) - 1) + else: + del indexes[object_type.name] if read_from_netbox is False: log.info(f"Created new {new_object.name} object: {new_object.get_display_name()}") @@ -326,12 +776,12 @@ def get_all_interfaces(self, this_object: (NBVM, NBDevice)): interfaces = list() if isinstance(this_object, NBVM): - for interface in self.get_all_items(NBVMInterface): + for interface in self.get_index_candidates(NBVMInterface, "virtual_machine", [this_object]): if grab(interface, "data.virtual_machine") == this_object: interfaces.append(interface) if isinstance(this_object, NBDevice): - for interface in self.get_all_items(NBInterface): + for interface in self.get_index_candidates(NBInterface, "device", [this_object]): if grab(interface, "data.device") == this_object: interfaces.append(interface) diff --git a/module/netbox/object_classes.py b/module/netbox/object_classes.py index 654b4e6e..9b13b459 100644 --- a/module/netbox/object_classes.py +++ b/module/netbox/object_classes.py @@ -261,6 +261,62 @@ def scopes_object_types(self, scopes_list) -> list: return result +class NBDataDict(dict): + """ + The data dict of a NetBoxObject. Behaves like a plain dict but reports every change to its owner, + which keeps the indexes of the inventory up to date (see module/netbox/inventory.py). + """ + + __slots__ = ("_owner",) + + def __init__(self, owner, *args, **kwargs): + super().__init__(*args, **kwargs) + self._owner = owner + + def _changed(self, key=None): + owner = getattr(self, "_owner", None) + if owner is not None: + owner.data_changed(key) + + def __setitem__(self, key, value): + super().__setitem__(key, value) + self._changed(key) + + def __delitem__(self, key): + super().__delitem__(key) + self._changed(key) + + def __ior__(self, other): + result = super().__ior__(other) + self._changed() + return result + + def clear(self): + super().clear() + self._changed() + + def pop(self, key, *args): + result = super().pop(key, *args) + self._changed(key) + return result + + def popitem(self): + result = super().popitem() + self._changed(result[0]) + return result + + def setdefault(self, key, default=None): + if key in self: + return self[key] + result = super().setdefault(key, default) + self._changed(key) + return result + + def update(self, *args, **kwargs): + super().update(*args, **kwargs) + self._changed() + + class NetBoxObject: """ Base class for all NetBox object types. Implements all methods used on a NetBox object. @@ -353,6 +409,34 @@ def __init__(self, data=None, read_from_netbox=False, inventory=None, source=Non # add data to this object self.update(data=data, read_from_netbox=read_from_netbox, source=source) + def __setattr__(self, key, value): + + # every change to data or the NetBox ID needs to be reported to the inventory indexes + if key == "data" and isinstance(value, dict) and \ + not (isinstance(value, NBDataDict) and getattr(value, "_owner", None) is self): + value = NBDataDict(self, value) + + super().__setattr__(key, value) + + if key in ("data", "nb_id"): + self.data_changed(None if key == "data" else "__nb_id__") + + def data_changed(self, key=None): + """ + reports a change of the data (key) or the NetBox ID ("__nb_id__") of this object to the inventory + + Parameters + ---------- + key: str + name of the changed data key, None if the whole data set changed + """ + + inventory = self.inventory + if inventory is not None: + object_data_changed = getattr(inventory, "object_data_changed", None) + if object_data_changed is not None: + object_data_changed(self, key) + def __repr__(self): return "<%s instance '%s' at %s>" % (self.__class__.__name__, self.get_display_name(), id(self)) @@ -2048,7 +2132,7 @@ def __init__(self, *args, **kwargs): def get_virtual_disks(self): result_list = list() - for ip_object in self.inventory.get_all_items(NBVirtualDisk): + for ip_object in self.inventory.get_index_candidates(NBVirtualDisk, "virtual_machine", [self]): if grab(ip_object, "data.virtual_machine") == self: result_list.append(ip_object) @@ -2084,7 +2168,7 @@ def __init__(self, *args, **kwargs): def get_ip_addresses(self): result_list = list() - for ip_object in self.inventory.get_all_items(NBIPAddress): + for ip_object in self.inventory.get_index_candidates(NBIPAddress, "assigned_object_id", [self]): if ip_object.data.get("assigned_object_id") == self: result_list.append(ip_object) @@ -2093,7 +2177,7 @@ def get_ip_addresses(self): def get_mac_addresses(self): result_list = list() - for mac_object in self.inventory.get_all_items(NBMACAddress): + for mac_object in self.inventory.get_index_candidates(NBMACAddress, "assigned_object_id", [self]): if mac_object.data.get("assigned_object_id") == self: result_list.append(mac_object) @@ -2138,7 +2222,7 @@ def __init__(self, *args, **kwargs): def get_ip_addresses(self): result_list = list() - for ip_object in self.inventory.get_all_items(NBIPAddress): + for ip_object in self.inventory.get_index_candidates(NBIPAddress, "assigned_object_id", [self]): if ip_object.data.get("assigned_object_id") == self: result_list.append(ip_object) @@ -2147,7 +2231,7 @@ def get_ip_addresses(self): def get_mac_addresses(self): result_list = list() - for mac_object in self.inventory.get_all_items(NBMACAddress): + for mac_object in self.inventory.get_index_candidates(NBMACAddress, "assigned_object_id", [self]): if mac_object.data.get("assigned_object_id") == self: result_list.append(mac_object) diff --git a/module/sources/common/source_base.py b/module/sources/common/source_base.py index e44de2b5..4e96fcfe 100644 --- a/module/sources/common/source_base.py +++ b/module/sources/common/source_base.py @@ -34,20 +34,11 @@ class SourceBase: def ip_address_objects_for(self, address: str) -> list: """ Return the inventory's IP address objects for an address (without prefix length), in - inventory order. The inventory only ever appends, so the index catches up with the - objects added since the last call instead of scanning every IP address per lookup. + inventory order, from the inventory's address index instead of scanning every IP address + per lookup. The index follows every change of an object's address, see the indexes in + module/netbox/inventory.py. """ - all_ips = self.inventory.get_all_items(NBIPAddress) - index = self.__dict__.setdefault("_ip_index", dict()) - indexed = self.__dict__.get("_ip_index_size", 0) - if indexed > len(all_ips): # a different inventory list: start over - index.clear() - indexed = 0 - for ip in all_ips[indexed:]: - key = grab(ip, "data.address", fallback="").split("/")[0] - index.setdefault(key, list()).append(ip) - self._ip_index_size = len(all_ips) - return index.get(address, list()) + return self.inventory.get_index_candidates(NBIPAddress, "address", [address]) def set_source_tag(self): self.source_tag = f"Source: {self.name}" @@ -388,7 +379,7 @@ def return_longest_matching_prefix_for_ip(self, ip_to_match=None, site_name=None current_longest_matching_prefix_length = 0 current_longest_matching_prefix = None - for prefix in self.inventory.get_all_items(NBPrefix): + for prefix in self.inventory.get_prefix_candidates(ip_to_match): if not prefix.matches_site(site_object): continue @@ -523,7 +514,8 @@ def add_update_interface(self, interface_object, device_object, interface_data, # if a new interface or not matching assigned MAC address, try to find an existing unassigned mac address if primary_mac_address_object is None: - for mac_address_object in self.inventory.get_all_items(NBMACAddress): + for mac_address_object in self.inventory.get_index_candidates(NBMACAddress, "mac_address", + [interface_mac_address]): # an object already assigned to this very interface is the one we want, not a duplicate if (grab(mac_address_object, "data.mac_address") == interface_mac_address and grab(mac_address_object, "data.assigned_object_id") in (None, interface_object)): diff --git a/module/sources/vmware/connection.py b/module/sources/vmware/connection.py index 123260cb..525984e1 100644 --- a/module/sources/vmware/connection.py +++ b/module/sources/vmware/connection.py @@ -1180,7 +1180,7 @@ def get_object_based_on_macs(self, object_type, mac_list=None): objects_with_matching_macs = dict() matching_object = None - for interface in self.inventory.get_all_items(interface_typ): + for interface in self.inventory.get_index_candidates(interface_typ, "mac_address", mac_list): if grab(interface, "data.mac_address") in mac_list: @@ -1286,7 +1286,7 @@ def _matches_device_primary_ip(device_primary_ip, ip_needle): if primary_ip6 is not None: primary_ip6 = str(primary_ip6).split("/")[0] - for device in self.inventory.get_all_items(object_type): + for device in self.inventory.get_primary_ip_candidates(object_type, [primary_ip4, primary_ip6]): if _matches_device_primary_ip(grab(device, "data.primary_ip4"), primary_ip4) is True: log.debug2(f"Found existing host '{device.get_display_name()}' " diff --git a/tests/test_inventory_indexes.py b/tests/test_inventory_indexes.py new file mode 100644 index 00000000..f16af5e9 --- /dev/null +++ b/tests/test_inventory_indexes.py @@ -0,0 +1,379 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2020 - 2026 netbox-sync team. All rights reserved. +# +# netbox-sync.py +# +# This work is licensed under the terms of the MIT license. +# For a copy, see file LICENSE.txt included in this +# repository or visit: . + +""" +The inventory keeps indexes so lookups don't have to compare every object of a type. These +tests check that every indexed lookup returns exactly what the plain scan over all objects +returned before, also after objects were renamed, moved, re-addressed or replaced behind +the inventory's back. +""" +import json +import random +from ipaddress import ip_address, ip_network +from types import SimpleNamespace + +import pytest + +from module.common.misc import grab +from module.netbox.object_classes import ( + NBDataDict, NBSite, NBCluster, NBVM, NBVMInterface, NBIPAddress, NBMACAddress, NBPrefix, NBDevice, + NBInterface, NBVirtualDisk, NBVLAN, NBClusterType, +) +from module.sources.common.source_base import SourceBase + +TS = "2026-01-01T00:00:00Z" + + +# reference implementations: the scans the inventory used before it had indexes + +def scan_get_by_data(inv, object_type, data): + object_name_to_find = None + for this_object in inv.get_all_items(object_type): + if object_name_to_find is None: + object_name_to_find = this_object.get_display_name(data, including_second_key=True) + if f"{object_name_to_find}".lower() == f"{this_object.get_display_name(including_second_key=True)}".lower(): + return this_object + return None + + +def scan_get_by_id(inv, object_type, nb_id): + for this_object in inv.get_all_items(object_type): + if this_object.nb_id == nb_id: + return this_object + return None + + +def scan_get_all_interfaces(inv, this_object): + if isinstance(this_object, NBVM): + return [i for i in inv.get_all_items(NBVMInterface) if grab(i, "data.virtual_machine") == this_object] + return [i for i in inv.get_all_items(NBInterface) if grab(i, "data.device") == this_object] + + +def scan_ips(inv, address): + return [ip for ip in inv.get_all_items(NBIPAddress) + if grab(ip, "data.address", fallback="").startswith(f"{address}/")] + + +def scan_longest_prefix(inv, ip, site_object): + longest_length, longest = 0, None + for prefix in inv.get_all_items(NBPrefix): + if not prefix.matches_site(site_object): + continue + network = grab(prefix, "data.prefix") + if network is None: + continue + if ip in network and network.prefixlen >= longest_length: + longest_length, longest = network.prefixlen, prefix + return longest + + +def indexed_longest_prefix(inv, ip, site_name): + return SourceBase.return_longest_matching_prefix_for_ip(SimpleNamespace(inventory=inv), ip, site_name) + + +@pytest.fixture +def base(inventory): + inventory.load_from_netbox(NBSite, [{"id": 1, "name": "S1", "slug": "s1", "last_updated": TS}, + {"id": 2, "name": "S2", "slug": "s2", "last_updated": TS}]) + inventory.load_from_netbox(NBClusterType, [{"id": 1, "name": "T", "slug": "t", "last_updated": TS}]) + inventory.load_from_netbox(NBCluster, [ + {"id": 1, "name": "C1", "type": {"id": 1}, "scope_type": "dcim.site", "scope_id": 1, "last_updated": TS}, + {"id": 2, "name": "C2", "type": {"id": 1}, "scope_type": "dcim.site", "scope_id": 2, "last_updated": TS}, + ]) + inventory.resolve_relations() + return SimpleNamespace(inventory=inventory, + c1=inventory.get_by_id(NBCluster, nb_id=1), c2=inventory.get_by_id(NBCluster, nb_id=2), + s1=inventory.get_by_id(NBSite, nb_id=1), s2=inventory.get_by_id(NBSite, nb_id=2)) + + +def add_vm(inv, name, cluster, nb_id): + return inv.load_from_netbox(NBVM, [{"id": nb_id, "name": name, "cluster": cluster, "last_updated": TS}]) or \ + inv.get_all_items(NBVM)[-1] + + +def test_data_is_a_tracking_dict(base): + vm = add_vm(base.inventory, "vm1", {"id": 1}, 10) + assert isinstance(vm.data, NBDataDict) and isinstance(vm.data, dict) + assert vm.data == {"id": 10, "name": "vm1", "cluster": {"id": 1}, "last_updated": TS} + assert json.loads(json.dumps(vm.data))["name"] == "vm1" + vm.data = {"id": 10, "name": "other"} + assert isinstance(vm.data, NBDataDict) + + +def test_lookup_is_case_insensitive(base): + inv = base.inventory + add_vm(inv, "MyVM", {"id": 1}, 10) + inv.resolve_relations() + found = inv.get_by_data(NBVM, data={"name": "myvm", "cluster": base.c1}) + assert found is not None and found.data["name"] == "MyVM" + assert inv.get_by_data(NBVM, data={"name": "MYVM", "cluster": base.c2}) is None + + +def test_rename_moves_object_in_index(base): + inv = base.inventory + vm = add_vm(inv, "old", {"id": 1}, 10) + inv.resolve_relations() + assert inv.get_by_data(NBVM, data={"name": "old", "cluster": base.c1}) is vm + + vm.update(data={"name": "new"}) + assert inv.get_by_data(NBVM, data={"name": "new", "cluster": base.c1}) is vm + assert inv.get_by_data(NBVM, data={"name": "old", "cluster": base.c1}) is None + + # direct writes to the data dict are tracked as well + vm.data["name"] = "direct" + assert inv.get_by_data(NBVM, data={"name": "direct", "cluster": base.c1}) is vm + vm.data = dict(vm.data, name="replaced") + assert inv.get_by_data(NBVM, data={"name": "replaced", "cluster": base.c1}) is vm + assert inv.get_by_data(NBVM, data={"name": "direct", "cluster": base.c1}) is None + + +def test_duplicate_names_return_first_in_inventory_order(base): + inv = base.inventory + first = add_vm(inv, "dup", {"id": 1}, 10) + second = add_vm(inv, "DUP", {"id": 1}, 11) + inv.resolve_relations() + assert inv.get_by_data(NBVM, data={"name": "dup", "cluster": base.c1}) is first + + # renaming the first one away and back keeps the inventory order, not the order of renames + first.update(data={"name": "away"}) + assert inv.get_by_data(NBVM, data={"name": "dup", "cluster": base.c1}) is second + first.update(data={"name": "dup"}) + assert inv.get_by_data(NBVM, data={"name": "dup", "cluster": base.c1}) is first + + +def test_secondary_key_change_is_found(base): + inv = base.inventory + vm = add_vm(inv, "vm1", {"id": 1}, 10) + inv.resolve_relations() + vm.update(data={"cluster": base.c2}) + assert inv.get_by_data(NBVM, data={"name": "vm1", "cluster": base.c2}) is vm + assert inv.get_by_data(NBVM, data={"name": "vm1", "cluster": base.c1}) is None + + # renaming the referenced object changes the display name of the VM without touching the VM + base.c2.update(data={"name": "C2-renamed"}) + assert inv.get_by_data(NBVM, data={"name": "vm1", "cluster": base.c2}) is vm + + +def test_names_with_parentheses(base): + inv = base.inventory + tricky = add_vm(inv, "a (C1", {"id": 1}, 10) + plain = add_vm(inv, "a", {"id": 1}, 11) + inv.resolve_relations() + for data in ({"name": "a (C1", "cluster": base.c1}, {"name": "a", "cluster": base.c1}, + {"name": "a (c1)", "cluster": base.c1}): + assert inv.get_by_data(NBVM, data=data) is scan_get_by_data(inv, NBVM, data) + assert inv.get_by_data(NBVM, data={"name": "a", "cluster": base.c1}) is plain + assert inv.get_by_data(NBVM, data={"name": "a (C1", "cluster": base.c1}) is tricky + + +def test_vlan_display_names(base): + inv = base.inventory + inv.load_from_netbox(NBVLAN, [{"id": 1, "vid": 100, "name": "one", "site": {"id": 1}, "last_updated": TS}, + {"id": 2, "vid": 100, "name": "two", "site": {"id": 2}, "last_updated": TS}, + {"id": 3, "vid": 200, "name": "three", "last_updated": TS}]) + inv.resolve_relations() + for data in ({"vid": 100, "name": "x", "site": base.s2}, {"vid": 100, "name": "x", "site": base.s1}, + {"vid": 200, "name": "three"}, {"vid": 200, "name": "THREE"}, {"vid": 300, "name": "x"}): + assert inv.get_by_data(NBVLAN, data=data) is scan_get_by_data(inv, NBVLAN, data) + + +def test_get_by_id_follows_id_changes(base): + inv = base.inventory + vm = add_vm(inv, "vm1", {"id": 1}, 10) + assert inv.get_by_id(NBVM, nb_id=10) is vm + vm.update(data={"id": 20, "name": "vm1", "cluster": {"id": 1}}, read_from_netbox=True) + assert inv.get_by_id(NBVM, nb_id=10) is None + assert inv.get_by_id(NBVM, nb_id=20) is vm + + +def test_interfaces_loaded_from_netbox_then_resolved(base): + inv = base.inventory + inv.load_from_netbox(NBVM, [{"id": 10, "name": "vm1", "cluster": {"id": 1}, "last_updated": TS}, + {"id": 11, "name": "vm2", "cluster": {"id": 1}, "last_updated": TS}]) + inv.load_from_netbox(NBVMInterface, [ + {"id": 1, "name": "eth0", "virtual_machine": {"id": 10}, "last_updated": TS}, + {"id": 2, "name": "eth1", "virtual_machine": {"id": 10}, "last_updated": TS}, + {"id": 3, "name": "eth0", "virtual_machine": {"id": 11}, "last_updated": TS}, + ]) + vm1, vm2 = inv.get_by_id(NBVM, nb_id=10), inv.get_by_id(NBVM, nb_id=11) + + # not resolved yet, a dict never equals an object + assert inv.get_all_interfaces(vm1) == scan_get_all_interfaces(inv, vm1) == [] + + inv.resolve_relations() + assert [i.nb_id for i in inv.get_all_interfaces(vm1)] == [1, 2] + assert [i.nb_id for i in inv.get_all_interfaces(vm2)] == [3] + + # re-parent an interface + interface = inv.get_by_id(NBVMInterface, nb_id=1) + interface.update(data={"virtual_machine": vm2}) + assert [i.nb_id for i in inv.get_all_interfaces(vm1)] == [2] + assert [i.nb_id for i in inv.get_all_interfaces(vm2)] == [1, 3] + + +def test_ip_reassignment_and_readdressing(base): + inv = base.inventory + inv.load_from_netbox(NBVM, [{"id": 10, "name": "vm1", "cluster": {"id": 1}, "last_updated": TS}]) + inv.load_from_netbox(NBVMInterface, [ + {"id": 1, "name": "eth0", "virtual_machine": {"id": 10}, "last_updated": TS}, + {"id": 2, "name": "eth1", "virtual_machine": {"id": 10}, "last_updated": TS}, + ]) + inv.load_from_netbox(NBIPAddress, [ + {"id": 1, "address": "10.0.0.1/24", "assigned_object_type": "virtualization.vminterface", + "assigned_object_id": 1, "last_updated": TS}, + {"id": 2, "address": "10.0.0.10/24", "last_updated": TS}, + ]) + inv.resolve_relations() + eth0, eth1 = inv.get_by_id(NBVMInterface, nb_id=1), inv.get_by_id(NBVMInterface, nb_id=2) + ip1 = inv.get_by_id(NBIPAddress, nb_id=1) + + # a prefix match of the string must not count: 10.0.0.1 is no candidate for 10.0.0.10 + assert inv.get_index_candidates(NBIPAddress, "address", ["10.0.0.1"]) == scan_ips(inv, "10.0.0.1") == [ip1] + assert eth0.get_ip_addresses() == [ip1] and eth1.get_ip_addresses() == [] + + ip1.update(data={"assigned_object_id": eth1}) + assert eth0.get_ip_addresses() == [] and eth1.get_ip_addresses() == [ip1] + + ip1.update(data={"address": "10.0.0.99/24"}) + assert inv.get_index_candidates(NBIPAddress, "address", ["10.0.0.1"]) == [] + assert inv.get_index_candidates(NBIPAddress, "address", ["10.0.0.99"]) == [ip1] + + +def test_list_replaced_behind_the_inventory(base): + inv = base.inventory + add_vm(inv, "vm1", {"id": 1}, 10) + assert inv.get_by_id(NBVM, nb_id=10) is not None + + # tests reset single object types like this + inv.base_structure[NBVM.name] = [] + assert inv.get_by_id(NBVM, nb_id=10) is None + vm = add_vm(inv, "vm1", {"id": 1}, 10) + assert inv.get_by_id(NBVM, nb_id=10) is vm + + +def test_virtual_disks_follow_their_vm(base): + inv = base.inventory + inv.load_from_netbox(NBVM, [{"id": 10, "name": "vm1", "cluster": {"id": 1}, "last_updated": TS}, + {"id": 11, "name": "vm2", "cluster": {"id": 1}, "last_updated": TS}]) + inv.load_from_netbox(NBVirtualDisk, [{"id": 1, "name": "d1", "size": 1, "virtual_machine": {"id": 10}, + "last_updated": TS}]) + inv.resolve_relations() + vm1, vm2 = inv.get_by_id(NBVM, nb_id=10), inv.get_by_id(NBVM, nb_id=11) + disk = inv.get_by_id(NBVirtualDisk, nb_id=1) + assert vm1.get_virtual_disks() == [disk] and vm2.get_virtual_disks() == [] + disk.update(data={"virtual_machine": vm2}) + assert vm1.get_virtual_disks() == [] and vm2.get_virtual_disks() == [disk] + + +def test_prefix_longest_match_equals_scan(base): + inv = base.inventory + prefixes = ["10.0.0.0/8", "10.1.0.0/16", "10.1.2.0/24", "10.1.2.0/24", "192.168.0.0/16", "0.0.0.0/0", + "2001:db8::/32", "2001:db8:1::/48"] + inv.load_from_netbox(NBPrefix, [ + {"id": i + 1, "prefix": p, "scope_type": "dcim.site" if i % 3 == 0 else None, + "scope_id": 1 if i % 3 == 0 else None, "last_updated": TS} for i, p in enumerate(prefixes)]) + inv.resolve_relations() + + for ip in ["10.1.2.3", "10.1.3.3", "10.2.0.1", "192.168.1.1", "8.8.8.8", "2001:db8:1::1", "2001:db8:2::1", + "2001:db9::1"]: + for site_name, site in [(None, None), ("S1", base.s1)]: + assert indexed_longest_prefix(inv, ip_address(ip), site_name) is \ + scan_longest_prefix(inv, ip_address(ip), site), (ip, site_name) + + # changing a prefix re-files it (prefixes are read only, so only a direct change is possible) + prefix = inv.get_by_id(NBPrefix, nb_id=5) + prefix.data["prefix"] = ip_network("10.1.2.0/25") + assert indexed_longest_prefix(inv, ip_address("10.1.2.3"), None) is \ + scan_longest_prefix(inv, ip_address("10.1.2.3"), None) + + +def test_mac_and_primary_ip_candidates(base): + inv = base.inventory + inv.load_from_netbox(NBVM, [ + {"id": 10, "name": "vm1", "cluster": {"id": 1}, "primary_ip4": {"id": 1}, "last_updated": TS}, + {"id": 11, "name": "vm2", "cluster": {"id": 1}, "last_updated": TS}]) + inv.load_from_netbox(NBVMInterface, [ + {"id": 1, "name": "eth0", "virtual_machine": {"id": 10}, "mac_address": "AA:AA:AA:AA:AA:01", + "last_updated": TS}, + {"id": 2, "name": "eth0", "virtual_machine": {"id": 11}, "mac_address": "AA:AA:AA:AA:AA:02", + "last_updated": TS}, + ]) + inv.load_from_netbox(NBIPAddress, [{"id": 1, "address": "10.0.0.1/24", "last_updated": TS}]) + inv.load_from_netbox(NBMACAddress, [{"id": 1, "mac_address": "AA:AA:AA:AA:AA:01", "last_updated": TS}]) + inv.resolve_relations() + vm1 = inv.get_by_id(NBVM, nb_id=10) + + assert [i.nb_id for i in inv.get_index_candidates(NBVMInterface, "mac_address", + ["AA:AA:AA:AA:AA:02", "AA:AA:AA:AA:AA:01"])] == [1, 2] + assert [m.nb_id for m in inv.get_index_candidates(NBMACAddress, "mac_address", ["AA:AA:AA:AA:AA:01"])] == [1] + assert inv.get_primary_ip_candidates(NBVM, ["10.0.0.1", None]) == [vm1] + assert inv.get_primary_ip_candidates(NBVM, ["10.0.0.2"]) == [] + + # a primary IP which is not resolved to an object is always a candidate + vm2 = inv.get_by_id(NBVM, nb_id=11) + vm2.data["primary_ip4"] = {"id": 99} + assert inv.get_primary_ip_candidates(NBVM, ["10.0.0.2"]) == [vm2] + + +def test_randomized_mutations_match_scans(base): + """mutate objects in every possible way and compare all indexed lookups with the scans""" + inv = base.inventory + rnd = random.Random(4711) + names = ["vm", "VM", "web (1)", "web", "db", "a (b) (c", "None", "x"] + addresses = [f"10.0.{a}.{b}" for a in range(2) for b in range(3)] + ["2001:db8::1", "2001:db8::10"] + clusters = [base.c1, base.c2] + + def new_vm(): + return inv.add_object(NBVM, data={"name": rnd.choice(names), "cluster": rnd.choice(clusters)}) + + vms = [new_vm() for _ in range(15)] + interfaces = [inv.add_object(NBVMInterface, data={"name": f"vNIC {rnd.randint(1, 3)}", + "virtual_machine": rnd.choice(vms)}) for _ in range(25)] + ips = [inv.add_object(NBIPAddress, data={"address": f"{rnd.choice(addresses)}/24", + "assigned_object_id": rnd.choice(interfaces)}) for _ in range(25)] + + def check(): + for name in names + [n.upper() for n in names]: + for cluster in clusters: + data = {"name": name, "cluster": cluster} + assert inv.get_by_data(NBVM, data=data) is scan_get_by_data(inv, NBVM, data), data + for vm in vms: + assert inv.get_all_interfaces(vm) == scan_get_all_interfaces(inv, vm) + assert inv.get_by_id(NBVM, nb_id=vm.nb_id) is scan_get_by_id(inv, NBVM, vm.nb_id) + for address in addresses: + assert [ip for ip in inv.get_index_candidates(NBIPAddress, "address", [address]) + if ip.data["address"].startswith(f"{address}/")] == scan_ips(inv, address) + for interface in interfaces: + assert interface.get_ip_addresses() == \ + [ip for ip in inv.get_all_items(NBIPAddress) if ip.data.get("assigned_object_id") == interface] + + check() + for step in range(300): + action = rnd.randrange(9) + if action == 0: + rnd.choice(vms).update(data={"name": rnd.choice(names)}) + elif action == 1: + rnd.choice(vms).update(data={"cluster": rnd.choice(clusters)}) + elif action == 2: + rnd.choice(interfaces).update(data={"virtual_machine": rnd.choice(vms)}) + elif action == 3: + rnd.choice(ips).update(data={"address": f"{rnd.choice(addresses)}/24"}) + elif action == 4: + rnd.choice(ips).update(data={"assigned_object_id": rnd.choice(interfaces)}) + elif action == 5: + vm = rnd.choice(vms) + vm.update(data={"id": rnd.randint(1, 20), "name": rnd.choice(names), "cluster": rnd.choice(clusters)}, + read_from_netbox=True) + elif action == 6: + vms.append(new_vm()) + elif action == 7: + rnd.choice(vms).data["name"] = rnd.choice(names) + else: + base.c1.update(data={"name": rnd.choice(["C1", "C1 (x)", "c2"])}) + check() diff --git a/tests/test_ip_address_index.py b/tests/test_ip_address_index.py index e3e1a4c3..bdd3eb03 100644 --- a/tests/test_ip_address_index.py +++ b/tests/test_ip_address_index.py @@ -53,3 +53,15 @@ def test_lookup_sees_objects_added_after_the_first_call(inventory): assert src.ip_address_objects_for("10.0.0.5") == [first, second] assert len(src.ip_address_objects_for("10.0.0.9")) == 1 + + +def test_lookup_follows_an_address_change_after_the_first_call(inventory): + moved = _add(inventory, "10.0.0.5/24") + other = _add(inventory, "10.0.0.6/24") + src = _source(inventory) + assert src.ip_address_objects_for("10.0.0.5") == [moved] + + moved.update(data={"address": "10.0.0.6/24"}) + + assert src.ip_address_objects_for("10.0.0.5") == _scan(inventory, "10.0.0.5") == [] + assert src.ip_address_objects_for("10.0.0.6") == _scan(inventory, "10.0.0.6") == [moved, other]