diff --git a/module/netbox/inventory.py b/module/netbox/inventory.py index 5900bd02..ded81700 100644 --- a/module/netbox/inventory.py +++ b/module/netbox/inventory.py @@ -9,6 +9,7 @@ import json import asyncio +from ipaddress import IPv4Address, IPv6Address, IPv4Network, IPv6Network, IPv4Interface, IPv6Interface from module.netbox import * from module.common.misc import grab @@ -17,6 +18,245 @@ log = get_logger() +""" +Indexes of the inventory + +Most lookups in the inventory used to compare every object of a type with the value to find, which makes +syncing N objects cost N^2. The indexes below map an attribute value to the objects carrying it, so a +lookup only needs to look at a handful of candidates. + +An index is only ever used to pick candidates. Every lookup still applies its original comparison to the +candidates, in the order the objects were added to the inventory, so the result is exactly the one the +full scan returned. For that to hold an index has to list every object which could match, which is +guaranteed by these invariants: + + * an index key is derived only from the data of the object itself (never from objects it references), + * every write to NetBoxObject.data and NetBoxObject.nb_id is reported to the inventory (see NBDataDict and + NetBoxObject.__setattr__) and the object is re-keyed straight away, + * a value which can't be turned into a reliable key is filed as "volatile" and returned for every lookup, + * a value which can never match a lookup (i.e. an interface parent which isn't an object yet) isn't indexed. + +The index of an object type is built on its first lookup. It is rebuilt whenever its list in +'base_structure' was replaced or changed behind the inventory's back (tests do that). +""" + +# key of the bucket holding objects which have to be considered for every lookup +_VOLATILE = object() + +# key for values which can never match a lookup, objects with this key are not indexed +_SKIP = object() + +# primary key values whose string representation is stable and can be used as key +_STABLE_PRIMARY_KEY_TYPES = (str, int, float, bool, IPv4Address, IPv6Address, IPv4Network, IPv6Network, + IPv4Interface, IPv6Interface) + + +def _display_name_key(value): + """ + Key of the primary key index. The display name of every object type starts with the string of its + primary key value, followed by nothing or " (" and its secondary key, so the lower-cased primary key + string is a prefix of every lower-cased display name this object can produce. + """ + if value is None or isinstance(value, _STABLE_PRIMARY_KEY_TYPES): + return f"{value}".lower() + # i.e. a NetBoxObject as primary key value, its string changes with the referenced object + return _VOLATILE + + +def _value_key(value): + """key for plain equality lookups""" + try: + hash(value) + except TypeError: + return _VOLATILE + return value + + +def _reference_key(value): + """key for lookups by referenced object, only objects can be equal to an object""" + if isinstance(value, NetBoxObject): + return value + return _SKIP + + +def _ip_address_key(value): + """key for IP address strings, the address without prefix length""" + if isinstance(value, str): + return value.split("/")[0] + return _SKIP + + +def _ip_network_key(value): + """key for IP prefixes, an IP can be matched against every possible network of all prefix lengths""" + if isinstance(value, (IPv4Network, IPv6Network)): + return value.version, value.prefixlen, int(value.network_address) + if value is None: + return _SKIP + return _VOLATILE + + +def _primary_ip_key(value): + """key for primary IPs of devices and VMs, they can be an object, a NetBox dict or an ID""" + if isinstance(value, NBIPAddress): + return value + if isinstance(value, (dict, int)): + return _VOLATILE + return _SKIP + + +class _IndexSpec: + """ + describes one index of an object type + + name: name of the index + watch: name of the data key which changes the key of this index ('__nb_id__' for the NetBox ID) + key_function: turns the watched value into an index key + """ + + __slots__ = ("name", "watch", "key_function") + + nb_id_key = "__nb_id__" + + def __init__(self, name, watch, key_function): + self.name = name + self.watch = watch + self.key_function = key_function + + def key_of(self, this_object): + + if self.watch == self.nb_id_key: + return self.key_function(this_object.nb_id) + + data = this_object.data + if not isinstance(data, dict): + return _SKIP + + return self.key_function(data.get(self.watch)) + + +# additional indexes on top of NetBox ID and primary key, data key as index name +_type_index_definitions = { + NBVM: {"serial": _value_key, "primary_ip4": _primary_ip_key, "primary_ip6": _primary_ip_key}, + NBDevice: {"serial": _value_key, "asset_tag": _value_key, + "primary_ip4": _primary_ip_key, "primary_ip6": _primary_ip_key}, + NBVMInterface: {"virtual_machine": _reference_key, "mac_address": _value_key}, + NBInterface: {"device": _reference_key, "mac_address": _value_key}, + NBIPAddress: {"address": _ip_address_key, "assigned_object_id": _reference_key}, + NBMACAddress: {"mac_address": _value_key, "assigned_object_id": _reference_key}, + NBPrefix: {"prefix": _ip_network_key}, + NBVirtualDisk: {"virtual_machine": _reference_key} +} + +_index_spec_cache = dict() + + +def _get_index_specs(object_type): + + specs = _index_spec_cache.get(object_type) + if specs is None: + specs = [ + _IndexSpec("__nb_id__", _IndexSpec.nb_id_key, _value_key), + _IndexSpec("__primary_key__", object_type.primary_key, _display_name_key) + ] + for data_key, key_function in _type_index_definitions.get(object_type, dict()).items(): + specs.append(_IndexSpec(data_key, data_key, key_function)) + + _index_spec_cache[object_type] = specs + + return specs + + +class _TypeIndex: + """ + All indexes of one object type. + + position: inventory position of every indexed object, used to return candidates in inventory order + keys: index name -> object -> key of this object + buckets: index name -> key -> objects with this key (dict used as ordered set) + """ + + def __init__(self, object_type, items): + + self.items = items + self.specs = _get_index_specs(object_type) + self.watch = dict() + for spec in self.specs: + self.watch.setdefault(spec.watch, list()).append(spec) + + self.position = dict() + self.keys = {spec.name: dict() for spec in self.specs} + self.buckets = {spec.name: dict() for spec in self.specs} + + for position, this_object in enumerate(items): + self.add(this_object, position) + + def add(self, this_object, position): + + # first occurrence defines the position, like a scan would + if this_object in self.position: + return + + self.position[this_object] = position + for spec in self.specs: + self._file(this_object, spec, spec.key_of(this_object)) + + def _file(self, this_object, spec, key): + + self.keys[spec.name][this_object] = key + if key is not _SKIP: + self.buckets[spec.name].setdefault(key, dict())[this_object] = None + + def update(self, this_object, watch_key=None): + + if this_object not in self.position: + return + + specs = self.specs if watch_key is None else self.watch.get(watch_key) + if not specs: + return + + for spec in specs: + old_key = self.keys[spec.name].get(this_object, _SKIP) + new_key = spec.key_of(this_object) + + if old_key is new_key or (type(old_key) is type(new_key) and old_key == new_key): + continue + + if old_key is not _SKIP: + bucket = self.buckets[spec.name].get(old_key) + if bucket is not None: + bucket.pop(this_object, None) + if len(bucket) == 0: + del self.buckets[spec.name][old_key] + + self._file(this_object, spec, new_key) + + def candidates(self, index_name, keys): + """ + return all objects filed under one of the keys plus all volatile ones in inventory order. + Returns None if a key can't be looked up, the caller needs to scan all objects then. + """ + + buckets = self.buckets[index_name] + + result = dict() + for key in keys: + try: + bucket = buckets.get(key) + except TypeError: + return None + if bucket: + result.update(bucket) + + volatile = buckets.get(_VOLATILE) + if volatile: + result.update(volatile) + + if len(result) <= 1: + return list(result) + + return sorted(result, key=self.position.__getitem__) + class NetBoxInventory: """ @@ -43,10 +283,167 @@ def init(self): # object types read from NetBox in this run self.loaded_types = set() + # indexes per object type name, built on first lookup + self._indexes = dict() + for object_type in NetBoxObject.__subclasses__(): self.base_structure[object_type.name] = list() + def _get_index(self, object_type): + """ + return the index of an object type, (re)build it if it doesn't match the current list of objects + + Parameters + ---------- + object_type: NetBoxObject subclass + object type to return the index for + + Returns + ------- + _TypeIndex: index of this object type + """ + + items = self.base_structure.get(object_type.name) + if items is None: + return _TypeIndex(object_type, list()) + + indexes = self.__dict__.get("_indexes") + if indexes is None: + indexes = self._indexes = dict() + + type_index = indexes.get(object_type.name) + if type_index is None or type_index.items is not items or len(type_index.position) != len(items): + type_index = indexes[object_type.name] = _TypeIndex(object_type, items) + + return type_index + + def object_data_changed(self, this_object, data_key=None): + """ + Called by a NetBoxObject whenever its data or its NetBox ID changes, keeps the indexes up to date. + + Parameters + ---------- + this_object: NetBoxObject + the changed object + data_key: str + the changed data key, None if the whole data set changed + """ + + indexes = self.__dict__.get("_indexes") + if not indexes: + return + + type_index = indexes.get(this_object.name) + if type_index is not None: + type_index.update(this_object, data_key) + + def get_index_candidates(self, object_type, index_name, values): + """ + Return all objects of $object_type which possibly carry one of $values in the indexed + attribute $index_name, in the order of the inventory. The result can contain objects which don't + match, callers have to apply their own comparison to each object. + + Parameters + ---------- + object_type: NetBoxObject subclass + object type to find + index_name: str + name of the index, the data key of one of the indexes in '_type_index_definitions' + values: list, tuple + values to look for + + Returns + ------- + list: of candidate objects + """ + + type_index = self._get_index(object_type) + + key_function = None + for spec in type_index.specs: + if spec.name == index_name: + key_function = spec.key_function + break + + if key_function is None: + raise ValueError(f"No index '{index_name}' defined for {object_type.name} objects") + + keys = list() + for value in values: + key = key_function(value) + if key is _VOLATILE: + return list(type_index.items) + if key is not _SKIP: + keys.append(key) + + candidates = type_index.candidates(index_name, keys) + if candidates is None: + return list(type_index.items) + + return candidates + + def get_prefix_candidates(self, ip_to_match): + """ + Return all prefixes which possibly contain $ip_to_match, in the order of the inventory. + Callers have to apply their own comparison to each prefix. + + Parameters + ---------- + ip_to_match: IPv4Address, IPv6Address + IP address to find prefixes for + + Returns + ------- + list: of candidate NBPrefix objects + """ + + if not isinstance(ip_to_match, (IPv4Address, IPv6Address)): + return list(self.get_all_items(NBPrefix)) + + max_prefix_length = ip_to_match.max_prefixlen + all_ones = (1 << max_prefix_length) - 1 + ip_int = int(ip_to_match) + + keys = list() + for prefix_length in range(max_prefix_length + 1): + netmask = all_ones ^ (all_ones >> prefix_length) + keys.append((ip_to_match.version, prefix_length, ip_int & netmask)) + + candidates = self._get_index(NBPrefix).candidates("prefix", keys) + if candidates is None: + return list(self.get_all_items(NBPrefix)) + + return candidates + + def get_primary_ip_candidates(self, object_type, ip_addresses): + """ + Return all objects of $object_type which possibly use one of $ip_addresses (without prefix length) + as primary IPv4 or IPv6 address, in the order of the inventory. Callers have to apply their own + comparison to each object. + + Parameters + ---------- + object_type: NBDevice, NBVM + object type to find + ip_addresses: list + IP address strings without prefix length + + Returns + ------- + list: of candidate objects + """ + + ip_objects = self.get_index_candidates(NBIPAddress, "address", [x for x in ip_addresses if x is not None]) + + result = dict() + for index_name in ["primary_ip4", "primary_ip6"]: + for this_object in self.get_index_candidates(object_type, index_name, ip_objects): + result[this_object] = None + + position = self._get_index(object_type).position + return sorted(result, key=position.__getitem__) + def add_source(self, source_handler=None): """ adds $source_tag to list of disabled sources @@ -83,7 +480,11 @@ def get_by_id(self, object_type, nb_id=None): if nb_id is None or self.base_structure[object_type.name] is None: return None - for this_object in self.base_structure[object_type.name]: + candidates = self._get_index(object_type).candidates("__nb_id__", [_value_key(nb_id)]) + if candidates is None: + candidates = self.base_structure[object_type.name] + + for this_object in candidates: if this_object.nb_id == nb_id: return this_object @@ -130,28 +531,34 @@ def get_by_data(self, object_type, data=None): elif data.get(object_type.primary_key) is not None: if object_type in (NBVMInterface, NBVirtualDisk) and isinstance(data.get("virtual_machine"), NBVM): # VM names are only unique within a cluster; use the resolved parent object. - for this_object in self.get_all_items(object_type): + for this_object in self.get_index_candidates(object_type, "virtual_machine", + [data["virtual_machine"]]): if this_object.data.get("virtual_machine") is data["virtual_machine"] and \ str(this_object.data.get(object_type.primary_key)).lower() == \ str(data[object_type.primary_key]).lower(): return this_object return None - object_name_to_find = None - for this_object in self.get_all_items(object_type): + object_name_to_find = self.get_all_items(object_type)[0].get_display_name(data, including_second_key=True) + name_to_find = f"{object_name_to_find}".lower() - if object_name_to_find is None: - object_name_to_find = this_object.get_display_name(data, including_second_key=True) + for this_object in self._get_display_name_candidates(object_type, name_to_find): # compare lower key - if f"{object_name_to_find}".lower() == \ - f"{this_object.get_display_name(including_second_key=True)}".lower(): + if name_to_find == f"{this_object.get_display_name(including_second_key=True)}".lower(): return this_object # try to match all data attributes else: - for this_object in self.get_all_items(object_type): + candidates = None + for attr_name, attr_value in data.items(): + if attr_name in ("serial", "asset_tag") and \ + attr_name in _type_index_definitions.get(object_type, dict()).keys(): + candidates = self.get_index_candidates(object_type, attr_name, [attr_value]) + break + + for this_object in candidates if candidates is not None else self.get_all_items(object_type): all_items_match = True for attr_name, attr_value in data.items(): @@ -164,6 +571,39 @@ def get_by_data(self, object_type, data=None): return None + def _get_display_name_candidates(self, object_type, name_to_find): + """ + Return all objects of $object_type whose lower-cased display name (including second key) can be + $name_to_find, in the order of the inventory. + + A display name is the primary key value, optionally followed by " (" and the secondary key. So the + primary key of a matching object is either the whole name or the part in front of one of the " (" + in it. + + Parameters + ---------- + object_type: NetBoxObject subclass + object type to find + name_to_find: str + lower-cased display name to find + + Returns + ------- + list: of candidate objects + """ + + keys = [name_to_find] + separator_position = name_to_find.find(" (") + while separator_position != -1: + keys.append(name_to_find[:separator_position]) + separator_position = name_to_find.find(" (", separator_position + 1) + + candidates = self._get_index(object_type).candidates("__primary_key__", keys) + if candidates is None: + return list(self.get_all_items(object_type)) + + return candidates + def slug_used(self, object_type: NetBoxObject, slug: str) -> bool: """ Determine if a slug for an object_tpe is already used @@ -215,7 +655,17 @@ def add_object(self, object_type, data=None, read_from_netbox=False, source=None new_object = object_type(data, read_from_netbox=read_from_netbox, inventory=self, source=source) # add to inventory - self.base_structure[object_type.name].append(new_object) + items = self.base_structure[object_type.name] + items.append(new_object) + + # add to index if one exists, otherwise it gets built with the next lookup + indexes = self.__dict__.get("_indexes") + type_index = indexes.get(object_type.name) if indexes else None + if type_index is not None: + if type_index.items is items and len(type_index.position) == len(items) - 1: + type_index.add(new_object, len(items) - 1) + else: + del indexes[object_type.name] if read_from_netbox is False: log.info(f"Created new {new_object.name} object: {new_object.get_display_name()}") @@ -326,12 +776,12 @@ def get_all_interfaces(self, this_object: (NBVM, NBDevice)): interfaces = list() if isinstance(this_object, NBVM): - for interface in self.get_all_items(NBVMInterface): + for interface in self.get_index_candidates(NBVMInterface, "virtual_machine", [this_object]): if grab(interface, "data.virtual_machine") == this_object: interfaces.append(interface) if isinstance(this_object, NBDevice): - for interface in self.get_all_items(NBInterface): + for interface in self.get_index_candidates(NBInterface, "device", [this_object]): if grab(interface, "data.device") == this_object: interfaces.append(interface) diff --git a/module/netbox/object_classes.py b/module/netbox/object_classes.py index 654b4e6e..9b13b459 100644 --- a/module/netbox/object_classes.py +++ b/module/netbox/object_classes.py @@ -261,6 +261,62 @@ def scopes_object_types(self, scopes_list) -> list: return result +class NBDataDict(dict): + """ + The data dict of a NetBoxObject. Behaves like a plain dict but reports every change to its owner, + which keeps the indexes of the inventory up to date (see module/netbox/inventory.py). + """ + + __slots__ = ("_owner",) + + def __init__(self, owner, *args, **kwargs): + super().__init__(*args, **kwargs) + self._owner = owner + + def _changed(self, key=None): + owner = getattr(self, "_owner", None) + if owner is not None: + owner.data_changed(key) + + def __setitem__(self, key, value): + super().__setitem__(key, value) + self._changed(key) + + def __delitem__(self, key): + super().__delitem__(key) + self._changed(key) + + def __ior__(self, other): + result = super().__ior__(other) + self._changed() + return result + + def clear(self): + super().clear() + self._changed() + + def pop(self, key, *args): + result = super().pop(key, *args) + self._changed(key) + return result + + def popitem(self): + result = super().popitem() + self._changed(result[0]) + return result + + def setdefault(self, key, default=None): + if key in self: + return self[key] + result = super().setdefault(key, default) + self._changed(key) + return result + + def update(self, *args, **kwargs): + super().update(*args, **kwargs) + self._changed() + + class NetBoxObject: """ Base class for all NetBox object types. Implements all methods used on a NetBox object. @@ -353,6 +409,34 @@ def __init__(self, data=None, read_from_netbox=False, inventory=None, source=Non # add data to this object self.update(data=data, read_from_netbox=read_from_netbox, source=source) + def __setattr__(self, key, value): + + # every change to data or the NetBox ID needs to be reported to the inventory indexes + if key == "data" and isinstance(value, dict) and \ + not (isinstance(value, NBDataDict) and getattr(value, "_owner", None) is self): + value = NBDataDict(self, value) + + super().__setattr__(key, value) + + if key in ("data", "nb_id"): + self.data_changed(None if key == "data" else "__nb_id__") + + def data_changed(self, key=None): + """ + reports a change of the data (key) or the NetBox ID ("__nb_id__") of this object to the inventory + + Parameters + ---------- + key: str + name of the changed data key, None if the whole data set changed + """ + + inventory = self.inventory + if inventory is not None: + object_data_changed = getattr(inventory, "object_data_changed", None) + if object_data_changed is not None: + object_data_changed(self, key) + def __repr__(self): return "<%s instance '%s' at %s>" % (self.__class__.__name__, self.get_display_name(), id(self)) @@ -2048,7 +2132,7 @@ def __init__(self, *args, **kwargs): def get_virtual_disks(self): result_list = list() - for ip_object in self.inventory.get_all_items(NBVirtualDisk): + for ip_object in self.inventory.get_index_candidates(NBVirtualDisk, "virtual_machine", [self]): if grab(ip_object, "data.virtual_machine") == self: result_list.append(ip_object) @@ -2084,7 +2168,7 @@ def __init__(self, *args, **kwargs): def get_ip_addresses(self): result_list = list() - for ip_object in self.inventory.get_all_items(NBIPAddress): + for ip_object in self.inventory.get_index_candidates(NBIPAddress, "assigned_object_id", [self]): if ip_object.data.get("assigned_object_id") == self: result_list.append(ip_object) @@ -2093,7 +2177,7 @@ def get_ip_addresses(self): def get_mac_addresses(self): result_list = list() - for mac_object in self.inventory.get_all_items(NBMACAddress): + for mac_object in self.inventory.get_index_candidates(NBMACAddress, "assigned_object_id", [self]): if mac_object.data.get("assigned_object_id") == self: result_list.append(mac_object) @@ -2138,7 +2222,7 @@ def __init__(self, *args, **kwargs): def get_ip_addresses(self): result_list = list() - for ip_object in self.inventory.get_all_items(NBIPAddress): + for ip_object in self.inventory.get_index_candidates(NBIPAddress, "assigned_object_id", [self]): if ip_object.data.get("assigned_object_id") == self: result_list.append(ip_object) @@ -2147,7 +2231,7 @@ def get_ip_addresses(self): def get_mac_addresses(self): result_list = list() - for mac_object in self.inventory.get_all_items(NBMACAddress): + for mac_object in self.inventory.get_index_candidates(NBMACAddress, "assigned_object_id", [self]): if mac_object.data.get("assigned_object_id") == self: result_list.append(mac_object) diff --git a/module/sources/common/source_base.py b/module/sources/common/source_base.py index e44de2b5..4e96fcfe 100644 --- a/module/sources/common/source_base.py +++ b/module/sources/common/source_base.py @@ -34,20 +34,11 @@ class SourceBase: def ip_address_objects_for(self, address: str) -> list: """ Return the inventory's IP address objects for an address (without prefix length), in - inventory order. The inventory only ever appends, so the index catches up with the - objects added since the last call instead of scanning every IP address per lookup. + inventory order, from the inventory's address index instead of scanning every IP address + per lookup. The index follows every change of an object's address, see the indexes in + module/netbox/inventory.py. """ - all_ips = self.inventory.get_all_items(NBIPAddress) - index = self.__dict__.setdefault("_ip_index", dict()) - indexed = self.__dict__.get("_ip_index_size", 0) - if indexed > len(all_ips): # a different inventory list: start over - index.clear() - indexed = 0 - for ip in all_ips[indexed:]: - key = grab(ip, "data.address", fallback="").split("/")[0] - index.setdefault(key, list()).append(ip) - self._ip_index_size = len(all_ips) - return index.get(address, list()) + return self.inventory.get_index_candidates(NBIPAddress, "address", [address]) def set_source_tag(self): self.source_tag = f"Source: {self.name}" @@ -388,7 +379,7 @@ def return_longest_matching_prefix_for_ip(self, ip_to_match=None, site_name=None current_longest_matching_prefix_length = 0 current_longest_matching_prefix = None - for prefix in self.inventory.get_all_items(NBPrefix): + for prefix in self.inventory.get_prefix_candidates(ip_to_match): if not prefix.matches_site(site_object): continue @@ -523,7 +514,8 @@ def add_update_interface(self, interface_object, device_object, interface_data, # if a new interface or not matching assigned MAC address, try to find an existing unassigned mac address if primary_mac_address_object is None: - for mac_address_object in self.inventory.get_all_items(NBMACAddress): + for mac_address_object in self.inventory.get_index_candidates(NBMACAddress, "mac_address", + [interface_mac_address]): # an object already assigned to this very interface is the one we want, not a duplicate if (grab(mac_address_object, "data.mac_address") == interface_mac_address and grab(mac_address_object, "data.assigned_object_id") in (None, interface_object)): diff --git a/module/sources/vmware/connection.py b/module/sources/vmware/connection.py index 123260cb..525984e1 100644 --- a/module/sources/vmware/connection.py +++ b/module/sources/vmware/connection.py @@ -1180,7 +1180,7 @@ def get_object_based_on_macs(self, object_type, mac_list=None): objects_with_matching_macs = dict() matching_object = None - for interface in self.inventory.get_all_items(interface_typ): + for interface in self.inventory.get_index_candidates(interface_typ, "mac_address", mac_list): if grab(interface, "data.mac_address") in mac_list: @@ -1286,7 +1286,7 @@ def _matches_device_primary_ip(device_primary_ip, ip_needle): if primary_ip6 is not None: primary_ip6 = str(primary_ip6).split("/")[0] - for device in self.inventory.get_all_items(object_type): + for device in self.inventory.get_primary_ip_candidates(object_type, [primary_ip4, primary_ip6]): if _matches_device_primary_ip(grab(device, "data.primary_ip4"), primary_ip4) is True: log.debug2(f"Found existing host '{device.get_display_name()}' " diff --git a/tests/test_inventory_indexes.py b/tests/test_inventory_indexes.py new file mode 100644 index 00000000..f16af5e9 --- /dev/null +++ b/tests/test_inventory_indexes.py @@ -0,0 +1,379 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2020 - 2026 netbox-sync team. All rights reserved. +# +# netbox-sync.py +# +# This work is licensed under the terms of the MIT license. +# For a copy, see file LICENSE.txt included in this +# repository or visit: . + +""" +The inventory keeps indexes so lookups don't have to compare every object of a type. These +tests check that every indexed lookup returns exactly what the plain scan over all objects +returned before, also after objects were renamed, moved, re-addressed or replaced behind +the inventory's back. +""" +import json +import random +from ipaddress import ip_address, ip_network +from types import SimpleNamespace + +import pytest + +from module.common.misc import grab +from module.netbox.object_classes import ( + NBDataDict, NBSite, NBCluster, NBVM, NBVMInterface, NBIPAddress, NBMACAddress, NBPrefix, NBDevice, + NBInterface, NBVirtualDisk, NBVLAN, NBClusterType, +) +from module.sources.common.source_base import SourceBase + +TS = "2026-01-01T00:00:00Z" + + +# reference implementations: the scans the inventory used before it had indexes + +def scan_get_by_data(inv, object_type, data): + object_name_to_find = None + for this_object in inv.get_all_items(object_type): + if object_name_to_find is None: + object_name_to_find = this_object.get_display_name(data, including_second_key=True) + if f"{object_name_to_find}".lower() == f"{this_object.get_display_name(including_second_key=True)}".lower(): + return this_object + return None + + +def scan_get_by_id(inv, object_type, nb_id): + for this_object in inv.get_all_items(object_type): + if this_object.nb_id == nb_id: + return this_object + return None + + +def scan_get_all_interfaces(inv, this_object): + if isinstance(this_object, NBVM): + return [i for i in inv.get_all_items(NBVMInterface) if grab(i, "data.virtual_machine") == this_object] + return [i for i in inv.get_all_items(NBInterface) if grab(i, "data.device") == this_object] + + +def scan_ips(inv, address): + return [ip for ip in inv.get_all_items(NBIPAddress) + if grab(ip, "data.address", fallback="").startswith(f"{address}/")] + + +def scan_longest_prefix(inv, ip, site_object): + longest_length, longest = 0, None + for prefix in inv.get_all_items(NBPrefix): + if not prefix.matches_site(site_object): + continue + network = grab(prefix, "data.prefix") + if network is None: + continue + if ip in network and network.prefixlen >= longest_length: + longest_length, longest = network.prefixlen, prefix + return longest + + +def indexed_longest_prefix(inv, ip, site_name): + return SourceBase.return_longest_matching_prefix_for_ip(SimpleNamespace(inventory=inv), ip, site_name) + + +@pytest.fixture +def base(inventory): + inventory.load_from_netbox(NBSite, [{"id": 1, "name": "S1", "slug": "s1", "last_updated": TS}, + {"id": 2, "name": "S2", "slug": "s2", "last_updated": TS}]) + inventory.load_from_netbox(NBClusterType, [{"id": 1, "name": "T", "slug": "t", "last_updated": TS}]) + inventory.load_from_netbox(NBCluster, [ + {"id": 1, "name": "C1", "type": {"id": 1}, "scope_type": "dcim.site", "scope_id": 1, "last_updated": TS}, + {"id": 2, "name": "C2", "type": {"id": 1}, "scope_type": "dcim.site", "scope_id": 2, "last_updated": TS}, + ]) + inventory.resolve_relations() + return SimpleNamespace(inventory=inventory, + c1=inventory.get_by_id(NBCluster, nb_id=1), c2=inventory.get_by_id(NBCluster, nb_id=2), + s1=inventory.get_by_id(NBSite, nb_id=1), s2=inventory.get_by_id(NBSite, nb_id=2)) + + +def add_vm(inv, name, cluster, nb_id): + return inv.load_from_netbox(NBVM, [{"id": nb_id, "name": name, "cluster": cluster, "last_updated": TS}]) or \ + inv.get_all_items(NBVM)[-1] + + +def test_data_is_a_tracking_dict(base): + vm = add_vm(base.inventory, "vm1", {"id": 1}, 10) + assert isinstance(vm.data, NBDataDict) and isinstance(vm.data, dict) + assert vm.data == {"id": 10, "name": "vm1", "cluster": {"id": 1}, "last_updated": TS} + assert json.loads(json.dumps(vm.data))["name"] == "vm1" + vm.data = {"id": 10, "name": "other"} + assert isinstance(vm.data, NBDataDict) + + +def test_lookup_is_case_insensitive(base): + inv = base.inventory + add_vm(inv, "MyVM", {"id": 1}, 10) + inv.resolve_relations() + found = inv.get_by_data(NBVM, data={"name": "myvm", "cluster": base.c1}) + assert found is not None and found.data["name"] == "MyVM" + assert inv.get_by_data(NBVM, data={"name": "MYVM", "cluster": base.c2}) is None + + +def test_rename_moves_object_in_index(base): + inv = base.inventory + vm = add_vm(inv, "old", {"id": 1}, 10) + inv.resolve_relations() + assert inv.get_by_data(NBVM, data={"name": "old", "cluster": base.c1}) is vm + + vm.update(data={"name": "new"}) + assert inv.get_by_data(NBVM, data={"name": "new", "cluster": base.c1}) is vm + assert inv.get_by_data(NBVM, data={"name": "old", "cluster": base.c1}) is None + + # direct writes to the data dict are tracked as well + vm.data["name"] = "direct" + assert inv.get_by_data(NBVM, data={"name": "direct", "cluster": base.c1}) is vm + vm.data = dict(vm.data, name="replaced") + assert inv.get_by_data(NBVM, data={"name": "replaced", "cluster": base.c1}) is vm + assert inv.get_by_data(NBVM, data={"name": "direct", "cluster": base.c1}) is None + + +def test_duplicate_names_return_first_in_inventory_order(base): + inv = base.inventory + first = add_vm(inv, "dup", {"id": 1}, 10) + second = add_vm(inv, "DUP", {"id": 1}, 11) + inv.resolve_relations() + assert inv.get_by_data(NBVM, data={"name": "dup", "cluster": base.c1}) is first + + # renaming the first one away and back keeps the inventory order, not the order of renames + first.update(data={"name": "away"}) + assert inv.get_by_data(NBVM, data={"name": "dup", "cluster": base.c1}) is second + first.update(data={"name": "dup"}) + assert inv.get_by_data(NBVM, data={"name": "dup", "cluster": base.c1}) is first + + +def test_secondary_key_change_is_found(base): + inv = base.inventory + vm = add_vm(inv, "vm1", {"id": 1}, 10) + inv.resolve_relations() + vm.update(data={"cluster": base.c2}) + assert inv.get_by_data(NBVM, data={"name": "vm1", "cluster": base.c2}) is vm + assert inv.get_by_data(NBVM, data={"name": "vm1", "cluster": base.c1}) is None + + # renaming the referenced object changes the display name of the VM without touching the VM + base.c2.update(data={"name": "C2-renamed"}) + assert inv.get_by_data(NBVM, data={"name": "vm1", "cluster": base.c2}) is vm + + +def test_names_with_parentheses(base): + inv = base.inventory + tricky = add_vm(inv, "a (C1", {"id": 1}, 10) + plain = add_vm(inv, "a", {"id": 1}, 11) + inv.resolve_relations() + for data in ({"name": "a (C1", "cluster": base.c1}, {"name": "a", "cluster": base.c1}, + {"name": "a (c1)", "cluster": base.c1}): + assert inv.get_by_data(NBVM, data=data) is scan_get_by_data(inv, NBVM, data) + assert inv.get_by_data(NBVM, data={"name": "a", "cluster": base.c1}) is plain + assert inv.get_by_data(NBVM, data={"name": "a (C1", "cluster": base.c1}) is tricky + + +def test_vlan_display_names(base): + inv = base.inventory + inv.load_from_netbox(NBVLAN, [{"id": 1, "vid": 100, "name": "one", "site": {"id": 1}, "last_updated": TS}, + {"id": 2, "vid": 100, "name": "two", "site": {"id": 2}, "last_updated": TS}, + {"id": 3, "vid": 200, "name": "three", "last_updated": TS}]) + inv.resolve_relations() + for data in ({"vid": 100, "name": "x", "site": base.s2}, {"vid": 100, "name": "x", "site": base.s1}, + {"vid": 200, "name": "three"}, {"vid": 200, "name": "THREE"}, {"vid": 300, "name": "x"}): + assert inv.get_by_data(NBVLAN, data=data) is scan_get_by_data(inv, NBVLAN, data) + + +def test_get_by_id_follows_id_changes(base): + inv = base.inventory + vm = add_vm(inv, "vm1", {"id": 1}, 10) + assert inv.get_by_id(NBVM, nb_id=10) is vm + vm.update(data={"id": 20, "name": "vm1", "cluster": {"id": 1}}, read_from_netbox=True) + assert inv.get_by_id(NBVM, nb_id=10) is None + assert inv.get_by_id(NBVM, nb_id=20) is vm + + +def test_interfaces_loaded_from_netbox_then_resolved(base): + inv = base.inventory + inv.load_from_netbox(NBVM, [{"id": 10, "name": "vm1", "cluster": {"id": 1}, "last_updated": TS}, + {"id": 11, "name": "vm2", "cluster": {"id": 1}, "last_updated": TS}]) + inv.load_from_netbox(NBVMInterface, [ + {"id": 1, "name": "eth0", "virtual_machine": {"id": 10}, "last_updated": TS}, + {"id": 2, "name": "eth1", "virtual_machine": {"id": 10}, "last_updated": TS}, + {"id": 3, "name": "eth0", "virtual_machine": {"id": 11}, "last_updated": TS}, + ]) + vm1, vm2 = inv.get_by_id(NBVM, nb_id=10), inv.get_by_id(NBVM, nb_id=11) + + # not resolved yet, a dict never equals an object + assert inv.get_all_interfaces(vm1) == scan_get_all_interfaces(inv, vm1) == [] + + inv.resolve_relations() + assert [i.nb_id for i in inv.get_all_interfaces(vm1)] == [1, 2] + assert [i.nb_id for i in inv.get_all_interfaces(vm2)] == [3] + + # re-parent an interface + interface = inv.get_by_id(NBVMInterface, nb_id=1) + interface.update(data={"virtual_machine": vm2}) + assert [i.nb_id for i in inv.get_all_interfaces(vm1)] == [2] + assert [i.nb_id for i in inv.get_all_interfaces(vm2)] == [1, 3] + + +def test_ip_reassignment_and_readdressing(base): + inv = base.inventory + inv.load_from_netbox(NBVM, [{"id": 10, "name": "vm1", "cluster": {"id": 1}, "last_updated": TS}]) + inv.load_from_netbox(NBVMInterface, [ + {"id": 1, "name": "eth0", "virtual_machine": {"id": 10}, "last_updated": TS}, + {"id": 2, "name": "eth1", "virtual_machine": {"id": 10}, "last_updated": TS}, + ]) + inv.load_from_netbox(NBIPAddress, [ + {"id": 1, "address": "10.0.0.1/24", "assigned_object_type": "virtualization.vminterface", + "assigned_object_id": 1, "last_updated": TS}, + {"id": 2, "address": "10.0.0.10/24", "last_updated": TS}, + ]) + inv.resolve_relations() + eth0, eth1 = inv.get_by_id(NBVMInterface, nb_id=1), inv.get_by_id(NBVMInterface, nb_id=2) + ip1 = inv.get_by_id(NBIPAddress, nb_id=1) + + # a prefix match of the string must not count: 10.0.0.1 is no candidate for 10.0.0.10 + assert inv.get_index_candidates(NBIPAddress, "address", ["10.0.0.1"]) == scan_ips(inv, "10.0.0.1") == [ip1] + assert eth0.get_ip_addresses() == [ip1] and eth1.get_ip_addresses() == [] + + ip1.update(data={"assigned_object_id": eth1}) + assert eth0.get_ip_addresses() == [] and eth1.get_ip_addresses() == [ip1] + + ip1.update(data={"address": "10.0.0.99/24"}) + assert inv.get_index_candidates(NBIPAddress, "address", ["10.0.0.1"]) == [] + assert inv.get_index_candidates(NBIPAddress, "address", ["10.0.0.99"]) == [ip1] + + +def test_list_replaced_behind_the_inventory(base): + inv = base.inventory + add_vm(inv, "vm1", {"id": 1}, 10) + assert inv.get_by_id(NBVM, nb_id=10) is not None + + # tests reset single object types like this + inv.base_structure[NBVM.name] = [] + assert inv.get_by_id(NBVM, nb_id=10) is None + vm = add_vm(inv, "vm1", {"id": 1}, 10) + assert inv.get_by_id(NBVM, nb_id=10) is vm + + +def test_virtual_disks_follow_their_vm(base): + inv = base.inventory + inv.load_from_netbox(NBVM, [{"id": 10, "name": "vm1", "cluster": {"id": 1}, "last_updated": TS}, + {"id": 11, "name": "vm2", "cluster": {"id": 1}, "last_updated": TS}]) + inv.load_from_netbox(NBVirtualDisk, [{"id": 1, "name": "d1", "size": 1, "virtual_machine": {"id": 10}, + "last_updated": TS}]) + inv.resolve_relations() + vm1, vm2 = inv.get_by_id(NBVM, nb_id=10), inv.get_by_id(NBVM, nb_id=11) + disk = inv.get_by_id(NBVirtualDisk, nb_id=1) + assert vm1.get_virtual_disks() == [disk] and vm2.get_virtual_disks() == [] + disk.update(data={"virtual_machine": vm2}) + assert vm1.get_virtual_disks() == [] and vm2.get_virtual_disks() == [disk] + + +def test_prefix_longest_match_equals_scan(base): + inv = base.inventory + prefixes = ["10.0.0.0/8", "10.1.0.0/16", "10.1.2.0/24", "10.1.2.0/24", "192.168.0.0/16", "0.0.0.0/0", + "2001:db8::/32", "2001:db8:1::/48"] + inv.load_from_netbox(NBPrefix, [ + {"id": i + 1, "prefix": p, "scope_type": "dcim.site" if i % 3 == 0 else None, + "scope_id": 1 if i % 3 == 0 else None, "last_updated": TS} for i, p in enumerate(prefixes)]) + inv.resolve_relations() + + for ip in ["10.1.2.3", "10.1.3.3", "10.2.0.1", "192.168.1.1", "8.8.8.8", "2001:db8:1::1", "2001:db8:2::1", + "2001:db9::1"]: + for site_name, site in [(None, None), ("S1", base.s1)]: + assert indexed_longest_prefix(inv, ip_address(ip), site_name) is \ + scan_longest_prefix(inv, ip_address(ip), site), (ip, site_name) + + # changing a prefix re-files it (prefixes are read only, so only a direct change is possible) + prefix = inv.get_by_id(NBPrefix, nb_id=5) + prefix.data["prefix"] = ip_network("10.1.2.0/25") + assert indexed_longest_prefix(inv, ip_address("10.1.2.3"), None) is \ + scan_longest_prefix(inv, ip_address("10.1.2.3"), None) + + +def test_mac_and_primary_ip_candidates(base): + inv = base.inventory + inv.load_from_netbox(NBVM, [ + {"id": 10, "name": "vm1", "cluster": {"id": 1}, "primary_ip4": {"id": 1}, "last_updated": TS}, + {"id": 11, "name": "vm2", "cluster": {"id": 1}, "last_updated": TS}]) + inv.load_from_netbox(NBVMInterface, [ + {"id": 1, "name": "eth0", "virtual_machine": {"id": 10}, "mac_address": "AA:AA:AA:AA:AA:01", + "last_updated": TS}, + {"id": 2, "name": "eth0", "virtual_machine": {"id": 11}, "mac_address": "AA:AA:AA:AA:AA:02", + "last_updated": TS}, + ]) + inv.load_from_netbox(NBIPAddress, [{"id": 1, "address": "10.0.0.1/24", "last_updated": TS}]) + inv.load_from_netbox(NBMACAddress, [{"id": 1, "mac_address": "AA:AA:AA:AA:AA:01", "last_updated": TS}]) + inv.resolve_relations() + vm1 = inv.get_by_id(NBVM, nb_id=10) + + assert [i.nb_id for i in inv.get_index_candidates(NBVMInterface, "mac_address", + ["AA:AA:AA:AA:AA:02", "AA:AA:AA:AA:AA:01"])] == [1, 2] + assert [m.nb_id for m in inv.get_index_candidates(NBMACAddress, "mac_address", ["AA:AA:AA:AA:AA:01"])] == [1] + assert inv.get_primary_ip_candidates(NBVM, ["10.0.0.1", None]) == [vm1] + assert inv.get_primary_ip_candidates(NBVM, ["10.0.0.2"]) == [] + + # a primary IP which is not resolved to an object is always a candidate + vm2 = inv.get_by_id(NBVM, nb_id=11) + vm2.data["primary_ip4"] = {"id": 99} + assert inv.get_primary_ip_candidates(NBVM, ["10.0.0.2"]) == [vm2] + + +def test_randomized_mutations_match_scans(base): + """mutate objects in every possible way and compare all indexed lookups with the scans""" + inv = base.inventory + rnd = random.Random(4711) + names = ["vm", "VM", "web (1)", "web", "db", "a (b) (c", "None", "x"] + addresses = [f"10.0.{a}.{b}" for a in range(2) for b in range(3)] + ["2001:db8::1", "2001:db8::10"] + clusters = [base.c1, base.c2] + + def new_vm(): + return inv.add_object(NBVM, data={"name": rnd.choice(names), "cluster": rnd.choice(clusters)}) + + vms = [new_vm() for _ in range(15)] + interfaces = [inv.add_object(NBVMInterface, data={"name": f"vNIC {rnd.randint(1, 3)}", + "virtual_machine": rnd.choice(vms)}) for _ in range(25)] + ips = [inv.add_object(NBIPAddress, data={"address": f"{rnd.choice(addresses)}/24", + "assigned_object_id": rnd.choice(interfaces)}) for _ in range(25)] + + def check(): + for name in names + [n.upper() for n in names]: + for cluster in clusters: + data = {"name": name, "cluster": cluster} + assert inv.get_by_data(NBVM, data=data) is scan_get_by_data(inv, NBVM, data), data + for vm in vms: + assert inv.get_all_interfaces(vm) == scan_get_all_interfaces(inv, vm) + assert inv.get_by_id(NBVM, nb_id=vm.nb_id) is scan_get_by_id(inv, NBVM, vm.nb_id) + for address in addresses: + assert [ip for ip in inv.get_index_candidates(NBIPAddress, "address", [address]) + if ip.data["address"].startswith(f"{address}/")] == scan_ips(inv, address) + for interface in interfaces: + assert interface.get_ip_addresses() == \ + [ip for ip in inv.get_all_items(NBIPAddress) if ip.data.get("assigned_object_id") == interface] + + check() + for step in range(300): + action = rnd.randrange(9) + if action == 0: + rnd.choice(vms).update(data={"name": rnd.choice(names)}) + elif action == 1: + rnd.choice(vms).update(data={"cluster": rnd.choice(clusters)}) + elif action == 2: + rnd.choice(interfaces).update(data={"virtual_machine": rnd.choice(vms)}) + elif action == 3: + rnd.choice(ips).update(data={"address": f"{rnd.choice(addresses)}/24"}) + elif action == 4: + rnd.choice(ips).update(data={"assigned_object_id": rnd.choice(interfaces)}) + elif action == 5: + vm = rnd.choice(vms) + vm.update(data={"id": rnd.randint(1, 20), "name": rnd.choice(names), "cluster": rnd.choice(clusters)}, + read_from_netbox=True) + elif action == 6: + vms.append(new_vm()) + elif action == 7: + rnd.choice(vms).data["name"] = rnd.choice(names) + else: + base.c1.update(data={"name": rnd.choice(["C1", "C1 (x)", "c2"])}) + check() diff --git a/tests/test_ip_address_index.py b/tests/test_ip_address_index.py index e3e1a4c3..bdd3eb03 100644 --- a/tests/test_ip_address_index.py +++ b/tests/test_ip_address_index.py @@ -53,3 +53,15 @@ def test_lookup_sees_objects_added_after_the_first_call(inventory): assert src.ip_address_objects_for("10.0.0.5") == [first, second] assert len(src.ip_address_objects_for("10.0.0.9")) == 1 + + +def test_lookup_follows_an_address_change_after_the_first_call(inventory): + moved = _add(inventory, "10.0.0.5/24") + other = _add(inventory, "10.0.0.6/24") + src = _source(inventory) + assert src.ip_address_objects_for("10.0.0.5") == [moved] + + moved.update(data={"address": "10.0.0.6/24"}) + + assert src.ip_address_objects_for("10.0.0.5") == _scan(inventory, "10.0.0.5") == [] + assert src.ip_address_objects_for("10.0.0.6") == _scan(inventory, "10.0.0.6") == [moved, other]