From a2b1681d548a780eeec1ad210cc209c12d3630b7 Mon Sep 17 00:00:00 2001 From: tyderian1978 Date: Tue, 17 Nov 2026 22:46:28 +1100 Subject: [PATCH] vmware: reduce repeated requests for the same vCenter data Every read of a property of a pyVmomi managed object is a SOAP round trip which returns the whole top level property. The VMware source read e.g. a VM's 'config' about 9 times, its 'guest' once per NIC, the ~1 MB 'config' of a host 6 times per host, and walked the same folders and resource pools again for every VM. * read 'config', 'runtime', 'guest' of a VM and 'summary', 'config' of a host once per object * skip VMs already parsed in the first run over all VMs by their managed object id instead of requesting name and config again * cache names, parents and resource pools of objects for the run; the folder and resource pool walks for VM paths, tenants and resource pool filters use these caches * cache the tags attached to an object for the run (the tags of parent folders, clusters and datacenters are requested for every VM); tag and category definitions are already cached since ec1ba92 * cache custom attribute definitions per object type and only request them if the object has custom values * hand the already retrieved guest info to add_update_interface() through the new SourceBase.get_vmware_guest_info() hook get_parent_object_by_class() kept its depth in an instance attribute which was not reset when a walk ended at the root, so later lookups could give up early. The rewrite walks iteratively and has no such state. Measured against vcsim with the captured inventories, synced inventory unchanged: vchvr: 1469 -> 323 SOAP calls, 20.1 -> 3.6 MB vc001: 1707 -> 355 SOAP calls, 30.1 -> 6.0 MB --- module/sources/common/source_base.py | 25 +- module/sources/vmware/connection.py | 325 +++++++++++++++++++------- tests/test_vmware_vcenter_requests.py | 211 +++++++++++++++++ 3 files changed, 474 insertions(+), 87 deletions(-) create mode 100644 tests/test_vmware_vcenter_requests.py diff --git a/module/sources/common/source_base.py b/module/sources/common/source_base.py index e44de2b5..7ca4aa09 100644 --- a/module/sources/common/source_base.py +++ b/module/sources/common/source_base.py @@ -353,6 +353,23 @@ def map_object_interfaces_to_current_interfaces(self, device_vm_object, interfac return return_data + def get_vmware_guest_info(self, vmware_object): + """ + Return the guest info of a VMware VM. Reading it is a round trip to vCenter, sources which already + retrieved it can return their copy. + + Parameters + ---------- + vmware_object: vim.VirtualMachine + the VM + + Returns + ------- + vim.vm.GuestInfo, None: the guest info + """ + + return grab(vmware_object, "guest") + def return_longest_matching_prefix_for_ip(self, ip_to_match=None, site_name=None) -> Optional[NBPrefix]: """ This is a lazy approach to find the longest matching prefix to an IP address. @@ -539,10 +556,14 @@ def add_update_interface(self, interface_object, device_object, interface_data, interface_object.update(data={"primary_mac_address": primary_mac_address_object}, source=self) + vmware_guest_info = None + if type(device_object) == NBVM: + vmware_guest_info = self.get_vmware_guest_info(vmware_object) + # skip handling of IPs for VMs with not installed/running guest tools skip_ip_handling = False if type(device_object) == NBVM and vmware_object is not None and \ - grab(vmware_object, 'guest.toolsRunningStatus') != "guestToolsRunning": + grab(vmware_guest_info, 'toolsRunningStatus') != "guestToolsRunning": log.debug(f"VM '{device_object.name}' guest tool status is 'NotRunning', skipping IP handling") skip_ip_handling = True @@ -818,7 +839,7 @@ def add_update_interface(self, interface_object, device_object, interface_data, # (seen on old TMOS releases), not a statement that every address is gone. Keep what is # in NetBox instead of tearing it off on every run. A real removal still reports the # interface, just without an address, so that case is unaffected - reported_interfaces = grab(vmware_object, "guest.net") + reported_interfaces = grab(vmware_guest_info, "net") if type(device_object) == NBVM and isinstance(reported_interfaces, list) and \ len(reported_interfaces) == 0: log.debug(f"VM '{device_object.name}' guest tools report no network interface at all, " diff --git a/module/sources/vmware/connection.py b/module/sources/vmware/connection.py index 123260cb..591f1171 100644 --- a/module/sources/vmware/connection.py +++ b/module/sources/vmware/connection.py @@ -109,8 +109,6 @@ class VMWareHandler(SourceBase): vm_name_max_length = 64 vm_description_max_length = 200 - recursion_level = 0 - # internal vars session = None tag_session = None @@ -188,6 +186,7 @@ def __init__(self, name=None): self.processed_host_names = dict() self.processed_vm_names = dict() self.processed_vm_uuid = list() + self.processed_vm_object_keys = set() self.object_cache = dict() self.object_path_cache = dict() self.parsing_vms_the_first_time = True @@ -1318,13 +1317,22 @@ def get_vmware_object_tags(self, obj): tag_list = list() if self.tag_session is not None: - # noinspection PyBroadException - try: - object_tag_ids = self.tag_session.tagging.TagAssociation.list_attached_tags( - DynamicID(type=grab(obj, "_wsdlName"), id=grab(obj, "_moId"))) - except Exception as e: - log.error(f"Unable to retrieve vCenter tags for '{obj.name}': {e}") - return + # the tags of parent objects (folders, clusters, datacenters) are requested for each + # VM and host, ask vCenter only once per object + attached_tags_cache = self.get_run_cache("attached_tags") + object_key = (grab(obj, "_wsdlName"), grab(obj, "_moId")) + + object_tag_ids = attached_tags_cache.get(object_key) + if object_tag_ids is None: + # noinspection PyBroadException + try: + object_tag_ids = self.tag_session.tagging.TagAssociation.list_attached_tags( + DynamicID(type=object_key[0], id=object_key[1])) + except Exception as e: + log.error(f"Unable to retrieve vCenter tags for '{obj.name}': {e}") + return + + attached_tags_cache[object_key] = object_tag_ids # vCenter tags and categories by id: most objects share a few hundred tags, so each # is fetched once per run (per source) instead of once per object it's attached to @@ -1404,7 +1412,7 @@ def collect_object_tags(self, obj): if tag_source is None or self.tag_session is None: return tag_list - log.debug2(f"Collecting tags for {obj.name}") + log.debug2(f"Collecting tags for {self.get_object_name(obj)}") if "object" in tag_source: tag_list.extend(self.get_vmware_object_tags(obj)) @@ -1463,9 +1471,10 @@ def get_object_custom_fields(self, obj): # add basic host data to device if object_type == "dcim.device": - num_cpu_cores = grab(obj, "summary.hardware.numCpuCores") - cpu_model = grab(obj, "summary.hardware.cpuModel") - memory_size = grab(obj, "summary.hardware.memorySize") + host_hardware = grab(obj, "summary.hardware") + num_cpu_cores = grab(host_hardware, "numCpuCores") + cpu_model = grab(host_hardware, "cpuModel") + memory_size = grab(host_hardware, "memorySize") if num_cpu_cores is not None: custom_field = self.add_update_custom_field({ @@ -1514,11 +1523,22 @@ def get_object_custom_fields(self, obj): return_custom_fields[grab(custom_field, "data.name")] = guest_hostname else: - log.debug2(f"VM '{grab(obj, 'name')}' guest hostname not reported by VMware Tools " + log.debug2(f"VM '{self.get_object_name(obj)}' guest hostname not reported by VMware Tools " "(not installed, not running or no data available yet). Keeping current " f"value of custom field '{guest_hostname_field_name}' untouched.") - field_definition = {grab(k, "key"): grab(k, "name") for k in grab(obj, "availableField", fallback=list())} + # the custom attribute definitions are the same for all objects of a type, + # request them only once per type and only if there are custom values to label + field_definition = dict() + if len(custom_value) > 0: + field_definition_cache = self.get_run_cache("custom_field_definitions") + object_wsdl_name = grab(obj, "_wsdlName") + field_definition = field_definition_cache.get(object_wsdl_name) + if field_definition is None: + field_definition = {grab(k, "key"): grab(k, "name") + for k in grab(obj, "availableField", fallback=list())} + if object_wsdl_name is not None: + field_definition_cache[object_wsdl_name] = field_definition for obj_custom_field in custom_value: key = grab(obj_custom_field, "key") @@ -2034,34 +2054,139 @@ def add_device_vm_to_inventory(self, object_type, object_data, pnic_data=None, v return device_vm_object, interface_objects + def get_run_cache(self, name): + """ + Return a dict which caches vCenter data for the duration of this run. Reading a property of a + pyVmomi managed object is a round trip to vCenter, the caches avoid asking for the same data + again and again (parents and names of clusters, folders and datacenters, tag definitions, ...). + + The caches are created on first use, handlers which were built without running __init__ + (as the unit tests do) get them as well. + + Parameters + ---------- + name: str + name of the cache + + Returns + ------- + dict: the cache + """ + + caches = self.__dict__.setdefault("_run_caches", dict()) + + return caches.setdefault(name, dict()) + + @staticmethod + def get_vmware_object_key(obj): + """ + Return a key which identifies a vCenter managed object without a round trip to vCenter + + Parameters + ---------- + obj: pyVmomi managed object + + Returns + ------- + tuple, None: type name and managed object id or None if obj is not a managed object + """ + + object_id = getattr(obj, "_moId", None) + + if object_id is None: + return None + + return obj.__class__.__name__, object_id + + def get_object_name(self, obj): + """ + Return the name of a vCenter managed object. The name is only requested once per object and run. + + Parameters + ---------- + obj: pyVmomi managed object + + Returns + ------- + str, None: name of the object + """ + + return self.get_cached_property(obj, "name") + + def get_cached_property(self, obj, property_name): + """ + Return a top level property of a vCenter managed object which doesn't change during a run, like + its name, parent or resource pool. It is only requested once per object and run. + + Parameters + ---------- + obj: pyVmomi managed object + property_name: str + name of the property + + Returns + ------- + the value of the property, None if it can't be read + """ + + object_key = self.get_vmware_object_key(obj) + + if object_key is None: + return grab(obj, property_name) + + property_cache = self.get_run_cache(f"property:{property_name}") + + if object_key not in property_cache: + property_cache[object_key] = grab(obj, property_name) + + return property_cache[object_key] + + def get_vmware_guest_info(self, vmware_object): + """ + Return the guest info of the VM which is currently parsed by add_virtual_machine() without + asking vCenter again, other objects are passed on to SourceBase.get_vmware_guest_info(). + """ + + object_key = self.get_vmware_object_key(vmware_object) + guest_info_cache = self.get_run_cache("vm_guest") + + if object_key is not None and object_key in guest_info_cache: + return guest_info_cache[object_key] + + return super().get_vmware_guest_info(vmware_object) + def get_parent_object_by_class(self, obj, object_class_to_find): + """ + Walk up the vCenter inventory tree until an object of the requested class is found. + The parent of each object is only requested once per run. + + Parameters + ---------- + obj: pyVmomi managed object + object to start with, returned itself if it is of the requested class + object_class_to_find: type + the pyVmomi class to find + + Returns + ------- + pyVmomi managed object, None: the found object or None if there is none + """ if obj is None or object_class_to_find is None: return - if isinstance(obj, object_class_to_find): - self.recursion_level = 0 - return obj - max_recursion = 20 - while True: - if self.recursion_level >= max_recursion: - self.recursion_level = 0 - return None + for _ in range(max_recursion + 1): - # noinspection PyBroadException - try: - parent = obj.parent - except Exception: - self.recursion_level = 0 - return None + if isinstance(obj, object_class_to_find): + return obj + + obj = self.get_cached_property(obj, "parent") - if isinstance(parent, object_class_to_find): - self.recursion_level = 0 - return parent + if obj is None: + return None - self.recursion_level += 1 - return self.get_parent_object_by_class(parent, object_class_to_find) + return None def add_object_to_cache(self, vm_object, netbox_object): @@ -2189,7 +2314,7 @@ def get_resource_pool_path(self, vm): otherwise None (i.e. for template VMs) """ - if grab(vm, "resourcePool") is None: + if self.get_cached_property(vm, "resourcePool") is None: return None cached_path = self.get_object_path_from_cache(vm, "resource_pool_path") @@ -2197,28 +2322,28 @@ def get_resource_pool_path(self, vm): return cached_path path_parts = list() - pool = grab(vm, "resourcePool") + pool = self.get_cached_property(vm, "resourcePool") # walk up the pool hierarchy, the cluster root pool ends the walk: it is the pool whose # parent is the cluster itself rather than another pool (vCenter names it "Resources", # but a child pool may carry that name too) while pool is not None and len(path_parts) < self.object_path_max_depth: - if not isinstance(grab(pool, "parent"), vim.ResourcePool): + if not isinstance(self.get_cached_property(pool, "parent"), vim.ResourcePool): break - pool_name = get_string_or_none(grab(pool, "name")) + pool_name = get_string_or_none(self.get_object_name(pool)) if pool_name is None: break path_parts.append(pool_name) - pool = grab(pool, "parent") + pool = self.get_cached_property(pool, "parent") - if pool is not None and isinstance(grab(pool, "parent"), vim.ResourcePool): - log.warning(f"Resource pool hierarchy of VM '{get_string_or_none(grab(vm, 'name'))}' is deeper " + if pool is not None and isinstance(self.get_cached_property(pool, "parent"), vim.ResourcePool): + log.warning(f"Resource pool hierarchy of VM '{get_string_or_none(self.get_object_name(vm))}' is deeper " f"than {self.object_path_max_depth} levels, the pool path is cut off at the top") path = "/".join(reversed(path_parts)) - log.debug2(f"VM '{get_string_or_none(grab(vm, 'name'))}' has resource pool path '{path}'") + log.debug2(f"VM '{get_string_or_none(self.get_object_name(vm))}' has resource pool path '{path}'") self.add_object_paths_to_cache(vm, {"resource_pool_path": path}) @@ -2242,7 +2367,7 @@ def get_vm_folder_path(self, vm): str, None: VM folder path if the VM is organized in a folder, otherwise None """ - if grab(vm, "parent") is None or not isinstance(grab(vm, "parent"), vim.Folder): + if not isinstance(self.get_cached_property(vm, "parent"), vim.Folder): return None cached_path = self.get_object_path_from_cache(vm, "vm_folder_path") @@ -2250,28 +2375,29 @@ def get_vm_folder_path(self, vm): return cached_path path_parts = list() - folder = grab(vm, "parent") + folder = self.get_cached_property(vm, "parent") # walk up the folder hierarchy, the datacenter's VM folder ends the walk while isinstance(folder, vim.Folder) and len(path_parts) < self.object_path_max_depth: # the folder directly below the datacenter is the datacenter's VM folder - if isinstance(grab(folder, "parent"), vim.Datacenter): + if isinstance(self.get_cached_property(folder, "parent"), vim.Datacenter): break - folder_name = get_string_or_none(grab(folder, "name")) + folder_name = get_string_or_none(self.get_object_name(folder)) if folder_name is None: break path_parts.append(folder_name) - folder = grab(folder, "parent") + folder = self.get_cached_property(folder, "parent") - if isinstance(folder, vim.Folder) and not isinstance(grab(folder, "parent"), vim.Datacenter): - log.warning(f"Folder hierarchy of VM '{get_string_or_none(grab(vm, 'name'))}' is deeper than " + if isinstance(folder, vim.Folder) and not isinstance(self.get_cached_property(folder, "parent"), + vim.Datacenter): + log.warning(f"Folder hierarchy of VM '{get_string_or_none(self.get_object_name(vm))}' is deeper than " f"{self.object_path_max_depth} levels, the folder path is cut off at the top") path = "/".join(reversed(path_parts)) - log.debug2(f"VM '{get_string_or_none(grab(vm, 'name'))}' has VM folder path '{path}'") + log.debug2(f"VM '{get_string_or_none(self.get_object_name(vm))}' has VM folder path '{path}'") self.add_object_paths_to_cache(vm, {"vm_folder_path": path}) @@ -2317,7 +2443,7 @@ def get_vm_tenant_name(self, vm, vm_name, nb_cluster_object, cluster_full_name): tenant_name = self.get_object_relation(resource_pool_path, "vm_tenant_resource_pool_relation") if tenant_name is None: - resource_pool_name = get_string_or_none(grab(vm, "resourcePool.name")) + resource_pool_name = get_string_or_none(self.get_object_name(self.get_cached_property(vm, "resourcePool"))) if resource_pool_name is not None: tenant_name = self.get_object_relation(resource_pool_name, "vm_tenant_resource_pool_relation") @@ -2330,7 +2456,7 @@ def get_vm_tenant_name(self, vm, vm_name, nb_cluster_object, cluster_full_name): tenant_name = self.get_object_relation(vm_folder_path, "vm_tenant_folder_relation") if tenant_name is None: - vm_folder_name = get_string_or_none(grab(vm, "parent.name")) + vm_folder_name = get_string_or_none(self.get_object_name(self.get_cached_property(vm, "parent"))) if vm_folder_name is not None: tenant_name = self.get_object_relation(vm_folder_name, "vm_tenant_folder_relation") @@ -2377,7 +2503,7 @@ def vm_passes_resource_pool_filter(self, vm): return True resource_pool_path = self.get_resource_pool_path(vm) - resource_pool_name = get_string_or_none(grab(vm, "resourcePool.name")) + resource_pool_name = get_string_or_none(self.get_object_name(self.get_cached_property(vm, "resourcePool"))) pool_identifiers = [x for x in dict.fromkeys([resource_pool_path, resource_pool_name]) if x not in [None, ""]] @@ -2769,11 +2895,15 @@ def add_host(self, obj): # Collecting data # + # each read of a property of 'obj' is a round trip to vCenter which returns the whole + # property, so 'summary' and 'config' (which is large) are requested only once + host_summary = grab(obj, "summary") + # collect all necessary data - manufacturer = get_string_or_none(grab(obj, "summary.hardware.vendor")) - model = get_string_or_none(grab(obj, "summary.hardware.model")) - product_name = get_string_or_none(grab(obj, "summary.config.product.name")) - product_version = get_string_or_none(grab(obj, "summary.config.product.version")) + manufacturer = get_string_or_none(grab(host_summary, "hardware.vendor")) + model = get_string_or_none(grab(host_summary, "hardware.model")) + product_name = get_string_or_none(grab(host_summary, "config.product.name")) + product_version = get_string_or_none(grab(host_summary, "config.product.version")) # collect platform platform = f"{product_name} {product_version}" @@ -2788,11 +2918,11 @@ def add_host(self, obj): # get status status = "offline" - if get_string_or_none(grab(obj, "summary.runtime.connectionState")) == "connected": + if get_string_or_none(grab(host_summary, "runtime.connectionState")) == "connected": status = "active" # prepare identifiers to find asset tag and serial number - identifiers = grab(obj, "summary.hardware.otherIdentifyingInfo", fallback=list()) + identifiers = grab(host_summary, "hardware.otherIdentifyingInfo", fallback=list()) identifier_dict = dict() for item in identifiers: value = grab(item, "identifierValue", fallback="") @@ -2858,9 +2988,12 @@ def add_host(self, obj): if len(host_custom_fields) > 0: host_data["custom_fields"] = host_custom_fields + host_config = grab(obj, "config") + host_network = grab(host_config, "network") + # iterate over hosts virtual switches, needed to enrich data on physical interfaces self.network_data["vswitch"][name] = dict() - for vswitch in grab(obj, "config.network.vswitch", fallback=list()): + for vswitch in grab(host_network, "vswitch", fallback=list()): vswitch_name = unquote(grab(vswitch, "name")) @@ -2878,7 +3011,7 @@ def add_host(self, obj): # iterate over hosts proxy switches, needed to enrich data on physical interfaces # also stores data on proxy switch configured mtu which is used for VM interfaces self.network_data["pswitch"][name] = dict() - for pswitch in grab(obj, "config.network.proxySwitch", fallback=list()): + for pswitch in grab(host_network, "proxySwitch", fallback=list()): pswitch_uuid = grab(pswitch, "dvsUuid") pswitch_name = unquote(grab(pswitch, "dvsName")) @@ -2896,7 +3029,7 @@ def add_host(self, obj): # iterate over hosts port groups, needed to enrich data on physical interfaces self.network_data["host_pgroup"][name] = dict() - for pgroup in grab(obj, "config.network.portgroup", fallback=list()): + for pgroup in grab(host_network, "portgroup", fallback=list()): pgroup_name = grab(pgroup, "spec.name") @@ -2928,7 +3061,7 @@ def add_host(self, obj): except Exception as e: log.debug(f"Unable to query network hints (CDP/LLDP) of host '{name}': {e}") - pnic_list = grab(obj, "config.network.pnic", fallback=list()) + pnic_list = grab(host_network, "pnic", fallback=list()) if self.settings.skip_host_nics is True: log.debug(f"Skipping physical interfaces of host '{name}' (skip_host_nics)") pnic_list = list() @@ -3079,7 +3212,7 @@ def add_host(self, obj): fc_hba_data_dict = dict() if self.settings.sync_host_fc_adapters is True: # only Fibre Channel adapters are synced, block and PCIe adapters are ignored - for hba in self.fc_hba_ports(grab(obj, "config.storageDevice.hostBusAdapter", fallback=list()), name): + for hba in self.fc_hba_ports(grab(host_config, "storageDevice.hostBusAdapter", fallback=list()), name): hba_device = get_string_or_none(grab(hba, "device")) if hba_device is None: @@ -3121,7 +3254,7 @@ def add_host(self, obj): # now iterate over all virtual interfaces and collect data vnic_data_dict = dict() vnic_ips = dict() - for vnic in grab(obj, "config.network.vnic", fallback=list()): + for vnic in grab(host_network, "vnic", fallback=list()): vnic_name = grab(vnic, "device") @@ -3357,34 +3490,46 @@ def add_virtual_machine(self, obj): virtual machine object to parse """ - name = get_string_or_none(grab(obj, "name")) + reevaluating_this_vm = self.parsing_objects_to_reevaluate is True and obj in self.objects_to_reevaluate + + # a VM parsed in the first run over all VMs is skipped in the second one by its UUID. Skip it by + # its managed object id before anything is requested from vCenter. + vm_object_key = self.get_vmware_object_key(obj) + if vm_object_key is not None and vm_object_key in self.processed_vm_object_keys and not reevaluating_this_vm: + return + + name = get_string_or_none(self.get_object_name(obj)) name, vm_description = self.get_vm_name_and_description(name) + # each read of a property of 'obj' is a round trip to vCenter which returns the whole property, + # so 'config' and 'runtime' (and 'guest' further down) are requested only once + vm_config = grab(obj, "config") + vm_runtime = grab(obj, "runtime") + # # Filtering # # get VM UUID - vm_uuid = grab(obj, "config.instanceUuid") + vm_uuid = grab(vm_config, "instanceUuid") - if (vm_uuid is None or vm_uuid in self.processed_vm_uuid) and \ - not (self.parsing_objects_to_reevaluate is True and obj in self.objects_to_reevaluate): + if (vm_uuid is None or vm_uuid in self.processed_vm_uuid) and not reevaluating_this_vm: return log.debug(f"Parsing vCenter VM: {name}") # get VM power state - status = "active" if get_string_or_none(grab(obj, "runtime.powerState")) == "poweredOn" else "offline" + status = "active" if get_string_or_none(grab(vm_runtime, "powerState")) == "poweredOn" else "offline" # check if vm is template - template = grab(obj, "config.template") + template = grab(vm_config, "template") if bool(self.settings.skip_vm_templates) is True and template is True: log.debug2(f"VM '{name}' is a template. Skipping") return if bool(self.settings.skip_srm_placeholder_vms) is True \ - and f"{grab(obj, 'config.managedBy.extensionKey')}".startswith("com.vmware.vcDr"): + and f"{grab(vm_config, 'managedBy.extensionKey')}".startswith("com.vmware.vcDr"): log.debug2(f"VM '{name}' is a SRM placeholder VM. Skipping") return @@ -3395,8 +3540,10 @@ def add_virtual_machine(self, obj): # add to processed VMs self.processed_vm_uuid.append(vm_uuid) + if vm_object_key is not None: + self.processed_vm_object_keys.add(vm_object_key) - parent_host = self.get_parent_object_by_class(grab(obj, "runtime.host"), vim.HostSystem) + parent_host = self.get_parent_object_by_class(grab(vm_runtime, "host"), vim.HostSystem) cluster_object = self.get_parent_object_by_class(parent_host, vim.ClusterComputeResource) # get single host 'cluster' if VM runs on one @@ -3419,9 +3566,9 @@ def add_virtual_machine(self, obj): log.debug(f"Virtual machine '{name}' is not part of a permitted cluster. Skipping") return - parent_name = grab(parent_host, "name") + parent_name = self.get_object_name(parent_host) cluster_name = grab(nb_cluster_object, "data.name") - cluster_full_name = f"{group.name}/{cluster_name}" + cluster_full_name = f"{self.get_object_name(group)}/{cluster_name}" if name in self.processed_vm_names.get(cluster_full_name, list()) and obj not in self.objects_to_reevaluate: if self.settings.vm_name_regex is not None: @@ -3455,12 +3602,20 @@ def add_virtual_machine(self, obj): if site_name is None: site_name = self.get_site_name(NBCluster, cluster_full_name) + vm_guest = grab(obj, "guest") + vm_guest_nics = grab(vm_guest, "net", fallback=list()) + + # source_base.add_update_interface() checks the guest tools state for every interface of this VM + self.get_run_cache("vm_guest").clear() + if vm_object_key is not None: + self.get_run_cache("vm_guest")[vm_object_key] = vm_guest + # first check against vm_platform_relation - platform = get_string_or_none(grab(obj, "config.guestFullName")) - platform = get_string_or_none(grab(obj, "guest.guestFullName", fallback=platform)) + platform = get_string_or_none(grab(vm_config, "guestFullName")) + platform = get_string_or_none(grab(vm_guest, "guestFullName", fallback=platform)) # extract prettyName from extraConfig exposed by guest tools - extra_config = {x.key: x.value for x in grab(obj, "config.extraConfig", fallback=[]) + extra_config = {x.key: x.value for x in grab(vm_config, "extraConfig", fallback=[]) if x.key in ["guestOS.detailed.data", "guestInfo.detailed.data"]} # first try 'guestInfo.detailed.data' and then 'guestOS.detailed.data' @@ -3493,11 +3648,11 @@ def add_virtual_machine(self, obj): if platform is not None: platform = self.get_object_relation(platform, "vm_platform_relation", fallback=platform) - hardware_devices = grab(obj, "config.hardware.device", fallback=list()) + hardware_devices = grab(vm_config, "hardware.device", fallback=list()) annotation = None if self.settings.skip_vm_comments is False: - annotation = get_string_or_none(grab(obj, "config.annotation")) + annotation = get_string_or_none(grab(vm_config, "annotation")) # resolve tenant for this VM tenant_name = self.get_vm_tenant_name(obj, name, nb_cluster_object, cluster_full_name) @@ -3519,7 +3674,7 @@ def add_virtual_machine(self, obj): vm_tags.extend(vcenter_tags) # vm memory depending on setting - vm_memory = grab(obj, "config.hardware.memoryMB", fallback=0) + vm_memory = grab(vm_config, "hardware.memoryMB", fallback=0) if self.settings.vm_disk_and_ram_in_decimal is True: vm_memory = int(vm_memory / 1024 * 1000) @@ -3529,7 +3684,7 @@ def add_virtual_machine(self, obj): "cluster": nb_cluster_object, "status": status, "memory": vm_memory, - "vcpus": grab(obj, "config.hardware.numCPU") + "vcpus": grab(vm_config, "hardware.numCPU") } # Add adaption for change in NetBox 3.3.0 VM model @@ -3574,7 +3729,7 @@ def add_virtual_machine(self, obj): vm_default_gateway_ip6 = None # check vm routing to determine which is the default interface for each IP version - for route in grab(obj, "guest.ipStack.0.ipRouteConfig.ipRoute", fallback=list()): + for route in grab(vm_guest, "ipStack.0.ipRouteConfig.ipRoute", fallback=list()): # we found a default route if grab(route, "prefixLength") == 0: @@ -3737,7 +3892,7 @@ def add_virtual_machine(self, obj): int_description = f"{int_description} ({vlan_description})" # find corresponding guest NIC and get IP addresses and connected status - for guest_nic in grab(obj, "guest.net", fallback=list()): + for guest_nic in vm_guest_nics: # get matching guest NIC if int_mac != normalize_mac_address(grab(guest_nic, "macAddress")): @@ -3820,7 +3975,7 @@ def add_virtual_machine(self, obj): dummy_interfaces_filter = self.settings.vm_dummy_interfaces_include_filter if self.settings.sync_vm_dummy_interfaces is True or \ (dummy_interfaces_filter is not None and dummy_interfaces_filter.match(name)): - for guest_nic in grab(obj, "guest.net", fallback=list()): + for guest_nic in vm_guest_nics: # get matching guest NIC MAC guest_nic_mac = normalize_mac_address(grab(guest_nic, "macAddress")) diff --git a/tests/test_vmware_vcenter_requests.py b/tests/test_vmware_vcenter_requests.py new file mode 100644 index 00000000..007f98b5 --- /dev/null +++ b/tests/test_vmware_vcenter_requests.py @@ -0,0 +1,211 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2020 - 2026 netbox-sync team. All rights reserved. +# +# netbox-sync.py +# +# This work is licensed under the terms of the MIT license. +# For a copy, see file LICENSE.txt included in this +# repository or visit: . + +""" +Reading a property of a pyVmomi managed object is a round trip to vCenter which returns the whole +property ('config' of a host is about 1 MB). The VMware source must not ask vCenter for the same data +over and over again. +""" +import collections +from types import SimpleNamespace + +import pytest + +from module.sources.vmware import connection as vmware_connection +from module.sources.vmware.connection import VMWareHandler + + +@pytest.fixture +def fetch_counter(monkeypatch): + """Counts every property read (SOAP 'Fetch') per managed object and property.""" + from pyVmomi import SoapAdapter + + counts = collections.Counter() + original = SoapAdapter.SoapStubAdapter.InvokeMethod + + def invoke(self, mo, info, args, outerStub=None): + if info.wsdlName == "Fetch": + counts[(mo._wsdlName, mo._moId, args[0])] += 1 + return original(self, mo, info, args, outerStub) + + monkeypatch.setattr(SoapAdapter.SoapStubAdapter, "InvokeMethod", invoke) + return counts + + +def test_host_and_vm_properties_are_not_requested_repeatedly(vmware_source, fetch_counter): + vmware_source.apply() + + assert len(fetch_counter) > 0, "no property reads were counted" + + # an offline VM is looked at in both runs over all VMs, so twice is fine + repeated = {key: count for key, count in fetch_counter.items() + if key[0] in ("VirtualMachine", "HostSystem") and count > 2} + + assert repeated == {}, "properties requested again and again from vCenter" + + +def test_a_vm_parsed_in_the_first_run_is_not_requested_again_in_the_second(vmware_source, fetch_counter, sdk): + from pyVmomi import vim + + view = sdk.viewManager.CreateContainerView(sdk.rootFolder, [vim.VirtualMachine], True) + powered_on = {vm._moId for vm in view.view if vm.runtime.powerState == "poweredOn"} + view.Destroy() + + # the reads above went through the counted stub as well + fetch_counter.clear() + vmware_source.apply() + + reevaluated = {obj._moId for obj in vmware_source.objects_to_reevaluate} + config_reads = [fetch_counter[("VirtualMachine", mo_id, "config")] for mo_id in powered_on - reevaluated] + + assert len(config_reads) > 0 + assert set(config_reads) == {1} + + +# +# unit tests without vcsim +# + +@pytest.fixture(autouse=True) +def dynamic_id(monkeypatch): + """The vSphere automation SDK is optional and absent here, the tag call needs the type.""" + monkeypatch.setattr(vmware_connection, "DynamicID", + lambda **kwargs: SimpleNamespace(**kwargs), raising=False) + + +class _CountingTagging: + """The parts of the vSphere tagging API get_vmware_object_tags() uses, counting each call.""" + + def __init__(self, attached): + self.calls = collections.Counter() + outer = self + + class _Tag: + @staticmethod + def get(tag_id): + outer.calls["Tag.get"] += 1 + return SimpleNamespace(name=f"name-{tag_id}", description="", category_id="cat-1") + + class _Category: + @staticmethod + def get(_category_id): + outer.calls["Category.get"] += 1 + return SimpleNamespace(name="env") + + class _TagAssociation: + @staticmethod + def list_attached_tags(dynamic_id): + outer.calls["list_attached_tags"] += 1 + return attached[dynamic_id.id] + + self.Tag, self.Category, self.TagAssociation = _Tag, _Category, _TagAssociation + + +def _bare_handler(inventory, **settings): + handler = object.__new__(VMWareHandler) + handler.inventory = inventory + handler.name = "test" + handler.source_tag = "Source: test" + handler.settings = SimpleNamespace(**settings) + return handler + + +def test_tag_definitions_and_attached_tags_are_requested_once(inventory): + tagging = _CountingTagging({"vm-1": ["t1", "t2"], "vm-2": ["t1"], "group-v1": ["t2"]}) + handler = _bare_handler(inventory, tag_name_include_category=True) + handler.tag_session = SimpleNamespace(tagging=tagging) + + def vm(mo_id): + return SimpleNamespace(name=mo_id, _wsdlName="VirtualMachine", _moId=mo_id) + + folder = SimpleNamespace(name="folder", _wsdlName="Folder", _moId="group-v1") + + results = list() + for obj in [vm("vm-1"), folder, vm("vm-2"), folder, vm("vm-1")]: + results.append(sorted(t.get_display_name() for t in handler.get_vmware_object_tags(obj))) + + assert results == [["env:name-t1", "env:name-t2"], ["env:name-t2"], ["env:name-t1"], + ["env:name-t2"], ["env:name-t1", "env:name-t2"]] + assert tagging.calls == {"list_attached_tags": 3, "Tag.get": 2, "Category.get": 1} + + +class _Node(SimpleNamespace): + """Fake managed object counting how often its parent is requested.""" + + def __init__(self, mo_id, parent, counter): + super().__init__(_moId=mo_id, _parent=parent, _counter=counter) + + @property + def parent(self): + self._counter[self._moId] += 1 + return self._parent + + +class _Datacenter(_Node): + pass + + +def test_parents_are_requested_once_per_object(): + counter = collections.Counter() + handler = object.__new__(VMWareHandler) + + datacenter = _Datacenter("datacenter-1", None, counter) + folder = _Node("group-h1", datacenter, counter) + cluster = _Node("domain-c1", folder, counter) + + for _ in range(5): + assert handler.get_parent_object_by_class(cluster, _Datacenter) is datacenter + + assert counter == {"domain-c1": 1, "group-h1": 1} + + +def test_unsuccessful_parent_lookups_do_not_break_later_ones(): + # a walk which ended at the root used to leave its depth (11 here) behind, the next walk then + # gave up after 20 - 11 levels and didn't find the datacenter 10 levels up + counter = collections.Counter() + handler = object.__new__(VMWareHandler) + + datacenter = _Datacenter("datacenter-1", None, counter) + chain = datacenter + for level in range(10): + chain = _Node(f"node-{level}", chain, counter) + + class _NotInTree: + pass + + assert handler.get_parent_object_by_class(chain, _NotInTree) is None + + assert handler.get_parent_object_by_class(chain, _Datacenter) is datacenter + + +def test_custom_field_definitions_are_only_requested_for_custom_values(inventory): + handler = _bare_handler(inventory, sync_custom_attributes=True, vm_custom_object_attributes=[], + host_custom_object_attributes=[], custom_attribute_exclude=None, + vm_guest_hostname_custom_field=None) + inventory.netbox_api_version = "4.3.0" + + class _VM(SimpleNamespace): + reads = 0 + + @property + def availableField(self): + _VM.reads += 1 + return [SimpleNamespace(key=1, name="Owner")] + + without_values = _VM(_wsdlName="VirtualMachine", customValue=[]) + with_values = [_VM(_wsdlName="VirtualMachine", customValue=[SimpleNamespace(key=1, value=f"user{i}")]) + for i in range(3)] + + assert handler.get_object_custom_fields(without_values) == dict() + assert _VM.reads == 0 + + results = [handler.get_object_custom_fields(vm) for vm in with_values] + + assert results == [{"vcsa_owner": f"user{i}"} for i in range(3)] + assert _VM.reads == 1