diff --git a/module/sources/common/source_base.py b/module/sources/common/source_base.py index e44de2b5..7ca4aa09 100644 --- a/module/sources/common/source_base.py +++ b/module/sources/common/source_base.py @@ -353,6 +353,23 @@ def map_object_interfaces_to_current_interfaces(self, device_vm_object, interfac return return_data + def get_vmware_guest_info(self, vmware_object): + """ + Return the guest info of a VMware VM. Reading it is a round trip to vCenter, sources which already + retrieved it can return their copy. + + Parameters + ---------- + vmware_object: vim.VirtualMachine + the VM + + Returns + ------- + vim.vm.GuestInfo, None: the guest info + """ + + return grab(vmware_object, "guest") + def return_longest_matching_prefix_for_ip(self, ip_to_match=None, site_name=None) -> Optional[NBPrefix]: """ This is a lazy approach to find the longest matching prefix to an IP address. @@ -539,10 +556,14 @@ def add_update_interface(self, interface_object, device_object, interface_data, interface_object.update(data={"primary_mac_address": primary_mac_address_object}, source=self) + vmware_guest_info = None + if type(device_object) == NBVM: + vmware_guest_info = self.get_vmware_guest_info(vmware_object) + # skip handling of IPs for VMs with not installed/running guest tools skip_ip_handling = False if type(device_object) == NBVM and vmware_object is not None and \ - grab(vmware_object, 'guest.toolsRunningStatus') != "guestToolsRunning": + grab(vmware_guest_info, 'toolsRunningStatus') != "guestToolsRunning": log.debug(f"VM '{device_object.name}' guest tool status is 'NotRunning', skipping IP handling") skip_ip_handling = True @@ -818,7 +839,7 @@ def add_update_interface(self, interface_object, device_object, interface_data, # (seen on old TMOS releases), not a statement that every address is gone. Keep what is # in NetBox instead of tearing it off on every run. A real removal still reports the # interface, just without an address, so that case is unaffected - reported_interfaces = grab(vmware_object, "guest.net") + reported_interfaces = grab(vmware_guest_info, "net") if type(device_object) == NBVM and isinstance(reported_interfaces, list) and \ len(reported_interfaces) == 0: log.debug(f"VM '{device_object.name}' guest tools report no network interface at all, " diff --git a/module/sources/vmware/connection.py b/module/sources/vmware/connection.py index 123260cb..591f1171 100644 --- a/module/sources/vmware/connection.py +++ b/module/sources/vmware/connection.py @@ -109,8 +109,6 @@ class VMWareHandler(SourceBase): vm_name_max_length = 64 vm_description_max_length = 200 - recursion_level = 0 - # internal vars session = None tag_session = None @@ -188,6 +186,7 @@ def __init__(self, name=None): self.processed_host_names = dict() self.processed_vm_names = dict() self.processed_vm_uuid = list() + self.processed_vm_object_keys = set() self.object_cache = dict() self.object_path_cache = dict() self.parsing_vms_the_first_time = True @@ -1318,13 +1317,22 @@ def get_vmware_object_tags(self, obj): tag_list = list() if self.tag_session is not None: - # noinspection PyBroadException - try: - object_tag_ids = self.tag_session.tagging.TagAssociation.list_attached_tags( - DynamicID(type=grab(obj, "_wsdlName"), id=grab(obj, "_moId"))) - except Exception as e: - log.error(f"Unable to retrieve vCenter tags for '{obj.name}': {e}") - return + # the tags of parent objects (folders, clusters, datacenters) are requested for each + # VM and host, ask vCenter only once per object + attached_tags_cache = self.get_run_cache("attached_tags") + object_key = (grab(obj, "_wsdlName"), grab(obj, "_moId")) + + object_tag_ids = attached_tags_cache.get(object_key) + if object_tag_ids is None: + # noinspection PyBroadException + try: + object_tag_ids = self.tag_session.tagging.TagAssociation.list_attached_tags( + DynamicID(type=object_key[0], id=object_key[1])) + except Exception as e: + log.error(f"Unable to retrieve vCenter tags for '{obj.name}': {e}") + return + + attached_tags_cache[object_key] = object_tag_ids # vCenter tags and categories by id: most objects share a few hundred tags, so each # is fetched once per run (per source) instead of once per object it's attached to @@ -1404,7 +1412,7 @@ def collect_object_tags(self, obj): if tag_source is None or self.tag_session is None: return tag_list - log.debug2(f"Collecting tags for {obj.name}") + log.debug2(f"Collecting tags for {self.get_object_name(obj)}") if "object" in tag_source: tag_list.extend(self.get_vmware_object_tags(obj)) @@ -1463,9 +1471,10 @@ def get_object_custom_fields(self, obj): # add basic host data to device if object_type == "dcim.device": - num_cpu_cores = grab(obj, "summary.hardware.numCpuCores") - cpu_model = grab(obj, "summary.hardware.cpuModel") - memory_size = grab(obj, "summary.hardware.memorySize") + host_hardware = grab(obj, "summary.hardware") + num_cpu_cores = grab(host_hardware, "numCpuCores") + cpu_model = grab(host_hardware, "cpuModel") + memory_size = grab(host_hardware, "memorySize") if num_cpu_cores is not None: custom_field = self.add_update_custom_field({ @@ -1514,11 +1523,22 @@ def get_object_custom_fields(self, obj): return_custom_fields[grab(custom_field, "data.name")] = guest_hostname else: - log.debug2(f"VM '{grab(obj, 'name')}' guest hostname not reported by VMware Tools " + log.debug2(f"VM '{self.get_object_name(obj)}' guest hostname not reported by VMware Tools " "(not installed, not running or no data available yet). Keeping current " f"value of custom field '{guest_hostname_field_name}' untouched.") - field_definition = {grab(k, "key"): grab(k, "name") for k in grab(obj, "availableField", fallback=list())} + # the custom attribute definitions are the same for all objects of a type, + # request them only once per type and only if there are custom values to label + field_definition = dict() + if len(custom_value) > 0: + field_definition_cache = self.get_run_cache("custom_field_definitions") + object_wsdl_name = grab(obj, "_wsdlName") + field_definition = field_definition_cache.get(object_wsdl_name) + if field_definition is None: + field_definition = {grab(k, "key"): grab(k, "name") + for k in grab(obj, "availableField", fallback=list())} + if object_wsdl_name is not None: + field_definition_cache[object_wsdl_name] = field_definition for obj_custom_field in custom_value: key = grab(obj_custom_field, "key") @@ -2034,34 +2054,139 @@ def add_device_vm_to_inventory(self, object_type, object_data, pnic_data=None, v return device_vm_object, interface_objects + def get_run_cache(self, name): + """ + Return a dict which caches vCenter data for the duration of this run. Reading a property of a + pyVmomi managed object is a round trip to vCenter, the caches avoid asking for the same data + again and again (parents and names of clusters, folders and datacenters, tag definitions, ...). + + The caches are created on first use, handlers which were built without running __init__ + (as the unit tests do) get them as well. + + Parameters + ---------- + name: str + name of the cache + + Returns + ------- + dict: the cache + """ + + caches = self.__dict__.setdefault("_run_caches", dict()) + + return caches.setdefault(name, dict()) + + @staticmethod + def get_vmware_object_key(obj): + """ + Return a key which identifies a vCenter managed object without a round trip to vCenter + + Parameters + ---------- + obj: pyVmomi managed object + + Returns + ------- + tuple, None: type name and managed object id or None if obj is not a managed object + """ + + object_id = getattr(obj, "_moId", None) + + if object_id is None: + return None + + return obj.__class__.__name__, object_id + + def get_object_name(self, obj): + """ + Return the name of a vCenter managed object. The name is only requested once per object and run. + + Parameters + ---------- + obj: pyVmomi managed object + + Returns + ------- + str, None: name of the object + """ + + return self.get_cached_property(obj, "name") + + def get_cached_property(self, obj, property_name): + """ + Return a top level property of a vCenter managed object which doesn't change during a run, like + its name, parent or resource pool. It is only requested once per object and run. + + Parameters + ---------- + obj: pyVmomi managed object + property_name: str + name of the property + + Returns + ------- + the value of the property, None if it can't be read + """ + + object_key = self.get_vmware_object_key(obj) + + if object_key is None: + return grab(obj, property_name) + + property_cache = self.get_run_cache(f"property:{property_name}") + + if object_key not in property_cache: + property_cache[object_key] = grab(obj, property_name) + + return property_cache[object_key] + + def get_vmware_guest_info(self, vmware_object): + """ + Return the guest info of the VM which is currently parsed by add_virtual_machine() without + asking vCenter again, other objects are passed on to SourceBase.get_vmware_guest_info(). + """ + + object_key = self.get_vmware_object_key(vmware_object) + guest_info_cache = self.get_run_cache("vm_guest") + + if object_key is not None and object_key in guest_info_cache: + return guest_info_cache[object_key] + + return super().get_vmware_guest_info(vmware_object) + def get_parent_object_by_class(self, obj, object_class_to_find): + """ + Walk up the vCenter inventory tree until an object of the requested class is found. + The parent of each object is only requested once per run. + + Parameters + ---------- + obj: pyVmomi managed object + object to start with, returned itself if it is of the requested class + object_class_to_find: type + the pyVmomi class to find + + Returns + ------- + pyVmomi managed object, None: the found object or None if there is none + """ if obj is None or object_class_to_find is None: return - if isinstance(obj, object_class_to_find): - self.recursion_level = 0 - return obj - max_recursion = 20 - while True: - if self.recursion_level >= max_recursion: - self.recursion_level = 0 - return None + for _ in range(max_recursion + 1): - # noinspection PyBroadException - try: - parent = obj.parent - except Exception: - self.recursion_level = 0 - return None + if isinstance(obj, object_class_to_find): + return obj + + obj = self.get_cached_property(obj, "parent") - if isinstance(parent, object_class_to_find): - self.recursion_level = 0 - return parent + if obj is None: + return None - self.recursion_level += 1 - return self.get_parent_object_by_class(parent, object_class_to_find) + return None def add_object_to_cache(self, vm_object, netbox_object): @@ -2189,7 +2314,7 @@ def get_resource_pool_path(self, vm): otherwise None (i.e. for template VMs) """ - if grab(vm, "resourcePool") is None: + if self.get_cached_property(vm, "resourcePool") is None: return None cached_path = self.get_object_path_from_cache(vm, "resource_pool_path") @@ -2197,28 +2322,28 @@ def get_resource_pool_path(self, vm): return cached_path path_parts = list() - pool = grab(vm, "resourcePool") + pool = self.get_cached_property(vm, "resourcePool") # walk up the pool hierarchy, the cluster root pool ends the walk: it is the pool whose # parent is the cluster itself rather than another pool (vCenter names it "Resources", # but a child pool may carry that name too) while pool is not None and len(path_parts) < self.object_path_max_depth: - if not isinstance(grab(pool, "parent"), vim.ResourcePool): + if not isinstance(self.get_cached_property(pool, "parent"), vim.ResourcePool): break - pool_name = get_string_or_none(grab(pool, "name")) + pool_name = get_string_or_none(self.get_object_name(pool)) if pool_name is None: break path_parts.append(pool_name) - pool = grab(pool, "parent") + pool = self.get_cached_property(pool, "parent") - if pool is not None and isinstance(grab(pool, "parent"), vim.ResourcePool): - log.warning(f"Resource pool hierarchy of VM '{get_string_or_none(grab(vm, 'name'))}' is deeper " + if pool is not None and isinstance(self.get_cached_property(pool, "parent"), vim.ResourcePool): + log.warning(f"Resource pool hierarchy of VM '{get_string_or_none(self.get_object_name(vm))}' is deeper " f"than {self.object_path_max_depth} levels, the pool path is cut off at the top") path = "/".join(reversed(path_parts)) - log.debug2(f"VM '{get_string_or_none(grab(vm, 'name'))}' has resource pool path '{path}'") + log.debug2(f"VM '{get_string_or_none(self.get_object_name(vm))}' has resource pool path '{path}'") self.add_object_paths_to_cache(vm, {"resource_pool_path": path}) @@ -2242,7 +2367,7 @@ def get_vm_folder_path(self, vm): str, None: VM folder path if the VM is organized in a folder, otherwise None """ - if grab(vm, "parent") is None or not isinstance(grab(vm, "parent"), vim.Folder): + if not isinstance(self.get_cached_property(vm, "parent"), vim.Folder): return None cached_path = self.get_object_path_from_cache(vm, "vm_folder_path") @@ -2250,28 +2375,29 @@ def get_vm_folder_path(self, vm): return cached_path path_parts = list() - folder = grab(vm, "parent") + folder = self.get_cached_property(vm, "parent") # walk up the folder hierarchy, the datacenter's VM folder ends the walk while isinstance(folder, vim.Folder) and len(path_parts) < self.object_path_max_depth: # the folder directly below the datacenter is the datacenter's VM folder - if isinstance(grab(folder, "parent"), vim.Datacenter): + if isinstance(self.get_cached_property(folder, "parent"), vim.Datacenter): break - folder_name = get_string_or_none(grab(folder, "name")) + folder_name = get_string_or_none(self.get_object_name(folder)) if folder_name is None: break path_parts.append(folder_name) - folder = grab(folder, "parent") + folder = self.get_cached_property(folder, "parent") - if isinstance(folder, vim.Folder) and not isinstance(grab(folder, "parent"), vim.Datacenter): - log.warning(f"Folder hierarchy of VM '{get_string_or_none(grab(vm, 'name'))}' is deeper than " + if isinstance(folder, vim.Folder) and not isinstance(self.get_cached_property(folder, "parent"), + vim.Datacenter): + log.warning(f"Folder hierarchy of VM '{get_string_or_none(self.get_object_name(vm))}' is deeper than " f"{self.object_path_max_depth} levels, the folder path is cut off at the top") path = "/".join(reversed(path_parts)) - log.debug2(f"VM '{get_string_or_none(grab(vm, 'name'))}' has VM folder path '{path}'") + log.debug2(f"VM '{get_string_or_none(self.get_object_name(vm))}' has VM folder path '{path}'") self.add_object_paths_to_cache(vm, {"vm_folder_path": path}) @@ -2317,7 +2443,7 @@ def get_vm_tenant_name(self, vm, vm_name, nb_cluster_object, cluster_full_name): tenant_name = self.get_object_relation(resource_pool_path, "vm_tenant_resource_pool_relation") if tenant_name is None: - resource_pool_name = get_string_or_none(grab(vm, "resourcePool.name")) + resource_pool_name = get_string_or_none(self.get_object_name(self.get_cached_property(vm, "resourcePool"))) if resource_pool_name is not None: tenant_name = self.get_object_relation(resource_pool_name, "vm_tenant_resource_pool_relation") @@ -2330,7 +2456,7 @@ def get_vm_tenant_name(self, vm, vm_name, nb_cluster_object, cluster_full_name): tenant_name = self.get_object_relation(vm_folder_path, "vm_tenant_folder_relation") if tenant_name is None: - vm_folder_name = get_string_or_none(grab(vm, "parent.name")) + vm_folder_name = get_string_or_none(self.get_object_name(self.get_cached_property(vm, "parent"))) if vm_folder_name is not None: tenant_name = self.get_object_relation(vm_folder_name, "vm_tenant_folder_relation") @@ -2377,7 +2503,7 @@ def vm_passes_resource_pool_filter(self, vm): return True resource_pool_path = self.get_resource_pool_path(vm) - resource_pool_name = get_string_or_none(grab(vm, "resourcePool.name")) + resource_pool_name = get_string_or_none(self.get_object_name(self.get_cached_property(vm, "resourcePool"))) pool_identifiers = [x for x in dict.fromkeys([resource_pool_path, resource_pool_name]) if x not in [None, ""]] @@ -2769,11 +2895,15 @@ def add_host(self, obj): # Collecting data # + # each read of a property of 'obj' is a round trip to vCenter which returns the whole + # property, so 'summary' and 'config' (which is large) are requested only once + host_summary = grab(obj, "summary") + # collect all necessary data - manufacturer = get_string_or_none(grab(obj, "summary.hardware.vendor")) - model = get_string_or_none(grab(obj, "summary.hardware.model")) - product_name = get_string_or_none(grab(obj, "summary.config.product.name")) - product_version = get_string_or_none(grab(obj, "summary.config.product.version")) + manufacturer = get_string_or_none(grab(host_summary, "hardware.vendor")) + model = get_string_or_none(grab(host_summary, "hardware.model")) + product_name = get_string_or_none(grab(host_summary, "config.product.name")) + product_version = get_string_or_none(grab(host_summary, "config.product.version")) # collect platform platform = f"{product_name} {product_version}" @@ -2788,11 +2918,11 @@ def add_host(self, obj): # get status status = "offline" - if get_string_or_none(grab(obj, "summary.runtime.connectionState")) == "connected": + if get_string_or_none(grab(host_summary, "runtime.connectionState")) == "connected": status = "active" # prepare identifiers to find asset tag and serial number - identifiers = grab(obj, "summary.hardware.otherIdentifyingInfo", fallback=list()) + identifiers = grab(host_summary, "hardware.otherIdentifyingInfo", fallback=list()) identifier_dict = dict() for item in identifiers: value = grab(item, "identifierValue", fallback="") @@ -2858,9 +2988,12 @@ def add_host(self, obj): if len(host_custom_fields) > 0: host_data["custom_fields"] = host_custom_fields + host_config = grab(obj, "config") + host_network = grab(host_config, "network") + # iterate over hosts virtual switches, needed to enrich data on physical interfaces self.network_data["vswitch"][name] = dict() - for vswitch in grab(obj, "config.network.vswitch", fallback=list()): + for vswitch in grab(host_network, "vswitch", fallback=list()): vswitch_name = unquote(grab(vswitch, "name")) @@ -2878,7 +3011,7 @@ def add_host(self, obj): # iterate over hosts proxy switches, needed to enrich data on physical interfaces # also stores data on proxy switch configured mtu which is used for VM interfaces self.network_data["pswitch"][name] = dict() - for pswitch in grab(obj, "config.network.proxySwitch", fallback=list()): + for pswitch in grab(host_network, "proxySwitch", fallback=list()): pswitch_uuid = grab(pswitch, "dvsUuid") pswitch_name = unquote(grab(pswitch, "dvsName")) @@ -2896,7 +3029,7 @@ def add_host(self, obj): # iterate over hosts port groups, needed to enrich data on physical interfaces self.network_data["host_pgroup"][name] = dict() - for pgroup in grab(obj, "config.network.portgroup", fallback=list()): + for pgroup in grab(host_network, "portgroup", fallback=list()): pgroup_name = grab(pgroup, "spec.name") @@ -2928,7 +3061,7 @@ def add_host(self, obj): except Exception as e: log.debug(f"Unable to query network hints (CDP/LLDP) of host '{name}': {e}") - pnic_list = grab(obj, "config.network.pnic", fallback=list()) + pnic_list = grab(host_network, "pnic", fallback=list()) if self.settings.skip_host_nics is True: log.debug(f"Skipping physical interfaces of host '{name}' (skip_host_nics)") pnic_list = list() @@ -3079,7 +3212,7 @@ def add_host(self, obj): fc_hba_data_dict = dict() if self.settings.sync_host_fc_adapters is True: # only Fibre Channel adapters are synced, block and PCIe adapters are ignored - for hba in self.fc_hba_ports(grab(obj, "config.storageDevice.hostBusAdapter", fallback=list()), name): + for hba in self.fc_hba_ports(grab(host_config, "storageDevice.hostBusAdapter", fallback=list()), name): hba_device = get_string_or_none(grab(hba, "device")) if hba_device is None: @@ -3121,7 +3254,7 @@ def add_host(self, obj): # now iterate over all virtual interfaces and collect data vnic_data_dict = dict() vnic_ips = dict() - for vnic in grab(obj, "config.network.vnic", fallback=list()): + for vnic in grab(host_network, "vnic", fallback=list()): vnic_name = grab(vnic, "device") @@ -3357,34 +3490,46 @@ def add_virtual_machine(self, obj): virtual machine object to parse """ - name = get_string_or_none(grab(obj, "name")) + reevaluating_this_vm = self.parsing_objects_to_reevaluate is True and obj in self.objects_to_reevaluate + + # a VM parsed in the first run over all VMs is skipped in the second one by its UUID. Skip it by + # its managed object id before anything is requested from vCenter. + vm_object_key = self.get_vmware_object_key(obj) + if vm_object_key is not None and vm_object_key in self.processed_vm_object_keys and not reevaluating_this_vm: + return + + name = get_string_or_none(self.get_object_name(obj)) name, vm_description = self.get_vm_name_and_description(name) + # each read of a property of 'obj' is a round trip to vCenter which returns the whole property, + # so 'config' and 'runtime' (and 'guest' further down) are requested only once + vm_config = grab(obj, "config") + vm_runtime = grab(obj, "runtime") + # # Filtering # # get VM UUID - vm_uuid = grab(obj, "config.instanceUuid") + vm_uuid = grab(vm_config, "instanceUuid") - if (vm_uuid is None or vm_uuid in self.processed_vm_uuid) and \ - not (self.parsing_objects_to_reevaluate is True and obj in self.objects_to_reevaluate): + if (vm_uuid is None or vm_uuid in self.processed_vm_uuid) and not reevaluating_this_vm: return log.debug(f"Parsing vCenter VM: {name}") # get VM power state - status = "active" if get_string_or_none(grab(obj, "runtime.powerState")) == "poweredOn" else "offline" + status = "active" if get_string_or_none(grab(vm_runtime, "powerState")) == "poweredOn" else "offline" # check if vm is template - template = grab(obj, "config.template") + template = grab(vm_config, "template") if bool(self.settings.skip_vm_templates) is True and template is True: log.debug2(f"VM '{name}' is a template. Skipping") return if bool(self.settings.skip_srm_placeholder_vms) is True \ - and f"{grab(obj, 'config.managedBy.extensionKey')}".startswith("com.vmware.vcDr"): + and f"{grab(vm_config, 'managedBy.extensionKey')}".startswith("com.vmware.vcDr"): log.debug2(f"VM '{name}' is a SRM placeholder VM. Skipping") return @@ -3395,8 +3540,10 @@ def add_virtual_machine(self, obj): # add to processed VMs self.processed_vm_uuid.append(vm_uuid) + if vm_object_key is not None: + self.processed_vm_object_keys.add(vm_object_key) - parent_host = self.get_parent_object_by_class(grab(obj, "runtime.host"), vim.HostSystem) + parent_host = self.get_parent_object_by_class(grab(vm_runtime, "host"), vim.HostSystem) cluster_object = self.get_parent_object_by_class(parent_host, vim.ClusterComputeResource) # get single host 'cluster' if VM runs on one @@ -3419,9 +3566,9 @@ def add_virtual_machine(self, obj): log.debug(f"Virtual machine '{name}' is not part of a permitted cluster. Skipping") return - parent_name = grab(parent_host, "name") + parent_name = self.get_object_name(parent_host) cluster_name = grab(nb_cluster_object, "data.name") - cluster_full_name = f"{group.name}/{cluster_name}" + cluster_full_name = f"{self.get_object_name(group)}/{cluster_name}" if name in self.processed_vm_names.get(cluster_full_name, list()) and obj not in self.objects_to_reevaluate: if self.settings.vm_name_regex is not None: @@ -3455,12 +3602,20 @@ def add_virtual_machine(self, obj): if site_name is None: site_name = self.get_site_name(NBCluster, cluster_full_name) + vm_guest = grab(obj, "guest") + vm_guest_nics = grab(vm_guest, "net", fallback=list()) + + # source_base.add_update_interface() checks the guest tools state for every interface of this VM + self.get_run_cache("vm_guest").clear() + if vm_object_key is not None: + self.get_run_cache("vm_guest")[vm_object_key] = vm_guest + # first check against vm_platform_relation - platform = get_string_or_none(grab(obj, "config.guestFullName")) - platform = get_string_or_none(grab(obj, "guest.guestFullName", fallback=platform)) + platform = get_string_or_none(grab(vm_config, "guestFullName")) + platform = get_string_or_none(grab(vm_guest, "guestFullName", fallback=platform)) # extract prettyName from extraConfig exposed by guest tools - extra_config = {x.key: x.value for x in grab(obj, "config.extraConfig", fallback=[]) + extra_config = {x.key: x.value for x in grab(vm_config, "extraConfig", fallback=[]) if x.key in ["guestOS.detailed.data", "guestInfo.detailed.data"]} # first try 'guestInfo.detailed.data' and then 'guestOS.detailed.data' @@ -3493,11 +3648,11 @@ def add_virtual_machine(self, obj): if platform is not None: platform = self.get_object_relation(platform, "vm_platform_relation", fallback=platform) - hardware_devices = grab(obj, "config.hardware.device", fallback=list()) + hardware_devices = grab(vm_config, "hardware.device", fallback=list()) annotation = None if self.settings.skip_vm_comments is False: - annotation = get_string_or_none(grab(obj, "config.annotation")) + annotation = get_string_or_none(grab(vm_config, "annotation")) # resolve tenant for this VM tenant_name = self.get_vm_tenant_name(obj, name, nb_cluster_object, cluster_full_name) @@ -3519,7 +3674,7 @@ def add_virtual_machine(self, obj): vm_tags.extend(vcenter_tags) # vm memory depending on setting - vm_memory = grab(obj, "config.hardware.memoryMB", fallback=0) + vm_memory = grab(vm_config, "hardware.memoryMB", fallback=0) if self.settings.vm_disk_and_ram_in_decimal is True: vm_memory = int(vm_memory / 1024 * 1000) @@ -3529,7 +3684,7 @@ def add_virtual_machine(self, obj): "cluster": nb_cluster_object, "status": status, "memory": vm_memory, - "vcpus": grab(obj, "config.hardware.numCPU") + "vcpus": grab(vm_config, "hardware.numCPU") } # Add adaption for change in NetBox 3.3.0 VM model @@ -3574,7 +3729,7 @@ def add_virtual_machine(self, obj): vm_default_gateway_ip6 = None # check vm routing to determine which is the default interface for each IP version - for route in grab(obj, "guest.ipStack.0.ipRouteConfig.ipRoute", fallback=list()): + for route in grab(vm_guest, "ipStack.0.ipRouteConfig.ipRoute", fallback=list()): # we found a default route if grab(route, "prefixLength") == 0: @@ -3737,7 +3892,7 @@ def add_virtual_machine(self, obj): int_description = f"{int_description} ({vlan_description})" # find corresponding guest NIC and get IP addresses and connected status - for guest_nic in grab(obj, "guest.net", fallback=list()): + for guest_nic in vm_guest_nics: # get matching guest NIC if int_mac != normalize_mac_address(grab(guest_nic, "macAddress")): @@ -3820,7 +3975,7 @@ def add_virtual_machine(self, obj): dummy_interfaces_filter = self.settings.vm_dummy_interfaces_include_filter if self.settings.sync_vm_dummy_interfaces is True or \ (dummy_interfaces_filter is not None and dummy_interfaces_filter.match(name)): - for guest_nic in grab(obj, "guest.net", fallback=list()): + for guest_nic in vm_guest_nics: # get matching guest NIC MAC guest_nic_mac = normalize_mac_address(grab(guest_nic, "macAddress")) diff --git a/tests/test_vmware_vcenter_requests.py b/tests/test_vmware_vcenter_requests.py new file mode 100644 index 00000000..007f98b5 --- /dev/null +++ b/tests/test_vmware_vcenter_requests.py @@ -0,0 +1,211 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2020 - 2026 netbox-sync team. All rights reserved. +# +# netbox-sync.py +# +# This work is licensed under the terms of the MIT license. +# For a copy, see file LICENSE.txt included in this +# repository or visit: . + +""" +Reading a property of a pyVmomi managed object is a round trip to vCenter which returns the whole +property ('config' of a host is about 1 MB). The VMware source must not ask vCenter for the same data +over and over again. +""" +import collections +from types import SimpleNamespace + +import pytest + +from module.sources.vmware import connection as vmware_connection +from module.sources.vmware.connection import VMWareHandler + + +@pytest.fixture +def fetch_counter(monkeypatch): + """Counts every property read (SOAP 'Fetch') per managed object and property.""" + from pyVmomi import SoapAdapter + + counts = collections.Counter() + original = SoapAdapter.SoapStubAdapter.InvokeMethod + + def invoke(self, mo, info, args, outerStub=None): + if info.wsdlName == "Fetch": + counts[(mo._wsdlName, mo._moId, args[0])] += 1 + return original(self, mo, info, args, outerStub) + + monkeypatch.setattr(SoapAdapter.SoapStubAdapter, "InvokeMethod", invoke) + return counts + + +def test_host_and_vm_properties_are_not_requested_repeatedly(vmware_source, fetch_counter): + vmware_source.apply() + + assert len(fetch_counter) > 0, "no property reads were counted" + + # an offline VM is looked at in both runs over all VMs, so twice is fine + repeated = {key: count for key, count in fetch_counter.items() + if key[0] in ("VirtualMachine", "HostSystem") and count > 2} + + assert repeated == {}, "properties requested again and again from vCenter" + + +def test_a_vm_parsed_in_the_first_run_is_not_requested_again_in_the_second(vmware_source, fetch_counter, sdk): + from pyVmomi import vim + + view = sdk.viewManager.CreateContainerView(sdk.rootFolder, [vim.VirtualMachine], True) + powered_on = {vm._moId for vm in view.view if vm.runtime.powerState == "poweredOn"} + view.Destroy() + + # the reads above went through the counted stub as well + fetch_counter.clear() + vmware_source.apply() + + reevaluated = {obj._moId for obj in vmware_source.objects_to_reevaluate} + config_reads = [fetch_counter[("VirtualMachine", mo_id, "config")] for mo_id in powered_on - reevaluated] + + assert len(config_reads) > 0 + assert set(config_reads) == {1} + + +# +# unit tests without vcsim +# + +@pytest.fixture(autouse=True) +def dynamic_id(monkeypatch): + """The vSphere automation SDK is optional and absent here, the tag call needs the type.""" + monkeypatch.setattr(vmware_connection, "DynamicID", + lambda **kwargs: SimpleNamespace(**kwargs), raising=False) + + +class _CountingTagging: + """The parts of the vSphere tagging API get_vmware_object_tags() uses, counting each call.""" + + def __init__(self, attached): + self.calls = collections.Counter() + outer = self + + class _Tag: + @staticmethod + def get(tag_id): + outer.calls["Tag.get"] += 1 + return SimpleNamespace(name=f"name-{tag_id}", description="", category_id="cat-1") + + class _Category: + @staticmethod + def get(_category_id): + outer.calls["Category.get"] += 1 + return SimpleNamespace(name="env") + + class _TagAssociation: + @staticmethod + def list_attached_tags(dynamic_id): + outer.calls["list_attached_tags"] += 1 + return attached[dynamic_id.id] + + self.Tag, self.Category, self.TagAssociation = _Tag, _Category, _TagAssociation + + +def _bare_handler(inventory, **settings): + handler = object.__new__(VMWareHandler) + handler.inventory = inventory + handler.name = "test" + handler.source_tag = "Source: test" + handler.settings = SimpleNamespace(**settings) + return handler + + +def test_tag_definitions_and_attached_tags_are_requested_once(inventory): + tagging = _CountingTagging({"vm-1": ["t1", "t2"], "vm-2": ["t1"], "group-v1": ["t2"]}) + handler = _bare_handler(inventory, tag_name_include_category=True) + handler.tag_session = SimpleNamespace(tagging=tagging) + + def vm(mo_id): + return SimpleNamespace(name=mo_id, _wsdlName="VirtualMachine", _moId=mo_id) + + folder = SimpleNamespace(name="folder", _wsdlName="Folder", _moId="group-v1") + + results = list() + for obj in [vm("vm-1"), folder, vm("vm-2"), folder, vm("vm-1")]: + results.append(sorted(t.get_display_name() for t in handler.get_vmware_object_tags(obj))) + + assert results == [["env:name-t1", "env:name-t2"], ["env:name-t2"], ["env:name-t1"], + ["env:name-t2"], ["env:name-t1", "env:name-t2"]] + assert tagging.calls == {"list_attached_tags": 3, "Tag.get": 2, "Category.get": 1} + + +class _Node(SimpleNamespace): + """Fake managed object counting how often its parent is requested.""" + + def __init__(self, mo_id, parent, counter): + super().__init__(_moId=mo_id, _parent=parent, _counter=counter) + + @property + def parent(self): + self._counter[self._moId] += 1 + return self._parent + + +class _Datacenter(_Node): + pass + + +def test_parents_are_requested_once_per_object(): + counter = collections.Counter() + handler = object.__new__(VMWareHandler) + + datacenter = _Datacenter("datacenter-1", None, counter) + folder = _Node("group-h1", datacenter, counter) + cluster = _Node("domain-c1", folder, counter) + + for _ in range(5): + assert handler.get_parent_object_by_class(cluster, _Datacenter) is datacenter + + assert counter == {"domain-c1": 1, "group-h1": 1} + + +def test_unsuccessful_parent_lookups_do_not_break_later_ones(): + # a walk which ended at the root used to leave its depth (11 here) behind, the next walk then + # gave up after 20 - 11 levels and didn't find the datacenter 10 levels up + counter = collections.Counter() + handler = object.__new__(VMWareHandler) + + datacenter = _Datacenter("datacenter-1", None, counter) + chain = datacenter + for level in range(10): + chain = _Node(f"node-{level}", chain, counter) + + class _NotInTree: + pass + + assert handler.get_parent_object_by_class(chain, _NotInTree) is None + + assert handler.get_parent_object_by_class(chain, _Datacenter) is datacenter + + +def test_custom_field_definitions_are_only_requested_for_custom_values(inventory): + handler = _bare_handler(inventory, sync_custom_attributes=True, vm_custom_object_attributes=[], + host_custom_object_attributes=[], custom_attribute_exclude=None, + vm_guest_hostname_custom_field=None) + inventory.netbox_api_version = "4.3.0" + + class _VM(SimpleNamespace): + reads = 0 + + @property + def availableField(self): + _VM.reads += 1 + return [SimpleNamespace(key=1, name="Owner")] + + without_values = _VM(_wsdlName="VirtualMachine", customValue=[]) + with_values = [_VM(_wsdlName="VirtualMachine", customValue=[SimpleNamespace(key=1, value=f"user{i}")]) + for i in range(3)] + + assert handler.get_object_custom_fields(without_values) == dict() + assert _VM.reads == 0 + + results = [handler.get_object_custom_fields(vm) for vm in with_values] + + assert results == [{"vcsa_owner": f"user{i}"} for i in range(3)] + assert _VM.reads == 1