From f86b5dff1ab3162b2ac3350d04301f0bc2fab1af Mon Sep 17 00:00:00 2001 From: Eric Krichbaum Date: Wed, 11 Nov 2026 15:53:30 -0600 Subject: [PATCH 01/21] Add VMware source integration documentation Document the setup and configuration for VMware source integration with NetBox, including user roles, object synchronization, and filtering options. --- docs/source_openstack.md | 234 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 234 insertions(+) create mode 100644 docs/source_openstack.md diff --git a/docs/source_openstack.md b/docs/source_openstack.md new file mode 100644 index 00000000..a8334f44 --- /dev/null +++ b/docs/source_openstack.md @@ -0,0 +1,234 @@ +# Source: vmware + +## Setup +You need to have a source section in your `settings.ini` file with following type: +```ini +type = vmware +``` +All options for this source are described in the [settings-example.ini](../settings-example.ini) file. + +If you have multiple vCenter instances just add another source with the same type in the **same** file. + +***IMPORTANT:*** For VMware source you should define the var `cluster_site_relation` which maps a vCenter cluster to an +exiting Site in NetBox. If undefined a placeholder site will be created. + +### vCenter user +You need a user account with "Read-only" role on vCenter root scope. +The "Propagate to children" setting must also be checked. + +## Adding objects from a vCenter to NetBox + +To match new/updated objects to existing objects in NetBox a few different steps are taken to find the correct object. + +You might be interested in this [description](common_concepts.md). This describes how discovered +IP addresses and interfaces will be added to NetBox. + +### Add/update device/VM object in inventory based on gathered data + +Try to find object first based on the object data, interface MAC addresses and primary IPs. +1. try to find by name and cluster/site +2. try to find by mac addresses interfaces +3. try to find by serial number (1st) or asset tag (2nd) +4. try to find by primary IP + +### Finding hosts and VMs from discovered data + +#### 1. try to find exact name match including cluster/site + +If this NetBox object (device, vm) matches, the found object will be used. + +#### 2. Try to find a NetBox object based on list of MAC addresses + +Iterate over all NetBox interfaces of an object (device, vm) and compare MAC address with list +of MAC addresses discovered for this object. If a match was found, count for every object (device, vm) +how many MAC addresses are matching. + +If exactly one NetBox object (device, vm) with matching interface MACs was found then this one will be used. + +If two or more NetBox object (device, vm) with matching MACs were found, compare the two +NetBox object (device, vm) with the highest amount of matching interfaces. If the ratio of matching interface +MAC addresses exceeds 2.0 then the top matching NetBox object (device, vm) is chosen as desired object. + +If the ratio is below 2.0 then None will be chosen. The probability is too low that +this one is the correct one. + +#### 3. Try to find a NetBox object based on the primary IP (v4 or v6) address + +If an exact matching NetBox object (device, vm) was found the object will be used +immediately without checking the other primary IP address (if defined). + +### Objects read and parsed from vCenter + +#### 1. Add a vCenter datacenter as a Cluster Group to NetBox +Simple, nothing special going on here + +#### 2. Add a vCenter cluster as a Cluster to NetBox. + +Cluster name is checked against `cluster_include_filter` and `cluster_exclude_filter config` setting. + +#### 3. Parse distributed virtual port group +This is done to extract name and VLAN IDs from each port group + +#### 4. Parse a vCenter (ESXi) host + +First host is filtered: +* host has a cluster and this cluster is permitted +* skip host with same name and site, we already parsed it (use unique host names) +* does the host pass the `host_include_filter` and `host_exclude_filter` + +Then all necessary host data will be collected:
+host model, manufacturer, serial, physical interfaces, virtual interfaces, +virtual switches, proxy switches, host port groups, interface VLANs, IP addresses + +Primary IPv4/6 will be determined by +1. if the interface port group name contains "management" or "mngt" +2. interface with this IP is the default route of this host + +#### 5. Parse a vCenter VM + +Iterate over all VMs twice! + +To handle VMs with the same name in a cluster we first iterate over all VMs and look only at the +active (online) ones and parse these first. Then we iterate a second time to catch the rest (also offline). + +This has been implemented to support migration scenarios where you create/copy the same +machine with a different setup like a new version or something. This way NetBox will be +updated primarily with the actual active VM data. + +First VM is filtered: +* VM has a cluster and is it permitted +* skip VMs with same name and cluster if already parsed +* does the VM pass the `vm_include_filter` and `vm_exclude_filter` +* does the VM pass the `vm_include_by_resource_pool_filter` and `vm_exclude_by_resource_pool_filter` + +Then all necessary VM data will be collected:
+platform, virtual interfaces, virtual cpu/disk/memory interface VLANs, IP addresses + +Primary IPv4/6 will be determined by interface that provides the default route for this VM. + +The resource pool filters are applied to the resource pool path (i.e. `Parent/Child`, the +cluster root pool `Resources` excluded) and to the plain resource pool name of the VM: the VM +is kept when either matches the include filter and skipped when either matches the exclude +filter. A VM placed directly in the cluster root pool has an empty pool path and is matched by +its plain pool name `Resources` only. A VM without a resource pool (a template) passes both +filters. + +**Note:**
+IP address information can only be extracted if guest tools are installed and running. + +#### 5b. Resolve the tenant of a VM + +The tenant of a VM is resolved from different sources. The first match wins: + +1. `vm_tenant_relation` matched against the VM name +2. `vm_tenant_resource_pool_relation` matched against the resource pool path and then + against the plain resource pool name of the VM +3. `vm_tenant_folder_relation` matched against the VM folder path (relative to the + datacenter's VM folder) and then against the plain folder name of the VM + + An empty path (a VM in the cluster root pool, or directly in the datacenter's VM folder) + is not matched; such a VM is matched by its plain pool name `Resources` or folder name + `vm` only. +4. if `vm_tenant_inherit_from_cluster` is enabled: the tenant resolved via + `cluster_tenant_relation` for this cluster or, if that relation matched nothing, the + tenant assigned to the cluster in NetBox + +A tenant already set on an existing NetBox VM is kept if `overwrite_vm_tenant` is disabled. +If a tenant was resolved for the VM, its IP addresses inherit it through the `device` step +of `ip_tenant_inheritance_order`. + +#### 6. Sync VMware Tools guest hostname to a custom field + +The vCenter VM name (used as the NetBox VM `name`) and the hostname reported from inside the +guest OS by VMware Tools are not always identical. This can happen if: +* a VM was renamed in vCenter but the OS hostname was not changed (or vice versa) +* an OS administrator changed the hostname without informing the virtualization team +* a VM was cloned and the guest hostname was not adjusted afterwards + +To make this drift visible in NetBox, the option `vm_guest_hostname_custom_field` can be set to +the name of a NetBox custom field. If set, the guest hostname reported by VMware Tools (vCenter +property `guest.hostName`) is synced into that custom field on every synced VM, while the NetBox +VM `name` keeps reflecting the vCenter inventory name unchanged. This allows administrators to +compare the NetBox VM name against the custom field value on the same VM record to spot naming +drift. + +```ini +vm_guest_hostname_custom_field = vmware_guest_hostname +``` + +If the custom field does not exist in NetBox yet it will be created automatically (type `Text`, +assigned to the `Virtual Machine` object type), the same way other custom fields managed by +netbox-sync (e.g. `vcsa_*` custom attributes) are created. An administrator can also pre-create +the field beforehand; netbox-sync will then just use the existing field. + +This option is unset by default, so the feature is disabled and no additional custom field is +created unless explicitly configured. + +VMware Tools may not always report a hostname, for example if Tools are not installed, not +running, outdated, or simply have not reported guest information yet. In that case +netbox-sync leaves the custom field untouched and keeps the last known value, instead of +clearing it. A message is logged at debug level (`-l DEBUG2`) whenever this happens. + +Example result in NetBox for a VM named `APPPRD01` in vCenter whose guest OS reports +`appprd01.corp.example.com` as its hostname: + +```text +Virtual Machine Name: APPPRD01 +Custom Fields: + VMware Guest Hostname: appprd01.corp.example.com +``` + +### Cables to CDP/LLDP neighbors + +An ESXi host reports the switch and the switch port each of its physical interfaces (pNICs) is +connected to, if CDP or LLDP is enabled on the switch. With the option `sync_host_cables` enabled +netbox-sync uses this information to create cables in NetBox between the host interface and the +switch port. + +```ini +sync_host_cables = True +``` + +Cables are objects which are usually maintained by hand, that's why this option is disabled by +default. With the option disabled no cable is read from or written to NetBox at all. NetBox 3.3 or +newer is needed, on older versions the option is ignored. + +netbox-sync only connects things it can find, it never creates the other end of a cable: + +* the device the neighbor reports as its system name must already exist in NetBox. The name is + matched exactly first, a short name is only matched against a FQDN if that match is unambiguous +* the port the neighbor reports must already exist as an interface of that device. Long and short + interface names are matched against each other, so a reported `FastEthernet0/16` also matches an + interface named `Fa0/16` in NetBox. CDP reports the port ID, LLDP additionally reports a port + description and both are tried +* both interfaces must already exist in NetBox. An interface which was just discovered gets its + cable during the next run +* neither of the two interfaces may be connected already. A cable which was created by hand or which + connects to a different port is never changed or deleted, it is reported at log level `DEBUG` + instead + +Cables created by this source are tagged like every other object and are marked as orphaned and +pruned once the host stops reporting that neighbor (see `prune_enabled`). Disabling the option again +leaves all previously created cables untouched in NetBox. + +### Filtering VM Disk Information +VM disks are synchronized between vCenter and NetBox. Since NetBox 3.7.0, virtual disks are tracked as separate objects +linked to VMs. In some scenarios, such as when temporary disks are attached to VMs during backup operations +(e.g., "Independent-nonpersistent" disks from Veeam), you might want to exclude these changes from synchronization +to avoid cluttering your NetBox change log. + +You can use the following filter options to exclude disk synchronization for specific VMs: + +1. **`vm_exclude_disk_sync`**: A regex pattern matching VM names where disk synchronization should be excluded. + ```ini + vm_exclude_disk_sync = backup-.*, veeam-.* + ``` + +2. **`vm_exclude_disk_sync_by_tag`**: A comma-separated list of vCenter tags. VMs with any of these tags will have + their disk information excluded from synchronization. + ```ini + vm_exclude_disk_sync_by_tag = backup-vm, veeam-job + ``` + +When a VM matches these filters, it will still be synchronized to NetBox with all its other information +(CPU, memory, interfaces, IP addresses, etc.), but changes to disk information will be ignored. From 904b2b64f1bb1bd8609472edf0c025feb241d6df Mon Sep 17 00:00:00 2001 From: Eric Krichbaum Date: Wed, 11 Nov 2026 15:54:45 -0600 Subject: [PATCH 02/21] Add OpenStack configuration options --- module/sources/openstack/config.py | 394 +++++++++++++++++++++++++++++ 1 file changed, 394 insertions(+) create mode 100644 module/sources/openstack/config.py diff --git a/module/sources/openstack/config.py b/module/sources/openstack/config.py new file mode 100644 index 00000000..63238580 --- /dev/null +++ b/module/sources/openstack/config.py @@ -0,0 +1,394 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2020 - 2023 Ricardo Bartels. All rights reserved. +# +# netbox-sync.py +# +# This work is licensed under the terms of the MIT license. +# For a copy, see file LICENSE.txt included in this +# repository or visit: . + +import re +from ipaddress import ip_address + +from module.common.misc import quoted_split +from module.config import source_config_section_name +from module.config.base import ConfigBase +from module.config.option import ConfigOption +from module.config.group import ConfigOptionGroup +from module.sources.common.config import * +from module.sources.common.permitted_subnets import PermittedSubnets +from module.common.logging import get_logger + +log = get_logger() + + +class OpenStackConfig(ConfigBase): + + section_name = source_config_section_name + source_name = None + source_name_example = "my-openstack-example" + + def __init__(self): + self.options = [ + ConfigOption(**config_option_enabled_definition), + + ConfigOption(**{**config_option_type_definition, "config_example": "openstack"}), + + ConfigOption("auth_url", + str, + description="OpenStack Identity v3 authentication URL", + config_example="https://api.openstack-instance.local:5000/v3", + mandatory=True), + + ConfigOption("project", + str, + description="Name of the OpenStack project", + config_example="MyProject", + mandatory=True), + + ConfigOption("username", + str, + description="username to use to log into OpenStack", + config_example="openstackuser", + mandatory=True), + + ConfigOption("password", + str, + description="password to use to log into OpenStack", + config_example="supersecret", + sensitive=True, + mandatory=True), + + ConfigOption("region", + str, + description="Which OpenStack region to use", + config_example="RegionOne", + mandatory=True), + + ConfigOption("user_domain", + str, + description="Which OpenStack user domain to use", + config_example="Default", + mandatory=True), + + ConfigOption("project_domain", + str, + description="Which OpenStack project domain to use", + config_example="Default", + mandatory=True), + + ConfigOption("group_name", + str, + description="OpenStack group name", + default_value="Openstack"), + + ConfigOption("validate_tls_certs", + bool, + description="""Enforces TLS certificate validation. + Disable only for an OpenStack endpoint with a self-signed certificate.""", + default_value=True), + + ConfigOption("all_projects", bool, default_value=True, + description="Discover servers and volumes across projects (requires administrative access)."), + + ConfigOption(**config_option_permitted_subnets_definition), + ConfigOption(**config_option_ip_tenant_inheritance_order_definition), + ConfigOption("preserve_primary_ips", bool, default_value=False, + description="Keep existing primary IP assignments when an address is no longer discovered."), + ConfigOption("skip_fhrp_group_ips", bool, default_value=True, + description="Do not assign addresses belonging to an existing FHRP group."), + ConfigOption("host_interface_exclude_filter", str, + description="Regular expression matching host interface names to ignore."), + ConfigOption("vm_interface_exclude_filter", str, + description="Regular expression matching VM interface names to ignore."), + ConfigOption("disable_vlan_sync", bool, default_value=False, + description="Do not synchronize VLANs discovered through existing prefixes."), + + ConfigOptionGroup(title="filter", + description="""filters can be used to include/exclude certain objects from importing + into NetBox. Include filters are checked first and exclude filters after. + An object name has to pass both filters to be synced to NetBox. + If a filter is unset it will be ignored. Filters are all treated as regex expressions! + If more then one expression should match, a '|' needs to be used + """, + config_example="""Example: (exclude all VMs with "replica" in their name + and all VMs starting with "backup"): vm_exclude_filter = .*replica.*|^backup.*""", + options=[ + ConfigOption("cluster_exclude_filter", + str, + description="""If a cluster is excluded from sync then ALL VMs and HOSTS + inside the availability zone will be ignored!"""), + ConfigOption("cluster_include_filter", str), + ConfigOption("host_exclude_filter", + str, + description="""This will only include/exclude the host, + not the VM if Host is part of a multi host cluster"""), + ConfigOption("host_include_filter", str), + ConfigOption("vm_exclude_filter", + str, description="simply include/exclude VMs"), + ConfigOption("vm_include_filter", str) + ]), + ConfigOptionGroup(title="relations", + options=[ + ConfigOption("cluster_site_relation", + str, + description="""\ + This option defines which OpenStack availability zone is part of a NetBox site. + This is done with a comma separated key = value list. + key: defines the cluster name as regex + value: defines the NetBox site name (use quotes if name contains commas) + This is a quite important config setting as IP addresses, prefixes, VLANs + and VRFs are site dependent. In order to assign the correct prefix to an IP + address it is important to pick the correct site. + A VM always depends on the cluster site relation + """, + config_example="nova = Cloud Site"), + ConfigOption("host_site_relation", + str, + description="""Same as cluster site but on host level. + If unset it will fall back to cluster_site_relation""", + config_example="nyc02.* = New York, ffm01.* = Frankfurt"), + ConfigOption("cluster_tenant_relation", + str, + description="""\ + This option defines which cluster/host/VM belongs to which tenant. + This is done with a comma separated key = value list. + key: defines a hosts/VM name as regex + value: defines the NetBox tenant name (use quotes if name contains commas) + a cluster can be specified as "Cluster-name" or + "availability-zone-name" + """, + config_example="Cluster_NYC.* = Customer A"), + ConfigOption("host_tenant_relation", str, config_example="compute.* = Infrastructure"), + ConfigOption("vm_tenant_relation", str, config_example="grafana.* = Infrastructure"), + ConfigOption("vm_platform_relation", + str, + description="""\ + This option defines custom platforms if the VMWare created platforms are not suitable. + Pretty much a mapping of VMWare platform name to your own platform name. + This is done with a comma separated key = value list. + key: defines a VMWare returned platform name + value: defines the desired NetBox platform name""", + config_example="centos-7.* = centos7, microsoft-windows-server-2016.* = Windows2016"), + ConfigOption("host_role_relation", + str, + description="""\ + Define the NetBox device role used for hosts. The default is + set to "Server". This is done with a comma separated key = value list. + key: defines host(s) name as regex + value: defines the NetBox role name (use quotes if name contains commas) + """, + default_value=".* = Server"), + ConfigOption("vm_role_relation", + str, + description="""\ + Define the NetBox device role used for VMs. This is done with a + comma separated key = value list, same as 'host_role_relation'. + key: defines VM(s) name as regex + value: defines the NetBox role name (use quotes if name contains commas) + """, + config_example=".* = Server"), + ConfigOption("cluster_tag_relation", + str, + description="""\ + Define NetBox tags which are assigned to a cluster, host or VM. This is + done with a comma separated key = value list. + key: defines a hosts/VM name as regex + value: defines the NetBox tag (use quotes if name contains commas) + Match the availability zone name to assign cluster tags.""", + config_example="nova = Infrastructure"), + ConfigOption("host_tag_relation", str, config_example="compute.* = Infrastructure"), + ConfigOption("vm_tag_relation", str, config_example="grafana.* = Infrastructure") + ]), + ConfigOption("dns_name_lookup", + bool, + description="""Perform a reverse lookup for all collected IP addresses. + If a dns name was found it will be added to the IP address object in NetBox + """, + default_value=True), + ConfigOption("custom_dns_servers", + str, + description="use custom DNS server to do the reverse lookups", + config_example="192.168.1.11, 192.168.1.12"), + ConfigOption("set_primary_ip", + str, + description="""\ + define how the primary IPs should be set + possible values: + + always: will remove primary IP from the object where this address is + currently set as primary and moves it to new object + + when-undefined: + only sets primary IP if undefined, will cause ERRORs if same IP is + assigned more then once to different hosts and IP is set as the + objects primary IP + + never: don't set any primary IPs, will cause the same ERRORs + as "when-undefined" + """, + default_value="when-undefined"), + ConfigOption("skip_vm_comments", + bool, + description="do not set notes to the UUID or name of the VM", + default_value=False), + ConfigOption("skip_vm_platform", + bool, + description="Do not use the OpenStack flavor name as the NetBox VM platform", + default_value=False), + ConfigOption("strip_host_domain_name", + bool, + description="strip domain part from host name before syncing device to NetBox", + default_value=False), + ConfigOption("strip_vm_domain_name", + bool, + description="strip domain part from VM name before syncing VM to NetBox", + default_value=False), + ConfigOptionGroup(title="custom object attributes", + description="""\ + add arbitrary host/vm object attributes as custom fields to NetBox. + multiple attributes can be defined comma separated. + use OpenStack SDK resource attribute names; dotted paths are supported. + """, + options=[ + ConfigOption("host_custom_object_attributes", + str, + config_example="id"), + ConfigOption("vm_custom_object_attributes", + str, + config_example="id, project_id") + ]), + ConfigOption("set_vm_name_to_uuid", + bool, + description="Set the name in Netbox to the VM UUID instead of name", + default_value=False), + + # removed settings + ConfigOption("netbox_host_device_role", + str, + deprecation_message="You need to switch to 'host_role_relation'.", + removed=True), + ConfigOption("netbox_vm_device_role", + str, + deprecation_message="You need to switch to 'vm_role_relation'.", + removed=True), + ConfigOption("sync_tags", + bool, + deprecation_message="You need to switch to 'host_tag_relation', " + + "'vm_tag_relation' or 'cluster_tag_relation'", + removed=True), + ConfigOption("sync_parent_tags", + bool, + deprecation_message="You need to switch to 'host_tag_relation', " + + "'vm_tag_relation' or 'cluster_tag_relation'", + removed=True) + ] + + super().__init__() + + def validate_options(self): + + for option in self.options: + + if option.value is None: + continue + + if "filter" in option.key: + + re_compiled = None + try: + re_compiled = re.compile(option.value) + except Exception as e: + log.error(f"Problem parsing regular expression for '{self.source_name}.{option.key}': {e}") + self.set_validation_failed() + + option.set_value(re_compiled) + + continue + + if "relation" in option.key: + + relation_data = list() + + relation_type = option.key.split("_")[1] + + for relation in quoted_split(option.value): + + object_name = relation.split("=")[0].strip(' "') + if "=" not in relation: + log.error(f"Config relation '{self.source_name}.{option.key}' must contain '='.") + self.set_validation_failed() + continue + relation_name = relation.split("=", 1)[1].strip(' "') + + if len(object_name) == 0 or len(relation_name) == 0: + log.error(f"Config option '{relation}' malformed got '{object_name}' for " + f"object name and '{relation_name}' for {relation_type} name.") + self.set_validation_failed() + continue + + try: + re_compiled = re.compile(object_name) + except Exception as e: + log.error(f"Problem parsing regular expression '{object_name}' for '{relation}': {e}") + self.set_validation_failed() + continue + + relation_data.append({ + "object_regex": re_compiled, + "assigned_name": relation_name + }) + + option.set_value(relation_data) + + continue + + if "custom_object_attributes" in option.key: + + option.set_value(quoted_split(option.value)) + + continue + + if option.key == "ip_tenant_inheritance_order": + order = quoted_split(option.value) + if any(value not in ("device", "prefix", "disabled") for value in order): + log.error(f"Invalid IP tenant inheritance order: '{option.value}'") + self.set_validation_failed() + option.set_value(order) + continue + + if option.key == "set_primary_ip": + if option.value not in ["always", "when-undefined", "never"]: + log.error(f"Primary IP option '{option.key}' value '{option.value}' invalid.") + self.set_validation_failed() + + if option.key == "custom_dns_servers": + + dns_name_lookup = self.get_option_by_name("dns_name_lookup") + + if not isinstance(dns_name_lookup, ConfigOption) or dns_name_lookup.value is False: + continue + + custom_dns_servers = quoted_split(option.value) + + tested_custom_dns_servers = list() + for custom_dns_server in custom_dns_servers: + try: + tested_custom_dns_servers.append(str(ip_address(custom_dns_server))) + except ValueError: + log.error(f"Config option 'custom_dns_servers' value '{custom_dns_server}' " + f"does not appear to be an IP address.") + self.set_validation_failed() + + option.set_value(tested_custom_dns_servers) + + continue + + permitted_subnets_option = self.get_option_by_name("permitted_subnets") + + if permitted_subnets_option is not None: + permitted_subnets = PermittedSubnets(permitted_subnets_option.value) + if permitted_subnets.validation_failed is True: + self.set_validation_failed() + + permitted_subnets_option.set_value(permitted_subnets) From a4c6340993abdc76cfc4895cebfe8cde06160605 Mon Sep 17 00:00:00 2001 From: Eric Krichbaum Date: Wed, 11 Nov 2026 15:56:13 -0600 Subject: [PATCH 03/21] connection for openstack source --- module/sources/openstack/connection.py | 1299 ++++++++++++++++++++++++ 1 file changed, 1299 insertions(+) create mode 100644 module/sources/openstack/connection.py diff --git a/module/sources/openstack/connection.py b/module/sources/openstack/connection.py new file mode 100644 index 00000000..01122ac4 --- /dev/null +++ b/module/sources/openstack/connection.py @@ -0,0 +1,1299 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2020 - 2021 Ricardo Bartels. All rights reserved. +# +# netbox-sync.py +# +# This work is licensed under the terms of the MIT license. +# For a copy, see file LICENSE.txt included in this +# repository or visit: . + +import pprint +from ipaddress import ip_interface +from importlib import import_module +from urllib.parse import unquote + +from packaging import version + +from module.sources.common.source_base import SourceBase +from module.sources.openstack.config import OpenStackConfig +from module.common.logging import get_logger, DEBUG3 +from module.common.misc import grab, dump, get_string_or_none +from module.common.support import normalize_mac_address +from module.netbox.inventory import NetBoxInventory +from module.netbox import * + +log = get_logger() + + +# noinspection PyTypeChecker +class OpenStackHandler(SourceBase): + """ + Source class to import data from a Openstack instance and add/update NetBox objects based on gathered information + """ + + dependent_netbox_objects = [ + NBTag, + NBManufacturer, + NBDeviceType, + NBPlatform, + NBClusterType, + NBClusterGroup, + NBDeviceRole, + NBSite, + NBCluster, + NBDevice, + NBVM, + NBVirtualDisk, + NBVMInterface, + NBInterface, + NBIPAddress, + NBPrefix, + NBTenant, + NBVRF, + NBVLAN, + NBVLANGroup, + NBMACAddress, + NBFHRPGroupItem, + NBCustomField + ] + + source_type = "openstack" + + # internal vars + session = None + site_name = None + + def __init__(self, name=None): + + if name is None: + raise ValueError(f"Invalid value for attribute 'name': '{name}'.") + + self.inventory = NetBoxInventory() + self.name = name + self.session = None + self.init_successful = False + + # parse settings + settings_handler = OpenStackConfig() + settings_handler.source_name = self.name + self.settings = settings_handler.parse() + + self.set_source_tag() + self.site_name = f"OpenStack: {name}" + + if self.settings.enabled is False: + log.info(f"Source '{name}' is currently disabled. Skipping") + return + + self.create_openstack_session() + + if self.session is None: + log.info(f"Source '{name}' is currently unavailable. Skipping") + return + + self.init_successful = True + self.permitted_clusters = dict() + self.cluster_host_map = dict() + self.volume_map = dict() + self.processed_host_names = dict() + self.processed_vm_names = dict() + self.object_cache = dict() + + def create_openstack_session(self): + """ + Initialize session with OpenStack + + Returns + ------- + bool: if initialization was successful or not + """ + + if self.session is not None: + return True + + log.debug(f"Starting OpenStack connection to '{self.settings.auth_url}'") + + try: + openstack = import_module("openstack") + except ImportError: + log.error("OpenStack source requires openstacksdk. Install with " + "'pip install -r requirements-openstack.txt' or 'uv sync --extra openstack'.") + return False + + try: + self.session = openstack.connect( + auth_url=self.settings.auth_url, + project_name=self.settings.project, + username=self.settings.username, + password=self.settings.password, + region_name=self.settings.region, + user_domain_name=self.settings.user_domain, + project_domain_name=self.settings.project_domain, + verify=self.settings.validate_tls_certs, + app_name='netbox-sync', + app_version='0.1', + ) + self.session.authorize() + except Exception as e: + log.error(f"Unable to connect to OpenStack instance '{self.settings.auth_url}': {e}") + self.finish() + return False + + log.info(f"Successfully connected to OpenStack '{self.settings.auth_url}'") + + return True + + def finish(self): + if self.session is not None: + self.session.close() + self.session = None + + def apply(self): + """ + Main source handler method. This method is called for each source from "main" program + to retrieve data from it source and apply it to the NetBox inventory. + + Every update of new/existing objects fot this source has to happen here. + """ + + log.info(f"Query data from OpenStack: '{self.settings.auth_url}'") + self.permitted_clusters.clear() + self.cluster_host_map.clear() + self.volume_map.clear() + self.processed_host_names.clear() + self.processed_vm_names.clear() + self.object_cache.clear() + + self.inventory.add_update_object(NBClusterGroup, data={"name": self.settings.group_name}, source=self) + + availability_zones = self.session.compute.availability_zones(details=True) + for availability_zone in availability_zones: + self.add_cluster(availability_zone) + + hypervisors = self.session.compute.hypervisors(details=True) + for hypervisor in hypervisors: + self.add_host(hypervisor) + + volumes = self.session.block_storage.volumes(details=True, all_projects=self.settings.all_projects) + for volume in volumes: + self.add_volume(volume) + + servers = self.session.compute.servers(details=True, all_projects=self.settings.all_projects) + # Prefer an already-associated server before using status to break ties. + for server in sorted(servers, key=self.get_server_priority): + self.add_virtual_machine(server) + + self.update_basic_data() + + @staticmethod + def passes_filter(name, include_filter, exclude_filter): + """ + checks if object name passes a defined object filter. + + Parameters + ---------- + name: str + name of the object to check + include_filter: regex object + regex object of include filter + exclude_filter: regex object + regex object of exclude filter + + Returns + ------- + bool: True if all filter passed, otherwise False + """ + + # first includes + if include_filter is not None and not include_filter.match(name): + log.debug(f"Object '{name}' did not match include filter '{include_filter.pattern}'. Skipping") + return False + + # second excludes + if exclude_filter is not None and exclude_filter.match(name): + log.debug(f"Object '{name}' matched exclude filter '{exclude_filter.pattern}'. Skipping") + return False + + return True + + def get_site_name(self, object_type, object_name, cluster_name=""): + """ + Return a site name for a NBCluster or NBDevice depending on config options + host_site_relation and cluster_site_relation + + Parameters + ---------- + object_type: (NBCluster, NBDevice) + object type to check site relation for + object_name: str + object name to check site relation for + cluster_name: str + cluster name of NBDevice to check for site name + + Returns + ------- + str: site name if a relation was found + """ + + if object_type not in [NBCluster, NBDevice]: + raise ValueError(f"Object must be a '{NBCluster.name}' or '{NBDevice.name}'.") + + log.debug2(f"Trying to find site name for {object_type.name} '{object_name}'") + + # check if site was provided in config + relation_name = "host_site_relation" if object_type == NBDevice else "cluster_site_relation" + + site_name = self.get_object_relation(object_name, relation_name) + + if object_type == NBDevice and site_name is None: + site_name = self.get_site_name(NBCluster, cluster_name) + if site_name is not None: + log.debug2(f"Found a matching cluster site for {object_name}, using site '{site_name}'") + + # set default site name + if site_name is None: + site_name = self.site_name + log.debug(f"No site relation for '{object_name}' found, using default site '{site_name}'") + + return site_name + + def get_object_based_on_macs(self, object_type, mac_list=None): + """ + Try to find a NetBox object based on list of MAC addresses. + + Iterate over all interfaces of this object type and compare MAC address with list of desired MAC + addresses. If match was found store related machine object and count every correct match. + + If exactly one machine with matching interfaces was found then this one will be returned. + + If two or more machines with matching MACs are found compare the two machines with + the highest amount of matching interfaces. If the ration of matching interfaces + exceeds 2.0 then the top matching machine is chosen as desired object. + + If the ration is below 2.0 then None will be returned. The probability is too low that + this one is the correct one. + + None will also be returned if no machine was found at all. + + Parameters + ---------- + object_type: (NBDevice, NBVM) + type of NetBox device to find in inventory + mac_list: list + list of MAC addresses to compare against NetBox interface objects + + Returns + ------- + (NBDevice, NBVM, None): object instance of found device, otherwise None + """ + + object_to_return = None + + if object_type not in [NBDevice, NBVM]: + raise ValueError(f"Object must be a '{NBVM.name}' or '{NBDevice.name}'.") + + if mac_list is None or not isinstance(mac_list, list) or len(mac_list) == 0: + return + + interface_typ = NBInterface if object_type == NBDevice else NBVMInterface + + objects_with_matching_macs = dict() + matching_object = None + + for interface in self.inventory.get_all_items(interface_typ): + + mac_address = grab(interface, "data.primary_mac_address.data.mac_address") or \ + grab(interface, "data.mac_address") + if mac_address in mac_list: + + matching_object = grab(interface, f"data.{interface.secondary_key}") + if not isinstance(matching_object, (NBDevice, NBVM)): + continue + + log.debug2("Found matching MAC '%s' on %s '%s'" % + (grab(interface, "data.mac_address"), object_type.name, + matching_object.get_display_name(including_second_key=True))) + + if objects_with_matching_macs.get(matching_object) is None: + objects_with_matching_macs[matching_object] = 1 + else: + objects_with_matching_macs[matching_object] += 1 + + # try to find object based on amount of matching MAC addresses + num_devices_witch_matching_macs = len(objects_with_matching_macs.keys()) + + if num_devices_witch_matching_macs == 1 and isinstance(matching_object, (NBDevice, NBVM)): + + log.debug2("Found one %s '%s' based on MAC addresses and using it" % + (object_type.name, matching_object.get_display_name(including_second_key=True))) + + object_to_return = list(objects_with_matching_macs.keys())[0] + + elif num_devices_witch_matching_macs > 1: + + log.debug2(f"Found {num_devices_witch_matching_macs} {object_type.name}s with matching MAC addresses") + + # now select the two top matches + first_choice, second_choice = \ + sorted(objects_with_matching_macs, key=objects_with_matching_macs.get, reverse=True)[0:2] + + first_choice_matches = objects_with_matching_macs.get(first_choice) + second_choice_matches = objects_with_matching_macs.get(second_choice) + + log.debug2(f"The top candidate {first_choice.get_display_name()} with {first_choice_matches} matches") + log.debug2(f"The second candidate {second_choice.get_display_name()} with {second_choice_matches} matches") + + # get ratio between + matching_ration = first_choice_matches / second_choice_matches + + # only pick the first one if the ration exceeds 2 + if matching_ration >= 2.0: + log.debug2(f"The matching ratio of {matching_ration} is high enough " + f"to select {first_choice.get_display_name()} as desired {object_type.name}") + object_to_return = first_choice + else: + log.debug2("Both candidates have a similar amount of " + "matching interface MAC addresses. Using NONE of them!") + + return object_to_return + + def get_object_based_on_primary_ip(self, object_type, primary_ip4=None, primary_ip6=None): + """ + Try to find a NBDevice or NBVM based on the primary IP address. If an exact + match was found the device/vm object will be returned immediately without + checking of the other primary IP address (if defined). + + Parameters + ---------- + object_type: (NBDevice, NBVM) + object type to look for + primary_ip4: str + primary IPv4 address of object to find + primary_ip6: str + primary IPv6 address of object to find + + Returns + ------- + + """ + + def _matches_device_primary_ip(device_primary_ip, ip_needle): + + ip = None + if device_primary_ip is not None and ip_needle is not None: + if isinstance(device_primary_ip, dict): + ip = grab(device_primary_ip, "address") + + elif isinstance(device_primary_ip, NBIPAddress): + ip = device_primary_ip.data.get("address") + elif isinstance(device_primary_ip, int): + ip = self.inventory.get_by_id(NBIPAddress, nb_id=device_primary_ip) + ip = grab(ip, "data.address") + + if ip is not None and ip.split("/")[0] == ip_needle: + return True + + return False + + if object_type not in [NBDevice, NBVM]: + raise ValueError(f"Object must be a '{NBVM.name}' or '{NBDevice.name}'.") + + if primary_ip4 is None and primary_ip6 is None: + return + + if primary_ip4 is not None: + primary_ip4 = str(primary_ip4).split("/")[0] + + if primary_ip6 is not None: + primary_ip6 = str(primary_ip6).split("/")[0] + + for device in self.inventory.get_all_items(object_type): + + if _matches_device_primary_ip(grab(device, "data.primary_ip4"), primary_ip4) is True: + log.debug2(f"Found existing host '{device.get_display_name()}' " + f"based on the primary IPv4 '{primary_ip4}'") + return device + + if _matches_device_primary_ip(grab(device, "data.primary_ip6"), primary_ip6) is True: + log.debug2(f"Found existing host '{device.get_display_name()}' " + f"based on the primary IPv6 '{primary_ip6}'") + return device + + def get_object_relation(self, name, relation, fallback=None): + """ + + Parameters + ---------- + name: str + name of the object to find a relation for + relation: str + name of the config variable relation (i.e: vm_tag_relation) + fallback: str + fallback string if no relation matched + + Returns + ------- + data: str, list, None + string of matching relation or list of matching tags + """ + + resolved_list = list() + for single_relation in grab(self.settings, relation, fallback=list()): + object_regex = single_relation.get("object_regex") + match_found = False + if object_regex.match(name): + resolved_name = single_relation.get("assigned_name") + log.debug2(f"Found a matching {relation} '{resolved_name}' ({object_regex.pattern}) for {name}") + resolved_list.append(resolved_name) + match_found = True + + # special cluster condition + if match_found is False and grab(f"{relation}".split("_"), "0") == "cluster": + + stripped_name = "/".join(name.split("/")[1:]) + if object_regex.match(stripped_name): + + resolved_name = single_relation.get("assigned_name") + log.debug2(f"Found a matching {relation} '{resolved_name}' ({object_regex.pattern}) " + f"for {stripped_name}") + resolved_list.append(resolved_name) + + if grab(f"{relation}".split("_"), "1") == "tag": + return resolved_list + + else: + resolved_name = fallback + if len(resolved_list) >= 1: + resolved_name = resolved_list[0] + if len(resolved_list) > 1: + log.debug(f"Found {len(resolved_list)} matches for {name} in {relation}." + f" Using first on: {resolved_name}") + + return resolved_name + + def get_cluster_for_host(self, hostname): + for cluster, hosts in self.cluster_host_map.items(): + if hostname in hosts: + return cluster + return None + + def get_server_priority(self, server): + name = get_string_or_none(grab(server, "name")) + if name is not None and self.settings.strip_vm_domain_name: + name = name.split(".")[0] + if self.settings.set_vm_name_to_uuid: + name = get_string_or_none(grab(server, "id")) + + cluster_name = get_string_or_none(grab(server, "availability_zone")) + cluster = self.get_object_from_cache(NBCluster, cluster_name) + if name is not None and cluster is not None and not self.settings.skip_vm_comments: + existing_vm = self.inventory.get_by_data( + NBVM, data={"name": name, "cluster": cluster} + ) + if grab(existing_vm, "data.comments") == get_string_or_none(grab(server, "id")): + return 0, server.status != "ACTIVE" + + return 1, server.status != "ACTIVE" + + @staticmethod + def get_compute_host_names(hosts): + compute_host_names = [] + for hostname, services in (hosts or {}).items(): + if not isinstance(services, dict) or not services: + compute_host_names.append(hostname) + continue + + service_names = {str(name).lower() for name in services} + service_names.update( + str(grab(service, "binary", fallback="")).lower() + for service in services.values() + ) + has_nova_services = any(name.startswith("nova-") for name in service_names) + if not has_nova_services or "nova-compute" in service_names: + compute_host_names.append(hostname) + + return compute_host_names + + def object_synced_by_other_source(self, nb_object): + if nb_object.source is not None: + return nb_object.source is not self + other_tags = [source.source_tag for source in self.inventory.source_list if source is not self] + return bool(set(nb_object.get_tags()).intersection(other_tags)) + + def add_custom_attributes(self, object_type, obj, data): + attributes = self.settings.host_custom_object_attributes if object_type == NBDevice else \ + self.settings.vm_custom_object_attributes + for attribute in attributes or []: + value = grab(obj, attribute) + if value is None: + continue + field = self.add_update_custom_field({ + "name": f"openstack_{attribute}", "label": attribute[:50], + "object_types": [object_type.object_type], "type": "text", + "description": f"OpenStack resource attribute '{attribute}'", + }) + data.setdefault("custom_fields", {})[field.data["name"]] = str(value) + + def add_device_vm_to_inventory(self, object_type, object_data, pnic_data=None, vnic_data=None, + nic_ips=None, p_ipv4=None, p_ipv6=None): + """ + Add/update device/VM object in inventory based on gathered data. + + Try to find object first based on the object data, interface MAC addresses and primary IPs. + 1. try to find by name and cluster/site + 2. try to find by mac addresses interfaces + 3. try to find by primary IP + + IP addresses for each interface are added here as well. First they will be checked and added + if all checks pass. For each IP address a matching IP prefix will be searched for. First we + look for longest matching IP Prefix in the same site. If this failed we try to find the longest + matching global IP Prefix. + + If a IP Prefix was found then we try to get the VRF and VLAN for this prefix. Now we compare + if interface VLAN and prefix VLAN match up and warn if they don't. Then we try to add data to + the IP address if not already set: + + add prefix VRF if VRF for this IP is undefined + add tenant if tenant for this IP is undefined + 1. try prefix tenant + 2. if prefix tenant is undefined try VLAN tenant + + And we also set primary IP4/6 for this object depending on the "set_primary_ip" setting. + + If a IP address is set as primary IP for another device then using this IP on another + device will be rejected by NetBox. + + Setting "always": + check all NBDevice and NBVM objects if this IP address is set as primary IP to any + other object then this one. If we found another object, then we unset the primary_ip* + for the found object and assign it to this object. + + This setting will also reset the primary IP if it has been changed in NetBox + + Setting "when-undefined": + Will set the primary IP for this object if primary_ip4/6 is undefined. Will cause a + NetBox error if IP has been assigned to a different object as well + + Setting "never": + Well, the attribute primary_ip4/6 will never be touched/changed. + + Parameters + ---------- + object_type: (NBDevice, NBVM) + NetBoxObject sub class of object to add + object_data: dict + data of object to add/update + pnic_data: dict + data of physical interfaces of this object, interface name as key + vnic_data: dict + data of virtual interfaces of this object, interface name as key + nic_ips: dict + dict of ips per interface of this object, interface name as key + p_ipv4: str + primary IPv4 as string including netmask/prefix + p_ipv6: str + primary IPv6 as string including netmask/prefix + + """ + + if object_type not in [NBDevice, NBVM]: + raise ValueError(f"Object must be a '{NBVM.name}' or '{NBDevice.name}'.") + + pnic_data = pnic_data or {} + vnic_data = vnic_data or {} + nic_ips = nic_ips or {} + + if log.level == DEBUG3: + + log.debug3("function: add_device_vm_to_inventory") + log.debug3(f"Object type {object_type}") + pprint.pprint(object_data) + pprint.pprint(pnic_data) + pprint.pprint(vnic_data) + pprint.pprint(nic_ips) + pprint.pprint(p_ipv4) + pprint.pprint(p_ipv6) + + # check existing Devices for matches + log.debug2(f"Trying to find a {object_type.name} based on the collected name, cluster, IP and MAC addresses") + + device_vm_object = self.inventory.get_by_data(object_type, data=object_data) + + if device_vm_object is not None: + log.debug2("Found a exact matching %s object: %s" % + (object_type.name, device_vm_object.get_display_name(including_second_key=True))) + + # keep searching if no exact match was found + else: + + log.debug2(f"No exact match found. Trying to find {object_type.name} based on MAC addresses") + + # on VMs vnic data is used, on physical devices pnic data is used + mac_source_data = vnic_data if object_type == NBVM else pnic_data + + nic_macs = [x.get("mac_address") for x in mac_source_data.values() if x.get("mac_address") is not None] + + device_vm_object = self.get_object_based_on_macs(object_type, nic_macs) + + if device_vm_object is not None: + log.debug2("Found a matching %s object: %s" % + (object_type.name, device_vm_object.get_display_name(including_second_key=True))) + + # keep looking for devices with the same primary IP + else: + + log.debug2(f"No match found. Trying to find {object_type.name} based on primary IP addresses") + + device_vm_object = self.get_object_based_on_primary_ip(object_type, p_ipv4, p_ipv6) + + if device_vm_object is None: + object_name = object_data.get(object_type.primary_key) + log.debug(f"No existing {object_type.name} object for {object_name}. Creating a new {object_type.name}.") + device_vm_object = self.inventory.add_object(object_type, data=object_data, source=self) + else: + device_vm_object.update(data=object_data, source=self) + + # update role according to config settings + object_name = object_data.get(object_type.primary_key) + role_name = self.get_object_relation(object_name, + "host_role_relation" if object_type == NBDevice else "vm_role_relation") + + if object_type == NBDevice: + if role_name is None: + role_name = "Server" + device_vm_object.update(data={"device_role": {"name": role_name}}) + if object_type == NBVM and role_name is not None: + device_vm_object.update(data={"role": {"name": role_name}}) + + # compile all nic data into one dictionary + if object_type == NBVM: + nic_data = vnic_data + else: + nic_data = {**pnic_data, **vnic_data} + exclude_filter = self.settings.vm_interface_exclude_filter if object_type == NBVM else \ + self.settings.host_interface_exclude_filter + if exclude_filter is not None: + nic_data = {name: data for name, data in nic_data.items() if not exclude_filter.match(name)} + + # map interfaces of existing object with discovered interfaces + nic_object_dict = self.map_object_interfaces_to_current_interfaces(device_vm_object, nic_data) + + if object_data.get("status", "") == "active" and (nic_ips is None or len(nic_ips.keys()) == 0): + log.debug(f"No IP addresses for '{object_name}' found!") + + primary_ipv4_object = None + primary_ipv6_object = None + + if p_ipv4 is not None: + try: + primary_ipv4_object = ip_interface(p_ipv4) + except ValueError: + log.error(f"Primary IPv4 ({p_ipv4}) does not appear to be a valid IP address (needs included suffix).") + + if p_ipv6 is not None: + try: + primary_ipv6_object = ip_interface(p_ipv6) + except ValueError: + log.error(f"Primary IPv6 ({p_ipv6}) does not appear to be a valid IP address (needs included suffix).") + + for int_name, int_data in nic_data.items(): + existing_interface = nic_object_dict.get(int_name) + if existing_interface is not None: + existing_name = grab(existing_interface, "data.name") + if existing_name is not None: + int_data["name"] = existing_name + if object_type == NBDevice: + int_data.pop("type", None) + + # add/update interface with retrieved data + nic_object, ip_address_objects = self.add_update_interface(nic_object_dict.get(int_name), device_vm_object, + int_data, nic_ips.get(int_name, list())) + + # add all interface IPs + for ip_object in ip_address_objects: + + if ip_object is None: + continue + + ip_interface_object = ip_interface(grab(ip_object, "data.address")) + + # continue if address is not a primary IP + if ip_interface_object not in [primary_ipv4_object, primary_ipv6_object]: + continue + + # set/update/remove primary IP addresses + set_this_primary_ip = False + ip_version = ip_interface_object.ip.version + if self.settings.set_primary_ip == "always": + + for object_type in [NBDevice, NBVM]: + + # new IPs don't need to be removed from other devices/VMs + if ip_object.is_new is True: + break + + for devices_vms in self.inventory.get_all_items(object_type): + + # device has no primary IP of this version + this_primary_ip = grab(devices_vms, f"data.primary_ip{ip_version}") + + # we found this exact object + if devices_vms == device_vm_object: + continue + + # device has the same object assigned + if this_primary_ip == ip_object: + devices_vms.unset_attribute(f"primary_ip{ip_version}") + + set_this_primary_ip = True + + elif self.settings.set_primary_ip != "never" and \ + grab(device_vm_object, f"data.primary_ip{ip_version}") is None: + set_this_primary_ip = True + + if set_this_primary_ip is True: + + log.debug(f"Setting IP '{grab(ip_object, 'data.address')}' as primary IPv{ip_version} for " + f"'{device_vm_object.get_display_name()}'") + device_vm_object.update(data={f"primary_ip{ip_version}": ip_object}) + + return device_vm_object + + def add_object_to_cache(self, obj_type, key, netbox_object): + + if obj_type is None or key is None or netbox_object is None: + return + + # noinspection PyBroadException + try: + vm_class_name = obj_type.__name__ + except Exception: + return + + if self.object_cache.get(vm_class_name) is None: + self.object_cache[vm_class_name] = dict() + + self.object_cache[vm_class_name][key] = netbox_object + + def get_object_from_cache(self, obj_type, key): + + if obj_type is None or key is None: + return + + # noinspection PyBroadException + try: + vm_class_name = obj_type.__name__ + except Exception: + return + + if self.object_cache.get(vm_class_name) is None: + return + + return self.object_cache[vm_class_name].get(key) + + def add_cluster(self, obj): + """ + Add a OpenStack Availability Zone as a NBCluster to NetBox. Cluster name is checked against + cluster_include_filter and cluster_exclude_filter config setting. Also adds + cluster and site_name to "self.permitted_clusters" so hosts and VMs can be + checked if they are part of a permitted cluster. + + Parameters + ---------- + obj: openstack.compute.v2.availability_zone.AvailabilityZone + cluster to add + """ + + name = get_string_or_none(grab(obj, "name")) + group = self.inventory.get_by_data(NBClusterGroup, data={"name": self.settings.group_name}) + + if name is None or group is None: + return + + group_name = grab(group, "data.name") + + log.debug(f"Parsing OpenStack AZ: {name}") + + if self.passes_filter(name, self.settings.cluster_include_filter, + self.settings.cluster_exclude_filter) is False: + return + + site_name = self.get_site_name(NBCluster, name) + + data = { + "name": name, + "type": {"name": "Openstack"}, + "group": group, + "site": {"name": site_name} + } + if version.parse(self.inventory.netbox_api_version) >= version.parse("4.2.0"): + data.pop("site") + data["scope_type"] = NBSite.object_type + data["scope_id"] = self.inventory.add_update_object(NBSite, data={"name": site_name}, source=self) + tags = self.get_object_relation(name, "cluster_tag_relation") + if tags: + data["tags"] = tags + + tenant_name = self.get_object_relation(name, "cluster_tenant_relation") + if tenant_name is not None: + data["tenant"] = {"name": tenant_name} + + # try to find cluster including cluster group + log.debug2("Trying to find a matching existing cluster") + cluster_object = None + for cluster_candidate in self.inventory.get_all_items(NBCluster): + if grab(cluster_candidate, "data.name") != name: + continue + if cluster_candidate.get_site_name() not in (None, site_name) and \ + self.object_synced_by_other_source(cluster_candidate): + continue + + # try to find a cluster with matching site + candidate_group = grab(cluster_candidate, "data.group.data.name") + if candidate_group not in (None, group_name): + continue + + if cluster_candidate.get_site_name() == site_name: + cluster_object = cluster_candidate + log.debug2("Found an existing cluster where 'name' and 'site' are matching") + break + + if grab(cluster_candidate, "data.group") is not None and \ + grab(cluster_candidate, "data.group.data.name") == group_name: + cluster_object = cluster_candidate + log.debug2("Found an existing cluster where 'name' and 'cluster group' are matching") + break + + if grab(cluster_candidate, "data.tenant") is not None and \ + tenant_name is not None and \ + grab(cluster_candidate, "data.tenant.data.name") == tenant_name: + cluster_object = cluster_candidate + log.debug2("Found an existing cluster where 'name' and 'tenant' are matching") + break + + if cluster_object is not None: + cluster_object.update(data=data, source=self) + else: + cluster_object = self.inventory.add_update_object(NBCluster, data=data, source=self) + + self.add_object_to_cache(NBCluster, name, cluster_object) + + self.cluster_host_map[name] = self.get_compute_host_names(grab(obj, "hosts")) + log.debug2(f"Found compute hosts for availability zone '{name}': " + f"{self.cluster_host_map[name]}") + + self.permitted_clusters[name] = site_name + + def add_host(self, obj): + """ + Parse a Openstack host to NetBox once all data is gathered. + + First host is filtered: + host has a cluster and is it permitted + was host with same name and site already parsed + does the host pass the host_include_filter and host_exclude_filter + + Then all necessary host data will be collected. + + Primary IPv4/6 will be determined by 'host_ip' value + + Parameters + ---------- + obj: Hypervisor + host object to parse + """ + + name = get_string_or_none(grab(obj, "name")) + if name is None: + return + + if name is not None and self.settings.strip_host_domain_name is True: + name = name.split(".")[0] + + # parse data + log.debug(f"Parsing Openstack host: {name}") + + # + # Filtering + # + + # manage site and cluster + short_name = get_string_or_none(obj.service_details["host"]) + cluster_name = self.get_cluster_for_host(short_name) + + if cluster_name is None: + # Nova omits disabled compute services from availability zone host lists + if get_string_or_none(grab(obj, "status")) == "disabled": + log.info(f"Host '{name}' is disabled in Nova and not part of any availability zone. Skipping.") + else: + log.error(f"Unable to find an availability zone for host '{name}'. Skipping.") + return + + if log.level == DEBUG3: + try: + log.info("Cluster data") + dump(obj.service_details.to_dict()) + except Exception as e: + log.error(e) + + if self.permitted_clusters.get(cluster_name) is None: + log.debug(f"Host '{name}' is not part of a permitted cluster. Skipping") + return + + # get cluster object + nb_cluster_object = self.get_object_from_cache(NBCluster, cluster_name) + + # get a site for this host + site_name = self.get_site_name(NBDevice, name, cluster_name) + + if name in self.processed_host_names.get(site_name, list()): + log.warning(f"Host '{name}' for site '{site_name}' already parsed. " + "Make sure to use unique host names. Skipping") + return + + # add host to processed list + if self.processed_host_names.get(site_name) is None: + self.processed_host_names[site_name] = list() + + self.processed_host_names[site_name].append(name) + + # filter hosts by name + if self.passes_filter(name, self.settings.host_include_filter, self.settings.host_exclude_filter) is False: + return + + # + # Collecting data + # + + # collect all necessary data + manufacturer = None + model = None + platform = get_string_or_none(obj.hypervisor_type) + + # if the device vendor/model cannot be retrieved (due to problem on the host), + # set a dummy value so the host still gets synced + if manufacturer is None: + manufacturer = "Generic Vendor" + if model is None: + model = "Generic Model" + + # get status + status = "offline" + if get_string_or_none(obj.status) == "enabled" and grab(obj, "state", fallback="up") == "up": + status = "active" + + # get host_tenant_relation + tenant_name = self.get_object_relation(name, "host_tenant_relation") + + # get host_tag_relation + host_tags = self.get_object_relation(name, "host_tag_relation") + + # prepare host data model + host_data = { + "name": name, + "device_type": { + "model": model, + "manufacturer": { + "name": manufacturer + } + }, + "site": {"name": site_name}, + "cluster": nb_cluster_object, + "status": status + } + + # add data if present + if platform is not None: + host_data["platform"] = {"name": platform} + if tenant_name is not None: + host_data["tenant"] = {"name": tenant_name} + if len(host_tags) > 0: + host_data["tags"] = host_tags + self.add_custom_attributes(NBDevice, obj, host_data) + + pnic_data_dict = dict() + pnic_data = { + "name": "eth0", + "enabled": True, + "type": "virtual" + } + pnic_data_dict["eth0"] = pnic_data + + ip_addr = grab(obj, "host_ip") + if ip_addr is not None and not self.settings.permitted_subnets.permitted(ip_addr, interface_name="eth0"): + ip_addr = None + prefix = None + matched_prefix = None + if ip_addr is not None: + matched_prefix = self.return_longest_matching_prefix_for_ip(ip_interface(ip_addr), site_name) + if matched_prefix is None: + matched_prefix = self.return_longest_matching_prefix_for_ip(ip_interface(ip_addr)) + if matched_prefix is not None: + prefix = matched_prefix.data["prefix"].prefixlen + ip_addr = f"{ip_addr}/{prefix}" + + vnic_ips = dict() + vnic_ips["eth0"] = list() + if ip_addr is not None: + ip_addr = str(ip_interface(ip_addr)) + vnic_ips["eth0"].append(ip_addr) + + host_primary_ip4 = ip_addr if ip_addr is not None and ip_interface(ip_addr).version == 4 else None + host_primary_ip6 = ip_addr if ip_addr is not None and ip_interface(ip_addr).version == 6 else None + + # add host to inventory + host = self.add_device_vm_to_inventory(NBDevice, object_data=host_data, pnic_data=pnic_data_dict, + vnic_data=dict(), nic_ips=vnic_ips, + p_ipv4=host_primary_ip4, p_ipv6=host_primary_ip6) + self.add_object_to_cache(NBDevice, short_name, host) + + return + + def add_volume(self, obj): + """ + Parse OpenStack volume and store in in a map. + """ + + id = obj.id + size = obj.size + + self.volume_map[id] = size + + def add_virtual_machine(self, obj): + """ + Parse a OpenStack VM add to NetBox once all data is gathered. + + Parameters + ---------- + obj: openstack.compute.v2.server.Server + virtual machine object to parse + """ + + name = get_string_or_none(grab(obj, "name")) + if name is None: + return + + if name is not None and self.settings.strip_vm_domain_name is True: + name = name.split(".")[0] + + if self.settings.set_vm_name_to_uuid: + name = get_string_or_none(obj.id) + + log.debug(f"Parsing OpenStack VM: {name}") + + # get VM power state + status = "active" if get_string_or_none(obj.status) == "ACTIVE" else "offline" + + cluster_name = get_string_or_none(obj.availability_zone) + + # check VM cluster + if cluster_name is None: + log.error(f"Requesting cluster for Virtual Machine '{name}' failed. Skipping.") + return + + elif self.permitted_clusters.get(cluster_name) is None: + log.debug(f"Virtual machine '{name}' is not part of a permitted cluster. Skipping") + return + + nb_cluster_object = self.get_object_from_cache(NBCluster, cluster_name) + + if name in self.processed_vm_names.get(cluster_name, list()): + log.warning(f"Virtual machine '{name}' for cluster '{cluster_name}' already parsed. " + "Make sure to use unique VM names. Skipping") + return + + # add host to processed list + if self.processed_vm_names.get(cluster_name) is None: + self.processed_vm_names[cluster_name] = list() + + self.processed_vm_names[cluster_name].append(name) + + # filter VMs by name + if self.passes_filter(name, self.settings.vm_include_filter, self.settings.vm_exclude_filter) is False: + return + + # + # Collect data + # + + site_name = self.get_site_name(NBCluster, cluster_name) + flavor = dict(grab(obj, "flavor") or {}) + if not all(key in flavor for key in ("vcpus", "ram", "disk")) and flavor.get("id"): + flavor = self.session.compute.get_flavor(flavor["id"]).to_dict() + # first check against vm_platform_relation + platform = None + if self.settings.skip_vm_platform is False: + platform = get_string_or_none(flavor.get("original_name") or flavor.get("name")) + + if platform is not None: + platform = self.get_object_relation(platform, "vm_platform_relation", fallback=platform) + + disks = {} + if grab(obj, "image") and flavor.get("disk"): + disks["root"] = int(flavor["disk"]) + if flavor.get("ephemeral"): + disks["ephemeral"] = int(flavor["ephemeral"]) + disk_complete = True + for volume in grab(obj, "attached_volumes", fallback=[]) or []: + size = self.volume_map.get(volume["id"]) + if size is None: + log.warning(f"Volume '{volume['id']}' of VM '{name}' was not discovered; preserving disk size.") + disk_complete = False + disks[volume["id"]] = int(size) if size is not None else None + + annotation = None + if self.settings.skip_vm_comments is False: + if not self.settings.set_vm_name_to_uuid: + annotation = get_string_or_none(obj.id) + + # assign vm_tenant_relation + tenant_name = self.get_object_relation(name, "vm_tenant_relation") + + vm_data = { + "name": name, + "cluster": nb_cluster_object, + "status": status, + } + if flavor.get("vcpus"): + vm_data["vcpus"] = float(flavor["vcpus"]) + if flavor.get("ram") is not None: + vm_data["memory"] = int(flavor["ram"]) + if disk_complete and version.parse(self.inventory.netbox_api_version) < version.parse("3.7.0"): + vm_data["disk"] = sum(disks.values()) + tags = self.get_object_relation(name, "vm_tag_relation") + if tags: + vm_data["tags"] = tags + host = self.get_object_from_cache(NBDevice, grab(obj, "compute_host")) + if host is not None and version.parse(self.inventory.netbox_api_version) >= version.parse("3.3.0"): + vm_data["device"] = host + + # Add adaption for change in NetBox 3.3.0 VM model + # issue: https://github.com/netbox-community/netbox/issues/10131#issuecomment-1225783758 + if version.parse(self.inventory.netbox_api_version) >= version.parse("3.3.0"): + vm_data["site"] = {"name": site_name} + if platform is not None: + vm_data["platform"] = {"name": platform} + if annotation is not None: + vm_data["comments"] = annotation + if tenant_name is not None: + vm_data["tenant"] = {"name": tenant_name} + + if self.settings.set_vm_name_to_uuid: + custom_field = self.add_update_custom_field({ + "name": "openstack_vm_name", + "label": "name", + "object_types": ["virtualization.virtualmachine"], + "type": "text", + "description": f"Openstack '{self.name}' synced object attribute 'name'" + }) + vm_data["custom_fields"] = { + grab(custom_field, "data.name"): get_string_or_none(grab(obj, "name")) + } + self.add_custom_attributes(NBVM, obj, vm_data) + + vm_primary_ip4 = None + vm_primary_ip6 = None + vm_primary_ip4_is_fixed = False + vm_primary_ip6_is_fixed = False + vm_nic_dict = dict() + nic_ips = dict() + count = 0 + + for network, addresses in (grab(obj, "addresses") or {}).items(): + default_mac = next((address.get("OS-EXT-IPS-MAC:mac_addr") for address in addresses + if address.get("OS-EXT-IPS-MAC:mac_addr")), None) + interfaces = {} + for address in addresses: + mac = address.get("OS-EXT-IPS-MAC:mac_addr") or default_mac + if mac not in interfaces: + count += 1 + full_name = unquote(f"vNIC{count} ({network})") + interfaces[mac] = full_name + vm_nic_dict[full_name] = { + "name": full_name, + "description": full_name, + "enabled": True, + } + if mac: + vm_nic_dict[full_name]["mac_address"] = normalize_mac_address(mac) + nic_ips[full_name] = [] + full_name = interfaces[mac] + try: + ip_addr = str(ip_interface(address["addr"])) + except (KeyError, ValueError): + log.warning(f"Invalid address on OpenStack VM '{name}'; skipping.") + continue + if not self.settings.permitted_subnets.permitted(ip_addr, interface_name=full_name): + continue + + matched_prefix = self.return_longest_matching_prefix_for_ip(ip_interface(ip_addr), site_name) + if matched_prefix is None: + matched_prefix = self.return_longest_matching_prefix_for_ip(ip_interface(ip_addr)) + if matched_prefix is not None: + prefix = matched_prefix.data["prefix"].prefixlen + ip_addr = f"{ip_interface(ip_addr).ip}/{prefix}" + + nic_ips[full_name].append(ip_addr) + is_fixed = address.get("OS-EXT-IPS:type") == "fixed" + if ip_interface(ip_addr).version == 4 and ( + vm_primary_ip4 is None or (is_fixed and not vm_primary_ip4_is_fixed)): + vm_primary_ip4 = ip_addr + vm_primary_ip4_is_fixed = is_fixed + if ip_interface(ip_addr).version == 6 and ( + vm_primary_ip6 is None or (is_fixed and not vm_primary_ip6_is_fixed)): + vm_primary_ip6 = ip_addr + vm_primary_ip6_is_fixed = is_fixed + + # add VM to inventory + vm = self.add_device_vm_to_inventory(NBVM, object_data=vm_data, vnic_data=vm_nic_dict, + nic_ips=nic_ips, p_ipv4=vm_primary_ip4, p_ipv6=vm_primary_ip6) + if version.parse(self.inventory.netbox_api_version) >= version.parse("3.7.0"): + for disk_name, disk_size in disks.items(): + data = {"name": disk_name, "virtual_machine": vm} + if disk_size is None: + existing = self.inventory.get_by_data(NBVirtualDisk, data=data) + if existing is not None: + existing.update(data=data, source=self) + continue + multiplier = 1000 if version.parse(self.inventory.netbox_api_version) >= version.parse("4.1.0") else 1 + data["size"] = disk_size * multiplier + self.inventory.add_update_object(NBVirtualDisk, data=data, source=self) + + return + + def update_basic_data(self): + """ + + Returns + ------- + + """ + + # add source identification tag + self.inventory.add_update_object(NBTag, data={ + "name": self.source_tag, + "description": f"Marks objects synced from Openstack '{self.name}' " + f"({self.settings.auth_url}) to this NetBox Instance." + }) + + # update virtual site if present + this_site_object = self.inventory.get_by_data(NBSite, data={"name": self.site_name}) + + if this_site_object is not None: + this_site_object.update(data={ + "name": self.site_name, + "comments": "A default virtual site created to house objects " + "that have been synced from this Openstack instance " + "and have no predefined site assigned." + }) + + server_role_object = self.inventory.get_by_data(NBDeviceRole, data={"name": "Server"}) + + if server_role_object is not None: + role_data = {"name": "Server", "vm_role": True} + if server_role_object.is_new is True: + role_data["color"] = "9e9e9e" + + server_role_object.update(data=role_data) + +# EOF From 458af00996d87cc06228630ad8eddea9cc3843ed Mon Sep 17 00:00:00 2001 From: Eric Krichbaum Date: Wed, 11 Nov 2026 15:57:25 -0600 Subject: [PATCH 04/21] Add OpenStackHandler to valid sources list --- module/sources/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/module/sources/__init__.py b/module/sources/__init__.py index ee45e4f1..5e346da2 100644 --- a/module/sources/__init__.py +++ b/module/sources/__init__.py @@ -10,6 +10,7 @@ # define all available sources here from module.sources.vmware.connection import VMWareHandler from module.sources.hetzner.connection import HetznerHandler +from module.sources.openstack.connection import OpenStackHandler from module.sources.ovirt.connection import OVirtHandler from module.sources.aws.connection import AWSHandler from module.sources.digitalocean.connection import DigitalOceanHandler @@ -22,7 +23,7 @@ from module.config import source_config_section_name # list of valid sources -valid_sources = [VMWareHandler, CheckRedfish, HetznerHandler, AWSHandler, DigitalOceanHandler, OVirtHandler] +valid_sources = [VMWareHandler, CheckRedfish, HetznerHandler, AWSHandler, DigitalOceanHandler, OVirtHandler, OpenStackHandler] def validate_source(source_class_object=None, state="pre"): From a8442192943b9948ebdd1c1f6f6a623ea661359a Mon Sep 17 00:00:00 2001 From: Eric Krichbaum Date: Wed, 11 Nov 2026 15:59:33 -0600 Subject: [PATCH 05/21] Add tests for OpenStack source integration This test file implements various tests for OpenStack discovery against fake SDK resources and real NetBox models, including authentication, resource handling, and filtering. --- tests/test_openstack_source.py | 454 +++++++++++++++++++++++++++++++++ 1 file changed, 454 insertions(+) create mode 100644 tests/test_openstack_source.py diff --git a/tests/test_openstack_source.py b/tests/test_openstack_source.py new file mode 100644 index 00000000..2a80bfc8 --- /dev/null +++ b/tests/test_openstack_source.py @@ -0,0 +1,454 @@ +"""OpenStack discovery against fake SDK resources and the real NetBox models.""" + +import sys +from types import ModuleType, SimpleNamespace +from unittest.mock import Mock + +import pytest + +from module.config.file_output import ConfigFileOutput +from module.config.files import ConfigFileINI, ConfigFileYAML +from module.config.parser import ConfigParser +from module.netbox import ( + NBCluster, NBClusterGroup, NBCustomField, NBDevice, NBInterface, NBIPAddress, NBMACAddress, + NBPrefix, NBSite, NBTenant, NBVirtualDisk, NBVM, NBVMInterface, NBVRF, +) +from module.sources import instantiate_sources, valid_sources +from module.sources.openstack.config import OpenStackConfig +from module.sources.openstack.connection import OpenStackHandler + + +def resource(**values): + obj = SimpleNamespace(**values) + obj.to_dict = lambda: values.copy() + return obj + + +def server(**overrides): + values = dict( + name="app.example.org", id="vm-uuid", status="ACTIVE", availability_zone="zone.example", + flavor={"original_name": "m1.small", "vcpus": 2, "ram": 2048, "disk": 10}, + image={"id": "image"}, attached_volumes=[{"id": "volume-1"}], compute_host="compute1", + addresses={"private": [ + {"addr": "10.0.0.8", "version": 4, "OS-EXT-IPS-MAC:mac_addr": "fa:16:3e:00:00:01", + "OS-EXT-IPS:type": "fixed"}, + {"addr": "fd00::8", "version": 6, "OS-EXT-IPS-MAC:mac_addr": "fa:16:3e:00:00:01", + "OS-EXT-IPS:type": "fixed"}, + {"addr": "192.0.2.8", "version": 4, "OS-EXT-IPS:type": "floating"}, + ]}, + ) + values.update(overrides) + return resource(**values) + + +@pytest.fixture +def cloud(monkeypatch, inventory): + settings = dict( + type="openstack", auth_url="https://identity.example/v3", project="admin", + username="sync", password="test-only", region="RegionOne", user_domain="Default", + project_domain="Default", permitted_subnets="10.0.0.0/8, fd00::/8", + dns_name_lookup=False, cluster_site_relation=".* = Cloud", vm_tenant_relation=".* = Operations", + cluster_tag_relation=".* = OpenStack", vm_tag_relation=".* = Production", + ) + parser = ConfigParser() + monkeypatch.setattr(parser, "content", {"source": {"cloud": settings}}) + monkeypatch.setattr(parser, "parsing_finished", True) + sdk = ModuleType("openstack") + session = Mock() + session.compute.availability_zones.return_value = [ + resource(name="zone.example", hosts={"compute1": {}}), + resource(name="empty", hosts=None), + ] + session.compute.hypervisors.return_value = [ + resource(name="compute1.example", service_details={"host": "compute1"}, status="enabled", + state="up", host_ip="10.0.0.2", hypervisor_type="QEMU", hypervisor_version=8000000), + ] + session.block_storage.volumes.return_value = [resource(id="volume-1", size=20)] + session.compute.servers.return_value = [server()] + sdk.connect = Mock(return_value=session) + monkeypatch.setitem(sys.modules, "openstack", sdk) + return SimpleNamespace(settings=settings, sdk=sdk, session=session, inventory=inventory) + + +@pytest.mark.parametrize("netbox_version", ["3.6.0", "3.7.0", "4.0.0", "4.1.0", "4.2.0", "4.3.0"]) +def test_discovery_uses_current_inventory_apis(cloud, netbox_version): + inventory = cloud.inventory + inventory.netbox_api_version = netbox_version + site = inventory.add_object(NBSite, data={"id": 1, "name": "Cloud"}, read_from_netbox=True) + tenant = inventory.add_object(NBTenant, data={"id": 1, "name": "Network"}, read_from_netbox=True) + vrf = inventory.add_object(NBVRF, data={"id": 1, "name": "Cloud VRF"}, read_from_netbox=True) + prefix_data = {"prefix": "10.0.0.0/24", "tenant": tenant, "vrf": vrf} + if netbox_version >= "4.2.0": + prefix_data.update(scope_type=NBSite.object_type, scope_id=site) + else: + prefix_data["site"] = site + prefix = inventory.add_object(NBPrefix, data=prefix_data, read_from_netbox=True) + # Model the resolved relations returned by the real NetBox inventory loader. + prefix.data.update(tenant=tenant, vrf=vrf) + prefix.data["scope_id" if netbox_version >= "4.2.0" else "site"] = site + inventory.add_object(NBPrefix, data={"prefix": "fd00::/64"}, read_from_netbox=True) + handler = OpenStackHandler("cloud") + assert handler.init_successful + assert cloud.sdk.connect.call_args.kwargs["verify"] is True + handler.apply() + cluster = handler.get_object_from_cache(NBCluster, "zone.example") + assert cluster.get_site_name() == "Cloud" + if netbox_version >= "4.2.0": + assert cluster.data["scope_id"] is site + assert cluster.data["scope_type"] == "dcim.site" + assert "site" not in cluster.data + host = inventory.get_all_items(NBDevice)[0] + vm = inventory.get_all_items(NBVM)[0] + assert host.data["primary_ip4"].data["address"] == "10.0.0.2/24" + assert host.data["platform"].data["name"] == "QEMU" + assert vm.data["device"] is host + assert vm.data["status"] == "active" + assert vm.data["vcpus"] == 2 + assert vm.data["memory"] == 2048 + if netbox_version < "3.7.0": + assert vm.data["disk"] == 30 + assert not vm.get_virtual_disks() + else: + assert "disk" not in vm.data + disks = {disk.data["name"]: disk.data["size"] for disk in vm.get_virtual_disks()} + multiplier = 1000 if netbox_version >= "4.1.0" else 1 + assert disks == {"root": 10 * multiplier, "volume-1": 20 * multiplier} + assert vm.data["platform"].data["name"] == "m1.small" + assert "Production" in [tag.data["name"] for tag in vm.data["tags"]] + assert "OpenStack" in [tag.data["name"] for tag in cluster.data["tags"]] + nic = inventory.get_all_items(NBVMInterface)[0] + assert len(inventory.get_all_items(NBVMInterface)) == 1 + assert nic.data["name"] == "vNIC1 (private)" + assert {ip.data["address"] for ip in nic.get_ip_addresses()} == {"10.0.0.8/24", "fd00::8/64"} + assert vm.data["primary_ip4"].data["vrf"].data["name"] == "Cloud VRF" + assert vm.data["primary_ip4"].data["tenant"].data["name"] == "Operations" + assert vm.data["primary_ip6"].data["address"] == "fd00::8/64" + if netbox_version >= "4.2.0": + assert len(inventory.get_all_items(NBMACAddress)) == 1 + assert nic.data["primary_mac_address"].data["mac_address"] == "FA:16:3E:00:00:01" + else: + assert nic.data["mac_address"] == "FA:16:3E:00:00:01" + handler.apply() + assert len(inventory.get_all_items(NBVM)) == 1 + assert len(inventory.get_all_items(NBVMInterface)) == 1 + assert len(inventory.get_all_items(NBCluster)) == 2 + if netbox_version >= "3.7.0": + assert len(inventory.get_all_items(NBVirtualDisk)) == 2 + handler.finish() + cloud.session.close.assert_called_once() + + +def test_registration_authentication_and_project_scope(cloud): + assert OpenStackHandler in valid_sources + assert OpenStackConfig in ConfigFileOutput.source_config_list + cloud.settings.update(validate_tls_certs=False, all_projects=False) + handlers = instantiate_sources() + assert len(handlers) == 1 + handler = handlers[0] + cloud.sdk.connect.assert_called_once_with( + auth_url=cloud.settings["auth_url"], project_name="admin", username="sync", + password="test-only", region_name="RegionOne", user_domain_name="Default", + project_domain_name="Default", verify=False, app_name="netbox-sync", app_version="0.1", + ) + cloud.session.authorize.assert_called_once() + handler.apply() + cloud.session.compute.servers.assert_called_once_with(details=True, all_projects=False) + cloud.session.block_storage.volumes.assert_called_once_with(details=True, all_projects=False) + + +def test_missing_optional_sdk_is_explicit_and_disabled_sources_do_not_import(cloud, monkeypatch, caplog): + def missing_sdk(name): + raise ModuleNotFoundError(name) + monkeypatch.setattr("module.sources.openstack.connection.import_module", missing_sdk) + handler = OpenStackHandler("cloud") + assert not handler.init_successful + assert "requirements-openstack.txt" in caplog.text + assert handler.session is None + cloud.settings["enabled"] = False + import_mock = Mock(side_effect=AssertionError("disabled source imported SDK")) + monkeypatch.setattr("module.sources.openstack.connection.import_module", import_mock) + assert not OpenStackHandler("cloud").init_successful + import_mock.assert_not_called() + + +def test_authentication_failure_without_legacy_msg_attribute(cloud, caplog): + cloud.session.authorize.side_effect = RuntimeError("identity unavailable") + handler = OpenStackHandler("cloud") + assert not handler.init_successful + assert handler.session is None + assert "identity unavailable" in caplog.text + cloud.session.close.assert_called_once() + + +def test_filters_active_duplicates_and_uuid_custom_field(cloud): + cloud.settings.update( + strip_host_domain_name=True, strip_vm_domain_name=True, + vm_exclude_filter="ignore.*", cluster_exclude_filter="empty", + vm_role_relation=".* = Application", + ) + cloud.session.compute.servers.return_value = [ + server(status="SHUTOFF", id="inactive"), server(), + server(name="ignore.example", id="ignored"), + ] + handler = OpenStackHandler("cloud") + handler.apply() + vms = cloud.inventory.get_all_items(NBVM) + assert len(vms) == 1 + assert vms[0].data["name"] == "app" + assert vms[0].data["status"] == "active" + assert vms[0].data["role"].data["name"] == "Application" + assert cloud.inventory.get_all_items(NBDevice)[0].data["name"] == "compute1" + assert len(cloud.inventory.get_all_items(NBCluster)) == 1 + cloud.settings["set_vm_name_to_uuid"] = True + cloud.session.compute.servers.return_value = [server()] + handler = OpenStackHandler("cloud") + handler.apply() + vm = cloud.inventory.get_all_items(NBVM)[0] + assert vm.data["name"] == "vm-uuid" + assert vm.data["custom_fields"]["openstack_vm_name"] == "app.example.org" + field = cloud.inventory.get_all_items(NBCustomField)[0] + assert field.data["object_types"] == ["virtualization.virtualmachine"] + handler.apply() + assert field.data["object_types"] == ["virtualization.virtualmachine"] + + +def test_duplicate_vm_names_keep_the_existing_server_identity(cloud): + cloud.session.compute.servers.return_value = [ + server(id="server-a", status="SHUTOFF"), + server(id="server-b", status="ACTIVE"), + ] + handler = OpenStackHandler("cloud") + handler.apply() + vm = cloud.inventory.get_all_items(NBVM)[0] + assert vm.data["comments"] == "server-b" + + cloud.session.compute.servers.return_value = [ + server(id="server-a", status="ACTIVE"), + server(id="server-b", status="SHUTOFF"), + ] + handler.apply() + assert vm.data["comments"] == "server-b" + assert vm.data["status"] == "offline" + assert len(cloud.inventory.get_all_items(NBVM)) == 1 + + +def test_duplicate_zone_hosts_are_assigned_to_the_nova_compute_zone(cloud): + cloud.session.compute.availability_zones.return_value = [ + resource(name="internal", hosts={ + "compute1": {"nova-scheduler": {}, "nova-conductor": {}}, + }), + resource(name="nova", hosts={"compute1": {"nova-compute": {}}}), + ] + cloud.session.compute.servers.return_value = [server(availability_zone="nova")] + + handler = OpenStackHandler("cloud") + handler.apply() + + host = cloud.inventory.get_all_items(NBDevice)[0] + vm = cloud.inventory.get_all_items(NBVM)[0] + assert host.data["cluster"].data["name"] == "nova" + assert vm.data["cluster"].data["name"] == "nova" + assert vm.data["device"] is host + + +def test_interface_names_and_host_interface_type_are_preserved(cloud): + addresses = { + "private": [{ + "addr": "10.0.0.8", "version": 4, + "OS-EXT-IPS-MAC:mac_addr": "fa:16:3e:00:00:01", + }], + "secondary": [{ + "addr": "10.0.0.9", "version": 4, + "OS-EXT-IPS-MAC:mac_addr": "fa:16:3e:00:00:02", + }], + } + cloud.session.compute.servers.return_value = [server(addresses=addresses)] + handler = OpenStackHandler("cloud") + handler.apply() + vm_nic = next(nic for nic in cloud.inventory.get_all_items(NBVMInterface) + if nic.data["name"] == "vNIC2 (secondary)") + host_interface = cloud.inventory.get_all_items(NBInterface)[0] + host_interface.data["type"] = "1000base-t" + + cloud.session.compute.servers.return_value = [ + server(addresses={"secondary": addresses["secondary"]}), + ] + handler.apply() + + assert vm_nic.data["name"] == "vNIC2 (secondary)" + assert host_interface.data["type"] == "1000base-t" + + +def test_network_ports_invalid_addresses_and_floating_primary_preference(cloud): + cloud.settings["permitted_subnets"] += ", 192.0.2.0/24" + vm_resource = server() + vm_resource.addresses["private"].extend([ + {"addr": "invalid", "version": 4}, + {"addr": "10.0.0.9", "version": 4, "OS-EXT-IPS-MAC:mac_addr": "fa:16:3e:00:00:02"}, + ]) + vm_resource.addresses["without-mac"] = [{"addr": "10.0.1.9", "version": 4}] + cloud.session.compute.servers.return_value = [vm_resource] + OpenStackHandler("cloud").apply() + vm = cloud.inventory.get_all_items(NBVM)[0] + nics = cloud.inventory.get_all_items(NBVMInterface) + assert len(nics) == 3 + assert vm.data["primary_ip4"].data["address"] == "10.0.0.8/32" + assert {ip.data["address"] for ip in nics[0].get_ip_addresses()} == { + "10.0.0.8/32", "fd00::8/128", "192.0.2.8/32", + } + + +def test_primary_ip_uses_first_fixed_address(cloud): + cloud.settings["permitted_subnets"] += ", 192.0.2.0/24" + cloud.session.compute.servers.return_value = [server(addresses={ + "private": [ + {"addr": "192.0.2.8", "version": 4, "OS-EXT-IPS:type": "floating"}, + {"addr": "10.0.0.8", "version": 4, "OS-EXT-IPS:type": "fixed"}, + {"addr": "10.0.0.9", "version": 4, "OS-EXT-IPS:type": "fixed"}, + ], + })] + + OpenStackHandler("cloud").apply() + + vm = cloud.inventory.get_all_items(NBVM)[0] + assert vm.data["primary_ip4"].data["address"] == "10.0.0.8/32" + + +def test_flavor_lookup_unknown_volume_and_ipv6_hypervisor(cloud): + cloud.session.compute.servers.return_value = [server(flavor={"id": "flavor-1"}, + attached_volumes=[{"id": "missing"}])] + cloud.session.compute.get_flavor.return_value = resource(name="large", vcpus=4, ram=4096, disk=20) + cloud.session.compute.hypervisors.return_value[0].host_ip = "fd00::2" + handler = OpenStackHandler("cloud") + handler.apply() + vm = cloud.inventory.get_all_items(NBVM)[0] + assert vm.data["memory"] == 4096 + assert vm.data["vcpus"] == 4 + assert "disk" not in vm.data + host = cloud.inventory.get_all_items(NBDevice)[0] + assert "primary_ip4" not in host.data + assert host.data["primary_ip6"].data["address"] == "fd00::2/128" + cloud.session.compute.get_flavor.assert_called_once_with("flavor-1") + + +@pytest.mark.parametrize("status, level, message", [ + ("disabled", "INFO", "is disabled in Nova"), + ("enabled", "ERROR", "Unable to find an availability zone"), +]) +def test_host_without_availability_zone_is_skipped(cloud, caplog, status, level, message): + cloud.session.compute.hypervisors.return_value.append( + resource(name="compute2.example", service_details={"host": "compute2"}, status=status, + state="up", host_ip="10.0.0.3", hypervisor_type="QEMU", hypervisor_version=8000000)) + handler = OpenStackHandler("cloud") + with caplog.at_level("INFO"): + handler.apply() + names = [device.data["name"] for device in cloud.inventory.get_all_items(NBDevice)] + assert "compute2.example" not in names + records = [r for r in caplog.records if "compute2.example" in r.getMessage() and message in r.getMessage()] + assert [r.levelname for r in records] == [level] + + +def test_cluster_matching_does_not_reassign_another_cloud(cloud): + other_group = cloud.inventory.add_object(NBClusterGroup, data={"name": "Other"}) + other_site = cloud.inventory.add_object(NBSite, data={"name": "Other Site"}) + foreign = cloud.inventory.add_object(NBCluster, data={ + "name": "zone.example", "group": other_group, "type": {"name": "Openstack"}, + "scope_type": NBSite.object_type, "scope_id": other_site, + }) + handler = OpenStackHandler("cloud") + handler.apply() + assert foreign.data["group"] is other_group + assert foreign.get_site_name() == "Other Site" + assert handler.get_object_from_cache(NBCluster, "zone.example") is not foreign + + +def test_clusters_with_same_default_group_are_isolated_by_source_tags(cloud): + other_source = SimpleNamespace(source_tag="Source: other") + cloud.inventory.source_list = [other_source] + other_site = cloud.inventory.add_object(NBSite, data={"name": "Other Site"}) + foreign = cloud.inventory.add_object(NBCluster, data={ + "name": "zone.example", "group": {"name": "Openstack"}, "type": {"name": "Openstack"}, + "scope_type": NBSite.object_type, "scope_id": other_site, "tags": ["Source: other"], + }) + handler = OpenStackHandler("cloud") + handler.apply() + assert foreign.get_site_name() == "Other Site" + assert handler.get_object_from_cache(NBCluster, "zone.example") is not foreign + + +def test_missing_volume_retains_existing_virtual_disk_size(cloud): + handler = OpenStackHandler("cloud") + handler.apply() + vm = cloud.inventory.get_all_items(NBVM)[0] + disk = next(disk for disk in vm.get_virtual_disks() if disk.data["name"] == "volume-1") + cloud.session.block_storage.volumes.return_value = [] + handler.apply() + assert disk.data["size"] == 20000 + assert disk.source is handler + assert len(vm.get_virtual_disks()) == 2 + + +@pytest.mark.parametrize("setting,value", [ + ("vm_include_filter", "["), ("vm_tenant_relation", "missing separator"), + ("set_primary_ip", "sometimes"), ("ip_tenant_inheritance_order", "unknown"), +]) +def test_invalid_settings_fail_cleanly(cloud, setting, value): + cloud.settings[setting] = value + with pytest.raises(SystemExit): + OpenStackHandler("cloud") + + +def test_mac_and_primary_ip_matching_use_current_netbox_objects(cloud): + handler = OpenStackHandler("cloud") + handler.apply() + vm = cloud.inventory.get_all_items(NBVM)[0] + assert handler.get_object_based_on_macs(NBVM, ["FA:16:3E:00:00:01"]) is vm + assert handler.get_object_based_on_primary_ip(NBVM, primary_ip4="10.0.0.8") is vm + assert handler.get_object_from_cache(NBVM, "zone.example") is None + + +def test_interface_exclusions_and_resource_custom_attributes(cloud): + cloud.settings.update( + vm_interface_exclude_filter="vNIC1.*", host_interface_exclude_filter="eth0", + vm_custom_object_attributes="id, flavor.vcpus", host_custom_object_attributes="name", + ) + OpenStackHandler("cloud").apply() + assert not cloud.inventory.get_all_items(NBVMInterface) + assert not cloud.inventory.get_all_items(NBIPAddress) + vm = cloud.inventory.get_all_items(NBVM)[0] + assert vm.data["custom_fields"]["openstack_id"] == "vm-uuid" + assert vm.data["custom_fields"]["openstack_flavor_vcpus"] == "2" + host = cloud.inventory.get_all_items(NBDevice)[0] + assert host.data["custom_fields"]["openstack_name"] == "compute1.example" + + +def test_source_neutral_vm_ip_updates_remove_undiscovered_addresses(cloud): + handler = OpenStackHandler("cloud") + handler.apply() + vm = cloud.inventory.get_all_items(NBVM)[0] + old_ip = vm.data["primary_ip4"] + cloud.session.compute.servers.return_value = [server(addresses={ + "private": [{"addr": "10.0.0.9", "version": 4, + "OS-EXT-IPS-MAC:mac_addr": "fa:16:3e:00:00:01"}], + })] + handler.settings.set_primary_ip = "always" + handler.apply() + assert vm.data["primary_ip4"].data["address"] == "10.0.0.9/32" + assert "assigned_object_id" in old_ip.unset_items + assert len(cloud.inventory.get_all_items(NBVMInterface)) == 1 + + +@pytest.mark.parametrize("file_type", [ConfigFileINI, ConfigFileYAML]) +def test_config_generation_does_not_require_sdk(monkeypatch, file_type): + monkeypatch.setattr("module.sources.openstack.connection.import_module", + Mock(side_effect=AssertionError("config generation imported SDK"))) + output = ConfigFileOutput.__new__(ConfigFileOutput) + output.config_file_type = file_type + output.comment_prefix = file_type.comment_prefix + output.lines = [] + output.format() + text = "\n".join(output.lines) + assert "my-openstack-example" in text + assert "openstack" in text + assert "all_projects" in text + assert "ip_tenant_inheritance_order" in text From db42607a9ff8ff9e3972a0bebaf1752dea0fad85 Mon Sep 17 00:00:00 2001 From: Eric Krichbaum Date: Wed, 11 Nov 2026 16:05:15 -0600 Subject: [PATCH 06/21] Add OpenStackConfig to source configuration list --- module/config/file_output.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/module/config/file_output.py b/module/config/file_output.py index 4210d86b..389825df 100644 --- a/module/config/file_output.py +++ b/module/config/file_output.py @@ -17,6 +17,7 @@ from module.sources.vmware.config import VMWareConfig from module.sources.check_redfish.config import CheckRedfishConfig from module.sources.hetzner.config import HetznerConfig +from module.sources.openstack.config import OpenStackConfig from module.sources.ovirt.config import OVirtConfig from module.sources.aws.config import AWSConfig from module.sources.digitalocean.config import DigitalOceanConfig @@ -39,6 +40,7 @@ class ConfigFileOutput(DescriptionFormatterMixin): VMWareConfig, CheckRedfishConfig, HetznerConfig, + OpenStackConfig, AWSConfig, DigitalOceanConfig, OVirtConfig From cac5182e90181b4eadb5f13c90323e2a9839edea Mon Sep 17 00:00:00 2001 From: Eric Krichbaum Date: Wed, 11 Nov 2026 16:08:01 -0600 Subject: [PATCH 07/21] Add OpenStack SDK requirements to requirements file --- requirements-openstack.txt | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 requirements-openstack.txt diff --git a/requirements-openstack.txt b/requirements-openstack.txt new file mode 100644 index 00000000..99abdacb --- /dev/null +++ b/requirements-openstack.txt @@ -0,0 +1,2 @@ +-r requirements.txt +openstacksdk>=4.0,<5 From 8401c35191ca05b673921ff9d1f7f0f2c4431e84 Mon Sep 17 00:00:00 2001 From: Eric Krichbaum Date: Wed, 11 Nov 2026 16:09:52 -0600 Subject: [PATCH 08/21] Add openstack optional dependency to pyproject.toml --- pyproject.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 6c169f7e..c8d6c1a1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,6 +33,9 @@ dependencies = [ aws = [ "boto3>=1.40,<2", ] +openstack = [ + "openstacksdk>=4.0,<5", +] ovirt = [ "ovirt-engine-sdk-python==4.6.3", ] From 083d75785fae325419f19c5f3a4728d19a8ef408 Mon Sep 17 00:00:00 2001 From: Eric Krichbaum Date: Wed, 11 Nov 2026 16:12:13 -0600 Subject: [PATCH 09/21] Add OpenStack source configuration to settings-example.ini Added OpenStack source configuration with various settings. --- settings-example.ini | 192 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 192 insertions(+) diff --git a/settings-example.ini b/settings-example.ini index f85e1a78..2732e8c8 100644 --- a/settings-example.ini +++ b/settings-example.ini @@ -828,4 +828,196 @@ password = ;host_custom_object_attributes = id ;vm_custom_object_attributes = id +[source/my-openstack-example] + +; Defines if this source is enabled or not +;enabled = True + +; type of source. This defines which source handler to use +type = openstack + +; OpenStack Identity v3 authentication URL +auth_url = https://api.openstack-instance.local:5000/v3 + +; Name of the OpenStack project +project = MyProject + +; username to use to log into OpenStack +username = openstackuser + +; password to use to log into OpenStack +password = supersecret + +; Which OpenStack region to use +region = RegionOne + +; Which OpenStack user domain to use +user_domain = Default + +; Which OpenStack project domain to use +project_domain = Default + +; OpenStack group name +;group_name = Openstack + +; Enforces TLS certificate validation. Disable only for an OpenStack endpoint with a self- +; signed certificate. +;validate_tls_certs = True + +; Discover servers and volumes across projects (requires administrative access). +;all_projects = True + +; IP networks eligible to be synced to NetBox. If an IP address is not part of this +; networks then it WON'T be synced to NetBox. To excluded small blocks from bigger IP +; blocks a leading '!' has to be added +;permitted_subnets = 172.16.0.0/12, 10.0.0.0/8, 192.168.0.0/16, fd00::/8, !10.23.42.0/24 + +; define in which order the IP address tenant will be assigned if tenant is undefined. +; possible values: +; * device : host or VM tenant will be assigned to the IP address +; * prefix : if the IP address belongs to an existing prefix and this prefix has a tenant assigned, then this one is used +; * disabled : no tenant assignment to the IP address will be performed +; the order of the definition is important, the default is "device, prefix" which means: +; If the device has a tenant then this one will be used. If not, the prefix tenant will be used if defined +;ip_tenant_inheritance_order = device, prefix + +; Keep existing primary IP assignments when an address is no longer discovered. +;preserve_primary_ips = False + +; Do not assign addresses belonging to an existing FHRP group. +;skip_fhrp_group_ips = True + +; Regular expression matching host interface names to ignore. +;host_interface_exclude_filter = + +; Regular expression matching VM interface names to ignore. +;vm_interface_exclude_filter = + +; Do not synchronize VLANs discovered through existing prefixes. +;disable_vlan_sync = False + +; filter options + +; filters can be used to include/exclude certain objects from importing into NetBox. +; Include filters are checked first and exclude filters after. An object name has to pass +; both filters to be synced to NetBox. If a filter is unset it will be ignored. Filters +; are all treated as regex expressions! If more then one expression should match, a '|' +; needs to be used +; +; Example: (exclude all VMs with "replica" in their name and all VMs starting with +; "backup"): vm_exclude_filter = .*replica.*|^backup.* + +; If a cluster is excluded from sync then ALL VMs and HOSTS inside the availability zone +; will be ignored! +;cluster_exclude_filter = +;cluster_include_filter = + +; This will only include/exclude the host, not the VM if Host is part of a multi host +; cluster +;host_exclude_filter = +;host_include_filter = + +; simply include/exclude VMs +;vm_exclude_filter = +;vm_include_filter = + +; relations options + +; This option defines which OpenStack availability zone is part of a NetBox site. +; This is done with a comma separated key = value list. +; key: defines the cluster name as regex +; value: defines the NetBox site name (use quotes if name contains commas) +; This is a quite important config setting as IP addresses, prefixes, VLANs +; and VRFs are site dependent. In order to assign the correct prefix to an IP +; address it is important to pick the correct site. +; A VM always depends on the cluster site relation +;cluster_site_relation = nova = Cloud Site + +; Same as cluster site but on host level. If unset it will fall back to +; cluster_site_relation +;host_site_relation = nyc02.* = New York, ffm01.* = Frankfurt + +; This option defines which cluster/host/VM belongs to which tenant. +; This is done with a comma separated key = value list. +; key: defines a hosts/VM name as regex +; value: defines the NetBox tenant name (use quotes if name contains commas) +; a cluster can be specified as "Cluster-name" or +; "availability-zone-name" +;cluster_tenant_relation = Cluster_NYC.* = Customer A +;host_tenant_relation = compute.* = Infrastructure +;vm_tenant_relation = grafana.* = Infrastructure + +; This option defines custom platforms if the VMWare created platforms are not suitable. +; Pretty much a mapping of VMWare platform name to your own platform name. +; This is done with a comma separated key = value list. +; key: defines a VMWare returned platform name +; value: defines the desired NetBox platform name +;vm_platform_relation = centos-7.* = centos7, microsoft-windows-server-2016.* = Windows2016 + +; Define the NetBox device role used for hosts. The default is +; set to "Server". This is done with a comma separated key = value list. +; key: defines host(s) name as regex +; value: defines the NetBox role name (use quotes if name contains commas) +;host_role_relation = .* = Server + +; Define the NetBox device role used for VMs. This is done with a +; comma separated key = value list, same as 'host_role_relation'. +; key: defines VM(s) name as regex +; value: defines the NetBox role name (use quotes if name contains commas) +;vm_role_relation = .* = Server + +; Define NetBox tags which are assigned to a cluster, host or VM. This is +; done with a comma separated key = value list. +; key: defines a hosts/VM name as regex +; value: defines the NetBox tag (use quotes if name contains commas) +; Match the availability zone name to assign cluster tags. +;cluster_tag_relation = nova = Infrastructure +;host_tag_relation = compute.* = Infrastructure +;vm_tag_relation = grafana.* = Infrastructure + +; Perform a reverse lookup for all collected IP addresses. If a dns name was found it will +; be added to the IP address object in NetBox +;dns_name_lookup = True + +; use custom DNS server to do the reverse lookups +;custom_dns_servers = 192.168.1.11, 192.168.1.12 + +; define how the primary IPs should be set +; possible values: +; +; always: will remove primary IP from the object where this address is +; currently set as primary and moves it to new object +; +; when-undefined: +; only sets primary IP if undefined, will cause ERRORs if same IP is +; assigned more then once to different hosts and IP is set as the +; objects primary IP +; +; never: don't set any primary IPs, will cause the same ERRORs +; as "when-undefined" +;set_primary_ip = when-undefined + +; do not set notes to the UUID or name of the VM +;skip_vm_comments = False + +; Do not use the OpenStack flavor name as the NetBox VM platform +;skip_vm_platform = False + +; strip domain part from host name before syncing device to NetBox +;strip_host_domain_name = False + +; strip domain part from VM name before syncing VM to NetBox +;strip_vm_domain_name = False + +; custom object attributes options + +; add arbitrary host/vm object attributes as custom fields to NetBox. +; multiple attributes can be defined comma separated. +; use OpenStack SDK resource attribute names; dotted paths are supported. +;host_custom_object_attributes = id +;vm_custom_object_attributes = id, project_id + +; Set the name in Netbox to the VM UUID instead of name +;set_vm_name_to_uuid = False + ;EOF From 9b08abd1b70b3b70780ffae255f968e18de1de45 Mon Sep 17 00:00:00 2001 From: Eric Krichbaum Date: Wed, 11 Nov 2026 16:16:08 -0600 Subject: [PATCH 10/21] Add OpenStack source type to README Added OpenStack as a new source type with installation details. --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index c403d1f6..3e2c882a 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,7 @@ Available source types: * VMware vCenter Server * AWS EC2, EBS, and network interfaces, with opt-in S3, Lambda, and EKS inventory (see [AWS source](docs/source_aws.md)) * oVirt / Red Hat Virtualization (see [oVirt source](docs/source_ovirt.md)) +* OpenStack (optional `openstacksdk` dependency; see [OpenStack source](docs/source_openstack.md)) * [bb-ricardo/check_redfish](https://github.com/bb-Ricardo/check_redfish) inventory files * [DigitalOcean](docs/source_digitalocean.md) @@ -48,6 +49,9 @@ This ensures stale objects are removed from NetBox keeping an accurate current s #### Source: oVirt (if used) * oVirt / Red Hat Virtualization Engine v4 API * Optional SDK and native libraries: see [installation and configuration](docs/source_ovirt.md) +#### Source: OpenStack (if used) +* Identity v3, Nova and Cinder APIs; `openstacksdk>=4.0,<5` +* See [OpenStack installation and configuration](docs/source_openstack.md) for credentials, permissions, and inventory mapping # Installing * here we assume we install in ```/opt``` From 6b131ee4e62889a5c664c1cb025428c18bfbfa37 Mon Sep 17 00:00:00 2001 From: Eric Krichbaum Date: Wed, 11 Nov 2026 16:17:15 -0600 Subject: [PATCH 11/21] Revise source_openstack.md for OpenStack integration Updated documentation to reflect OpenStack source configuration and usage, replacing VMware references with OpenStack-specific details. --- docs/source_openstack.md | 409 +++++++++++++++++++-------------------- 1 file changed, 202 insertions(+), 207 deletions(-) diff --git a/docs/source_openstack.md b/docs/source_openstack.md index a8334f44..3617eb1d 100644 --- a/docs/source_openstack.md +++ b/docs/source_openstack.md @@ -1,234 +1,229 @@ -# Source: vmware +# OpenStack source -## Setup -You need to have a source section in your `settings.ini` file with following type: -```ini -type = vmware -``` -All options for this source are described in the [settings-example.ini](../settings-example.ini) file. - -If you have multiple vCenter instances just add another source with the same type in the **same** file. - -***IMPORTANT:*** For VMware source you should define the var `cluster_site_relation` which maps a vCenter cluster to an -exiting Site in NetBox. If undefined a placeholder site will be created. - -### vCenter user -You need a user account with "Read-only" role on vCenter root scope. -The "Propagate to children" setting must also be checked. - -## Adding objects from a vCenter to NetBox - -To match new/updated objects to existing objects in NetBox a few different steps are taken to find the correct object. - -You might be interested in this [description](common_concepts.md). This describes how discovered -IP addresses and interfaces will be added to NetBox. - -### Add/update device/VM object in inventory based on gathered data - -Try to find object first based on the object data, interface MAC addresses and primary IPs. -1. try to find by name and cluster/site -2. try to find by mac addresses interfaces -3. try to find by serial number (1st) or asset tag (2nd) -4. try to find by primary IP - -### Finding hosts and VMs from discovered data - -#### 1. try to find exact name match including cluster/site - -If this NetBox object (device, vm) matches, the found object will be used. - -#### 2. Try to find a NetBox object based on list of MAC addresses - -Iterate over all NetBox interfaces of an object (device, vm) and compare MAC address with list -of MAC addresses discovered for this object. If a match was found, count for every object (device, vm) -how many MAC addresses are matching. - -If exactly one NetBox object (device, vm) with matching interface MACs was found then this one will be used. - -If two or more NetBox object (device, vm) with matching MACs were found, compare the two -NetBox object (device, vm) with the highest amount of matching interfaces. If the ratio of matching interface -MAC addresses exceeds 2.0 then the top matching NetBox object (device, vm) is chosen as desired object. - -If the ratio is below 2.0 then None will be chosen. The probability is too low that -this one is the correct one. - -#### 3. Try to find a NetBox object based on the primary IP (v4 or v6) address - -If an exact matching NetBox object (device, vm) was found the object will be used -immediately without checking the other primary IP address (if defined). +The `openstack` source reads availability zones, compute hypervisors, servers, +flavors, and Cinder volumes, then reconciles their inventory into NetBox. +OpenStack is queried read-only; inventory writes are made to NetBox. +All identifiers and credentials below are fictitious placeholders. -### Objects read and parsed from vCenter +## Installation -#### 1. Add a vCenter datacenter as a Cluster Group to NetBox -Simple, nothing special going on here +The SDK is optional; other sources do not require it: -#### 2. Add a vCenter cluster as a Cluster to NetBox. - -Cluster name is checked against `cluster_include_filter` and `cluster_exclude_filter config` setting. - -#### 3. Parse distributed virtual port group -This is done to extract name and VLAN IDs from each port group - -#### 4. Parse a vCenter (ESXi) host - -First host is filtered: -* host has a cluster and this cluster is permitted -* skip host with same name and site, we already parsed it (use unique host names) -* does the host pass the `host_include_filter` and `host_exclude_filter` - -Then all necessary host data will be collected:
-host model, manufacturer, serial, physical interfaces, virtual interfaces, -virtual switches, proxy switches, host port groups, interface VLANs, IP addresses - -Primary IPv4/6 will be determined by -1. if the interface port group name contains "management" or "mngt" -2. interface with this IP is the default route of this host - -#### 5. Parse a vCenter VM - -Iterate over all VMs twice! - -To handle VMs with the same name in a cluster we first iterate over all VMs and look only at the -active (online) ones and parse these first. Then we iterate a second time to catch the rest (also offline). - -This has been implemented to support migration scenarios where you create/copy the same -machine with a different setup like a new version or something. This way NetBox will be -updated primarily with the actual active VM data. - -First VM is filtered: -* VM has a cluster and is it permitted -* skip VMs with same name and cluster if already parsed -* does the VM pass the `vm_include_filter` and `vm_exclude_filter` -* does the VM pass the `vm_include_by_resource_pool_filter` and `vm_exclude_by_resource_pool_filter` - -Then all necessary VM data will be collected:
-platform, virtual interfaces, virtual cpu/disk/memory interface VLANs, IP addresses - -Primary IPv4/6 will be determined by interface that provides the default route for this VM. - -The resource pool filters are applied to the resource pool path (i.e. `Parent/Child`, the -cluster root pool `Resources` excluded) and to the plain resource pool name of the VM: the VM -is kept when either matches the include filter and skipped when either matches the exclude -filter. A VM placed directly in the cluster root pool has an empty pool path and is matched by -its plain pool name `Resources` only. A VM without a resource pool (a template) passes both -filters. +```sh +pip install -r requirements-openstack.txt +# Alternatively: +uv sync --extra openstack +``` -**Note:**
-IP address information can only be extracted if guest tools are installed and running. +The supported SDK range is `openstacksdk>=4.0,<5`. Selecting an OpenStack source +without the SDK logs an error and skips that source. -#### 5b. Resolve the tenant of a VM +## Authentication and permissions -The tenant of a VM is resolved from different sources. The first match wins: +The current source uses Keystone Identity v3 username/password authentication +with an explicit project, region, user domain, and project domain. It passes +these source settings directly to `openstacksdk`; it does not expose +`clouds.yaml` cloud selection or application-credential authentication. +SDK YAML configuration and `OS_*` environment loading are disabled, so +`OS_CLOUD`, `OS_INTERFACE`, and other OpenStack client settings cannot override +the configured source. Standard `NBS_SOURCE_*` configuration remains supported. -1. `vm_tenant_relation` matched against the VM name -2. `vm_tenant_resource_pool_relation` matched against the resource pool path and then - against the plain resource pool name of the VM -3. `vm_tenant_folder_relation` matched against the VM folder path (relative to the - datacenter's VM folder) and then against the plain folder name of the VM +The account must be allowed to read: - An empty path (a VM in the cluster root pool, or directly in the datacenter's VM folder) - is not matched; such a VM is matched by its plain pool name `Resources` or folder name - `vm` only. -4. if `vm_tenant_inherit_from_cluster` is enabled: the tenant resolved via - `cluster_tenant_relation` for this cluster or, if that relation matched nothing, the - tenant assigned to the cluster in NetBox +* Detailed Nova availability zones, including their host information. +* Detailed Nova hypervisors and servers. +* Nova flavors when server responses do not include their full specifications. +* Detailed Cinder volumes, including sizes for attached volumes. -A tenant already set on an existing NetBox VM is kept if `overwrite_vm_tenant` is disabled. -If a tenant was resolved for the VM, its IP addresses inherit it through the `device` step -of `ip_tenant_inheritance_order`. +`all_projects = True` is the default for server and volume discovery and +requires cross-project access, normally administrative or suitably delegated +read permissions. Exact roles depend on the cloud's Nova and Cinder policy. +`all_projects = False` requests servers and volumes within the authenticated +project, but does not remove the availability-zone or hypervisor queries; +project-only credentials may still lack the permissions needed by this source. -#### 6. Sync VMware Tools guest hostname to a custom field +Use TLS certificate validation (the default). Configure the runtime's trusted +CA certificates for private endpoints rather than disabling verification. +The sync host needs outbound access to Keystone and the Nova/Cinder endpoints +advertised in the service catalog. No guest SSH access or guest agent is required +for addresses returned by Nova. -The vCenter VM name (used as the NetBox VM `name`) and the hostname reported from inside the -guest OS by VMware Tools are not always identical. This can happen if: -* a VM was renamed in vCenter but the OS hostname was not changed (or vice versa) -* an OS administrator changed the hostname without informing the virtualization team -* a VM was cloned and the guest hostname was not adjusted afterwards +## Configuration -To make this drift visible in NetBox, the option `vm_guest_hostname_custom_field` can be set to -the name of a NetBox custom field. If set, the guest hostname reported by VMware Tools (vCenter -property `guest.hostName`) is synced into that custom field on every synced VM, while the NetBox -VM `name` keeps reflecting the vCenter inventory name unchanged. This allows administrators to -compare the NetBox VM name against the custom field value on the same VM record to spot naming -drift. +Add this section to the application's existing NetBox settings: ```ini -vm_guest_hostname_custom_field = vmware_guest_hostname +[source/example-openstack] +type = openstack +auth_url = https://identity.example.org:5000/v3 +project = inventory +username = inventory-reader +password = REPLACE_WITH_PASSWORD +region = RegionOne +user_domain = Default +project_domain = Default +validate_tls_certs = True +all_projects = True +group_name = Example OpenStack +cluster_site_relation = .* = Example Cloud Site +permitted_subnets = 10.0.0.0/8, fd00::/8 +dns_name_lookup = False ``` -If the custom field does not exist in NetBox yet it will be created automatically (type `Text`, -assigned to the `Virtual Machine` object type), the same way other custom fields managed by -netbox-sync (e.g. `vcsa_*` custom attributes) are created. An administrator can also pre-create -the field beforehand; netbox-sync will then just use the existing field. +All seven authentication/scope fields (`auth_url`, `project`, `username`, +`password`, `region`, `user_domain`, and `project_domain`) are mandatory. +`group_name` defaults to `Openstack`. Use separate named sources for other +regions or clouds; there is no automatic region enumeration. + +Keep real passwords out of version control and restrict access to configuration +files. The application also supports its standard source environment settings; +for example, a source defined entirely through environment variables can use: + +```sh +NBS_SOURCE_1_NAME=example-openstack +NBS_SOURCE_1_TYPE=openstack +NBS_SOURCE_1_AUTH_URL=https://identity.example.org:5000/v3 +NBS_SOURCE_1_PROJECT=inventory +NBS_SOURCE_1_USERNAME=inventory-reader +NBS_SOURCE_1_PASSWORD=REPLACE_WITH_PASSWORD +NBS_SOURCE_1_REGION=RegionOne +NBS_SOURCE_1_USER_DOMAIN=Default +NBS_SOURCE_1_PROJECT_DOMAIN=Default +``` -This option is unset by default, so the feature is disabled and no additional custom field is -created unless explicitly configured. +These are example variable assignments; supply secrets through your runtime's +secret mechanism rather than storing this block in a repository. +Generate the complete supported configuration with: -VMware Tools may not always report a hostname, for example if Tools are not installed, not -running, outdated, or simply have not reported guest information yet. In that case -netbox-sync leaves the custom field untouched and keeps the last known value, instead of -clearing it. A message is logged at debug level (`-l DEBUG2`) whenever this happens. +```sh +python netbox-sync.py -g +``` -Example result in NetBox for a VM named `APPPRD01` in vCenter whose guest OS reports -`appprd01.corp.example.com` as its hostname: +### Site, network, and tenant relations + +`cluster_site_relation` maps availability-zone names to NetBox sites using +regular expressions. `host_site_relation` can override sites for hypervisors. +Correct site mapping matters for prefix, VLAN, and VRF selection. + +Only addresses permitted by `permitted_subnets` are imported. Pre-create the +NetBox prefixes, VRFs, and VLANs appropriate to your cloud's networks. Address +prefix matching prefers the mapped site and can fall back to other matching +prefixes; without a matching prefix, a bare IPv4/IPv6 address retains its +host-length mask. Carefully inspect overlapping tenant address spaces before +permitting IP writes. + +Interfaces and addresses come from Nova server address data, not a separate +Neutron topology inventory. Fixed and floating addresses may both be imported +when permitted; fixed addresses are preferred for primary IP selection. +Within a resolved VM, existing interfaces with a discovered MAC keep their +names before positional interface matching is attempted. Detaching an earlier +port therefore does not renumber or steal the later port's interface, MAC, +or IPs; the detached interface is left for normal orphan handling. +An interface paired without a MAC match is renamed to the discovered network +name. Ports without MAC data still rely on positional/shared matching. +Virtual disks are matched by disk name within the resolved VM, so same-named +VMs in different availability zones retain separate disks. +For `all_projects = True`, the source can repair an IP still assigned to an +active but deleted VM after fully consuming the Nova server list. Normally both +the IP and old VM must have only this source's `Source:` tag; orphaned records +can qualify through the cluster-ownership recovery described below. The old VM's stored +server UUID (UUID name, or UUID comments with default naming) must be absent +from the complete snapshot, the target UUID must be present, and exactly one +server must report the IP. Existing VRF matching and shared-address-role +safeguards still apply. Repair is logged and does not disable the old +interface or claim the old VM; normal orphan handling remains responsible for +those objects. Unknown UUIDs, unverified/mixed ownership, incomplete server +discovery, duplicate IP discovery, and project-only discovery do not authorize +this repair. Other source types retain their existing reassignment behavior. +Because normal orphan handling removes Source tags, an IP or old VM with no +Source tags may instead qualify when it has the orphan tag and its old +cluster has exclusively this source's persisted tag, belongs to the configured +group, and is resolved and owned by this source in current discovery. +This does not permit recovery from unknown or competing clusters, non-orphaned +untagged records, or records with another source's tag. All UUID, complete +snapshot, unique-IP-claim, and VRF checks still apply. Successful repair restores +the IP's Source tag and removes its orphan tag through normal tagging, without +retagging or reviving the old VM. +Blocked stale-IP repair logs include the first failing prerequisite and relevant +NetBox IDs, ownership tags, or UUID state. A following shared-interface warning +confirms that the assignment was left unchanged. +`set_primary_ip`, `preserve_primary_ips`, `skip_fhrp_group_ips`, +`ip_tenant_inheritance_order`, interface exclusion filters, and +`disable_vlan_sync` control the shared network reconciliation behavior. + +Tenants and tags are configured through relations rather than automatically +mirroring Keystone project names: -```text -Virtual Machine Name: APPPRD01 -Custom Fields: - VMware Guest Hostname: appprd01.corp.example.com +```ini +cluster_site_relation = ^zone-a$ = Example Cloud Site +vm_tenant_relation = ^production-.* = Example Tenant +vm_tag_relation = .* = OpenStack ``` -### Cables to CDP/LLDP neighbors +Host, VM, and cluster site/tenant/role/tag options and platform mappings are +described in the generated configuration and [example settings](../settings-example.ini). + +## Object mapping + +| OpenStack | NetBox | +| --- | --- | +| Configured group name | Cluster group | +| Availability zone | Cluster | +| Hypervisor | Device and reported host interface/IP | +| Server | Virtual machine | +| Flavor vCPUs / RAM | VM CPU / memory | +| Flavor name | VM platform unless disabled or mapped | +| Image-backed root / ephemeral disks | VM disk capacity or virtual disks | +| Attached Cinder volume | VM disk capacity or virtual disk named by volume ID | +| Server network/MAC/address data | VM interfaces, MACs, and permitted IP addresses | + +Servers with status `ACTIVE` are active in NetBox; other states map to offline. +By default VM names are used, with duplicate names within an availability zone +logged and skipped. Consider `set_vm_name_to_uuid = True` for cross-project +inventories where names repeat; the original name is then stored in the +`openstack_vm_name` custom field. Changing naming settings can affect matching, +so review a dry run before applying them to existing inventory. + +`skip_vm_platform` disables flavor-to-platform mapping. `skip_vm_comments` +disables VM UUID comments. `strip_host_domain_name` and +`strip_vm_domain_name` control domain stripping. +`host_custom_object_attributes` and `vm_custom_object_attributes` copy selected +SDK resource attributes into custom fields; dotted paths are supported. + +NetBox 3.7+ uses separate virtual disks; older versions retain aggregate VM +disk capacity. OpenStack reports flavor and volume sizes in gigabytes. NetBox +versions before 4.1 store the reported value as gigabytes; from 4.1, the source +converts it to megabytes by multiplying by 1000. Missing attached-volume sizes +are logged rather than replaced with fabricated zero sizes. NetBox 4.2+ uses +scoped cluster sites and separate MAC address objects. + +## Filters and lifecycle + +Cluster, host, and VM include/exclude filters are regular expressions matched +from the start of the name. Include checks run before exclude checks. +Excluding an availability zone excludes its hosts and VMs; excluding a host +does not by itself exclude its VMs. + +Objects participate in the application's normal source tagging, orphaning, and +pruning lifecycle. Changing project scope, filters, naming, or permitted networks +can change which objects are considered discovered. Avoid overlapping sources +and inspect the dry-run log before allowing writes or pruning. + +## Validation + +Start with a dry run using credentials with the intended scope: + +```sh +python netbox-sync.py -n -c settings.ini +``` -An ESXi host reports the switch and the switch port each of its physical interfaces (pNICs) is -connected to, if CDP or LLDP is enabled on the switch. With the option `sync_host_cables` enabled -netbox-sync uses this information to create cables in NetBox between the host interface and the -switch port. +Check authentication, availability-zone coverage, hypervisor access, server and +volume counts, site mappings, duplicate names, and address/VRF assignments. +Unit tests use a fake SDK and the real in-memory NetBox models: -```ini -sync_host_cables = True +```sh +python -m pytest tests/test_openstack_source.py ``` -Cables are objects which are usually maintained by hand, that's why this option is disabled by -default. With the option disabled no cable is read from or written to NetBox at all. NetBox 3.3 or -newer is needed, on older versions the option is ignored. - -netbox-sync only connects things it can find, it never creates the other end of a cable: - -* the device the neighbor reports as its system name must already exist in NetBox. The name is - matched exactly first, a short name is only matched against a FQDN if that match is unambiguous -* the port the neighbor reports must already exist as an interface of that device. Long and short - interface names are matched against each other, so a reported `FastEthernet0/16` also matches an - interface named `Fa0/16` in NetBox. CDP reports the port ID, LLDP additionally reports a port - description and both are tried -* both interfaces must already exist in NetBox. An interface which was just discovered gets its - cable during the next run -* neither of the two interfaces may be connected already. A cable which was created by hand or which - connects to a different port is never changed or deleted, it is reported at log level `DEBUG` - instead - -Cables created by this source are tagged like every other object and are marked as orphaned and -pruned once the host stops reporting that neighbor (see `prune_enabled`). Disabling the option again -leaves all previously created cables untouched in NetBox. - -### Filtering VM Disk Information -VM disks are synchronized between vCenter and NetBox. Since NetBox 3.7.0, virtual disks are tracked as separate objects -linked to VMs. In some scenarios, such as when temporary disks are attached to VMs during backup operations -(e.g., "Independent-nonpersistent" disks from Veeam), you might want to exclude these changes from synchronization -to avoid cluttering your NetBox change log. - -You can use the following filter options to exclude disk synchronization for specific VMs: - -1. **`vm_exclude_disk_sync`**: A regex pattern matching VM names where disk synchronization should be excluded. - ```ini - vm_exclude_disk_sync = backup-.*, veeam-.* - ``` - -2. **`vm_exclude_disk_sync_by_tag`**: A comma-separated list of vCenter tags. VMs with any of these tags will have - their disk information excluded from synchronization. - ```ini - vm_exclude_disk_sync_by_tag = backup-vm, veeam-job - ``` - -When a VM matches these filters, it will still be synchronized to NetBox with all its other information -(CPU, memory, interfaces, IP addresses, etc.), but changes to disk information will be ignored. +These tests do not replace validation against your cloud's API versions and +authorization policies. From 7ad82cb2ee314801d8b6766879692eb3cf7f129a Mon Sep 17 00:00:00 2001 From: Eric Krichbaum Date: Wed, 11 Nov 2026 16:23:36 -0600 Subject: [PATCH 12/21] Add OpenStack requirements to Dockerfile Added support for OpenStack dependencies in the Dockerfile. --- Dockerfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Dockerfile b/Dockerfile index 07dcfedd..ad5ee0f5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,10 +3,12 @@ FROM python:3.14-slim AS builder COPY requirements.txt . COPY requirements-aws.txt . COPY requirements-ovirt.txt . +COPY requirements-openstack.txt . ARG VENV=/opt/netbox-sync/venv ARG INSTALL_AWS=false ARG INSTALL_OVIRT=false +ARG INSTALL_OPENSTACK=false # Install dependencies RUN if [ "$INSTALL_OVIRT" = "true" ]; then \ @@ -19,6 +21,7 @@ RUN if [ "$INSTALL_OVIRT" = "true" ]; then \ $VENV/bin/pip install -r requirements.txt && \ if [ "$INSTALL_AWS" = "true" ]; then $VENV/bin/pip install -r requirements-aws.txt; fi && \ if [ "$INSTALL_OVIRT" = "true" ]; then $VENV/bin/pip install -r requirements-ovirt.txt; fi && \ + if [ "$INSTALL_OPENSTACK" = "true" ]; then $VENV/bin/pip install -r requirements-openstack.txt; fi && \ $VENV/bin/pip install vmware-vcenter==9.1.1.0 && \ $VENV/bin/python3 -m pip uninstall -y pip && \ find $VENV -type d -name "__pycache__" -print0 | xargs -0 -n1 rm -rf From ddfab023ae7fda2d76206b8546c3160d478a6d53 Mon Sep 17 00:00:00 2001 From: Eric Krichbaum Date: Sat, 14 Nov 2026 15:36:52 -0600 Subject: [PATCH 13/21] adjustments for code that didn't get migrated during rebase MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SDK configuration: Prevents OS_* variables and clouds.yaml from overriding source settings. VM interfaces: Matches existing ports by MAC, preserving the surviving port’s IP when another is detached and updating the name when a port is replaced. Stale IPs: Checks the complete server snapshot, ownership and IP uniqueness before allowing reassignment. This part depends on the shared-source hook --- module/sources/aws/connection.py | 2216 +++++++++++++++++++----------- 1 file changed, 1395 insertions(+), 821 deletions(-) diff --git a/module/sources/aws/connection.py b/module/sources/aws/connection.py index 25ece72c..2586d021 100644 --- a/module/sources/aws/connection.py +++ b/module/sources/aws/connection.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright (c) 2020 - 2026 netbox-sync team. All rights reserved. +# Copyright (c) 2020 - 2021 Ricardo Bartels. All rights reserved. # # netbox-sync.py # @@ -7,865 +7,1439 @@ # For a copy, see file LICENSE.txt included in this # repository or visit: . -from datetime import datetime, timezone -from hashlib import sha256 -from ipaddress import ip_address, ip_interface, ip_network -from itertools import product +import pprint +from collections import Counter +from ipaddress import ip_interface +from importlib import import_module +from urllib.parse import unquote +from uuid import UUID -from packaging.version import Version +from packaging import version -from module.common.logging import get_logger -from module.common.misc import do_error_exit +from module.sources.common.source_base import SourceBase +from module.sources.openstack.config import OpenStackConfig +from module.common.logging import get_logger, DEBUG3 +from module.common.misc import grab, dump, get_string_or_none from module.common.support import normalize_mac_address -from module.netbox import ( - NBCustomField, NBCluster, NBClusterGroup, NBClusterType, NBIPAddress, - NBMACAddress, NBSite, NBTag, NBTenant, NBVirtualDisk, NBVM, NBVMInterface, NBVRF, -) from module.netbox.inventory import NetBoxInventory from module.netbox.connection import NetBoxHandler -from module.sources.aws.client import AWSClient, AWSDiscoveryError -from module.sources.aws.config import AWSConfig -from module.sources.common.source_base import SourceBase +from module.netbox import * log = get_logger() -def json_safe(value): +# noinspection PyTypeChecker +class OpenStackHandler(SourceBase): """ - boto3 returns timestamps (EKS createdAt, modifiedAt, ...) as datetime objects. A NetBox - JSON field only takes plain JSON, so store them as ISO 8601 strings in UTC; the local - timezone of the host running the sync must not change the stored value. + Source class to import data from a Openstack instance and add/update NetBox objects based on gathered information """ - if isinstance(value, dict): - return {key: json_safe(item) for key, item in value.items()} - if isinstance(value, (list, tuple)): - return [json_safe(item) for item in value] - if isinstance(value, datetime): - if value.tzinfo is not None: - value = value.astimezone(timezone.utc) - return value.isoformat() - return value - - -class AWSHandler(SourceBase): - source_type = "aws" + dependent_netbox_objects = [ - NBTag, NBCustomField, NBTenant, NBSite, NBClusterGroup, NBClusterType, - NBCluster, NBVM, NBVMInterface, NBMACAddress, NBIPAddress, NBVRF, NBVirtualDisk, + NBTag, + NBManufacturer, + NBDeviceType, + NBPlatform, + NBClusterType, + NBClusterGroup, + NBDeviceRole, + NBSite, + NBCluster, + NBDevice, + NBVM, + NBVirtualDisk, + NBVMInterface, + NBInterface, + NBIPAddress, + NBPrefix, + NBTenant, + NBVRF, + NBVLAN, + NBVLANGroup, + NBMACAddress, + NBFHRPGroupItem, + NBCustomField ] + source_type = "openstack" + + # internal vars + session = None + site_name = None + def __init__(self, name=None): - if not name: - raise ValueError(f"Invalid source name '{name}'.") - self.name = name + + if name is None: + raise ValueError(f"Invalid value for attribute 'name': '{name}'.") + self.inventory = NetBoxInventory() - config = AWSConfig() - config.source_name = name - self.settings = config.parse() - self.set_source_tag() + self.name = name + self.session = None + self._server_snapshot_ids = None + self._server_ip_counts = Counter() self.init_successful = False - if not self.settings.enabled: - log.info(f"Source '{name}' is disabled. Skipping.") + + # parse settings + settings_handler = OpenStackConfig() + settings_handler.source_name = self.name + self.settings = settings_handler.parse() + + self.set_source_tag() + self.site_name = f"OpenStack: {name}" + + if self.settings.enabled is False: + log.info(f"Source '{name}' is currently disabled. Skipping") return - try: - self.client = AWSClient(profile=self.settings.profile) - except AWSDiscoveryError as exc: - self.abort(exc) + + self.create_openstack_session() + + if self.session is None: + log.info(f"Source '{name}' is currently unavailable. Skipping") + return + self.init_successful = True + self.permitted_clusters = dict() + self.cluster_host_map = dict() + self.volume_map = dict() + self.processed_host_names = dict() + self.processed_vm_names = dict() + self.object_cache = dict() - def abort(self, error): - message = f"AWS source '{self.name}': {error}. No NetBox writes or pruning will be performed." - log.error(message) - do_error_exit(message) + def create_openstack_session(self): + """ + Initialize session with OpenStack + + Returns + ------- + bool: if initialization was successful or not + """ + + if self.session is not None: + return True + + log.debug(f"Starting OpenStack connection to '{self.settings.auth_url}'") - def apply(self): try: - if self.settings.sync_s3 and Version(self.inventory.netbox_api_version) < Version("3.7"): - raise ValueError("S3 bucket inventory requires NetBox 3.7 or newer") - services = tuple(service for service in ("s3", "lambda", "eks") - if getattr(self.settings, f"sync_{service}")) - options = {"services": services} if services else {} - if self.settings.organization: - snapshots = self.client.discover_organization( - self.settings.regions, self.settings.organization_role_name, - expected_account=self.settings.account_id, - discovery_region=self.settings.discovery_region, - include_management=self.settings.organization_include_management, - external_id=self.settings.organization_external_id, - account_ids=self.settings.organization_account_ids, **options) - account_names = getattr(self.client, "account_names", {}) - else: - account, snapshot = self.client.discover( - self.settings.regions, self.settings.account_id, self.settings.discovery_region, **options) - snapshots = {account: snapshot} if account is not None else {} - account_names = {} - if self.settings.organization and self.settings.sync_private_ips: - self.resolve_shared_networks(snapshots) - self.defer_inaccessible_network_instances(snapshots) - vrfs = {} - for account, snapshot in snapshots.items(): - self.validate_services(account, snapshot) - self.validate_snapshot(account, snapshot, vrfs) - self.retain_disabled_services() - if getattr(self.client, "access_denied", []): - self.retain_inaccessible_objects() - account_labels = { - account: account_names.get(account) or self.settings.account_name or account - for account in snapshots - } - label_counts = {} - for label in account_labels.values(): - key = label.casefold() - label_counts[key] = label_counts.get(key, 0) + 1 - for account, label in account_labels.items(): - if label_counts[label.casefold()] > 1: - account_labels[account] = f"{label} ({account})" - for account, snapshot in snapshots.items(): - self.sync_snapshot(account, snapshot, account_name=account_labels[account]) - self.reconcile_access_tags() - except (AWSDiscoveryError, ValueError, KeyError) as exc: - self.abort(exc) - - def validate_services(self, account, snapshot): - for region, data in snapshot.items(): - inaccessible = data.get("InaccessibleServices", []) - for enabled, key in ( - (self.settings.sync_s3 and "s3" not in inaccessible, "Buckets"), - (self.settings.sync_lambda and "lambda" not in inaccessible, "Functions"), - (self.settings.sync_eks and "eks" not in inaccessible, "EKSClusters"), - ): - if enabled and not isinstance(data.get(key), list): - raise ValueError(f"Missing AWS {key} discovery (account '{account}', region '{region}')") - if enabled: - identity_key = {"Buckets": "Name", "Functions": "FunctionArn", "EKSClusters": "arn"}[key] - identities = [resource[identity_key] for resource in data[key]] - if len(set(identities)) != len(identities): - raise ValueError(f"Duplicate AWS {key} discovery (account '{account}', region '{region}')") - for bucket in data["Buckets"] if self.settings.sync_s3 and "s3" not in inaccessible else []: - if bucket["Region"] != region or not bucket["Name"] or len(bucket["Name"]) > 63: - raise ValueError(f"Invalid S3 bucket '{bucket['Name']}' for region '{region}'") - self.service_tags({tag["Key"]: tag["Value"] for tag in bucket["Tags"]}) - if "Versioning" not in bucket or "Encryption" not in bucket: - raise ValueError(f"Incomplete S3 bucket metadata for '{bucket['Name']}'") - for function in data["Functions"] if self.settings.sync_lambda and "lambda" not in inaccessible else []: - self.validate_arn(function["FunctionArn"], "lambda", account, region) - if function["State"] not in ("Pending", "Active", "Inactive", "Failed"): - raise ValueError(f"Unknown Lambda state '{function['State']}' for '{function['FunctionArn']}'") - if not function["FunctionName"] or not isinstance(function["MemorySize"], int): - raise ValueError(f"Incomplete Lambda configuration for '{function['FunctionArn']}'") - self.service_tags(function["Tags"]) - if self.settings.sync_eks and "eks" not in inaccessible: - clusters = {cluster["name"]: cluster for cluster in data["EKSClusters"]} - if len(clusters) != len(data["EKSClusters"]): - raise ValueError(f"Duplicate EKS cluster discovery in account '{account}', region '{region}'") - asg_clusters = {} - for cluster in data["EKSClusters"]: - self.validate_arn(cluster["arn"], "eks", account, region) - for field in ("status", "version", "resourcesVpcConfig", "nodegroups", "fargateProfiles"): - if field not in cluster: - raise ValueError(f"Incomplete EKS {field} for '{cluster['arn']}'") - self.service_tags(cluster.get("tags", {})) - for group in cluster["nodegroups"]: - for asg in group.get("resources", {}).get("autoScalingGroups", []): - name = asg["name"] - if name in asg_clusters and asg_clusters[name] != cluster["name"]: - raise ValueError(f"EKS Auto Scaling group '{name}' belongs to multiple clusters") - asg_clusters[name] = cluster["name"] - data["EKSWorkers"] = {} - for instance in data["instances"]: - tags = {tag["Key"]: tag["Value"] for tag in instance.get("Tags", [])} - names = {tags[key] for key in ( - "aws:eks:cluster-name", "eks:cluster-name", "eksctl.cluster.k8s.io/v1alpha1/cluster-name", - ) if tags.get(key)} - asg_cluster = asg_clusters.get(tags.get("aws:autoscaling:groupName")) - if asg_cluster: - names.add(asg_cluster) - if len(names) > 1: - raise ValueError(f"Conflicting EKS cluster membership for instance '{instance['InstanceId']}'") - if names: - name = next(iter(names)) - if name in clusters: - data["EKSWorkers"][instance["InstanceId"]] = name - else: - log.warning( - f"AWS instance '{instance['InstanceId']}' references undiscovered EKS cluster " - f"'{name}' (account '{account}', region '{region}'); retaining EC2 placement.") - - def validate_arn(self, arn, service, account, region): - parts = arn.split(":", 5) - if len(parts) != 6 or parts[0] != "arn" or parts[2:5] != [service, region, account] or not parts[5]: - raise ValueError(f"Invalid {service} resource ARN '{arn}' for account '{account}', region '{region}'") - - def service_tags(self, tags): - result = [f"AWS:{key}={tags[key]}" for key in self.settings.tag_keys if key in tags] - if any(len(tag) > 100 for tag in result): - raise ValueError("AWS service tag exceeds the NetBox tag name limit (100 characters)") - return result - - def reconcile_tags(self, item, tags): - # Tag lookup can reuse an existing name with different capitalization. - desired_tags = {self.inventory.get_by_data(NBTag, data={"name": name}) for name in tags} - managed_prefixes = tuple(f"AWS:{key}=".casefold() for key in self.settings.tag_keys) - for tag in item.data.get("tags", []): - if tag.get_display_name().casefold().startswith(managed_prefixes) and tag not in desired_tags: - item.remove_tags(tag) - - def retain_disabled_services(self): - for object_type in (NBVM, NBVirtualDisk, NBCluster): - for item in self.inventory.get_all_items(object_type): - service = (item.data.get("custom_fields") or {}).get("aws_service") - if service in ("s3", "lambda", "eks") and not getattr(self.settings, f"sync_{service}") and \ - (item.source is self or self.source_tag in item.get_tags()): - item.set_source(self) - - def access_scope(self, item): - fields = item.data.get("custom_fields") or {} - metadata = fields.get("aws_metadata") or {} - identity = fields.get("aws_resource_id") or "" - service = "ec2" if fields.get("aws_instance_id") else fields.get("aws_service") - account = fields.get("aws_account_id") or metadata.get("account") - region = fields.get("aws_region") or metadata.get("region") - resource = None - if identity.startswith("arn:"): - parts = identity.split(":", 5) - if len(parts) == 6: - service, region, account, resource = parts[2:] - if service == "lambda": - resource = identity - elif service == "eks": - resource = resource.removeprefix("cluster/") - elif identity.startswith("s3/"): - parts = identity.split("/") - account = parts[1] - if isinstance(item, NBVirtualDisk): - resource = parts[2] - else: - region = parts[2] - for parent_key in ("virtual_machine", "cluster"): - parent = item.data.get(parent_key) - if parent is not None: - parent_account, parent_region, parent_service, _ = self.access_scope(parent) - account = account or parent_account - region = region or parent_region - service = service or parent_service + openstack = import_module("openstack") + except ImportError: + log.error("OpenStack source requires openstacksdk. Install with " + "'pip install -r requirements-openstack.txt' or 'uv sync --extra openstack'.") + return False + + try: + self.session = openstack.connect( + load_yaml_config=False, + load_envvars=False, + auth_url=self.settings.auth_url, + project_name=self.settings.project, + username=self.settings.username, + password=self.settings.password, + region_name=self.settings.region, + user_domain_name=self.settings.user_domain, + project_domain_name=self.settings.project_domain, + verify=self.settings.validate_tls_certs, + app_name='netbox-sync', + app_version='0.1', + ) + self.session.authorize() + except Exception as e: + log.error(f"Unable to connect to OpenStack instance '{self.settings.auth_url}': {e}") + self.finish() + return False + + log.info(f"Successfully connected to OpenStack '{self.settings.auth_url}'") + + return True + + def finish(self): + if self.session is not None: + self.session.close() + self.session = None + + def apply(self): + """ + Main source handler method. This method is called for each source from "main" program + to retrieve data from it source and apply it to the NetBox inventory. + + Every update of new/existing objects fot this source has to happen here. + """ + + self._server_snapshot_ids = None + self._server_ip_counts.clear() + log.info(f"Query data from OpenStack: '{self.settings.auth_url}'") + self.permitted_clusters.clear() + self.cluster_host_map.clear() + self.volume_map.clear() + self.processed_host_names.clear() + self.processed_vm_names.clear() + self.object_cache.clear() + + self.inventory.add_update_object(NBClusterGroup, data={"name": self.settings.group_name}, source=self) + + availability_zones = self.session.compute.availability_zones(details=True) + for availability_zone in availability_zones: + self.add_cluster(availability_zone) + + hypervisors = self.session.compute.hypervisors(details=True) + for hypervisor in hypervisors: + self.add_host(hypervisor) + + volumes = self.session.block_storage.volumes(details=True, all_projects=self.settings.all_projects) + for volume in volumes: + self.add_volume(volume) + + servers = list(self.session.compute.servers(details=True, all_projects=self.settings.all_projects)) + snapshot_ids = set() + for server in servers: + try: + snapshot_ids.add(str(UUID(str(server.id)))) + except (ValueError, AttributeError): + log.warning("OpenStack server snapshot contains an invalid UUID; stale IP repair is disabled.") break - if isinstance(item, (NBIPAddress, NBMACAddress)): - parent = item.get_device_vm() - if parent is not None: - account, region, service, _ = self.access_scope(parent) - if account is None: - group = item if isinstance(item, NBClusterGroup) else item.data.get("group") - name = group.data.get("name", "") if group is not None else "" - if name.startswith("AWS "): - label = name[4:] - account = next((key for key, value in getattr(self.client, "account_names", {}).items() - if label in (value, f"{value} ({key})")), None) - if account is None and label.isdigit(): - account = label - if region is None: - site = item.data.get("scope_id") or item.data.get("site") - name = site.data.get("name", "") if isinstance(site, NBSite) else "" - if name.startswith("AWS "): - region = name[4:] - return account, region, service or "ec2", resource - - def reconcile_access_tags(self): - checks = getattr(self.client, "access_checks", []) - if not checks: + else: + self._server_snapshot_ids = snapshot_ids + for server in servers: + addresses = set() + for network_addresses in (grab(server, "addresses") or {}).values(): + for address in network_addresses: + try: + addresses.add(str(ip_interface(address["addr"]).ip)) + except (KeyError, ValueError): + continue + self._server_ip_counts.update(addresses) + # Prefer an already-associated server before using status to break ties. + try: + for server in sorted(servers, key=self.get_server_priority): + self.add_virtual_machine(server) + + self.update_basic_data() + finally: + self._server_snapshot_ids = None + + def can_reassign_stale_ip(self, ip, device_vm_object): + def blocked(reason): + log.warning( + f"OpenStack source '{self.name}': stale IP repair blocked for " + f"'{ip.data['address']}' (NetBox IP ID {ip.nb_id}): {reason}." + ) + return False + + if self._server_snapshot_ids is None: + return blocked("no complete server UUID snapshot is available") + if not self.settings.all_projects: + return blocked("all_projects is False; absence from this project does not prove deletion") + old_vm = ip.get_device_vm() + if not isinstance(old_vm, NBVM) or not isinstance(device_vm_object, NBVM): + return blocked("both owners must be virtual machines") + if old_vm.source is not None: + return blocked( + f"old VM '{old_vm.get_display_name()}' (NetBox VM ID {old_vm.nb_id}) " + f"was claimed this run by source '{getattr(old_vm.source, 'name', None)}'" + ) + if device_vm_object.source is not self: + return blocked("the target VM is not owned by this source in this run") + recovered_orphan_ownership = False + for item in (ip, old_vm): + source_tags = {tag for tag in item.get_tags() if tag.startswith("Source: ")} + if source_tags != {self.source_tag}: + if source_tags or NetBoxHandler.orphaned_tag not in item.get_tags(): + return blocked( + f"{item.name} '{item.get_display_name()}' (NetBox ID {item.nb_id}) has " + f"source tags {sorted(source_tags)}; expected only '{self.source_tag}' " + "or an orphaned record with verified cluster ownership" + ) + recovered_orphan_ownership = True + if recovered_orphan_ownership: + cluster = old_vm.data.get("cluster") + cluster_tags = {tag for tag in cluster.get_tags() if tag.startswith("Source: ")} \ + if isinstance(cluster, NBCluster) else set() + if not isinstance(cluster, NBCluster) or cluster.source is not self or \ + cluster_tags != {self.source_tag} or \ + self.get_object_from_cache(NBCluster, cluster.data.get("name")) is not cluster or \ + grab(cluster, "data.group.data.name") != self.settings.group_name: + return blocked("orphaned records lack an exclusively source-owned cluster in current discovery") + if ip.source is not None and ip.source is not self: + return blocked(f"IP was claimed by another source '{getattr(ip.source, 'name', None)}'") + identity_key = "name" if self.settings.set_vm_name_to_uuid else "comments" + identities = [] + for vm in (old_vm, device_vm_object): + try: + identities.append(str(UUID(str(vm.data.get(identity_key))))) + except ValueError: + return blocked( + f"VM '{vm.get_display_name()}' (NetBox VM ID {vm.nb_id}) has no valid " + f"server UUID in '{identity_key}'" + ) + old_id, current_id = identities + address = str(ip_interface(ip.data["address"]).ip) + if old_id in self._server_snapshot_ids: + return blocked(f"old server UUID {old_id} is still present in discovery") + if current_id not in self._server_snapshot_ids: + return blocked(f"target server UUID {current_id} is absent from discovery") + if self._server_ip_counts[address] != 1: + return blocked(f"{self._server_ip_counts[address]} discovered servers report this IP; expected one") + log.info( + f"Reclaiming OpenStack IP '{ip.data['address']}' (NetBox ID {ip.nb_id}) from VM " + f"'{old_vm.get_display_name()}' (UUID {old_id}), absent from the complete server snapshot, " + f"to '{device_vm_object.get_display_name()}' (UUID {current_id}) for source '{self.name}'." + + (" Recovered missing orphan Source tags using verified cluster ownership." + if recovered_orphan_ownership else "") + ) + return True + + @staticmethod + def passes_filter(name, include_filter, exclude_filter): + """ + checks if object name passes a defined object filter. + + Parameters + ---------- + name: str + name of the object to check + include_filter: regex object + regex object of include filter + exclude_filter: regex object + regex object of exclude filter + + Returns + ------- + bool: True if all filter passed, otherwise False + """ + + # first includes + if include_filter is not None and not include_filter.match(name): + log.debug(f"Object '{name}' did not match include filter '{include_filter.pattern}'. Skipping") + return False + + # second excludes + if exclude_filter is not None and exclude_filter.match(name): + log.debug(f"Object '{name}' matched exclude filter '{exclude_filter.pattern}'. Skipping") + return False + + return True + + def get_site_name(self, object_type, object_name, cluster_name=""): + """ + Return a site name for a NBCluster or NBDevice depending on config options + host_site_relation and cluster_site_relation + + Parameters + ---------- + object_type: (NBCluster, NBDevice) + object type to check site relation for + object_name: str + object name to check site relation for + cluster_name: str + cluster name of NBDevice to check for site name + + Returns + ------- + str: site name if a relation was found + """ + + if object_type not in [NBCluster, NBDevice]: + raise ValueError(f"Object must be a '{NBCluster.name}' or '{NBDevice.name}'.") + + log.debug2(f"Trying to find site name for {object_type.name} '{object_name}'") + + # check if site was provided in config + relation_name = "host_site_relation" if object_type == NBDevice else "cluster_site_relation" + + site_name = self.get_object_relation(object_name, relation_name) + + if object_type == NBDevice and site_name is None: + site_name = self.get_site_name(NBCluster, cluster_name) + if site_name is not None: + log.debug2(f"Found a matching cluster site for {object_name}, using site '{site_name}'") + + # set default site name + if site_name is None: + site_name = self.site_name + log.debug(f"No site relation for '{object_name}' found, using default site '{site_name}'") + + return site_name + + def get_object_based_on_macs(self, object_type, mac_list=None): + """ + Try to find a NetBox object based on list of MAC addresses. + + Iterate over all interfaces of this object type and compare MAC address with list of desired MAC + addresses. If match was found store related machine object and count every correct match. + + If exactly one machine with matching interfaces was found then this one will be returned. + + If two or more machines with matching MACs are found compare the two machines with + the highest amount of matching interfaces. If the ration of matching interfaces + exceeds 2.0 then the top matching machine is chosen as desired object. + + If the ration is below 2.0 then None will be returned. The probability is too low that + this one is the correct one. + + None will also be returned if no machine was found at all. + + Parameters + ---------- + object_type: (NBDevice, NBVM) + type of NetBox device to find in inventory + mac_list: list + list of MAC addresses to compare against NetBox interface objects + + Returns + ------- + (NBDevice, NBVM, None): object instance of found device, otherwise None + """ + + object_to_return = None + + if object_type not in [NBDevice, NBVM]: + raise ValueError(f"Object must be a '{NBVM.name}' or '{NBDevice.name}'.") + + if mac_list is None or not isinstance(mac_list, list) or len(mac_list) == 0: return - prefix = "Sync:AccessDenied:" - scopes = {} - for check in checks: - outcomes = scopes.setdefault( - (check.account, check.region, check.service, check.resource), (set(), set())) - outcomes[0 if check.denied else 1].add(check.operation) - for object_type in self.dependent_netbox_objects: - for item in self.inventory.get_all_items(object_type): - if object_type in (NBTag, NBCustomField, NBTenant) or "tags" not in item.data_model or not ( - item.source is self or self.source_tag in item.get_tags()): + + interface_typ = NBInterface if object_type == NBDevice else NBVMInterface + + objects_with_matching_macs = dict() + matching_object = None + + for interface in self.inventory.get_all_items(interface_typ): + + mac_address = grab(interface, "data.primary_mac_address.data.mac_address") or \ + grab(interface, "data.mac_address") + if mac_address in mac_list: + + matching_object = grab(interface, f"data.{interface.secondary_key}") + if not isinstance(matching_object, (NBDevice, NBVM)): continue - account, region, service, resource = self.access_scope(item) - denied, succeeded = set(), set() - for key in product( - (None, account), (None, region), ("organizations", "sts", "account", service), - (None, resource)): - if key in scopes: - failures, successes = scopes[key] - denied.update(failures) - succeeded.update(successes) - for operation in denied: - item.add_tags(f"{prefix}{operation}") - for tag in item.get_tags(): - if tag.startswith(prefix) and tag[len(prefix):] in succeeded - denied: - item.remove_tags(tag) - - def retain_source_objects(self): - # Source-wide retention also protects children and cross-account shared-network dependencies. - for object_type in self.dependent_netbox_objects: - for item in self.inventory.get_all_items(object_type): - if self.source_tag in item.get_tags(): - item.set_source(self) - - def retain_inaccessible_objects(self): - self.retain_source_objects() - log.warning( - f"AWS source '{self.name}': authorization-denied discovery is incomplete. " - "Retaining all previously managed source objects this run; accessible resources will still be updated.") - - def defer_inaccessible_network_instances(self, snapshots): - if not any(service in ("ec2", "account", "sts", "organizations") - for _, _, service in getattr(self.client, "access_denied", [])): + + log.debug2("Found matching MAC '%s' on %s '%s'" % + (grab(interface, "data.mac_address"), object_type.name, + matching_object.get_display_name(including_second_key=True))) + + if objects_with_matching_macs.get(matching_object) is None: + objects_with_matching_macs[matching_object] = 1 + else: + objects_with_matching_macs[matching_object] += 1 + + # try to find object based on amount of matching MAC addresses + num_devices_witch_matching_macs = len(objects_with_matching_macs.keys()) + + if num_devices_witch_matching_macs == 1 and isinstance(matching_object, (NBDevice, NBVM)): + + log.debug2("Found one %s '%s' based on MAC addresses and using it" % + (object_type.name, matching_object.get_display_name(including_second_key=True))) + + object_to_return = list(objects_with_matching_macs.keys())[0] + + elif num_devices_witch_matching_macs > 1: + + log.debug2(f"Found {num_devices_witch_matching_macs} {object_type.name}s with matching MAC addresses") + + # now select the two top matches + first_choice, second_choice = \ + sorted(objects_with_matching_macs, key=objects_with_matching_macs.get, reverse=True)[0:2] + + first_choice_matches = objects_with_matching_macs.get(first_choice) + second_choice_matches = objects_with_matching_macs.get(second_choice) + + log.debug2(f"The top candidate {first_choice.get_display_name()} with {first_choice_matches} matches") + log.debug2(f"The second candidate {second_choice.get_display_name()} with {second_choice_matches} matches") + + # get ratio between + matching_ration = first_choice_matches / second_choice_matches + + # only pick the first one if the ration exceeds 2 + if matching_ration >= 2.0: + log.debug2(f"The matching ratio of {matching_ration} is high enough " + f"to select {first_choice.get_display_name()} as desired {object_type.name}") + object_to_return = first_choice + else: + log.debug2("Both candidates have a similar amount of " + "matching interface MAC addresses. Using NONE of them!") + + return object_to_return + + def get_object_based_on_primary_ip(self, object_type, primary_ip4=None, primary_ip6=None): + """ + Try to find a NBDevice or NBVM based on the primary IP address. If an exact + match was found the device/vm object will be returned immediately without + checking of the other primary IP address (if defined). + + Parameters + ---------- + object_type: (NBDevice, NBVM) + object type to look for + primary_ip4: str + primary IPv4 address of object to find + primary_ip6: str + primary IPv6 address of object to find + + Returns + ------- + + """ + + def _matches_device_primary_ip(device_primary_ip, ip_needle): + + ip = None + if device_primary_ip is not None and ip_needle is not None: + if isinstance(device_primary_ip, dict): + ip = grab(device_primary_ip, "address") + + elif isinstance(device_primary_ip, NBIPAddress): + ip = device_primary_ip.data.get("address") + elif isinstance(device_primary_ip, int): + ip = self.inventory.get_by_id(NBIPAddress, nb_id=device_primary_ip) + ip = grab(ip, "data.address") + + if ip is not None and ip.split("/")[0] == ip_needle: + return True + + return False + + if object_type not in [NBDevice, NBVM]: + raise ValueError(f"Object must be a '{NBVM.name}' or '{NBDevice.name}'.") + + if primary_ip4 is None and primary_ip6 is None: return - for account, snapshot in snapshots.items(): - for region, data in snapshot.items(): - retained = [] - for instance in data["instances"]: - missing_network = False - for reference in instance.get("NetworkInterfaces", []): - eni = data["NetworkInterfaces"].get(reference["NetworkInterfaceId"]) - if eni is not None and (eni["VpcId"] not in data["Vpcs"] or - eni["SubnetId"] not in data["Subnets"]): - missing_network = True - if missing_network: - log.warning( - f"AWS instance '{instance['InstanceId']}' (account '{account}', region '{region}') " - "has missing network dependencies during authorization-denied discovery. " - "Deferring its update and retaining existing objects.") - else: - retained.append(instance) - data["instances"] = retained - - def resource_name(self, name, identity, limit): - if len(name) <= limit: - return name - suffix = sha256(identity.encode("utf-8")).hexdigest()[:12] - return f"{name[:limit - 13]}-{suffix}" - - def service_object(self, object_type, identity, data): - fields = data["custom_fields"] - fields["aws_metadata"] = json_safe(fields["aws_metadata"]) - matches = [item for item in self.inventory.get_all_items(object_type) - if (item.data.get("custom_fields") or {}).get("aws_resource_id") == identity] - if len(matches) > 1: - raise ValueError(f"Multiple NetBox {object_type.name} objects match AWS resource '{identity}'") - if matches: - item = matches[0] - if item.source is not self and self.source_tag not in item.get_tags(): - raise ValueError(f"AWS resource '{identity}' belongs to another source") - item.update(data=data, source=self) + + if primary_ip4 is not None: + primary_ip4 = str(primary_ip4).split("/")[0] + + if primary_ip6 is not None: + primary_ip6 = str(primary_ip6).split("/")[0] + + for device in self.inventory.get_all_items(object_type): + + if _matches_device_primary_ip(grab(device, "data.primary_ip4"), primary_ip4) is True: + log.debug2(f"Found existing host '{device.get_display_name()}' " + f"based on the primary IPv4 '{primary_ip4}'") + return device + + if _matches_device_primary_ip(grab(device, "data.primary_ip6"), primary_ip6) is True: + log.debug2(f"Found existing host '{device.get_display_name()}' " + f"based on the primary IPv6 '{primary_ip6}'") + return device + + def get_object_relation(self, name, relation, fallback=None): + """ + + Parameters + ---------- + name: str + name of the object to find a relation for + relation: str + name of the config variable relation (i.e: vm_tag_relation) + fallback: str + fallback string if no relation matched + + Returns + ------- + data: str, list, None + string of matching relation or list of matching tags + """ + + resolved_list = list() + for single_relation in grab(self.settings, relation, fallback=list()): + object_regex = single_relation.get("object_regex") + match_found = False + if object_regex.match(name): + resolved_name = single_relation.get("assigned_name") + log.debug2(f"Found a matching {relation} '{resolved_name}' ({object_regex.pattern}) for {name}") + resolved_list.append(resolved_name) + match_found = True + + # special cluster condition + if match_found is False and grab(f"{relation}".split("_"), "0") == "cluster": + + stripped_name = "/".join(name.split("/")[1:]) + if object_regex.match(stripped_name): + + resolved_name = single_relation.get("assigned_name") + log.debug2(f"Found a matching {relation} '{resolved_name}' ({object_regex.pattern}) " + f"for {stripped_name}") + resolved_list.append(resolved_name) + + if grab(f"{relation}".split("_"), "1") == "tag": + return resolved_list + + else: + resolved_name = fallback + if len(resolved_list) >= 1: + resolved_name = resolved_list[0] + if len(resolved_list) > 1: + log.debug(f"Found {len(resolved_list)} matches for {name} in {relation}." + f" Using first on: {resolved_name}") + + return resolved_name + + def get_cluster_for_host(self, hostname): + for cluster, hosts in self.cluster_host_map.items(): + if hostname in hosts: + return cluster + return None + + def get_server_priority(self, server): + name = get_string_or_none(grab(server, "name")) + if name is not None and self.settings.strip_vm_domain_name: + name = name.split(".")[0] + if self.settings.set_vm_name_to_uuid: + name = get_string_or_none(grab(server, "id")) + + cluster_name = get_string_or_none(grab(server, "availability_zone")) + cluster = self.get_object_from_cache(NBCluster, cluster_name) + if name is not None and cluster is not None and not self.settings.skip_vm_comments: + existing_vm = self.inventory.get_by_data( + NBVM, data={"name": name, "cluster": cluster} + ) + if grab(existing_vm, "data.comments") == get_string_or_none(grab(server, "id")): + return 0, server.status != "ACTIVE" + + return 1, server.status != "ACTIVE" + + @staticmethod + def get_compute_host_names(hosts): + compute_host_names = [] + for hostname, services in (hosts or {}).items(): + if not isinstance(services, dict) or not services: + compute_host_names.append(hostname) + continue + + service_names = {str(name).lower() for name in services} + service_names.update( + str(grab(service, "binary", fallback="")).lower() + for service in services.values() + ) + has_nova_services = any(name.startswith("nova-") for name in service_names) + if not has_nova_services or "nova-compute" in service_names: + compute_host_names.append(hostname) + + return compute_host_names + + def object_synced_by_other_source(self, nb_object): + if nb_object.source is not None: + return nb_object.source is not self + other_tags = [source.source_tag for source in self.inventory.source_list if source is not self] + return bool(set(nb_object.get_tags()).intersection(other_tags)) + + def add_custom_attributes(self, object_type, obj, data): + attributes = self.settings.host_custom_object_attributes if object_type == NBDevice else \ + self.settings.vm_custom_object_attributes + for attribute in attributes or []: + value = grab(obj, attribute) + if value is None: + continue + field = self.add_update_custom_field({ + "name": f"openstack_{attribute}", "label": attribute[:50], + "object_types": [object_type.object_type], "type": "text", + "description": f"OpenStack resource attribute '{attribute}'", + }) + data.setdefault("custom_fields", {})[field.data["name"]] = str(value) + + def add_device_vm_to_inventory(self, object_type, object_data, pnic_data=None, vnic_data=None, + nic_ips=None, p_ipv4=None, p_ipv6=None): + """ + Add/update device/VM object in inventory based on gathered data. + + Try to find object first based on the object data, interface MAC addresses and primary IPs. + 1. try to find by name and cluster/site + 2. try to find by mac addresses interfaces + 3. try to find by primary IP + + IP addresses for each interface are added here as well. First they will be checked and added + if all checks pass. For each IP address a matching IP prefix will be searched for. First we + look for longest matching IP Prefix in the same site. If this failed we try to find the longest + matching global IP Prefix. + + If a IP Prefix was found then we try to get the VRF and VLAN for this prefix. Now we compare + if interface VLAN and prefix VLAN match up and warn if they don't. Then we try to add data to + the IP address if not already set: + + add prefix VRF if VRF for this IP is undefined + add tenant if tenant for this IP is undefined + 1. try prefix tenant + 2. if prefix tenant is undefined try VLAN tenant + + And we also set primary IP4/6 for this object depending on the "set_primary_ip" setting. + + If a IP address is set as primary IP for another device then using this IP on another + device will be rejected by NetBox. + + Setting "always": + check all NBDevice and NBVM objects if this IP address is set as primary IP to any + other object then this one. If we found another object, then we unset the primary_ip* + for the found object and assign it to this object. + + This setting will also reset the primary IP if it has been changed in NetBox + + Setting "when-undefined": + Will set the primary IP for this object if primary_ip4/6 is undefined. Will cause a + NetBox error if IP has been assigned to a different object as well + + Setting "never": + Well, the attribute primary_ip4/6 will never be touched/changed. + + Parameters + ---------- + object_type: (NBDevice, NBVM) + NetBoxObject sub class of object to add + object_data: dict + data of object to add/update + pnic_data: dict + data of physical interfaces of this object, interface name as key + vnic_data: dict + data of virtual interfaces of this object, interface name as key + nic_ips: dict + dict of ips per interface of this object, interface name as key + p_ipv4: str + primary IPv4 as string including netmask/prefix + p_ipv6: str + primary IPv6 as string including netmask/prefix + + """ + + if object_type not in [NBDevice, NBVM]: + raise ValueError(f"Object must be a '{NBVM.name}' or '{NBDevice.name}'.") + + pnic_data = pnic_data or {} + vnic_data = vnic_data or {} + nic_ips = nic_ips or {} + + if log.level == DEBUG3: + + log.debug3("function: add_device_vm_to_inventory") + log.debug3(f"Object type {object_type}") + pprint.pprint(object_data) + pprint.pprint(pnic_data) + pprint.pprint(vnic_data) + pprint.pprint(nic_ips) + pprint.pprint(p_ipv4) + pprint.pprint(p_ipv6) + + # check existing Devices for matches + log.debug2(f"Trying to find a {object_type.name} based on the collected name, cluster, IP and MAC addresses") + + device_vm_object = self.inventory.get_by_data(object_type, data=object_data) + + if device_vm_object is not None: + log.debug2("Found a exact matching %s object: %s" % + (object_type.name, device_vm_object.get_display_name(including_second_key=True))) + + # keep searching if no exact match was found + else: + + log.debug2(f"No exact match found. Trying to find {object_type.name} based on MAC addresses") + + # on VMs vnic data is used, on physical devices pnic data is used + mac_source_data = vnic_data if object_type == NBVM else pnic_data + + nic_macs = [x.get("mac_address") for x in mac_source_data.values() if x.get("mac_address") is not None] + + device_vm_object = self.get_object_based_on_macs(object_type, nic_macs) + + if device_vm_object is not None: + log.debug2("Found a matching %s object: %s" % + (object_type.name, device_vm_object.get_display_name(including_second_key=True))) + + # keep looking for devices with the same primary IP + else: + + log.debug2(f"No match found. Trying to find {object_type.name} based on primary IP addresses") + + device_vm_object = self.get_object_based_on_primary_ip(object_type, p_ipv4, p_ipv6) + + if device_vm_object is None: + object_name = object_data.get(object_type.primary_key) + log.debug(f"No existing {object_type.name} object for {object_name}. Creating a new {object_type.name}.") + device_vm_object = self.inventory.add_object(object_type, data=object_data, source=self) + else: + device_vm_object.update(data=object_data, source=self) + + # update role according to config settings + object_name = object_data.get(object_type.primary_key) + role_name = self.get_object_relation(object_name, + "host_role_relation" if object_type == NBDevice else "vm_role_relation") + + if object_type == NBDevice: + if role_name is None: + role_name = "Server" + device_vm_object.update(data={"device_role": {"name": role_name}}) + if object_type == NBVM and role_name is not None: + device_vm_object.update(data={"role": {"name": role_name}}) + + # compile all nic data into one dictionary + if object_type == NBVM: + nic_data = vnic_data else: - parent = "virtual_machine" if object_type is NBVirtualDisk else "cluster" if object_type is NBVM else "group" - if any(item.data.get("name") == data["name"] and item.data.get(parent) is data.get(parent) - for item in self.inventory.get_all_items(object_type)): - raise ValueError(f"NetBox {object_type.name} name collision for AWS resource '{identity}'") - item = self.inventory.add_object(object_type, data=data, source=self) - self.reconcile_tags(item, data["tags"]) - return item - - def service_cluster(self, account, region, site, group, service, identity, name, - metadata=None, tags=None): + nic_data = {**pnic_data, **vnic_data} + exclude_filter = self.settings.vm_interface_exclude_filter if object_type == NBVM else \ + self.settings.host_interface_exclude_filter + if exclude_filter is not None: + nic_data = {name: data for name, data in nic_data.items() if not exclude_filter.match(name)} + + mac_matched_interfaces = set() + if object_type == NBVM: + current_by_mac = {} + for interface in self.inventory.get_all_interfaces(device_vm_object): + existing_name = grab(interface, "data.name") + if exclude_filter is not None and existing_name is not None and exclude_filter.match(existing_name): + continue + if version.parse(self.inventory.netbox_api_version) >= version.parse("4.2.0"): + mac = grab(interface, "data.primary_mac_address.data.mac_address") + else: + mac = grab(interface, "data.mac_address") + if mac: + current_by_mac.setdefault(normalize_mac_address(mac), []).append(interface) + names_by_mac = {} + for int_name, int_data in nic_data.items(): + mac = int_data.get("mac_address") + matches = current_by_mac.get(normalize_mac_address(mac), []) if mac else [] + if len(matches) > 1: + message = f"Multiple interfaces of OpenStack VM '{object_name}' match MAC '{mac}'." + log.error(message) + raise ValueError(message) + if matches: + names_by_mac[int_name] = matches[0].data["name"] + mac_matched_interfaces.add(matches[0]) + used_names = set(names_by_mac.values()) + remapped_data = {} + remapped_ips = {} + for int_name, int_data in nic_data.items(): + target_name = names_by_mac.get(int_name, int_name) + if int_name not in names_by_mac: + index = 1 + network_label = int_name.partition(" ")[2] + while target_name in used_names: + target_name = f"vNIC{index} {network_label}" + index += 1 + used_names.add(target_name) + if int_name not in names_by_mac: + int_data["description"] = target_name + int_data["name"] = target_name + remapped_data[target_name] = int_data + remapped_ips[target_name] = nic_ips.get(int_name, []) + nic_data = remapped_data + nic_ips = remapped_ips + + # map interfaces of existing object with discovered interfaces + nic_object_dict = self.map_object_interfaces_to_current_interfaces(device_vm_object, nic_data) + + if object_data.get("status", "") == "active" and (nic_ips is None or len(nic_ips.keys()) == 0): + log.debug(f"No IP addresses for '{object_name}' found!") + + primary_ipv4_object = None + primary_ipv6_object = None + + if p_ipv4 is not None: + try: + primary_ipv4_object = ip_interface(p_ipv4) + except ValueError: + log.error(f"Primary IPv4 ({p_ipv4}) does not appear to be a valid IP address (needs included suffix).") + + if p_ipv6 is not None: + try: + primary_ipv6_object = ip_interface(p_ipv6) + except ValueError: + log.error(f"Primary IPv6 ({p_ipv6}) does not appear to be a valid IP address (needs included suffix).") + + for int_name, int_data in nic_data.items(): + existing_interface = nic_object_dict.get(int_name) + if existing_interface is not None: + existing_name = grab(existing_interface, "data.name") + if existing_name is not None and ( + object_type == NBDevice or existing_interface in mac_matched_interfaces): + int_data["name"] = existing_name + if object_type == NBDevice: + int_data.pop("type", None) + + # add/update interface with retrieved data + nic_object, ip_address_objects = self.add_update_interface(nic_object_dict.get(int_name), device_vm_object, + int_data, nic_ips.get(int_name, list())) + + # add all interface IPs + for ip_object in ip_address_objects: + + if ip_object is None: + continue + + ip_interface_object = ip_interface(grab(ip_object, "data.address")) + + # continue if address is not a primary IP + if ip_interface_object not in [primary_ipv4_object, primary_ipv6_object]: + continue + + # set/update/remove primary IP addresses + set_this_primary_ip = False + ip_version = ip_interface_object.ip.version + if self.settings.set_primary_ip == "always": + + for object_type in [NBDevice, NBVM]: + + # new IPs don't need to be removed from other devices/VMs + if ip_object.is_new is True: + break + + for devices_vms in self.inventory.get_all_items(object_type): + + # device has no primary IP of this version + this_primary_ip = grab(devices_vms, f"data.primary_ip{ip_version}") + + # we found this exact object + if devices_vms == device_vm_object: + continue + + # device has the same object assigned + if this_primary_ip == ip_object: + devices_vms.unset_attribute(f"primary_ip{ip_version}") + + set_this_primary_ip = True + + elif self.settings.set_primary_ip != "never" and \ + grab(device_vm_object, f"data.primary_ip{ip_version}") is None: + set_this_primary_ip = True + + if set_this_primary_ip is True: + + log.debug(f"Setting IP '{grab(ip_object, 'data.address')}' as primary IPv{ip_version} for " + f"'{device_vm_object.get_display_name()}'") + device_vm_object.update(data={f"primary_ip{ip_version}": ip_object}) + + return device_vm_object + + def add_object_to_cache(self, obj_type, key, netbox_object): + + if obj_type is None or key is None or netbox_object is None: + return + + # noinspection PyBroadException + try: + vm_class_name = obj_type.__name__ + except Exception: + return + + if self.object_cache.get(vm_class_name) is None: + self.object_cache[vm_class_name] = dict() + + self.object_cache[vm_class_name][key] = netbox_object + + def get_object_from_cache(self, obj_type, key): + + if obj_type is None or key is None: + return + + # noinspection PyBroadException + try: + vm_class_name = obj_type.__name__ + except Exception: + return + + if self.object_cache.get(vm_class_name) is None: + return + + return self.object_cache[vm_class_name].get(key) + + def add_cluster(self, obj): + """ + Add a OpenStack Availability Zone as a NBCluster to NetBox. Cluster name is checked against + cluster_include_filter and cluster_exclude_filter config setting. Also adds + cluster and site_name to "self.permitted_clusters" so hosts and VMs can be + checked if they are part of a permitted cluster. + + Parameters + ---------- + obj: openstack.compute.v2.availability_zone.AvailabilityZone + cluster to add + """ + + name = get_string_or_none(grab(obj, "name")) + group = self.inventory.get_by_data(NBClusterGroup, data={"name": self.settings.group_name}) + + if name is None or group is None: + return + + group_name = grab(group, "data.name") + + log.debug(f"Parsing OpenStack AZ: {name}") + + if self.passes_filter(name, self.settings.cluster_include_filter, + self.settings.cluster_exclude_filter) is False: + return + + site_name = self.get_site_name(NBCluster, name) + data = { - "name": self.resource_name(name, identity, 100), "group": group, - "type": {"name": f"AWS {service.upper() if service != 'lambda' else 'Lambda'}"}, - "custom_fields": {"aws_service": service, "aws_resource_id": identity, - "aws_metadata": metadata or {"account": account, "region": region}}, - "tags": self.service_tags(tags or {}), + "name": name, + "type": {"name": "Openstack"}, + "group": group, + "site": {"name": site_name} } - if Version(self.inventory.netbox_api_version) >= Version("4.2"): - data.update(scope_type="dcim.site", scope_id=site) + if version.parse(self.inventory.netbox_api_version) >= version.parse("4.2.0"): + data.pop("site") + data["scope_type"] = NBSite.object_type + data["scope_id"] = self.inventory.add_update_object(NBSite, data={"name": site_name}, source=self) + tags = self.get_object_relation(name, "cluster_tag_relation") + if tags: + data["tags"] = tags + + tenant_name = self.get_object_relation(name, "cluster_tenant_relation") + if tenant_name is not None: + data["tenant"] = {"name": tenant_name} + + # try to find cluster including cluster group + log.debug2("Trying to find a matching existing cluster") + cluster_object = None + for cluster_candidate in self.inventory.get_all_items(NBCluster): + if grab(cluster_candidate, "data.name") != name: + continue + if cluster_candidate.get_site_name() not in (None, site_name) and \ + self.object_synced_by_other_source(cluster_candidate): + continue + + # try to find a cluster with matching site + candidate_group = grab(cluster_candidate, "data.group.data.name") + if candidate_group not in (None, group_name): + continue + + if cluster_candidate.get_site_name() == site_name: + cluster_object = cluster_candidate + log.debug2("Found an existing cluster where 'name' and 'site' are matching") + break + + if grab(cluster_candidate, "data.group") is not None and \ + grab(cluster_candidate, "data.group.data.name") == group_name: + cluster_object = cluster_candidate + log.debug2("Found an existing cluster where 'name' and 'cluster group' are matching") + break + + if grab(cluster_candidate, "data.tenant") is not None and \ + tenant_name is not None and \ + grab(cluster_candidate, "data.tenant.data.name") == tenant_name: + cluster_object = cluster_candidate + log.debug2("Found an existing cluster where 'name' and 'tenant' are matching") + break + + if cluster_object is not None: + cluster_object.update(data=data, source=self) else: - data["site"] = site - return self.service_object(NBCluster, identity, data) + cluster_object = self.inventory.add_update_object(NBCluster, data=data, source=self) - def logical_vm(self, account, region, site, cluster, service, identity, name, metadata, tags, **attributes): - data = { - "name": self.resource_name(name, identity, 64), "cluster": cluster, "status": "active", - "comments": f"Logical AWS {service} inventory object; not an EC2 instance.", - "custom_fields": { - "aws_account_id": account, "aws_region": region, "aws_service": service, - "aws_resource_id": identity, "aws_metadata": metadata, + self.add_object_to_cache(NBCluster, name, cluster_object) + + self.cluster_host_map[name] = self.get_compute_host_names(grab(obj, "hosts")) + log.debug2(f"Found compute hosts for availability zone '{name}': " + f"{self.cluster_host_map[name]}") + + self.permitted_clusters[name] = site_name + + def add_host(self, obj): + """ + Parse a Openstack host to NetBox once all data is gathered. + + First host is filtered: + host has a cluster and is it permitted + was host with same name and site already parsed + does the host pass the host_include_filter and host_exclude_filter + + Then all necessary host data will be collected. + + Primary IPv4/6 will be determined by 'host_ip' value + + Parameters + ---------- + obj: Hypervisor + host object to parse + """ + + name = get_string_or_none(grab(obj, "name")) + if name is None: + return + + if name is not None and self.settings.strip_host_domain_name is True: + name = name.split(".")[0] + + # parse data + log.debug(f"Parsing Openstack host: {name}") + + # + # Filtering + # + + # manage site and cluster + short_name = get_string_or_none(obj.service_details["host"]) + cluster_name = self.get_cluster_for_host(short_name) + + if cluster_name is None: + # Nova omits disabled compute services from availability zone host lists + if get_string_or_none(grab(obj, "status")) == "disabled": + log.info(f"Host '{name}' is disabled in Nova and not part of any availability zone. Skipping.") + else: + log.error(f"Unable to find an availability zone for host '{name}'. Skipping.") + return + + if log.level == DEBUG3: + try: + log.info("Cluster data") + dump(obj.service_details.to_dict()) + except Exception as e: + log.error(e) + + if self.permitted_clusters.get(cluster_name) is None: + log.debug(f"Host '{name}' is not part of a permitted cluster. Skipping") + return + + # get cluster object + nb_cluster_object = self.get_object_from_cache(NBCluster, cluster_name) + + # get a site for this host + site_name = self.get_site_name(NBDevice, name, cluster_name) + + if name in self.processed_host_names.get(site_name, list()): + log.warning(f"Host '{name}' for site '{site_name}' already parsed. " + "Make sure to use unique host names. Skipping") + return + + # add host to processed list + if self.processed_host_names.get(site_name) is None: + self.processed_host_names[site_name] = list() + + self.processed_host_names[site_name].append(name) + + # filter hosts by name + if self.passes_filter(name, self.settings.host_include_filter, self.settings.host_exclude_filter) is False: + return + + # + # Collecting data + # + + # collect all necessary data + manufacturer = None + model = None + platform = get_string_or_none(obj.hypervisor_type) + + # if the device vendor/model cannot be retrieved (due to problem on the host), + # set a dummy value so the host still gets synced + if manufacturer is None: + manufacturer = "Generic Vendor" + if model is None: + model = "Generic Model" + + # get status + status = "offline" + if get_string_or_none(obj.status) == "enabled" and grab(obj, "state", fallback="up") == "up": + status = "active" + + # get host_tenant_relation + tenant_name = self.get_object_relation(name, "host_tenant_relation") + + # get host_tag_relation + host_tags = self.get_object_relation(name, "host_tag_relation") + + # prepare host data model + host_data = { + "name": name, + "device_type": { + "model": model, + "manufacturer": { + "name": manufacturer + } }, - "tags": [f"AWS:logical-{service}", *self.service_tags(tags)], - **attributes, + "site": {"name": site_name}, + "cluster": nb_cluster_object, + "status": status } - if Version(self.inventory.netbox_api_version) >= Version("3.3"): - data["site"] = site - if self.settings.tenant: - data["tenant"] = {"name": self.settings.tenant} - return self.service_object(NBVM, identity, data) - - def sync_lambda(self, account, region, site, cluster, function): - status = {"Pending": "planned", "Active": "active", "Inactive": "offline", "Failed": "failed"} - self.logical_vm( - account, region, site, cluster, "lambda", function["FunctionArn"], - f"Lambda {function['FunctionName']}", - {key: value for key, value in function.items() if key != "Tags"}, - function["Tags"], memory=function["MemorySize"], status=status[function["State"]]) - - def sync_s3(self, account, region, site, group, buckets): - if not buckets: + + # add data if present + if platform is not None: + host_data["platform"] = {"name": platform} + if tenant_name is not None: + host_data["tenant"] = {"name": tenant_name} + if len(host_tags) > 0: + host_data["tags"] = host_tags + self.add_custom_attributes(NBDevice, obj, host_data) + + pnic_data_dict = dict() + pnic_data = { + "name": "eth0", + "enabled": True, + "type": "virtual" + } + pnic_data_dict["eth0"] = pnic_data + + ip_addr = grab(obj, "host_ip") + if ip_addr is not None and not self.settings.permitted_subnets.permitted(ip_addr, interface_name="eth0"): + ip_addr = None + prefix = None + matched_prefix = None + if ip_addr is not None: + matched_prefix = self.return_longest_matching_prefix_for_ip(ip_interface(ip_addr), site_name) + if matched_prefix is None: + matched_prefix = self.return_longest_matching_prefix_for_ip(ip_interface(ip_addr)) + if matched_prefix is not None: + prefix = matched_prefix.data["prefix"].prefixlen + ip_addr = f"{ip_addr}/{prefix}" + + vnic_ips = dict() + vnic_ips["eth0"] = list() + if ip_addr is not None: + ip_addr = str(ip_interface(ip_addr)) + vnic_ips["eth0"].append(ip_addr) + + host_primary_ip4 = ip_addr if ip_addr is not None and ip_interface(ip_addr).version == 4 else None + host_primary_ip6 = ip_addr if ip_addr is not None and ip_interface(ip_addr).version == 6 else None + + # add host to inventory + host = self.add_device_vm_to_inventory(NBDevice, object_data=host_data, pnic_data=pnic_data_dict, + vnic_data=dict(), nic_ips=vnic_ips, + p_ipv4=host_primary_ip4, p_ipv6=host_primary_ip6) + self.add_object_to_cache(NBDevice, short_name, host) + + return + + def add_volume(self, obj): + """ + Parse OpenStack volume and store in in a map. + """ + + id = obj.id + size = obj.size + + self.volume_map[id] = size + + def add_virtual_machine(self, obj): + """ + Parse a OpenStack VM add to NetBox once all data is gathered. + + Parameters + ---------- + obj: openstack.compute.v2.server.Server + virtual machine object to parse + """ + + name = get_string_or_none(grab(obj, "name")) + if name is None: return - identity = f"s3/{account}/{region}" - try: - cluster = self.service_cluster( - account, region, site, group, "s3", identity, f"S3 {account}/{region}") - except ValueError as exc: - self.retain_source_objects() - log.warning(f"Skipping S3 buckets (account '{account}', region '{region}'): {exc}") + + if name is not None and self.settings.strip_vm_domain_name is True: + name = name.split(".")[0] + + if self.settings.set_vm_name_to_uuid: + name = get_string_or_none(obj.id) + + log.debug(f"Parsing OpenStack VM: {name}") + + # get VM power state + status = "active" if get_string_or_none(obj.status) == "ACTIVE" else "offline" + + cluster_name = get_string_or_none(obj.availability_zone) + + # check VM cluster + if cluster_name is None: + log.error(f"Requesting cluster for Virtual Machine '{name}' failed. Skipping.") return - try: - holder = self.logical_vm( - account, region, site, cluster, "s3", identity, f"S3 inventory {account}/{region}", - {"kind": "bucket-inventory-holder", "usage_status": "unknown"}, {}) - except ValueError as exc: - self.retain_source_objects() - log.warning(f"Skipping S3 buckets (account '{account}', region '{region}'): {exc}") + + elif self.permitted_clusters.get(cluster_name) is None: + log.debug(f"Virtual machine '{name}' is not part of a permitted cluster. Skipping") return - for bucket in buckets: - bucket_id = f"s3/{account}/{bucket['Name']}" - try: - self.service_object(NBVirtualDisk, bucket_id, { - "name": bucket["Name"], "virtual_machine": holder, - # NetBox requires size; zero is explicitly unknown, never a usage measurement. - "size": 0, - "description": "AWS S3 bucket; usage unknown (size 0 is a placeholder, not measured).", - "custom_fields": { - "aws_service": "s3", "aws_resource_id": bucket_id, - "aws_metadata": {**{key: value for key, value in bucket.items() if key != "Tags"}, - "usage_status": "unknown", "usage_bytes": None}, - }, - "tags": self.service_tags({tag["Key"]: tag["Value"] for tag in bucket["Tags"]}), - }) - except ValueError as exc: - log.warning(f"Skipping S3 bucket '{bucket['Name']}' " - f"(account '{account}', region '{region}'): {exc}") - - def resolve_shared_networks(self, snapshots): - networks = {} - for account, snapshot in snapshots.items(): - for region, data in snapshot.items(): - for kind in ("Vpcs", "Subnets"): - for resource_id, resource in data[kind].items(): - owner = resource.get("OwnerId", account) - key = (kind, region, resource_id) - owners = networks.setdefault(key, {}) - if resource.get("OwnerId") and any( - item.get("OwnerId") and previous_owner != owner - for previous_owner, item in owners.items()): - raise ValueError( - f"Conflicting AWS {kind} ownership for '{resource_id}' " - f"(region '{region}'): {', '.join(sorted(set(owners) | {owner}))}") - if owner in owners: - previous = owners[owner] - fields = ("VpcId", "CidrBlock") - if any(previous.get(field) != resource.get(field) for field in fields) or \ - set(self.ipv6_networks(previous)) != set(self.ipv6_networks(resource)): - raise ValueError( - f"Conflicting AWS {kind} discovery for '{resource_id}' " - f"(owner '{owner}', region '{region}')") - owners[owner] = resource - for account, snapshot in snapshots.items(): - for region, data in snapshot.items(): - data["VpcOwners"] = {} - for instance in data["instances"]: - for reference in instance.get("NetworkInterfaces", []): - eni = data["NetworkInterfaces"].get(reference["NetworkInterfaceId"]) - if eni is None: - continue - vpc_id, subnet_id = eni["VpcId"], eni["SubnetId"] - vpcs = networks.get(("Vpcs", region, vpc_id), {}) - local_vpc = data["Vpcs"].get(vpc_id) - owner = local_vpc.get("OwnerId", account) if local_vpc is not None else None - if owner is None and len(vpcs) == 1: - owner = next(iter(vpcs)) - if owner is None and vpcs: - raise ValueError( - f"Ambiguous AWS VPC ownership for '{vpc_id}' " - f"(account '{account}', region '{region}', " - f"ENI '{eni['NetworkInterfaceId']}'): {', '.join(sorted(vpcs))}") - if owner is None: - continue - data["VpcOwners"][vpc_id] = owner - data["Vpcs"][vpc_id] = dict(vpcs[owner], OwnerId=owner) - subnets = networks.get(("Subnets", region, subnet_id), {}) - if subnets and owner not in subnets: - raise ValueError( - f"Conflicting AWS subnet ownership for '{subnet_id}' " - f"(account '{account}', region '{region}', " - f"ENI '{eni['NetworkInterfaceId']}', VPC owner '{owner}'): " - f"{', '.join(sorted(subnets))}") - if owner in subnets: - data["Subnets"][subnet_id] = dict(subnets[owner], OwnerId=owner) - - def vrf_name(self, account, region, vpc): - return self.settings.vpc_vrf_relation.get( - f"{account}/{region}/{vpc}", self.settings.vpc_vrf_relation.get( - f"{region}/{vpc}", f"AWS {account}/{region}/{vpc}")) - - def get_vrf(self, account, region, vpc): - account_region_vpc = f"{account}/{region}/{vpc}" - region_vpc = f"{region}/{vpc}" - has_mapping = account_region_vpc in self.settings.vpc_vrf_relation or \ - region_vpc in self.settings.vpc_vrf_relation - name = self.vrf_name(account, region, vpc) - vrf = self.inventory.get_by_data(NBVRF, data={"name": name}) - if vrf is None: - if has_mapping: - raise ValueError(f"Create mapped NetBox VRF '{name}' for VPC '{vpc}' before importing its IPs") - vrf = self.inventory.add_object(NBVRF, data={"name": name}, source=self) - return vrf - - def ipv6_networks(self, resource): - return [ - ip_network(entry["Ipv6CidrBlock"]) - for entry in resource.get("Ipv6CidrBlockAssociationSet", []) - if entry.get("Ipv6CidrBlockState", {}).get("State") == "associated" - ] - - def addresses(self, eni, subnet): - networks = [ip_network(subnet["CidrBlock"]), *self.ipv6_networks(subnet)] - addresses = [ - (entry["PrivateIpAddress"], entry.get("Primary", False)) - for entry in eni.get("PrivateIpAddresses", []) - ] - addresses.extend( - (entry["Ipv6Address"], entry.get("IsPrimaryIpv6", False)) - for entry in eni.get("Ipv6Addresses", []) - ) - result = [] - for value, primary in addresses: - address = ip_address(value) - network = next((net for net in networks - if net.version == address.version and address in net), None) - if network is None: - raise ValueError(f"ENI '{eni['NetworkInterfaceId']}' IP '{value}' has no matching AWS subnet") - value = f"{address}/{network.prefixlen}" - if self.settings.permitted_subnets.permitted(value, interface_name=eni["NetworkInterfaceId"]): - result.append((value, primary)) - return result - - def validate_snapshot(self, account, snapshot, vrfs=None): - if vrfs is None: - vrfs = {} - for region, data in snapshot.items(): - for instance in data["instances"]: - spec = data["InstanceTypes"].get(instance["InstanceType"]) - if spec is None: - raise ValueError(f"Missing AWS instance type '{instance['InstanceType']}'") - if "DefaultVCpus" not in spec["VCpuInfo"] or "SizeInMiB" not in spec["MemoryInfo"]: - raise ValueError(f"Incomplete AWS specifications for '{instance['InstanceType']}'") - for tag in instance.get("Tags", []): - if tag["Key"] in self.settings.tag_keys and len(f"AWS:{tag['Key']}={tag['Value']}") > 100: - raise ValueError(f"AWS tag '{tag['Key']}' exceeds the NetBox tag name limit (100 characters)") - for attachment in instance.get("BlockDeviceMappings", []): - volume_id = attachment.get("Ebs", {}).get("VolumeId") - if volume_id and volume_id not in data["Volumes"]: - raise ValueError(f"Missing attached EBS volume '{volume_id}'") - for reference in instance.get("NetworkInterfaces", []): - eni = data["NetworkInterfaces"].get(reference["NetworkInterfaceId"]) - if eni is None: - raise ValueError(f"Missing attached ENI '{reference['NetworkInterfaceId']}'") - if eni.get("Attachment", {}).get("InstanceId") != instance["InstanceId"]: - raise ValueError(f"ENI '{eni['NetworkInterfaceId']}' changed attachment during discovery") - if not eni.get("MacAddress"): - raise ValueError(f"ENI '{eni['NetworkInterfaceId']}' has no MAC address") - if not self.settings.sync_private_ips: - continue - vpc, subnet = eni["VpcId"], data["Subnets"].get(eni["SubnetId"]) - network_errors = [] - if vpc not in data["Vpcs"]: - network_errors.append(f"VPC '{vpc}' was not returned by DescribeVpcs") - if subnet is None: - network_errors.append( - f"subnet '{eni['SubnetId']}' was not returned by DescribeSubnets") - elif subnet["VpcId"] != vpc: - network_errors.append( - f"subnet '{eni['SubnetId']}' belongs to VPC '{subnet['VpcId']}', " - f"not ENI VPC '{vpc}'") - if network_errors: - raise ValueError( - f"Incomplete VPC/subnet discovery for ENI '{eni['NetworkInterfaceId']}' " - f"(account '{account}', region '{region}', instance '{instance['InstanceId']}', " - f"VPC '{vpc}', subnet '{eni['SubnetId']}'): {'; '.join(network_errors)}") - if self.addresses(eni, subnet): - owner = data.get("VpcOwners", {}).get(vpc, account) - vrf = self.get_vrf(owner, region, vpc) - identity = (owner, region, vpc) - if vrf in vrfs and vrfs[vrf] != identity: - raise ValueError(f"VRF '{vrf.data['name']}' is mapped to more than one AWS VPC") - vrfs[vrf] = identity - - def custom_fields(self): - fields = ( - ("aws_account_id", "AWS account ID", [NBVM.object_type]), - ("aws_region", "AWS region", [NBVM.object_type]), - ("aws_instance_id", "AWS instance ID", [NBVM.object_type]), - ("aws_name", "AWS Name tag", [NBVM.object_type]), - ("aws_instance_type", "AWS instance type", [NBVM.object_type]), - ("aws_availability_zone", "AWS Availability Zone", [NBVM.object_type]), - ("aws_vpc_id", "AWS VPC ID", [NBVM.object_type, NBVMInterface.object_type]), - ("aws_eni_id", "AWS ENI ID", [NBVMInterface.object_type]), - ("aws_subnet_id", "AWS subnet ID", [NBVMInterface.object_type]), - ("aws_public_ipv4", "AWS public IPv4", [NBVMInterface.object_type]), - ) - for name, label, types in fields: - self.add_update_custom_field({ - "name": name, "label": label, "object_types": types, "type": "text", - "description": f"Reported by AWS ({self.name})", - }) - if any((self.settings.sync_s3, self.settings.sync_lambda, self.settings.sync_eks)): - types = [NBVM.object_type, NBCluster.object_type, NBVirtualDisk.object_type] - for name, field_type in ( - ("aws_service", "text"), ("aws_resource_id", "text"), - ("aws_metadata", "json"), - ): - field = self.add_update_custom_field({ - "name": name, "label": name.replace("_", " ").upper(), - "object_types": types, "type": field_type, - "description": f"Reported by AWS ({self.name})", + + nb_cluster_object = self.get_object_from_cache(NBCluster, cluster_name) + + if name in self.processed_vm_names.get(cluster_name, list()): + log.warning(f"Virtual machine '{name}' for cluster '{cluster_name}' already parsed. " + "Make sure to use unique VM names. Skipping") + return + + # add host to processed list + if self.processed_vm_names.get(cluster_name) is None: + self.processed_vm_names[cluster_name] = list() + + self.processed_vm_names[cluster_name].append(name) + + # filter VMs by name + if self.passes_filter(name, self.settings.vm_include_filter, self.settings.vm_exclude_filter) is False: + return + + # + # Collect data + # + + site_name = self.get_site_name(NBCluster, cluster_name) + flavor = dict(grab(obj, "flavor") or {}) + if not all(key in flavor for key in ("vcpus", "ram", "disk")) and flavor.get("id"): + flavor = self.session.compute.get_flavor(flavor["id"]).to_dict() + # first check against vm_platform_relation + platform = None + if self.settings.skip_vm_platform is False: + platform = get_string_or_none(flavor.get("original_name") or flavor.get("name")) + + if platform is not None: + platform = self.get_object_relation(platform, "vm_platform_relation", fallback=platform) + + disks = {} + if grab(obj, "image") and flavor.get("disk"): + disks["root"] = int(flavor["disk"]) + if flavor.get("ephemeral"): + disks["ephemeral"] = int(flavor["ephemeral"]) + disk_complete = True + for volume in grab(obj, "attached_volumes", fallback=[]) or []: + size = self.volume_map.get(volume["id"]) + if size is None: + log.warning(f"Volume '{volume['id']}' of VM '{name}' was not discovered; preserving disk size.") + disk_complete = False + disks[volume["id"]] = int(size) if size is not None else None + + annotation = None + if self.settings.skip_vm_comments is False: + if not self.settings.set_vm_name_to_uuid: + annotation = get_string_or_none(obj.id) + + # assign vm_tenant_relation + tenant_name = self.get_object_relation(name, "vm_tenant_relation") + + vm_data = { + "name": name, + "cluster": nb_cluster_object, + "status": status, + } + if flavor.get("vcpus"): + vm_data["vcpus"] = float(flavor["vcpus"]) + if flavor.get("ram") is not None: + vm_data["memory"] = int(flavor["ram"]) + if disk_complete and version.parse(self.inventory.netbox_api_version) < version.parse("3.7.0"): + vm_data["disk"] = sum(disks.values()) + tags = self.get_object_relation(name, "vm_tag_relation") + if tags: + vm_data["tags"] = tags + host = self.get_object_from_cache(NBDevice, grab(obj, "compute_host")) + if host is not None and version.parse(self.inventory.netbox_api_version) >= version.parse("3.3.0"): + vm_data["device"] = host + + # Add adaption for change in NetBox 3.3.0 VM model + # issue: https://github.com/netbox-community/netbox/issues/10131#issuecomment-1225783758 + if version.parse(self.inventory.netbox_api_version) >= version.parse("3.3.0"): + vm_data["site"] = {"name": site_name} + if platform is not None: + vm_data["platform"] = {"name": platform} + if annotation is not None: + vm_data["comments"] = annotation + if tenant_name is not None: + vm_data["tenant"] = {"name": tenant_name} + + if self.settings.set_vm_name_to_uuid: + custom_field = self.add_update_custom_field({ + "name": "openstack_vm_name", + "label": "name", + "object_types": ["virtualization.virtualmachine"], + "type": "text", + "description": f"Openstack '{self.name}' synced object attribute 'name'" }) - existing_type = field.data.get("type") - if isinstance(existing_type, dict): - existing_type = existing_type.get("value") - if existing_type != field_type: - raise ValueError( - f"NetBox custom field '{name}' must have type '{field_type}', not '{existing_type}'") - - def sync_snapshot(self, account, snapshot, account_name=None): - self.custom_fields() - account_name = account_name or account - group = self.inventory.add_update_object( - NBClusterGroup, data={"name": f"AWS {account_name}"}, source=self) - api_version = Version(self.inventory.netbox_api_version) - for region, data in snapshot.items(): - site = self.inventory.add_update_object( - NBSite, data={"name": f"AWS {region}"}, source=self) - cluster_data = { - "name": f"AWS {account_name}/{region}", "group": group, - "type": {"name": "AWS EC2"}, + vm_data["custom_fields"] = { + grab(custom_field, "data.name"): get_string_or_none(grab(obj, "name")) } - if api_version >= Version("4.2"): - cluster_data.update(scope_type="dcim.site", scope_id=site) - else: - cluster_data["site"] = site - cluster = self.inventory.add_update_object(NBCluster, data=cluster_data, source=self) - eks_clusters = {} - inaccessible = data.get("InaccessibleServices", []) - if self.settings.sync_eks and "eks" not in inaccessible: - for resource in data["EKSClusters"]: - try: - eks_cluster = self.service_cluster( - account, region, site, group, "eks", resource["arn"], - f"EKS {account}/{region}/{resource['name']}", - metadata={key: value for key, value in resource.items() if key != "tags"}, - tags=resource.get("tags", {})) - eks_clusters[resource["name"]] = eks_cluster - except ValueError as exc: - self.retain_source_objects() - log.warning(f"Skipping EKS cluster '{resource['name']}' " - f"(account '{account}', region '{region}'): {exc}") - if self.settings.sync_lambda and "lambda" not in inaccessible: + self.add_custom_attributes(NBVM, obj, vm_data) + + vm_primary_ip4 = None + vm_primary_ip6 = None + vm_primary_ip4_is_fixed = False + vm_primary_ip6_is_fixed = False + vm_nic_dict = dict() + nic_ips = dict() + count = 0 + + for network, addresses in (grab(obj, "addresses") or {}).items(): + default_mac = next((address.get("OS-EXT-IPS-MAC:mac_addr") for address in addresses + if address.get("OS-EXT-IPS-MAC:mac_addr")), None) + interfaces = {} + for address in addresses: + mac = address.get("OS-EXT-IPS-MAC:mac_addr") or default_mac + if mac not in interfaces: + count += 1 + full_name = unquote(f"vNIC{count} ({network})") + interfaces[mac] = full_name + vm_nic_dict[full_name] = { + "name": full_name, + "description": full_name, + "enabled": True, + } + if mac: + vm_nic_dict[full_name]["mac_address"] = normalize_mac_address(mac) + nic_ips[full_name] = [] + full_name = interfaces[mac] try: - lambda_cluster = self.service_cluster( - account, region, site, group, "lambda", f"lambda/{account}/{region}", - f"Lambda {account}/{region}") - except ValueError as exc: - self.retain_source_objects() - log.warning(f"Skipping Lambda functions (account '{account}', region '{region}'): {exc}") - else: - for function in data["Functions"]: - try: - self.sync_lambda(account, region, site, lambda_cluster, function) - except ValueError as exc: - self.retain_source_objects() - log.warning(f"Skipping function '{function['FunctionName']}' " - f"(account '{account}', region '{region}'): {exc}") - if self.settings.sync_s3 and "s3" not in inaccessible: - self.sync_s3(account, region, site, group, data["Buckets"]) - name_counts = {} - for instance in data["instances"]: - name = next((tag["Value"].strip() for tag in instance.get("Tags", []) - if tag["Key"] == "Name" and tag["Value"].strip()), instance["InstanceId"]) - key = name.casefold() - name_counts[key] = name_counts.get(key, 0) + 1 - for instance in data["instances"]: - tags = {tag["Key"]: tag["Value"] for tag in instance.get("Tags", [])} - vm_name = tags.get("Name", "").strip() or instance["InstanceId"] - if name_counts[vm_name.casefold()] > 1: - vm_name = f"{vm_name} ({instance['InstanceId']})" - eks_cluster_name = data.get("EKSWorkers", {}).get(instance["InstanceId"]) - if eks_cluster_name is not None and eks_cluster_name not in eks_clusters and \ - self.settings.sync_eks and "eks" not in inaccessible: - log.warning(f"Skipping instance '{instance['InstanceId']}' (account '{account}', " - f"region '{region}'): its EKS cluster '{eks_cluster_name}' was not synced.") + ip_addr = str(ip_interface(address["addr"])) + except (KeyError, ValueError): + log.warning(f"Invalid address on OpenStack VM '{name}'; skipping.") continue - self.sync_instance( - account, region, instance, data, site, - eks_clusters.get(eks_cluster_name, cluster), - api_version, vm_name=vm_name) - - def sync_instance(self, account, region, instance, data, site, cluster, api_version, vm_name=None): - instance_id = instance["InstanceId"] - if "eks" in data.get("InaccessibleServices", []): - existing = next((item for item in self.inventory.get_all_items(NBVM) - if (item.data.get("custom_fields") or {}).get("aws_account_id") == account and - (item.data.get("custom_fields") or {}).get("aws_instance_id") == instance_id), None) - if existing is not None and existing.data.get("cluster") is not None: - cluster = existing.data["cluster"] - tags = {tag["Key"]: tag["Value"] for tag in instance.get("Tags", [])} - spec = data["InstanceTypes"][instance["InstanceType"]] - vm_data = { - "name": vm_name or tags.get("Name", "").strip() or instance_id, "cluster": cluster, - "status": "active" if instance["State"]["Name"] == "running" else "offline", - "vcpus": spec["VCpuInfo"]["DefaultVCpus"], - "memory": spec["MemoryInfo"]["SizeInMiB"], - "custom_fields": { - "aws_account_id": account, "aws_region": region, - "aws_instance_id": instance_id, "aws_name": tags.get("Name", ""), - "aws_instance_type": instance["InstanceType"], - "aws_availability_zone": instance["Placement"]["AvailabilityZone"], - "aws_vpc_id": instance.get("VpcId", ""), - }, - "tags": [f"AWS:{key}={tags[key]}" for key in self.settings.tag_keys if key in tags], - } - cpu = instance.get("CpuOptions", {}) - if cpu.get("CoreCount") is not None and cpu.get("ThreadsPerCore") is not None: - vm_data["vcpus"] = cpu["CoreCount"] * cpu["ThreadsPerCore"] - if api_version >= Version("3.3"): - vm_data["site"] = site - if api_version >= Version("4.1"): - vm_data["serial"] = instance_id - if self.settings.tenant: - vm_data["tenant"] = {"name": self.settings.tenant} - disks = [ - data["Volumes"][entry["Ebs"]["VolumeId"]] - for entry in instance.get("BlockDeviceMappings", []) if "Ebs" in entry - ] - if api_version < Version("3.7"): - vm_data["disk"] = sum(volume["Size"] for volume in disks) - vm = next((item for item in self.inventory.get_all_items(NBVM) - if (item.data.get("custom_fields") or {}).get("aws_account_id") == account and - (item.data.get("custom_fields") or {}).get("aws_instance_id") == instance_id), None) - if vm is None: - vm = self.inventory.add_update_object(NBVM, data=vm_data, source=self) - else: - vm.update(data=vm_data, source=self) - self.reconcile_tags(vm, vm_data["tags"]) - if api_version >= Version("3.7"): - for volume in disks: - self.add_update_vm_child(NBVirtualDisk, vm, { - "name": volume["VolumeId"], "virtual_machine": vm, - "size": volume["Size"] * (1024 if api_version >= Version("4.1") else 1), - "description": f"AWS EBS {volume['VolumeType']}", - }) - for reference in instance.get("NetworkInterfaces", []): - eni = data["NetworkInterfaces"][reference["NetworkInterfaceId"]] - self.sync_interface(account, region, vm, eni, data, api_version) - - def add_update_vm_child(self, object_type, vm, data): - # Display-name matching loses the VM's account/cluster when names repeat. - existing = next((item for item in self.inventory.get_all_items(object_type) - if item.data.get("virtual_machine") is vm and - item.data.get("name") == data["name"]), None) - if existing is None: - return self.inventory.add_object(object_type, data=data, source=self) - existing.update(data=data, source=self) - return existing - - def sync_interface(self, account, region, vm, eni, data, api_version): - interface_data = { - "name": eni["NetworkInterfaceId"], "virtual_machine": vm, "enabled": True, - "description": eni.get("Description", ""), - "custom_fields": { - "aws_eni_id": eni["NetworkInterfaceId"], "aws_vpc_id": eni.get("VpcId", ""), - "aws_subnet_id": eni.get("SubnetId", ""), - "aws_public_ipv4": ", ".join(sorted({ - address["Association"]["PublicIp"] - for address in eni.get("PrivateIpAddresses", []) - if address.get("Association", {}).get("PublicIp") - })), - }, - } - mac = normalize_mac_address(eni.get("MacAddress")) - if api_version < Version("4.2"): - interface_data["mac_address"] = mac - interface = self.add_update_vm_child(NBVMInterface, vm, interface_data) - if api_version >= Version("4.2") and mac is not None: - mac_object = next((item for item in self.inventory.get_all_items(NBMACAddress) - if item.data.get("mac_address") == mac and - item.data.get("assigned_object_id") in (None, interface)), None) - mac_data = {"mac_address": mac, "assigned_object_id": interface, - "assigned_object_type": NBVMInterface} - if mac_object is None: - mac_object = self.inventory.add_object(NBMACAddress, data=mac_data, source=self) - else: - mac_object.update(data=mac_data, source=self) - for old in interface.get_mac_addresses(): - if old is not mac_object: - old.remove_interface_association() - interface.update(data={"primary_mac_address": mac_object}, source=self) - if not self.settings.sync_private_ips: - for address in interface.get_ip_addresses(): - if self.source_tag in address.get_tags(): - address.set_source(self) - return - subnet = data["Subnets"][eni["SubnetId"]] - addresses = self.addresses(eni, subnet) - discovered = [] - for address, primary in addresses: - owner = data.get("VpcOwners", {}).get(eni["VpcId"], account) - vrf = self.get_vrf(owner, region, eni["VpcId"]) - parsed = ip_interface(address) - candidates = [ - item for item in self.inventory.get_all_items(NBIPAddress) - if ip_interface(item.data["address"]).ip == parsed.ip and item.data.get("vrf") is vrf - ] - if len(candidates) > 1: - raise ValueError(f"Multiple NetBox IPs match '{address}' in VRF '{vrf.data['name']}'") - existing = candidates[0] if candidates else None - if existing is not None and existing.data.get("assigned_object_id") is not None and \ - existing.get_interface() is not interface: - owner = existing.get_device_vm() - owner_tags = owner.get_tags() if owner is not None else [] - same_source = owner is not None and ( - owner.source is self or self.source_tag in owner_tags) - managed_owner = same_source or NetBoxHandler.primary_tag in owner_tags - managed_orphan = NetBoxHandler.primary_tag in owner_tags and \ - NetBoxHandler.orphaned_tag in owner_tags - owner_status = owner.data.get("status") if owner is not None else None - if isinstance(owner_status, dict): - owner_status = owner_status.get("value") - if managed_owner and owner_status == "active" and vm.data["status"] == "offline": - log.error( - f"AWS source '{self.name}': IP '{address}' in VRF '{vrf.data['name']}' " - f"is assigned to active owner '{owner.get_display_name()}'. " - f"Skipping reassignment to offline ENI '{eni['NetworkInterfaceId']}'.") + if not self.settings.permitted_subnets.permitted(ip_addr, interface_name=full_name): continue - offline_to_active = managed_owner and owner_status == "offline" and vm.data["status"] == "active" - if not (same_source or managed_orphan or offline_to_active): - owner_name = owner.get_display_name() if owner is not None else "unresolved owner" - log.error( - f"AWS source '{self.name}': IP '{address}' in VRF '{vrf.data['name']}' " - f"belongs to another source (owner '{owner_name}'). " - f"Skipping reassignment to ENI '{eni['NetworkInterfaceId']}'.") + + matched_prefix = self.return_longest_matching_prefix_for_ip(ip_interface(ip_addr), site_name) + if matched_prefix is None: + matched_prefix = self.return_longest_matching_prefix_for_ip(ip_interface(ip_addr)) + if matched_prefix is not None: + prefix = matched_prefix.data["prefix"].prefixlen + ip_addr = f"{ip_interface(ip_addr).ip}/{prefix}" + + nic_ips[full_name].append(ip_addr) + is_fixed = address.get("OS-EXT-IPS:type") == "fixed" + if ip_interface(ip_addr).version == 4 and ( + vm_primary_ip4 is None or (is_fixed and not vm_primary_ip4_is_fixed)): + vm_primary_ip4 = ip_addr + vm_primary_ip4_is_fixed = is_fixed + if ip_interface(ip_addr).version == 6 and ( + vm_primary_ip6 is None or (is_fixed and not vm_primary_ip6_is_fixed)): + vm_primary_ip6 = ip_addr + vm_primary_ip6_is_fixed = is_fixed + + # add VM to inventory + vm = self.add_device_vm_to_inventory(NBVM, object_data=vm_data, vnic_data=vm_nic_dict, + nic_ips=nic_ips, p_ipv4=vm_primary_ip4, p_ipv6=vm_primary_ip6) + if version.parse(self.inventory.netbox_api_version) >= version.parse("3.7.0"): + for disk_name, disk_size in disks.items(): + data = {"name": disk_name, "virtual_machine": vm} + if disk_size is None: + existing = self.inventory.get_by_data(NBVirtualDisk, data=data) + if existing is not None: + existing.update(data=data, source=self) continue - if managed_orphan or offline_to_active: - reason = "orphaned" if managed_orphan else "offline" - log.warning( - f"AWS source '{self.name}': Reclaiming IP '{address}' in VRF '{vrf.data['name']}' " - f"from {reason} owner '{owner.get_display_name()}' " - f"for ENI '{eni['NetworkInterfaceId']}'.") - ip_data = {"address": address, "vrf": vrf, "assigned_object_id": interface, - "assigned_object_type": NBVMInterface} - if self.settings.tenant: - ip_data["tenant"] = {"name": self.settings.tenant} - if existing is None: - existing = self.inventory.add_object(NBIPAddress, data=ip_data, source=self) - else: - existing.update(data=ip_data, source=self) - discovered.append(existing) - if eni.get("Attachment", {}).get("DeviceIndex") == 0 and primary: - vm.update(data={f"primary_ip{parsed.version}": existing}, source=self) - for old in interface.get_ip_addresses(): - if old not in discovered and (old.source is self or self.source_tag in old.get_tags()): - old.remove_interface_association() + multiplier = 1000 if version.parse(self.inventory.netbox_api_version) >= version.parse("4.1.0") else 1 + data["size"] = disk_size * multiplier + self.inventory.add_update_object(NBVirtualDisk, data=data, source=self) + + return + + def update_basic_data(self): + """ + + Returns + ------- + + """ + + # add source identification tag + self.inventory.add_update_object(NBTag, data={ + "name": self.source_tag, + "description": f"Marks objects synced from Openstack '{self.name}' " + f"({self.settings.auth_url}) to this NetBox Instance." + }) + + # update virtual site if present + this_site_object = self.inventory.get_by_data(NBSite, data={"name": self.site_name}) + + if this_site_object is not None: + this_site_object.update(data={ + "name": self.site_name, + "comments": "A default virtual site created to house objects " + "that have been synced from this Openstack instance " + "and have no predefined site assigned." + }) + + server_role_object = self.inventory.get_by_data(NBDeviceRole, data={"name": "Server"}) + + if server_role_object is not None: + role_data = {"name": "Server", "vm_role": True} + if server_role_object.is_new is True: + role_data["color"] = "9e9e9e" + + server_role_object.update(data=role_data) + +# EOF From 469030cd5efb5b4b393dad9ca085aad59ffb67f5 Mon Sep 17 00:00:00 2001 From: Eric Krichbaum Date: Sat, 14 Nov 2026 15:37:28 -0600 Subject: [PATCH 14/21] Enhance OpenStack tests for port management Added tests for detaching ports and handling stale IPs. --- tests/test_openstack_source.py | 78 +++++++++++++++++++++++++++++++++- 1 file changed, 77 insertions(+), 1 deletion(-) diff --git a/tests/test_openstack_source.py b/tests/test_openstack_source.py index 2a80bfc8..79e3026b 100644 --- a/tests/test_openstack_source.py +++ b/tests/test_openstack_source.py @@ -90,6 +90,8 @@ def test_discovery_uses_current_inventory_apis(cloud, netbox_version): handler = OpenStackHandler("cloud") assert handler.init_successful assert cloud.sdk.connect.call_args.kwargs["verify"] is True + assert cloud.sdk.connect.call_args.kwargs["load_yaml_config"] is False + assert cloud.sdk.connect.call_args.kwargs["load_envvars"] is False handler.apply() cluster = handler.get_object_from_cache(NBCluster, "zone.example") assert cluster.get_site_name() == "Cloud" @@ -148,7 +150,8 @@ def test_registration_authentication_and_project_scope(cloud): cloud.sdk.connect.assert_called_once_with( auth_url=cloud.settings["auth_url"], project_name="admin", username="sync", password="test-only", region_name="RegionOne", user_domain_name="Default", - project_domain_name="Default", verify=False, app_name="netbox-sync", app_version="0.1", + project_domain_name="Default", verify=False, load_yaml_config=False, load_envvars=False, + app_name="netbox-sync", app_version="0.1", ) cloud.session.authorize.assert_called_once() handler.apply() @@ -279,6 +282,79 @@ def test_interface_names_and_host_interface_type_are_preserved(cloud): assert host_interface.data["type"] == "1000base-t" +def test_detaching_first_port_keeps_second_port_interface_and_ip(cloud): + first = {"addr": "10.0.0.41", "version": 4, "OS-EXT-IPS-MAC:mac_addr": "fa:16:3e:00:00:01"} + second = {"addr": "10.0.0.42", "version": 4, "OS-EXT-IPS-MAC:mac_addr": "fa:16:3e:00:00:02"} + cloud.session.compute.servers.return_value = [server(addresses={"private": [first, second]})] + handler = OpenStackHandler("cloud") + handler.apply() + + vm = cloud.inventory.get_all_items(NBVM)[0] + second_interface = next(interface for interface in cloud.inventory.get_all_interfaces(vm) + if interface.data["name"] == "vNIC2 (private)") + cloud.session.compute.servers.return_value = [server(addresses={"private": [second]})] + handler.apply() + + assert second_interface.data["name"] == "vNIC2 (private)" + assert {ip.data["address"] for ip in second_interface.get_ip_addresses()} == {"10.0.0.42/32"} + + +def test_replacing_private_port_with_public_port_renames_interface(cloud): + private = {"addr": "10.0.0.41", "version": 4, "OS-EXT-IPS-MAC:mac_addr": "fa:16:3e:00:00:01"} + public = {"addr": "10.0.0.42", "version": 4, "OS-EXT-IPS-MAC:mac_addr": "fa:16:3e:00:00:02"} + cloud.session.compute.servers.return_value = [server(addresses={"private": [private]})] + handler = OpenStackHandler("cloud") + handler.apply() + + vm = cloud.inventory.get_all_items(NBVM)[0] + interface = cloud.inventory.get_all_interfaces(vm)[0] + cloud.session.compute.servers.return_value = [server(addresses={"public": [public]})] + handler.apply() + + assert interface.data["name"] == "vNIC1 (public)" + assert interface.data["description"] == "vNIC1 (public)" + assert "10.0.0.42/32" in {ip.data["address"] for ip in interface.get_ip_addresses()} + + +@pytest.mark.parametrize("all_projects, expected_vm_name", [ + (True, "replacement.example.org"), + (False, "deleted.example.org"), +]) +def test_stale_ip_repair_requires_complete_all_projects_discovery(cloud, all_projects, expected_vm_name): + cloud.settings["all_projects"] = all_projects + old_id = "00000000-0000-0000-0000-000000000001" + new_id = "00000000-0000-0000-0000-000000000002" + old_port = { + "addr": "10.0.0.42", "version": 4, + "OS-EXT-IPS-MAC:mac_addr": "fa:16:3e:00:00:01", + } + cloud.session.compute.servers.return_value = [ + server(id=old_id, name="deleted.example.org", addresses={"private": [old_port]}) + ] + handler = OpenStackHandler("cloud") + handler.apply() + old_vm = cloud.inventory.get_all_items(NBVM)[0] + stale_ip = old_vm.data["primary_ip4"] + old_vm.add_tags(handler.source_tag) + stale_ip.add_tags(handler.source_tag) + old_vm.source = None + stale_ip.source = None + + replacement = [ + {"addr": "10.0.0.43", "version": 4, + "OS-EXT-IPS-MAC:mac_addr": "fa:16:3e:00:00:02"}, + {"addr": "10.0.0.42", "version": 4, + "OS-EXT-IPS-MAC:mac_addr": "fa:16:3e:00:00:03"}, + ] + cloud.session.compute.servers.return_value = [ + server(id=new_id, name="replacement.example.org", addresses={"private": replacement}) + ] + handler.apply() + + assigned_vm = stale_ip.get_device_vm() + assert assigned_vm.data["name"] == expected_vm_name + + def test_network_ports_invalid_addresses_and_floating_primary_preference(cloud): cloud.settings["permitted_subnets"] += ", 192.0.2.0/24" vm_resource = server() From 42a9f153b989f23290411c6dfd0f3d2458f260be Mon Sep 17 00:00:00 2001 From: Eric Krichbaum Date: Sat, 14 Nov 2026 15:39:51 -0600 Subject: [PATCH 15/21] Implement can_reassign_stale_ip method Add method to check if stale IP can be reassigned. --- module/sources/common/source_base.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/module/sources/common/source_base.py b/module/sources/common/source_base.py index 48d4af16..e44de2b5 100644 --- a/module/sources/common/source_base.py +++ b/module/sources/common/source_base.py @@ -52,6 +52,9 @@ def ip_address_objects_for(self, address: str) -> list: def set_source_tag(self): self.source_tag = f"Source: {self.name}" + def can_reassign_stale_ip(self, ip, device_vm_object): + return False + @classmethod def implements(cls, source_type): @@ -706,6 +709,11 @@ def add_update_interface(self, interface_object, device_object, interface_data, if "active" not in str(grab(current_ip_device, "data.status")): current_nic_enabled = False + if current_nic_enabled is True and this_nic_enabled is True and \ + self.can_reassign_stale_ip(ip, device_object): + this_ip_object = ip + break + if current_nic_enabled is True and this_nic_enabled is False: log.debug(f"Current interface '{current_ip_nic.get_display_name()}' for IP '{ip_object}'" f" is enabled and this one '{interface_object.get_display_name()}' is disabled. " From 04ad2ac968e624a046bf3ed64df8cfd1ae1c4bcc Mon Sep 17 00:00:00 2001 From: Eric Krichbaum Date: Sat, 14 Nov 2026 15:45:36 -0600 Subject: [PATCH 16/21] Add Counter for server IP counts and improve UUID handling --- module/sources/openstack/connection.py | 156 ++++++++++++++++++++++++- 1 file changed, 151 insertions(+), 5 deletions(-) diff --git a/module/sources/openstack/connection.py b/module/sources/openstack/connection.py index 01122ac4..2586d021 100644 --- a/module/sources/openstack/connection.py +++ b/module/sources/openstack/connection.py @@ -8,9 +8,11 @@ # repository or visit: . import pprint +from collections import Counter from ipaddress import ip_interface from importlib import import_module from urllib.parse import unquote +from uuid import UUID from packaging import version @@ -20,6 +22,7 @@ from module.common.misc import grab, dump, get_string_or_none from module.common.support import normalize_mac_address from module.netbox.inventory import NetBoxInventory +from module.netbox.connection import NetBoxHandler from module.netbox import * log = get_logger() @@ -71,6 +74,8 @@ def __init__(self, name=None): self.inventory = NetBoxInventory() self.name = name self.session = None + self._server_snapshot_ids = None + self._server_ip_counts = Counter() self.init_successful = False # parse settings @@ -122,6 +127,8 @@ def create_openstack_session(self): try: self.session = openstack.connect( + load_yaml_config=False, + load_envvars=False, auth_url=self.settings.auth_url, project_name=self.settings.project, username=self.settings.username, @@ -156,6 +163,8 @@ def apply(self): Every update of new/existing objects fot this source has to happen here. """ + self._server_snapshot_ids = None + self._server_ip_counts.clear() log.info(f"Query data from OpenStack: '{self.settings.auth_url}'") self.permitted_clusters.clear() self.cluster_host_map.clear() @@ -178,12 +187,104 @@ def apply(self): for volume in volumes: self.add_volume(volume) - servers = self.session.compute.servers(details=True, all_projects=self.settings.all_projects) + servers = list(self.session.compute.servers(details=True, all_projects=self.settings.all_projects)) + snapshot_ids = set() + for server in servers: + try: + snapshot_ids.add(str(UUID(str(server.id)))) + except (ValueError, AttributeError): + log.warning("OpenStack server snapshot contains an invalid UUID; stale IP repair is disabled.") + break + else: + self._server_snapshot_ids = snapshot_ids + for server in servers: + addresses = set() + for network_addresses in (grab(server, "addresses") or {}).values(): + for address in network_addresses: + try: + addresses.add(str(ip_interface(address["addr"]).ip)) + except (KeyError, ValueError): + continue + self._server_ip_counts.update(addresses) # Prefer an already-associated server before using status to break ties. - for server in sorted(servers, key=self.get_server_priority): - self.add_virtual_machine(server) + try: + for server in sorted(servers, key=self.get_server_priority): + self.add_virtual_machine(server) + + self.update_basic_data() + finally: + self._server_snapshot_ids = None + + def can_reassign_stale_ip(self, ip, device_vm_object): + def blocked(reason): + log.warning( + f"OpenStack source '{self.name}': stale IP repair blocked for " + f"'{ip.data['address']}' (NetBox IP ID {ip.nb_id}): {reason}." + ) + return False - self.update_basic_data() + if self._server_snapshot_ids is None: + return blocked("no complete server UUID snapshot is available") + if not self.settings.all_projects: + return blocked("all_projects is False; absence from this project does not prove deletion") + old_vm = ip.get_device_vm() + if not isinstance(old_vm, NBVM) or not isinstance(device_vm_object, NBVM): + return blocked("both owners must be virtual machines") + if old_vm.source is not None: + return blocked( + f"old VM '{old_vm.get_display_name()}' (NetBox VM ID {old_vm.nb_id}) " + f"was claimed this run by source '{getattr(old_vm.source, 'name', None)}'" + ) + if device_vm_object.source is not self: + return blocked("the target VM is not owned by this source in this run") + recovered_orphan_ownership = False + for item in (ip, old_vm): + source_tags = {tag for tag in item.get_tags() if tag.startswith("Source: ")} + if source_tags != {self.source_tag}: + if source_tags or NetBoxHandler.orphaned_tag not in item.get_tags(): + return blocked( + f"{item.name} '{item.get_display_name()}' (NetBox ID {item.nb_id}) has " + f"source tags {sorted(source_tags)}; expected only '{self.source_tag}' " + "or an orphaned record with verified cluster ownership" + ) + recovered_orphan_ownership = True + if recovered_orphan_ownership: + cluster = old_vm.data.get("cluster") + cluster_tags = {tag for tag in cluster.get_tags() if tag.startswith("Source: ")} \ + if isinstance(cluster, NBCluster) else set() + if not isinstance(cluster, NBCluster) or cluster.source is not self or \ + cluster_tags != {self.source_tag} or \ + self.get_object_from_cache(NBCluster, cluster.data.get("name")) is not cluster or \ + grab(cluster, "data.group.data.name") != self.settings.group_name: + return blocked("orphaned records lack an exclusively source-owned cluster in current discovery") + if ip.source is not None and ip.source is not self: + return blocked(f"IP was claimed by another source '{getattr(ip.source, 'name', None)}'") + identity_key = "name" if self.settings.set_vm_name_to_uuid else "comments" + identities = [] + for vm in (old_vm, device_vm_object): + try: + identities.append(str(UUID(str(vm.data.get(identity_key))))) + except ValueError: + return blocked( + f"VM '{vm.get_display_name()}' (NetBox VM ID {vm.nb_id}) has no valid " + f"server UUID in '{identity_key}'" + ) + old_id, current_id = identities + address = str(ip_interface(ip.data["address"]).ip) + if old_id in self._server_snapshot_ids: + return blocked(f"old server UUID {old_id} is still present in discovery") + if current_id not in self._server_snapshot_ids: + return blocked(f"target server UUID {current_id} is absent from discovery") + if self._server_ip_counts[address] != 1: + return blocked(f"{self._server_ip_counts[address]} discovered servers report this IP; expected one") + log.info( + f"Reclaiming OpenStack IP '{ip.data['address']}' (NetBox ID {ip.nb_id}) from VM " + f"'{old_vm.get_display_name()}' (UUID {old_id}), absent from the complete server snapshot, " + f"to '{device_vm_object.get_display_name()}' (UUID {current_id}) for source '{self.name}'." + + (" Recovered missing orphan Source tags using verified cluster ownership." + if recovered_orphan_ownership else "") + ) + return True @staticmethod def passes_filter(name, include_filter, exclude_filter): @@ -675,6 +776,50 @@ def add_device_vm_to_inventory(self, object_type, object_data, pnic_data=None, v if exclude_filter is not None: nic_data = {name: data for name, data in nic_data.items() if not exclude_filter.match(name)} + mac_matched_interfaces = set() + if object_type == NBVM: + current_by_mac = {} + for interface in self.inventory.get_all_interfaces(device_vm_object): + existing_name = grab(interface, "data.name") + if exclude_filter is not None and existing_name is not None and exclude_filter.match(existing_name): + continue + if version.parse(self.inventory.netbox_api_version) >= version.parse("4.2.0"): + mac = grab(interface, "data.primary_mac_address.data.mac_address") + else: + mac = grab(interface, "data.mac_address") + if mac: + current_by_mac.setdefault(normalize_mac_address(mac), []).append(interface) + names_by_mac = {} + for int_name, int_data in nic_data.items(): + mac = int_data.get("mac_address") + matches = current_by_mac.get(normalize_mac_address(mac), []) if mac else [] + if len(matches) > 1: + message = f"Multiple interfaces of OpenStack VM '{object_name}' match MAC '{mac}'." + log.error(message) + raise ValueError(message) + if matches: + names_by_mac[int_name] = matches[0].data["name"] + mac_matched_interfaces.add(matches[0]) + used_names = set(names_by_mac.values()) + remapped_data = {} + remapped_ips = {} + for int_name, int_data in nic_data.items(): + target_name = names_by_mac.get(int_name, int_name) + if int_name not in names_by_mac: + index = 1 + network_label = int_name.partition(" ")[2] + while target_name in used_names: + target_name = f"vNIC{index} {network_label}" + index += 1 + used_names.add(target_name) + if int_name not in names_by_mac: + int_data["description"] = target_name + int_data["name"] = target_name + remapped_data[target_name] = int_data + remapped_ips[target_name] = nic_ips.get(int_name, []) + nic_data = remapped_data + nic_ips = remapped_ips + # map interfaces of existing object with discovered interfaces nic_object_dict = self.map_object_interfaces_to_current_interfaces(device_vm_object, nic_data) @@ -700,7 +845,8 @@ def add_device_vm_to_inventory(self, object_type, object_data, pnic_data=None, v existing_interface = nic_object_dict.get(int_name) if existing_interface is not None: existing_name = grab(existing_interface, "data.name") - if existing_name is not None: + if existing_name is not None and ( + object_type == NBDevice or existing_interface in mac_matched_interfaces): int_data["name"] = existing_name if object_type == NBDevice: int_data.pop("type", None) From 6dcccc8706fe26859955164c9884b378d3596a42 Mon Sep 17 00:00:00 2001 From: Eric Krichbaum Date: Sat, 14 Nov 2026 15:46:24 -0600 Subject: [PATCH 17/21] replace correct AWS file from bade paste. --- module/sources/aws/connection.py | 2234 +++++++++++------------------- 1 file changed, 839 insertions(+), 1395 deletions(-) diff --git a/module/sources/aws/connection.py b/module/sources/aws/connection.py index 2586d021..8df4e270 100644 --- a/module/sources/aws/connection.py +++ b/module/sources/aws/connection.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright (c) 2020 - 2021 Ricardo Bartels. All rights reserved. +# Copyright (c) 2020 - 2026 netbox-sync team. All rights reserved. # # netbox-sync.py # @@ -7,1439 +7,883 @@ # For a copy, see file LICENSE.txt included in this # repository or visit: . -import pprint -from collections import Counter -from ipaddress import ip_interface -from importlib import import_module -from urllib.parse import unquote -from uuid import UUID +from datetime import datetime, timezone +from hashlib import sha256 +from ipaddress import ip_address, ip_interface, ip_network +from itertools import product -from packaging import version +from packaging.version import Version -from module.sources.common.source_base import SourceBase -from module.sources.openstack.config import OpenStackConfig -from module.common.logging import get_logger, DEBUG3 -from module.common.misc import grab, dump, get_string_or_none +from module.common.logging import get_logger +from module.common.misc import do_error_exit from module.common.support import normalize_mac_address +from module.netbox import ( + NBCustomField, NBCluster, NBClusterGroup, NBClusterType, NBIPAddress, + NBMACAddress, NBSite, NBTag, NBTenant, NBVirtualDisk, NBVM, NBVMInterface, NBVRF, +) from module.netbox.inventory import NetBoxInventory from module.netbox.connection import NetBoxHandler -from module.netbox import * +from module.sources.aws.client import AWSClient, AWSDiscoveryError +from module.sources.aws.config import AWSConfig +from module.sources.common.source_base import SourceBase log = get_logger() -# noinspection PyTypeChecker -class OpenStackHandler(SourceBase): +def json_safe(value): """ - Source class to import data from a Openstack instance and add/update NetBox objects based on gathered information + boto3 returns timestamps (EKS createdAt, modifiedAt, ...) as datetime objects. A NetBox + JSON field only takes plain JSON, so store them as ISO 8601 strings in UTC; the local + timezone of the host running the sync must not change the stored value. """ - + if isinstance(value, dict): + return {key: json_safe(item) for key, item in value.items()} + if isinstance(value, (list, tuple)): + return [json_safe(item) for item in value] + if isinstance(value, datetime): + if value.tzinfo is not None: + value = value.astimezone(timezone.utc) + return value.isoformat() + return value + + +class AWSHandler(SourceBase): + source_type = "aws" dependent_netbox_objects = [ - NBTag, - NBManufacturer, - NBDeviceType, - NBPlatform, - NBClusterType, - NBClusterGroup, - NBDeviceRole, - NBSite, - NBCluster, - NBDevice, - NBVM, - NBVirtualDisk, - NBVMInterface, - NBInterface, - NBIPAddress, - NBPrefix, - NBTenant, - NBVRF, - NBVLAN, - NBVLANGroup, - NBMACAddress, - NBFHRPGroupItem, - NBCustomField + NBTag, NBCustomField, NBTenant, NBSite, NBClusterGroup, NBClusterType, + NBCluster, NBVM, NBVMInterface, NBMACAddress, NBIPAddress, NBVRF, NBVirtualDisk, ] - source_type = "openstack" - - # internal vars - session = None - site_name = None - def __init__(self, name=None): - - if name is None: - raise ValueError(f"Invalid value for attribute 'name': '{name}'.") - - self.inventory = NetBoxInventory() + if not name: + raise ValueError(f"Invalid source name '{name}'.") self.name = name - self.session = None - self._server_snapshot_ids = None - self._server_ip_counts = Counter() - self.init_successful = False - - # parse settings - settings_handler = OpenStackConfig() - settings_handler.source_name = self.name - self.settings = settings_handler.parse() - + self.inventory = NetBoxInventory() + config = AWSConfig() + config.source_name = name + self.settings = config.parse() self.set_source_tag() - self.site_name = f"OpenStack: {name}" - - if self.settings.enabled is False: - log.info(f"Source '{name}' is currently disabled. Skipping") - return - - self.create_openstack_session() - - if self.session is None: - log.info(f"Source '{name}' is currently unavailable. Skipping") + self.init_successful = False + if not self.settings.enabled: + log.info(f"Source '{name}' is disabled. Skipping.") return - - self.init_successful = True - self.permitted_clusters = dict() - self.cluster_host_map = dict() - self.volume_map = dict() - self.processed_host_names = dict() - self.processed_vm_names = dict() - self.object_cache = dict() - - def create_openstack_session(self): - """ - Initialize session with OpenStack - - Returns - ------- - bool: if initialization was successful or not - """ - - if self.session is not None: - return True - - log.debug(f"Starting OpenStack connection to '{self.settings.auth_url}'") - - try: - openstack = import_module("openstack") - except ImportError: - log.error("OpenStack source requires openstacksdk. Install with " - "'pip install -r requirements-openstack.txt' or 'uv sync --extra openstack'.") - return False - try: - self.session = openstack.connect( - load_yaml_config=False, - load_envvars=False, - auth_url=self.settings.auth_url, - project_name=self.settings.project, - username=self.settings.username, - password=self.settings.password, - region_name=self.settings.region, - user_domain_name=self.settings.user_domain, - project_domain_name=self.settings.project_domain, - verify=self.settings.validate_tls_certs, - app_name='netbox-sync', - app_version='0.1', - ) - self.session.authorize() - except Exception as e: - log.error(f"Unable to connect to OpenStack instance '{self.settings.auth_url}': {e}") - self.finish() - return False - - log.info(f"Successfully connected to OpenStack '{self.settings.auth_url}'") - - return True + self.client = AWSClient(profile=self.settings.profile) + except AWSDiscoveryError as exc: + self.abort(exc) + self.init_successful = True - def finish(self): - if self.session is not None: - self.session.close() - self.session = None + def abort(self, error): + message = f"AWS source '{self.name}': {error}. No NetBox writes or pruning will be performed." + log.error(message) + do_error_exit(message) def apply(self): - """ - Main source handler method. This method is called for each source from "main" program - to retrieve data from it source and apply it to the NetBox inventory. - - Every update of new/existing objects fot this source has to happen here. - """ - - self._server_snapshot_ids = None - self._server_ip_counts.clear() - log.info(f"Query data from OpenStack: '{self.settings.auth_url}'") - self.permitted_clusters.clear() - self.cluster_host_map.clear() - self.volume_map.clear() - self.processed_host_names.clear() - self.processed_vm_names.clear() - self.object_cache.clear() - - self.inventory.add_update_object(NBClusterGroup, data={"name": self.settings.group_name}, source=self) - - availability_zones = self.session.compute.availability_zones(details=True) - for availability_zone in availability_zones: - self.add_cluster(availability_zone) - - hypervisors = self.session.compute.hypervisors(details=True) - for hypervisor in hypervisors: - self.add_host(hypervisor) - - volumes = self.session.block_storage.volumes(details=True, all_projects=self.settings.all_projects) - for volume in volumes: - self.add_volume(volume) - - servers = list(self.session.compute.servers(details=True, all_projects=self.settings.all_projects)) - snapshot_ids = set() - for server in servers: - try: - snapshot_ids.add(str(UUID(str(server.id)))) - except (ValueError, AttributeError): - log.warning("OpenStack server snapshot contains an invalid UUID; stale IP repair is disabled.") - break - else: - self._server_snapshot_ids = snapshot_ids - for server in servers: - addresses = set() - for network_addresses in (grab(server, "addresses") or {}).values(): - for address in network_addresses: - try: - addresses.add(str(ip_interface(address["addr"]).ip)) - except (KeyError, ValueError): - continue - self._server_ip_counts.update(addresses) - # Prefer an already-associated server before using status to break ties. try: - for server in sorted(servers, key=self.get_server_priority): - self.add_virtual_machine(server) - - self.update_basic_data() - finally: - self._server_snapshot_ids = None - - def can_reassign_stale_ip(self, ip, device_vm_object): - def blocked(reason): - log.warning( - f"OpenStack source '{self.name}': stale IP repair blocked for " - f"'{ip.data['address']}' (NetBox IP ID {ip.nb_id}): {reason}." - ) - return False - - if self._server_snapshot_ids is None: - return blocked("no complete server UUID snapshot is available") - if not self.settings.all_projects: - return blocked("all_projects is False; absence from this project does not prove deletion") - old_vm = ip.get_device_vm() - if not isinstance(old_vm, NBVM) or not isinstance(device_vm_object, NBVM): - return blocked("both owners must be virtual machines") - if old_vm.source is not None: - return blocked( - f"old VM '{old_vm.get_display_name()}' (NetBox VM ID {old_vm.nb_id}) " - f"was claimed this run by source '{getattr(old_vm.source, 'name', None)}'" - ) - if device_vm_object.source is not self: - return blocked("the target VM is not owned by this source in this run") - recovered_orphan_ownership = False - for item in (ip, old_vm): - source_tags = {tag for tag in item.get_tags() if tag.startswith("Source: ")} - if source_tags != {self.source_tag}: - if source_tags or NetBoxHandler.orphaned_tag not in item.get_tags(): - return blocked( - f"{item.name} '{item.get_display_name()}' (NetBox ID {item.nb_id}) has " - f"source tags {sorted(source_tags)}; expected only '{self.source_tag}' " - "or an orphaned record with verified cluster ownership" - ) - recovered_orphan_ownership = True - if recovered_orphan_ownership: - cluster = old_vm.data.get("cluster") - cluster_tags = {tag for tag in cluster.get_tags() if tag.startswith("Source: ")} \ - if isinstance(cluster, NBCluster) else set() - if not isinstance(cluster, NBCluster) or cluster.source is not self or \ - cluster_tags != {self.source_tag} or \ - self.get_object_from_cache(NBCluster, cluster.data.get("name")) is not cluster or \ - grab(cluster, "data.group.data.name") != self.settings.group_name: - return blocked("orphaned records lack an exclusively source-owned cluster in current discovery") - if ip.source is not None and ip.source is not self: - return blocked(f"IP was claimed by another source '{getattr(ip.source, 'name', None)}'") - identity_key = "name" if self.settings.set_vm_name_to_uuid else "comments" - identities = [] - for vm in (old_vm, device_vm_object): - try: - identities.append(str(UUID(str(vm.data.get(identity_key))))) - except ValueError: - return blocked( - f"VM '{vm.get_display_name()}' (NetBox VM ID {vm.nb_id}) has no valid " - f"server UUID in '{identity_key}'" - ) - old_id, current_id = identities - address = str(ip_interface(ip.data["address"]).ip) - if old_id in self._server_snapshot_ids: - return blocked(f"old server UUID {old_id} is still present in discovery") - if current_id not in self._server_snapshot_ids: - return blocked(f"target server UUID {current_id} is absent from discovery") - if self._server_ip_counts[address] != 1: - return blocked(f"{self._server_ip_counts[address]} discovered servers report this IP; expected one") - log.info( - f"Reclaiming OpenStack IP '{ip.data['address']}' (NetBox ID {ip.nb_id}) from VM " - f"'{old_vm.get_display_name()}' (UUID {old_id}), absent from the complete server snapshot, " - f"to '{device_vm_object.get_display_name()}' (UUID {current_id}) for source '{self.name}'." - + (" Recovered missing orphan Source tags using verified cluster ownership." - if recovered_orphan_ownership else "") - ) - return True - - @staticmethod - def passes_filter(name, include_filter, exclude_filter): - """ - checks if object name passes a defined object filter. - - Parameters - ---------- - name: str - name of the object to check - include_filter: regex object - regex object of include filter - exclude_filter: regex object - regex object of exclude filter - - Returns - ------- - bool: True if all filter passed, otherwise False - """ - - # first includes - if include_filter is not None and not include_filter.match(name): - log.debug(f"Object '{name}' did not match include filter '{include_filter.pattern}'. Skipping") - return False - - # second excludes - if exclude_filter is not None and exclude_filter.match(name): - log.debug(f"Object '{name}' matched exclude filter '{exclude_filter.pattern}'. Skipping") - return False - - return True - - def get_site_name(self, object_type, object_name, cluster_name=""): - """ - Return a site name for a NBCluster or NBDevice depending on config options - host_site_relation and cluster_site_relation - - Parameters - ---------- - object_type: (NBCluster, NBDevice) - object type to check site relation for - object_name: str - object name to check site relation for - cluster_name: str - cluster name of NBDevice to check for site name - - Returns - ------- - str: site name if a relation was found - """ - - if object_type not in [NBCluster, NBDevice]: - raise ValueError(f"Object must be a '{NBCluster.name}' or '{NBDevice.name}'.") - - log.debug2(f"Trying to find site name for {object_type.name} '{object_name}'") - - # check if site was provided in config - relation_name = "host_site_relation" if object_type == NBDevice else "cluster_site_relation" - - site_name = self.get_object_relation(object_name, relation_name) - - if object_type == NBDevice and site_name is None: - site_name = self.get_site_name(NBCluster, cluster_name) - if site_name is not None: - log.debug2(f"Found a matching cluster site for {object_name}, using site '{site_name}'") - - # set default site name - if site_name is None: - site_name = self.site_name - log.debug(f"No site relation for '{object_name}' found, using default site '{site_name}'") - - return site_name - - def get_object_based_on_macs(self, object_type, mac_list=None): - """ - Try to find a NetBox object based on list of MAC addresses. - - Iterate over all interfaces of this object type and compare MAC address with list of desired MAC - addresses. If match was found store related machine object and count every correct match. - - If exactly one machine with matching interfaces was found then this one will be returned. - - If two or more machines with matching MACs are found compare the two machines with - the highest amount of matching interfaces. If the ration of matching interfaces - exceeds 2.0 then the top matching machine is chosen as desired object. - - If the ration is below 2.0 then None will be returned. The probability is too low that - this one is the correct one. - - None will also be returned if no machine was found at all. - - Parameters - ---------- - object_type: (NBDevice, NBVM) - type of NetBox device to find in inventory - mac_list: list - list of MAC addresses to compare against NetBox interface objects - - Returns - ------- - (NBDevice, NBVM, None): object instance of found device, otherwise None - """ - - object_to_return = None - - if object_type not in [NBDevice, NBVM]: - raise ValueError(f"Object must be a '{NBVM.name}' or '{NBDevice.name}'.") - - if mac_list is None or not isinstance(mac_list, list) or len(mac_list) == 0: - return - - interface_typ = NBInterface if object_type == NBDevice else NBVMInterface - - objects_with_matching_macs = dict() - matching_object = None - - for interface in self.inventory.get_all_items(interface_typ): - - mac_address = grab(interface, "data.primary_mac_address.data.mac_address") or \ - grab(interface, "data.mac_address") - if mac_address in mac_list: - - matching_object = grab(interface, f"data.{interface.secondary_key}") - if not isinstance(matching_object, (NBDevice, NBVM)): - continue - - log.debug2("Found matching MAC '%s' on %s '%s'" % - (grab(interface, "data.mac_address"), object_type.name, - matching_object.get_display_name(including_second_key=True))) - - if objects_with_matching_macs.get(matching_object) is None: - objects_with_matching_macs[matching_object] = 1 - else: - objects_with_matching_macs[matching_object] += 1 - - # try to find object based on amount of matching MAC addresses - num_devices_witch_matching_macs = len(objects_with_matching_macs.keys()) - - if num_devices_witch_matching_macs == 1 and isinstance(matching_object, (NBDevice, NBVM)): - - log.debug2("Found one %s '%s' based on MAC addresses and using it" % - (object_type.name, matching_object.get_display_name(including_second_key=True))) - - object_to_return = list(objects_with_matching_macs.keys())[0] - - elif num_devices_witch_matching_macs > 1: - - log.debug2(f"Found {num_devices_witch_matching_macs} {object_type.name}s with matching MAC addresses") - - # now select the two top matches - first_choice, second_choice = \ - sorted(objects_with_matching_macs, key=objects_with_matching_macs.get, reverse=True)[0:2] - - first_choice_matches = objects_with_matching_macs.get(first_choice) - second_choice_matches = objects_with_matching_macs.get(second_choice) - - log.debug2(f"The top candidate {first_choice.get_display_name()} with {first_choice_matches} matches") - log.debug2(f"The second candidate {second_choice.get_display_name()} with {second_choice_matches} matches") - - # get ratio between - matching_ration = first_choice_matches / second_choice_matches - - # only pick the first one if the ration exceeds 2 - if matching_ration >= 2.0: - log.debug2(f"The matching ratio of {matching_ration} is high enough " - f"to select {first_choice.get_display_name()} as desired {object_type.name}") - object_to_return = first_choice + if self.settings.sync_s3 and Version(self.inventory.netbox_api_version) < Version("3.7"): + raise ValueError("S3 bucket inventory requires NetBox 3.7 or newer") + services = tuple(service for service in ("s3", "lambda", "eks") + if getattr(self.settings, f"sync_{service}")) + options = {"services": services} if services else {} + if self.settings.organization: + snapshots = self.client.discover_organization( + self.settings.regions, self.settings.organization_role_name, + expected_account=self.settings.account_id, + discovery_region=self.settings.discovery_region, + include_management=self.settings.organization_include_management, + external_id=self.settings.organization_external_id, + account_ids=self.settings.organization_account_ids, **options) + account_names = getattr(self.client, "account_names", {}) else: - log.debug2("Both candidates have a similar amount of " - "matching interface MAC addresses. Using NONE of them!") - - return object_to_return - - def get_object_based_on_primary_ip(self, object_type, primary_ip4=None, primary_ip6=None): - """ - Try to find a NBDevice or NBVM based on the primary IP address. If an exact - match was found the device/vm object will be returned immediately without - checking of the other primary IP address (if defined). - - Parameters - ---------- - object_type: (NBDevice, NBVM) - object type to look for - primary_ip4: str - primary IPv4 address of object to find - primary_ip6: str - primary IPv6 address of object to find - - Returns - ------- - - """ - - def _matches_device_primary_ip(device_primary_ip, ip_needle): - - ip = None - if device_primary_ip is not None and ip_needle is not None: - if isinstance(device_primary_ip, dict): - ip = grab(device_primary_ip, "address") - - elif isinstance(device_primary_ip, NBIPAddress): - ip = device_primary_ip.data.get("address") - elif isinstance(device_primary_ip, int): - ip = self.inventory.get_by_id(NBIPAddress, nb_id=device_primary_ip) - ip = grab(ip, "data.address") - - if ip is not None and ip.split("/")[0] == ip_needle: - return True - - return False - - if object_type not in [NBDevice, NBVM]: - raise ValueError(f"Object must be a '{NBVM.name}' or '{NBDevice.name}'.") - - if primary_ip4 is None and primary_ip6 is None: + account, snapshot = self.client.discover( + self.settings.regions, self.settings.account_id, self.settings.discovery_region, **options) + snapshots = {account: snapshot} if account is not None else {} + account_names = {} + if self.settings.organization and self.settings.sync_private_ips: + self.resolve_shared_networks(snapshots) + self.defer_inaccessible_network_instances(snapshots) + vrfs = {} + for account, snapshot in snapshots.items(): + self.validate_services(account, snapshot) + self.validate_snapshot(account, snapshot, vrfs) + self.retain_disabled_services() + if getattr(self.client, "access_denied", []): + self.retain_inaccessible_objects() + account_labels = { + account: account_names.get(account) or self.settings.account_name or account + for account in snapshots + } + label_counts = {} + for label in account_labels.values(): + key = label.casefold() + label_counts[key] = label_counts.get(key, 0) + 1 + for account, label in account_labels.items(): + if label_counts[label.casefold()] > 1: + account_labels[account] = f"{label} ({account})" + for account, snapshot in snapshots.items(): + self.sync_snapshot(account, snapshot, account_name=account_labels[account]) + self.reconcile_access_tags() + except (AWSDiscoveryError, ValueError, KeyError) as exc: + self.abort(exc) + + def validate_services(self, account, snapshot): + for region, data in snapshot.items(): + inaccessible = data.get("InaccessibleServices", []) + for enabled, key in ( + (self.settings.sync_s3 and "s3" not in inaccessible, "Buckets"), + (self.settings.sync_lambda and "lambda" not in inaccessible, "Functions"), + (self.settings.sync_eks and "eks" not in inaccessible, "EKSClusters"), + ): + if enabled and not isinstance(data.get(key), list): + raise ValueError(f"Missing AWS {key} discovery (account '{account}', region '{region}')") + if enabled: + identity_key = {"Buckets": "Name", "Functions": "FunctionArn", "EKSClusters": "arn"}[key] + identities = [resource[identity_key] for resource in data[key]] + if len(set(identities)) != len(identities): + raise ValueError(f"Duplicate AWS {key} discovery (account '{account}', region '{region}')") + for bucket in data["Buckets"] if self.settings.sync_s3 and "s3" not in inaccessible else []: + if bucket["Region"] != region or not bucket["Name"] or len(bucket["Name"]) > 63: + raise ValueError(f"Invalid S3 bucket '{bucket['Name']}' for region '{region}'") + self.service_tags({tag["Key"]: tag["Value"] for tag in bucket["Tags"]}) + if "Versioning" not in bucket or "Encryption" not in bucket: + raise ValueError(f"Incomplete S3 bucket metadata for '{bucket['Name']}'") + for function in data["Functions"] if self.settings.sync_lambda and "lambda" not in inaccessible else []: + self.validate_arn(function["FunctionArn"], "lambda", account, region) + if function["State"] not in ("Pending", "Active", "Inactive", "Failed"): + raise ValueError(f"Unknown Lambda state '{function['State']}' for '{function['FunctionArn']}'") + if not function["FunctionName"] or not isinstance(function["MemorySize"], int): + raise ValueError(f"Incomplete Lambda configuration for '{function['FunctionArn']}'") + self.service_tags(function["Tags"]) + if self.settings.sync_eks and "eks" not in inaccessible: + clusters = {cluster["name"]: cluster for cluster in data["EKSClusters"]} + if len(clusters) != len(data["EKSClusters"]): + raise ValueError(f"Duplicate EKS cluster discovery in account '{account}', region '{region}'") + asg_clusters = {} + for cluster in data["EKSClusters"]: + self.validate_arn(cluster["arn"], "eks", account, region) + for field in ("status", "version", "resourcesVpcConfig", "nodegroups", "fargateProfiles"): + if field not in cluster: + raise ValueError(f"Incomplete EKS {field} for '{cluster['arn']}'") + self.service_tags(cluster.get("tags", {})) + for group in cluster["nodegroups"]: + for asg in group.get("resources", {}).get("autoScalingGroups", []): + name = asg["name"] + if name in asg_clusters and asg_clusters[name] != cluster["name"]: + raise ValueError(f"EKS Auto Scaling group '{name}' belongs to multiple clusters") + asg_clusters[name] = cluster["name"] + data["EKSWorkers"] = {} + for instance in data["instances"]: + tags = {tag["Key"]: tag["Value"] for tag in instance.get("Tags", [])} + names = {tags[key] for key in ( + "aws:eks:cluster-name", "eks:cluster-name", "eksctl.cluster.k8s.io/v1alpha1/cluster-name", + ) if tags.get(key)} + asg_cluster = asg_clusters.get(tags.get("aws:autoscaling:groupName")) + if asg_cluster: + names.add(asg_cluster) + if len(names) > 1: + raise ValueError(f"Conflicting EKS cluster membership for instance '{instance['InstanceId']}'") + if names: + name = next(iter(names)) + if name in clusters: + data["EKSWorkers"][instance["InstanceId"]] = name + else: + log.warning( + f"AWS instance '{instance['InstanceId']}' references undiscovered EKS cluster " + f"'{name}' (account '{account}', region '{region}'); retaining EC2 placement.") + + def validate_arn(self, arn, service, account, region): + parts = arn.split(":", 5) + if len(parts) != 6 or parts[0] != "arn" or parts[2:5] != [service, region, account] or not parts[5]: + raise ValueError(f"Invalid {service} resource ARN '{arn}' for account '{account}', region '{region}'") + + def service_tags(self, tags): + result = [f"AWS:{key}={tags[key]}" for key in self.settings.tag_keys if key in tags] + if any(len(tag) > 100 for tag in result): + raise ValueError("AWS service tag exceeds the NetBox tag name limit (100 characters)") + return result + + def reconcile_tags(self, item, tags): + # Tag lookup can reuse an existing name with different capitalization. + desired_tags = {self.inventory.get_by_data(NBTag, data={"name": name}) for name in tags} + managed_prefixes = tuple(f"AWS:{key}=".casefold() for key in self.settings.tag_keys) + for tag in item.data.get("tags", []): + if tag.get_display_name().casefold().startswith(managed_prefixes) and tag not in desired_tags: + item.remove_tags(tag) + + def retain_disabled_services(self): + for object_type in (NBVM, NBVirtualDisk, NBCluster): + for item in self.inventory.get_all_items(object_type): + service = (item.data.get("custom_fields") or {}).get("aws_service") + if service in ("s3", "lambda", "eks") and not getattr(self.settings, f"sync_{service}") and \ + (item.source is self or self.source_tag in item.get_tags()): + item.set_source(self) + + def access_scope(self, item): + fields = item.data.get("custom_fields") or {} + metadata = fields.get("aws_metadata") or {} + identity = fields.get("aws_resource_id") or "" + service = "ec2" if fields.get("aws_instance_id") else fields.get("aws_service") + account = fields.get("aws_account_id") or metadata.get("account") + region = fields.get("aws_region") or metadata.get("region") + resource = None + if identity.startswith("arn:"): + parts = identity.split(":", 5) + if len(parts) == 6: + service, region, account, resource = parts[2:] + if service == "lambda": + resource = identity + elif service == "eks": + resource = resource.removeprefix("cluster/") + elif identity.startswith("s3/"): + parts = identity.split("/") + account = parts[1] + if isinstance(item, NBVirtualDisk): + resource = parts[2] + else: + region = parts[2] + for parent_key in ("virtual_machine", "cluster"): + parent = item.data.get(parent_key) + if parent is not None: + parent_account, parent_region, parent_service, _ = self.access_scope(parent) + account = account or parent_account + region = region or parent_region + service = service or parent_service + break + if isinstance(item, (NBIPAddress, NBMACAddress)): + parent = item.get_device_vm() + if parent is not None: + account, region, service, _ = self.access_scope(parent) + if account is None: + group = item if isinstance(item, NBClusterGroup) else item.data.get("group") + name = group.data.get("name", "") if group is not None else "" + if name.startswith("AWS "): + label = name[4:] + account = next((key for key, value in getattr(self.client, "account_names", {}).items() + if label in (value, f"{value} ({key})")), None) + if account is None and label.isdigit(): + account = label + if region is None: + site = item.data.get("scope_id") or item.data.get("site") + name = site.data.get("name", "") if isinstance(site, NBSite) else "" + if name.startswith("AWS "): + region = name[4:] + return account, region, service or "ec2", resource + + def reconcile_access_tags(self): + checks = getattr(self.client, "access_checks", []) + if not checks: return - - if primary_ip4 is not None: - primary_ip4 = str(primary_ip4).split("/")[0] - - if primary_ip6 is not None: - primary_ip6 = str(primary_ip6).split("/")[0] - - for device in self.inventory.get_all_items(object_type): - - if _matches_device_primary_ip(grab(device, "data.primary_ip4"), primary_ip4) is True: - log.debug2(f"Found existing host '{device.get_display_name()}' " - f"based on the primary IPv4 '{primary_ip4}'") - return device - - if _matches_device_primary_ip(grab(device, "data.primary_ip6"), primary_ip6) is True: - log.debug2(f"Found existing host '{device.get_display_name()}' " - f"based on the primary IPv6 '{primary_ip6}'") - return device - - def get_object_relation(self, name, relation, fallback=None): - """ - - Parameters - ---------- - name: str - name of the object to find a relation for - relation: str - name of the config variable relation (i.e: vm_tag_relation) - fallback: str - fallback string if no relation matched - - Returns - ------- - data: str, list, None - string of matching relation or list of matching tags - """ - - resolved_list = list() - for single_relation in grab(self.settings, relation, fallback=list()): - object_regex = single_relation.get("object_regex") - match_found = False - if object_regex.match(name): - resolved_name = single_relation.get("assigned_name") - log.debug2(f"Found a matching {relation} '{resolved_name}' ({object_regex.pattern}) for {name}") - resolved_list.append(resolved_name) - match_found = True - - # special cluster condition - if match_found is False and grab(f"{relation}".split("_"), "0") == "cluster": - - stripped_name = "/".join(name.split("/")[1:]) - if object_regex.match(stripped_name): - - resolved_name = single_relation.get("assigned_name") - log.debug2(f"Found a matching {relation} '{resolved_name}' ({object_regex.pattern}) " - f"for {stripped_name}") - resolved_list.append(resolved_name) - - if grab(f"{relation}".split("_"), "1") == "tag": - return resolved_list - - else: - resolved_name = fallback - if len(resolved_list) >= 1: - resolved_name = resolved_list[0] - if len(resolved_list) > 1: - log.debug(f"Found {len(resolved_list)} matches for {name} in {relation}." - f" Using first on: {resolved_name}") - - return resolved_name - - def get_cluster_for_host(self, hostname): - for cluster, hosts in self.cluster_host_map.items(): - if hostname in hosts: - return cluster - return None - - def get_server_priority(self, server): - name = get_string_or_none(grab(server, "name")) - if name is not None and self.settings.strip_vm_domain_name: - name = name.split(".")[0] - if self.settings.set_vm_name_to_uuid: - name = get_string_or_none(grab(server, "id")) - - cluster_name = get_string_or_none(grab(server, "availability_zone")) - cluster = self.get_object_from_cache(NBCluster, cluster_name) - if name is not None and cluster is not None and not self.settings.skip_vm_comments: - existing_vm = self.inventory.get_by_data( - NBVM, data={"name": name, "cluster": cluster} - ) - if grab(existing_vm, "data.comments") == get_string_or_none(grab(server, "id")): - return 0, server.status != "ACTIVE" - - return 1, server.status != "ACTIVE" - - @staticmethod - def get_compute_host_names(hosts): - compute_host_names = [] - for hostname, services in (hosts or {}).items(): - if not isinstance(services, dict) or not services: - compute_host_names.append(hostname) - continue - - service_names = {str(name).lower() for name in services} - service_names.update( - str(grab(service, "binary", fallback="")).lower() - for service in services.values() - ) - has_nova_services = any(name.startswith("nova-") for name in service_names) - if not has_nova_services or "nova-compute" in service_names: - compute_host_names.append(hostname) - - return compute_host_names - - def object_synced_by_other_source(self, nb_object): - if nb_object.source is not None: - return nb_object.source is not self - other_tags = [source.source_tag for source in self.inventory.source_list if source is not self] - return bool(set(nb_object.get_tags()).intersection(other_tags)) - - def add_custom_attributes(self, object_type, obj, data): - attributes = self.settings.host_custom_object_attributes if object_type == NBDevice else \ - self.settings.vm_custom_object_attributes - for attribute in attributes or []: - value = grab(obj, attribute) - if value is None: - continue - field = self.add_update_custom_field({ - "name": f"openstack_{attribute}", "label": attribute[:50], - "object_types": [object_type.object_type], "type": "text", - "description": f"OpenStack resource attribute '{attribute}'", - }) - data.setdefault("custom_fields", {})[field.data["name"]] = str(value) - - def add_device_vm_to_inventory(self, object_type, object_data, pnic_data=None, vnic_data=None, - nic_ips=None, p_ipv4=None, p_ipv6=None): - """ - Add/update device/VM object in inventory based on gathered data. - - Try to find object first based on the object data, interface MAC addresses and primary IPs. - 1. try to find by name and cluster/site - 2. try to find by mac addresses interfaces - 3. try to find by primary IP - - IP addresses for each interface are added here as well. First they will be checked and added - if all checks pass. For each IP address a matching IP prefix will be searched for. First we - look for longest matching IP Prefix in the same site. If this failed we try to find the longest - matching global IP Prefix. - - If a IP Prefix was found then we try to get the VRF and VLAN for this prefix. Now we compare - if interface VLAN and prefix VLAN match up and warn if they don't. Then we try to add data to - the IP address if not already set: - - add prefix VRF if VRF for this IP is undefined - add tenant if tenant for this IP is undefined - 1. try prefix tenant - 2. if prefix tenant is undefined try VLAN tenant - - And we also set primary IP4/6 for this object depending on the "set_primary_ip" setting. - - If a IP address is set as primary IP for another device then using this IP on another - device will be rejected by NetBox. - - Setting "always": - check all NBDevice and NBVM objects if this IP address is set as primary IP to any - other object then this one. If we found another object, then we unset the primary_ip* - for the found object and assign it to this object. - - This setting will also reset the primary IP if it has been changed in NetBox - - Setting "when-undefined": - Will set the primary IP for this object if primary_ip4/6 is undefined. Will cause a - NetBox error if IP has been assigned to a different object as well - - Setting "never": - Well, the attribute primary_ip4/6 will never be touched/changed. - - Parameters - ---------- - object_type: (NBDevice, NBVM) - NetBoxObject sub class of object to add - object_data: dict - data of object to add/update - pnic_data: dict - data of physical interfaces of this object, interface name as key - vnic_data: dict - data of virtual interfaces of this object, interface name as key - nic_ips: dict - dict of ips per interface of this object, interface name as key - p_ipv4: str - primary IPv4 as string including netmask/prefix - p_ipv6: str - primary IPv6 as string including netmask/prefix - - """ - - if object_type not in [NBDevice, NBVM]: - raise ValueError(f"Object must be a '{NBVM.name}' or '{NBDevice.name}'.") - - pnic_data = pnic_data or {} - vnic_data = vnic_data or {} - nic_ips = nic_ips or {} - - if log.level == DEBUG3: - - log.debug3("function: add_device_vm_to_inventory") - log.debug3(f"Object type {object_type}") - pprint.pprint(object_data) - pprint.pprint(pnic_data) - pprint.pprint(vnic_data) - pprint.pprint(nic_ips) - pprint.pprint(p_ipv4) - pprint.pprint(p_ipv6) - - # check existing Devices for matches - log.debug2(f"Trying to find a {object_type.name} based on the collected name, cluster, IP and MAC addresses") - - device_vm_object = self.inventory.get_by_data(object_type, data=object_data) - - if device_vm_object is not None: - log.debug2("Found a exact matching %s object: %s" % - (object_type.name, device_vm_object.get_display_name(including_second_key=True))) - - # keep searching if no exact match was found - else: - - log.debug2(f"No exact match found. Trying to find {object_type.name} based on MAC addresses") - - # on VMs vnic data is used, on physical devices pnic data is used - mac_source_data = vnic_data if object_type == NBVM else pnic_data - - nic_macs = [x.get("mac_address") for x in mac_source_data.values() if x.get("mac_address") is not None] - - device_vm_object = self.get_object_based_on_macs(object_type, nic_macs) - - if device_vm_object is not None: - log.debug2("Found a matching %s object: %s" % - (object_type.name, device_vm_object.get_display_name(including_second_key=True))) - - # keep looking for devices with the same primary IP - else: - - log.debug2(f"No match found. Trying to find {object_type.name} based on primary IP addresses") - - device_vm_object = self.get_object_based_on_primary_ip(object_type, p_ipv4, p_ipv6) - - if device_vm_object is None: - object_name = object_data.get(object_type.primary_key) - log.debug(f"No existing {object_type.name} object for {object_name}. Creating a new {object_type.name}.") - device_vm_object = self.inventory.add_object(object_type, data=object_data, source=self) - else: - device_vm_object.update(data=object_data, source=self) - - # update role according to config settings - object_name = object_data.get(object_type.primary_key) - role_name = self.get_object_relation(object_name, - "host_role_relation" if object_type == NBDevice else "vm_role_relation") - - if object_type == NBDevice: - if role_name is None: - role_name = "Server" - device_vm_object.update(data={"device_role": {"name": role_name}}) - if object_type == NBVM and role_name is not None: - device_vm_object.update(data={"role": {"name": role_name}}) - - # compile all nic data into one dictionary - if object_type == NBVM: - nic_data = vnic_data - else: - nic_data = {**pnic_data, **vnic_data} - exclude_filter = self.settings.vm_interface_exclude_filter if object_type == NBVM else \ - self.settings.host_interface_exclude_filter - if exclude_filter is not None: - nic_data = {name: data for name, data in nic_data.items() if not exclude_filter.match(name)} - - mac_matched_interfaces = set() - if object_type == NBVM: - current_by_mac = {} - for interface in self.inventory.get_all_interfaces(device_vm_object): - existing_name = grab(interface, "data.name") - if exclude_filter is not None and existing_name is not None and exclude_filter.match(existing_name): - continue - if version.parse(self.inventory.netbox_api_version) >= version.parse("4.2.0"): - mac = grab(interface, "data.primary_mac_address.data.mac_address") - else: - mac = grab(interface, "data.mac_address") - if mac: - current_by_mac.setdefault(normalize_mac_address(mac), []).append(interface) - names_by_mac = {} - for int_name, int_data in nic_data.items(): - mac = int_data.get("mac_address") - matches = current_by_mac.get(normalize_mac_address(mac), []) if mac else [] - if len(matches) > 1: - message = f"Multiple interfaces of OpenStack VM '{object_name}' match MAC '{mac}'." - log.error(message) - raise ValueError(message) - if matches: - names_by_mac[int_name] = matches[0].data["name"] - mac_matched_interfaces.add(matches[0]) - used_names = set(names_by_mac.values()) - remapped_data = {} - remapped_ips = {} - for int_name, int_data in nic_data.items(): - target_name = names_by_mac.get(int_name, int_name) - if int_name not in names_by_mac: - index = 1 - network_label = int_name.partition(" ")[2] - while target_name in used_names: - target_name = f"vNIC{index} {network_label}" - index += 1 - used_names.add(target_name) - if int_name not in names_by_mac: - int_data["description"] = target_name - int_data["name"] = target_name - remapped_data[target_name] = int_data - remapped_ips[target_name] = nic_ips.get(int_name, []) - nic_data = remapped_data - nic_ips = remapped_ips - - # map interfaces of existing object with discovered interfaces - nic_object_dict = self.map_object_interfaces_to_current_interfaces(device_vm_object, nic_data) - - if object_data.get("status", "") == "active" and (nic_ips is None or len(nic_ips.keys()) == 0): - log.debug(f"No IP addresses for '{object_name}' found!") - - primary_ipv4_object = None - primary_ipv6_object = None - - if p_ipv4 is not None: - try: - primary_ipv4_object = ip_interface(p_ipv4) - except ValueError: - log.error(f"Primary IPv4 ({p_ipv4}) does not appear to be a valid IP address (needs included suffix).") - - if p_ipv6 is not None: - try: - primary_ipv6_object = ip_interface(p_ipv6) - except ValueError: - log.error(f"Primary IPv6 ({p_ipv6}) does not appear to be a valid IP address (needs included suffix).") - - for int_name, int_data in nic_data.items(): - existing_interface = nic_object_dict.get(int_name) - if existing_interface is not None: - existing_name = grab(existing_interface, "data.name") - if existing_name is not None and ( - object_type == NBDevice or existing_interface in mac_matched_interfaces): - int_data["name"] = existing_name - if object_type == NBDevice: - int_data.pop("type", None) - - # add/update interface with retrieved data - nic_object, ip_address_objects = self.add_update_interface(nic_object_dict.get(int_name), device_vm_object, - int_data, nic_ips.get(int_name, list())) - - # add all interface IPs - for ip_object in ip_address_objects: - - if ip_object is None: - continue - - ip_interface_object = ip_interface(grab(ip_object, "data.address")) - - # continue if address is not a primary IP - if ip_interface_object not in [primary_ipv4_object, primary_ipv6_object]: + prefix = "Sync:AccessDenied:" + scopes = {} + for check in checks: + outcomes = scopes.setdefault( + (check.account, check.region, check.service, check.resource), (set(), set())) + outcomes[0 if check.denied else 1].add(check.operation) + for object_type in self.dependent_netbox_objects: + for item in self.inventory.get_all_items(object_type): + if object_type in (NBTag, NBCustomField, NBTenant) or "tags" not in item.data_model or not ( + item.source is self or self.source_tag in item.get_tags()): continue - - # set/update/remove primary IP addresses - set_this_primary_ip = False - ip_version = ip_interface_object.ip.version - if self.settings.set_primary_ip == "always": - - for object_type in [NBDevice, NBVM]: - - # new IPs don't need to be removed from other devices/VMs - if ip_object.is_new is True: - break - - for devices_vms in self.inventory.get_all_items(object_type): - - # device has no primary IP of this version - this_primary_ip = grab(devices_vms, f"data.primary_ip{ip_version}") - - # we found this exact object - if devices_vms == device_vm_object: - continue - - # device has the same object assigned - if this_primary_ip == ip_object: - devices_vms.unset_attribute(f"primary_ip{ip_version}") - - set_this_primary_ip = True - - elif self.settings.set_primary_ip != "never" and \ - grab(device_vm_object, f"data.primary_ip{ip_version}") is None: - set_this_primary_ip = True - - if set_this_primary_ip is True: - - log.debug(f"Setting IP '{grab(ip_object, 'data.address')}' as primary IPv{ip_version} for " - f"'{device_vm_object.get_display_name()}'") - device_vm_object.update(data={f"primary_ip{ip_version}": ip_object}) - - return device_vm_object - - def add_object_to_cache(self, obj_type, key, netbox_object): - - if obj_type is None or key is None or netbox_object is None: - return - - # noinspection PyBroadException - try: - vm_class_name = obj_type.__name__ - except Exception: - return - - if self.object_cache.get(vm_class_name) is None: - self.object_cache[vm_class_name] = dict() - - self.object_cache[vm_class_name][key] = netbox_object - - def get_object_from_cache(self, obj_type, key): - - if obj_type is None or key is None: - return - - # noinspection PyBroadException - try: - vm_class_name = obj_type.__name__ - except Exception: - return - - if self.object_cache.get(vm_class_name) is None: - return - - return self.object_cache[vm_class_name].get(key) - - def add_cluster(self, obj): - """ - Add a OpenStack Availability Zone as a NBCluster to NetBox. Cluster name is checked against - cluster_include_filter and cluster_exclude_filter config setting. Also adds - cluster and site_name to "self.permitted_clusters" so hosts and VMs can be - checked if they are part of a permitted cluster. - - Parameters - ---------- - obj: openstack.compute.v2.availability_zone.AvailabilityZone - cluster to add - """ - - name = get_string_or_none(grab(obj, "name")) - group = self.inventory.get_by_data(NBClusterGroup, data={"name": self.settings.group_name}) - - if name is None or group is None: - return - - group_name = grab(group, "data.name") - - log.debug(f"Parsing OpenStack AZ: {name}") - - if self.passes_filter(name, self.settings.cluster_include_filter, - self.settings.cluster_exclude_filter) is False: + account, region, service, resource = self.access_scope(item) + denied, succeeded = set(), set() + for key in product( + (None, account), (None, region), ("organizations", "sts", "account", service), + (None, resource)): + if key in scopes: + failures, successes = scopes[key] + denied.update(failures) + succeeded.update(successes) + for operation in denied: + item.add_tags(f"{prefix}{operation}") + for tag in item.get_tags(): + if tag.startswith(prefix) and tag[len(prefix):] in succeeded - denied: + item.remove_tags(tag) + + def retain_source_objects(self): + # Source-wide retention also protects children and cross-account shared-network dependencies. + for object_type in self.dependent_netbox_objects: + for item in self.inventory.get_all_items(object_type): + if self.source_tag in item.get_tags(): + item.set_source(self) + + def retain_inaccessible_objects(self): + self.retain_source_objects() + log.warning( + f"AWS source '{self.name}': authorization-denied discovery is incomplete. " + "Retaining all previously managed source objects this run; accessible resources will still be updated.") + + def defer_inaccessible_network_instances(self, snapshots): + if not any(service in ("ec2", "account", "sts", "organizations") + for _, _, service in getattr(self.client, "access_denied", [])): return - - site_name = self.get_site_name(NBCluster, name) - + for account, snapshot in snapshots.items(): + for region, data in snapshot.items(): + retained = [] + for instance in data["instances"]: + missing_network = False + for reference in instance.get("NetworkInterfaces", []): + eni = data["NetworkInterfaces"].get(reference["NetworkInterfaceId"]) + if eni is not None and (eni["VpcId"] not in data["Vpcs"] or + eni["SubnetId"] not in data["Subnets"]): + missing_network = True + if missing_network: + log.warning( + f"AWS instance '{instance['InstanceId']}' (account '{account}', region '{region}') " + "has missing network dependencies during authorization-denied discovery. " + "Deferring its update and retaining existing objects.") + else: + retained.append(instance) + data["instances"] = retained + + def resource_name(self, name, identity, limit): + if len(name) <= limit: + return name + suffix = sha256(identity.encode("utf-8")).hexdigest()[:12] + return f"{name[:limit - 13]}-{suffix}" + + def service_object(self, object_type, identity, data): + """Adopt tagged legacy objects; EC2 VMs carry aws_instance_id, not aws_resource_id.""" + fields = data["custom_fields"] + fields["aws_metadata"] = json_safe(fields["aws_metadata"]) + items = self.inventory.get_all_items(object_type) + matches = [item for item in items + if (item.data.get("custom_fields") or {}).get("aws_resource_id") == identity] + parent = "virtual_machine" if object_type is NBVirtualDisk else "cluster" if object_type is NBVM else "group" + if not matches: + name_matches = [item for item in items + if item.data.get("name") == data["name"] and + item.data.get(parent) is data.get(parent)] + if len(name_matches) == 1: + candidate = name_matches[0] + fields = candidate.data.get("custom_fields") or {} + candidate_identity = fields.get("aws_resource_id") + candidate_service = fields.get("aws_service") + identity_missing = candidate_identity is None or ( + isinstance(candidate_identity, str) and not candidate_identity.strip()) + service = (data.get("custom_fields") or {}).get("aws_service") + service_matches = candidate_service in (None, "", service) + source_managed = candidate.source is self or self.source_tag in candidate.get_tags() + if identity_missing and service_matches and source_managed: + matches = [candidate] + if len(matches) > 1: + raise ValueError(f"Multiple NetBox {object_type.name} objects match AWS resource '{identity}'") + if matches: + item = matches[0] + if item.source is not self and self.source_tag not in item.get_tags(): + raise ValueError(f"AWS resource '{identity}' belongs to another source") + item.update(data=data, source=self) + else: + if any(item.data.get("name") == data["name"] and item.data.get(parent) is data.get(parent) + for item in items): + raise ValueError(f"NetBox {object_type.name} name collision for AWS resource '{identity}'") + item = self.inventory.add_object(object_type, data=data, source=self) + self.reconcile_tags(item, data["tags"]) + return item + + def service_cluster(self, account, region, site, group, service, identity, name, + metadata=None, tags=None): data = { - "name": name, - "type": {"name": "Openstack"}, - "group": group, - "site": {"name": site_name} + "name": self.resource_name(name, identity, 100), "group": group, + "type": {"name": f"AWS {service.upper() if service != 'lambda' else 'Lambda'}"}, + "custom_fields": {"aws_service": service, "aws_resource_id": identity, + "aws_metadata": metadata or {"account": account, "region": region}}, + "tags": self.service_tags(tags or {}), } - if version.parse(self.inventory.netbox_api_version) >= version.parse("4.2.0"): - data.pop("site") - data["scope_type"] = NBSite.object_type - data["scope_id"] = self.inventory.add_update_object(NBSite, data={"name": site_name}, source=self) - tags = self.get_object_relation(name, "cluster_tag_relation") - if tags: - data["tags"] = tags - - tenant_name = self.get_object_relation(name, "cluster_tenant_relation") - if tenant_name is not None: - data["tenant"] = {"name": tenant_name} - - # try to find cluster including cluster group - log.debug2("Trying to find a matching existing cluster") - cluster_object = None - for cluster_candidate in self.inventory.get_all_items(NBCluster): - if grab(cluster_candidate, "data.name") != name: - continue - if cluster_candidate.get_site_name() not in (None, site_name) and \ - self.object_synced_by_other_source(cluster_candidate): - continue - - # try to find a cluster with matching site - candidate_group = grab(cluster_candidate, "data.group.data.name") - if candidate_group not in (None, group_name): - continue - - if cluster_candidate.get_site_name() == site_name: - cluster_object = cluster_candidate - log.debug2("Found an existing cluster where 'name' and 'site' are matching") - break - - if grab(cluster_candidate, "data.group") is not None and \ - grab(cluster_candidate, "data.group.data.name") == group_name: - cluster_object = cluster_candidate - log.debug2("Found an existing cluster where 'name' and 'cluster group' are matching") - break - - if grab(cluster_candidate, "data.tenant") is not None and \ - tenant_name is not None and \ - grab(cluster_candidate, "data.tenant.data.name") == tenant_name: - cluster_object = cluster_candidate - log.debug2("Found an existing cluster where 'name' and 'tenant' are matching") - break - - if cluster_object is not None: - cluster_object.update(data=data, source=self) + if Version(self.inventory.netbox_api_version) >= Version("4.2"): + data.update(scope_type="dcim.site", scope_id=site) else: - cluster_object = self.inventory.add_update_object(NBCluster, data=data, source=self) - - self.add_object_to_cache(NBCluster, name, cluster_object) - - self.cluster_host_map[name] = self.get_compute_host_names(grab(obj, "hosts")) - log.debug2(f"Found compute hosts for availability zone '{name}': " - f"{self.cluster_host_map[name]}") - - self.permitted_clusters[name] = site_name - - def add_host(self, obj): - """ - Parse a Openstack host to NetBox once all data is gathered. - - First host is filtered: - host has a cluster and is it permitted - was host with same name and site already parsed - does the host pass the host_include_filter and host_exclude_filter - - Then all necessary host data will be collected. - - Primary IPv4/6 will be determined by 'host_ip' value - - Parameters - ---------- - obj: Hypervisor - host object to parse - """ - - name = get_string_or_none(grab(obj, "name")) - if name is None: - return - - if name is not None and self.settings.strip_host_domain_name is True: - name = name.split(".")[0] - - # parse data - log.debug(f"Parsing Openstack host: {name}") - - # - # Filtering - # - - # manage site and cluster - short_name = get_string_or_none(obj.service_details["host"]) - cluster_name = self.get_cluster_for_host(short_name) - - if cluster_name is None: - # Nova omits disabled compute services from availability zone host lists - if get_string_or_none(grab(obj, "status")) == "disabled": - log.info(f"Host '{name}' is disabled in Nova and not part of any availability zone. Skipping.") - else: - log.error(f"Unable to find an availability zone for host '{name}'. Skipping.") - return - - if log.level == DEBUG3: - try: - log.info("Cluster data") - dump(obj.service_details.to_dict()) - except Exception as e: - log.error(e) - - if self.permitted_clusters.get(cluster_name) is None: - log.debug(f"Host '{name}' is not part of a permitted cluster. Skipping") - return - - # get cluster object - nb_cluster_object = self.get_object_from_cache(NBCluster, cluster_name) - - # get a site for this host - site_name = self.get_site_name(NBDevice, name, cluster_name) - - if name in self.processed_host_names.get(site_name, list()): - log.warning(f"Host '{name}' for site '{site_name}' already parsed. " - "Make sure to use unique host names. Skipping") - return + data["site"] = site + return self.service_object(NBCluster, identity, data) - # add host to processed list - if self.processed_host_names.get(site_name) is None: - self.processed_host_names[site_name] = list() - - self.processed_host_names[site_name].append(name) - - # filter hosts by name - if self.passes_filter(name, self.settings.host_include_filter, self.settings.host_exclude_filter) is False: - return - - # - # Collecting data - # - - # collect all necessary data - manufacturer = None - model = None - platform = get_string_or_none(obj.hypervisor_type) - - # if the device vendor/model cannot be retrieved (due to problem on the host), - # set a dummy value so the host still gets synced - if manufacturer is None: - manufacturer = "Generic Vendor" - if model is None: - model = "Generic Model" - - # get status - status = "offline" - if get_string_or_none(obj.status) == "enabled" and grab(obj, "state", fallback="up") == "up": - status = "active" - - # get host_tenant_relation - tenant_name = self.get_object_relation(name, "host_tenant_relation") - - # get host_tag_relation - host_tags = self.get_object_relation(name, "host_tag_relation") - - # prepare host data model - host_data = { - "name": name, - "device_type": { - "model": model, - "manufacturer": { - "name": manufacturer - } + def logical_vm(self, account, region, site, cluster, service, identity, name, metadata, tags, **attributes): + data = { + "name": self.resource_name(name, identity, 64), "cluster": cluster, "status": "active", + "comments": f"Logical AWS {service} inventory object; not an EC2 instance.", + "custom_fields": { + "aws_account_id": account, "aws_region": region, "aws_service": service, + "aws_resource_id": identity, "aws_metadata": metadata, }, - "site": {"name": site_name}, - "cluster": nb_cluster_object, - "status": status + "tags": [f"AWS:logical-{service}", *self.service_tags(tags)], + **attributes, } - - # add data if present - if platform is not None: - host_data["platform"] = {"name": platform} - if tenant_name is not None: - host_data["tenant"] = {"name": tenant_name} - if len(host_tags) > 0: - host_data["tags"] = host_tags - self.add_custom_attributes(NBDevice, obj, host_data) - - pnic_data_dict = dict() - pnic_data = { - "name": "eth0", - "enabled": True, - "type": "virtual" - } - pnic_data_dict["eth0"] = pnic_data - - ip_addr = grab(obj, "host_ip") - if ip_addr is not None and not self.settings.permitted_subnets.permitted(ip_addr, interface_name="eth0"): - ip_addr = None - prefix = None - matched_prefix = None - if ip_addr is not None: - matched_prefix = self.return_longest_matching_prefix_for_ip(ip_interface(ip_addr), site_name) - if matched_prefix is None: - matched_prefix = self.return_longest_matching_prefix_for_ip(ip_interface(ip_addr)) - if matched_prefix is not None: - prefix = matched_prefix.data["prefix"].prefixlen - ip_addr = f"{ip_addr}/{prefix}" - - vnic_ips = dict() - vnic_ips["eth0"] = list() - if ip_addr is not None: - ip_addr = str(ip_interface(ip_addr)) - vnic_ips["eth0"].append(ip_addr) - - host_primary_ip4 = ip_addr if ip_addr is not None and ip_interface(ip_addr).version == 4 else None - host_primary_ip6 = ip_addr if ip_addr is not None and ip_interface(ip_addr).version == 6 else None - - # add host to inventory - host = self.add_device_vm_to_inventory(NBDevice, object_data=host_data, pnic_data=pnic_data_dict, - vnic_data=dict(), nic_ips=vnic_ips, - p_ipv4=host_primary_ip4, p_ipv6=host_primary_ip6) - self.add_object_to_cache(NBDevice, short_name, host) - - return - - def add_volume(self, obj): - """ - Parse OpenStack volume and store in in a map. - """ - - id = obj.id - size = obj.size - - self.volume_map[id] = size - - def add_virtual_machine(self, obj): - """ - Parse a OpenStack VM add to NetBox once all data is gathered. - - Parameters - ---------- - obj: openstack.compute.v2.server.Server - virtual machine object to parse - """ - - name = get_string_or_none(grab(obj, "name")) - if name is None: - return - - if name is not None and self.settings.strip_vm_domain_name is True: - name = name.split(".")[0] - - if self.settings.set_vm_name_to_uuid: - name = get_string_or_none(obj.id) - - log.debug(f"Parsing OpenStack VM: {name}") - - # get VM power state - status = "active" if get_string_or_none(obj.status) == "ACTIVE" else "offline" - - cluster_name = get_string_or_none(obj.availability_zone) - - # check VM cluster - if cluster_name is None: - log.error(f"Requesting cluster for Virtual Machine '{name}' failed. Skipping.") - return - - elif self.permitted_clusters.get(cluster_name) is None: - log.debug(f"Virtual machine '{name}' is not part of a permitted cluster. Skipping") + if Version(self.inventory.netbox_api_version) >= Version("3.3"): + data["site"] = site + if self.settings.tenant: + data["tenant"] = {"name": self.settings.tenant} + return self.service_object(NBVM, identity, data) + + def sync_lambda(self, account, region, site, cluster, function): + status = {"Pending": "planned", "Active": "active", "Inactive": "offline", "Failed": "failed"} + self.logical_vm( + account, region, site, cluster, "lambda", function["FunctionArn"], + f"Lambda {function['FunctionName']}", + {key: value for key, value in function.items() if key != "Tags"}, + function["Tags"], memory=function["MemorySize"], status=status[function["State"]]) + + def sync_s3(self, account, region, site, group, buckets): + if not buckets: return - - nb_cluster_object = self.get_object_from_cache(NBCluster, cluster_name) - - if name in self.processed_vm_names.get(cluster_name, list()): - log.warning(f"Virtual machine '{name}' for cluster '{cluster_name}' already parsed. " - "Make sure to use unique VM names. Skipping") + identity = f"s3/{account}/{region}" + try: + cluster = self.service_cluster( + account, region, site, group, "s3", identity, f"S3 {account}/{region}") + except ValueError as exc: + self.retain_source_objects() + log.warning(f"Skipping S3 buckets (account '{account}', region '{region}'): {exc}") return - - # add host to processed list - if self.processed_vm_names.get(cluster_name) is None: - self.processed_vm_names[cluster_name] = list() - - self.processed_vm_names[cluster_name].append(name) - - # filter VMs by name - if self.passes_filter(name, self.settings.vm_include_filter, self.settings.vm_exclude_filter) is False: + try: + holder = self.logical_vm( + account, region, site, cluster, "s3", identity, f"S3 inventory {account}/{region}", + {"kind": "bucket-inventory-holder", "usage_status": "unknown"}, {}) + except ValueError as exc: + self.retain_source_objects() + log.warning(f"Skipping S3 buckets (account '{account}', region '{region}'): {exc}") return - - # - # Collect data - # - - site_name = self.get_site_name(NBCluster, cluster_name) - flavor = dict(grab(obj, "flavor") or {}) - if not all(key in flavor for key in ("vcpus", "ram", "disk")) and flavor.get("id"): - flavor = self.session.compute.get_flavor(flavor["id"]).to_dict() - # first check against vm_platform_relation - platform = None - if self.settings.skip_vm_platform is False: - platform = get_string_or_none(flavor.get("original_name") or flavor.get("name")) - - if platform is not None: - platform = self.get_object_relation(platform, "vm_platform_relation", fallback=platform) - - disks = {} - if grab(obj, "image") and flavor.get("disk"): - disks["root"] = int(flavor["disk"]) - if flavor.get("ephemeral"): - disks["ephemeral"] = int(flavor["ephemeral"]) - disk_complete = True - for volume in grab(obj, "attached_volumes", fallback=[]) or []: - size = self.volume_map.get(volume["id"]) - if size is None: - log.warning(f"Volume '{volume['id']}' of VM '{name}' was not discovered; preserving disk size.") - disk_complete = False - disks[volume["id"]] = int(size) if size is not None else None - - annotation = None - if self.settings.skip_vm_comments is False: - if not self.settings.set_vm_name_to_uuid: - annotation = get_string_or_none(obj.id) - - # assign vm_tenant_relation - tenant_name = self.get_object_relation(name, "vm_tenant_relation") - - vm_data = { - "name": name, - "cluster": nb_cluster_object, - "status": status, - } - if flavor.get("vcpus"): - vm_data["vcpus"] = float(flavor["vcpus"]) - if flavor.get("ram") is not None: - vm_data["memory"] = int(flavor["ram"]) - if disk_complete and version.parse(self.inventory.netbox_api_version) < version.parse("3.7.0"): - vm_data["disk"] = sum(disks.values()) - tags = self.get_object_relation(name, "vm_tag_relation") - if tags: - vm_data["tags"] = tags - host = self.get_object_from_cache(NBDevice, grab(obj, "compute_host")) - if host is not None and version.parse(self.inventory.netbox_api_version) >= version.parse("3.3.0"): - vm_data["device"] = host - - # Add adaption for change in NetBox 3.3.0 VM model - # issue: https://github.com/netbox-community/netbox/issues/10131#issuecomment-1225783758 - if version.parse(self.inventory.netbox_api_version) >= version.parse("3.3.0"): - vm_data["site"] = {"name": site_name} - if platform is not None: - vm_data["platform"] = {"name": platform} - if annotation is not None: - vm_data["comments"] = annotation - if tenant_name is not None: - vm_data["tenant"] = {"name": tenant_name} - - if self.settings.set_vm_name_to_uuid: - custom_field = self.add_update_custom_field({ - "name": "openstack_vm_name", - "label": "name", - "object_types": ["virtualization.virtualmachine"], - "type": "text", - "description": f"Openstack '{self.name}' synced object attribute 'name'" + for bucket in buckets: + bucket_id = f"s3/{account}/{bucket['Name']}" + try: + self.service_object(NBVirtualDisk, bucket_id, { + "name": bucket["Name"], "virtual_machine": holder, + # NetBox requires size; zero is explicitly unknown, never a usage measurement. + "size": 0, + "description": "AWS S3 bucket; usage unknown (size 0 is a placeholder, not measured).", + "custom_fields": { + "aws_service": "s3", "aws_resource_id": bucket_id, + "aws_metadata": {**{key: value for key, value in bucket.items() if key != "Tags"}, + "usage_status": "unknown", "usage_bytes": None}, + }, + "tags": self.service_tags({tag["Key"]: tag["Value"] for tag in bucket["Tags"]}), + }) + except ValueError as exc: + log.warning(f"Skipping S3 bucket '{bucket['Name']}' " + f"(account '{account}', region '{region}'): {exc}") + + def resolve_shared_networks(self, snapshots): + networks = {} + for account, snapshot in snapshots.items(): + for region, data in snapshot.items(): + for kind in ("Vpcs", "Subnets"): + for resource_id, resource in data[kind].items(): + owner = resource.get("OwnerId", account) + key = (kind, region, resource_id) + owners = networks.setdefault(key, {}) + if resource.get("OwnerId") and any( + item.get("OwnerId") and previous_owner != owner + for previous_owner, item in owners.items()): + raise ValueError( + f"Conflicting AWS {kind} ownership for '{resource_id}' " + f"(region '{region}'): {', '.join(sorted(set(owners) | {owner}))}") + if owner in owners: + previous = owners[owner] + fields = ("VpcId", "CidrBlock") + if any(previous.get(field) != resource.get(field) for field in fields) or \ + set(self.ipv6_networks(previous)) != set(self.ipv6_networks(resource)): + raise ValueError( + f"Conflicting AWS {kind} discovery for '{resource_id}' " + f"(owner '{owner}', region '{region}')") + owners[owner] = resource + for account, snapshot in snapshots.items(): + for region, data in snapshot.items(): + data["VpcOwners"] = {} + for instance in data["instances"]: + for reference in instance.get("NetworkInterfaces", []): + eni = data["NetworkInterfaces"].get(reference["NetworkInterfaceId"]) + if eni is None: + continue + vpc_id, subnet_id = eni["VpcId"], eni["SubnetId"] + vpcs = networks.get(("Vpcs", region, vpc_id), {}) + local_vpc = data["Vpcs"].get(vpc_id) + owner = local_vpc.get("OwnerId", account) if local_vpc is not None else None + if owner is None and len(vpcs) == 1: + owner = next(iter(vpcs)) + if owner is None and vpcs: + raise ValueError( + f"Ambiguous AWS VPC ownership for '{vpc_id}' " + f"(account '{account}', region '{region}', " + f"ENI '{eni['NetworkInterfaceId']}'): {', '.join(sorted(vpcs))}") + if owner is None: + continue + data["VpcOwners"][vpc_id] = owner + data["Vpcs"][vpc_id] = dict(vpcs[owner], OwnerId=owner) + subnets = networks.get(("Subnets", region, subnet_id), {}) + if subnets and owner not in subnets: + raise ValueError( + f"Conflicting AWS subnet ownership for '{subnet_id}' " + f"(account '{account}', region '{region}', " + f"ENI '{eni['NetworkInterfaceId']}', VPC owner '{owner}'): " + f"{', '.join(sorted(subnets))}") + if owner in subnets: + data["Subnets"][subnet_id] = dict(subnets[owner], OwnerId=owner) + + def vrf_name(self, account, region, vpc): + return self.settings.vpc_vrf_relation.get( + f"{account}/{region}/{vpc}", self.settings.vpc_vrf_relation.get( + f"{region}/{vpc}", f"AWS {account}/{region}/{vpc}")) + + def get_vrf(self, account, region, vpc): + account_region_vpc = f"{account}/{region}/{vpc}" + region_vpc = f"{region}/{vpc}" + has_mapping = account_region_vpc in self.settings.vpc_vrf_relation or \ + region_vpc in self.settings.vpc_vrf_relation + name = self.vrf_name(account, region, vpc) + vrf = self.inventory.get_by_data(NBVRF, data={"name": name}) + if vrf is None: + if has_mapping: + raise ValueError(f"Create mapped NetBox VRF '{name}' for VPC '{vpc}' before importing its IPs") + vrf = self.inventory.add_object(NBVRF, data={"name": name}, source=self) + return vrf + + def ipv6_networks(self, resource): + return [ + ip_network(entry["Ipv6CidrBlock"]) + for entry in resource.get("Ipv6CidrBlockAssociationSet", []) + if entry.get("Ipv6CidrBlockState", {}).get("State") == "associated" + ] + + def addresses(self, eni, subnet): + networks = [ip_network(subnet["CidrBlock"]), *self.ipv6_networks(subnet)] + addresses = [ + (entry["PrivateIpAddress"], entry.get("Primary", False)) + for entry in eni.get("PrivateIpAddresses", []) + ] + addresses.extend( + (entry["Ipv6Address"], entry.get("IsPrimaryIpv6", False)) + for entry in eni.get("Ipv6Addresses", []) + ) + result = [] + for value, primary in addresses: + address = ip_address(value) + network = next((net for net in networks + if net.version == address.version and address in net), None) + if network is None: + raise ValueError(f"ENI '{eni['NetworkInterfaceId']}' IP '{value}' has no matching AWS subnet") + value = f"{address}/{network.prefixlen}" + if self.settings.permitted_subnets.permitted(value, interface_name=eni["NetworkInterfaceId"]): + result.append((value, primary)) + return result + + def validate_snapshot(self, account, snapshot, vrfs=None): + if vrfs is None: + vrfs = {} + for region, data in snapshot.items(): + for instance in data["instances"]: + spec = data["InstanceTypes"].get(instance["InstanceType"]) + if spec is None: + raise ValueError(f"Missing AWS instance type '{instance['InstanceType']}'") + if "DefaultVCpus" not in spec["VCpuInfo"] or "SizeInMiB" not in spec["MemoryInfo"]: + raise ValueError(f"Incomplete AWS specifications for '{instance['InstanceType']}'") + for tag in instance.get("Tags", []): + if tag["Key"] in self.settings.tag_keys and len(f"AWS:{tag['Key']}={tag['Value']}") > 100: + raise ValueError(f"AWS tag '{tag['Key']}' exceeds the NetBox tag name limit (100 characters)") + for attachment in instance.get("BlockDeviceMappings", []): + volume_id = attachment.get("Ebs", {}).get("VolumeId") + if volume_id and volume_id not in data["Volumes"]: + raise ValueError(f"Missing attached EBS volume '{volume_id}'") + for reference in instance.get("NetworkInterfaces", []): + eni = data["NetworkInterfaces"].get(reference["NetworkInterfaceId"]) + if eni is None: + raise ValueError(f"Missing attached ENI '{reference['NetworkInterfaceId']}'") + if eni.get("Attachment", {}).get("InstanceId") != instance["InstanceId"]: + raise ValueError(f"ENI '{eni['NetworkInterfaceId']}' changed attachment during discovery") + if not eni.get("MacAddress"): + raise ValueError(f"ENI '{eni['NetworkInterfaceId']}' has no MAC address") + if not self.settings.sync_private_ips: + continue + vpc, subnet = eni["VpcId"], data["Subnets"].get(eni["SubnetId"]) + network_errors = [] + if vpc not in data["Vpcs"]: + network_errors.append(f"VPC '{vpc}' was not returned by DescribeVpcs") + if subnet is None: + network_errors.append( + f"subnet '{eni['SubnetId']}' was not returned by DescribeSubnets") + elif subnet["VpcId"] != vpc: + network_errors.append( + f"subnet '{eni['SubnetId']}' belongs to VPC '{subnet['VpcId']}', " + f"not ENI VPC '{vpc}'") + if network_errors: + raise ValueError( + f"Incomplete VPC/subnet discovery for ENI '{eni['NetworkInterfaceId']}' " + f"(account '{account}', region '{region}', instance '{instance['InstanceId']}', " + f"VPC '{vpc}', subnet '{eni['SubnetId']}'): {'; '.join(network_errors)}") + if self.addresses(eni, subnet): + owner = data.get("VpcOwners", {}).get(vpc, account) + vrf = self.get_vrf(owner, region, vpc) + identity = (owner, region, vpc) + if vrf in vrfs and vrfs[vrf] != identity: + raise ValueError(f"VRF '{vrf.data['name']}' is mapped to more than one AWS VPC") + vrfs[vrf] = identity + + def custom_fields(self): + fields = ( + ("aws_account_id", "AWS account ID", [NBVM.object_type]), + ("aws_region", "AWS region", [NBVM.object_type]), + ("aws_instance_id", "AWS instance ID", [NBVM.object_type]), + ("aws_name", "AWS Name tag", [NBVM.object_type]), + ("aws_instance_type", "AWS instance type", [NBVM.object_type]), + ("aws_availability_zone", "AWS Availability Zone", [NBVM.object_type]), + ("aws_vpc_id", "AWS VPC ID", [NBVM.object_type, NBVMInterface.object_type]), + ("aws_eni_id", "AWS ENI ID", [NBVMInterface.object_type]), + ("aws_subnet_id", "AWS subnet ID", [NBVMInterface.object_type]), + ("aws_public_ipv4", "AWS public IPv4", [NBVMInterface.object_type]), + ) + for name, label, types in fields: + self.add_update_custom_field({ + "name": name, "label": label, "object_types": types, "type": "text", + "description": f"Reported by AWS ({self.name})", + }) + if any((self.settings.sync_s3, self.settings.sync_lambda, self.settings.sync_eks)): + types = [NBVM.object_type, NBCluster.object_type, NBVirtualDisk.object_type] + for name, field_type in ( + ("aws_service", "text"), ("aws_resource_id", "text"), + ("aws_metadata", "json"), + ): + field = self.add_update_custom_field({ + "name": name, "label": name.replace("_", " ").upper(), + "object_types": types, "type": field_type, + "description": f"Reported by AWS ({self.name})", }) - vm_data["custom_fields"] = { - grab(custom_field, "data.name"): get_string_or_none(grab(obj, "name")) + existing_type = field.data.get("type") + if isinstance(existing_type, dict): + existing_type = existing_type.get("value") + if existing_type != field_type: + raise ValueError( + f"NetBox custom field '{name}' must have type '{field_type}', not '{existing_type}'") + + def sync_snapshot(self, account, snapshot, account_name=None): + self.custom_fields() + account_name = account_name or account + group = self.inventory.add_update_object( + NBClusterGroup, data={"name": f"AWS {account_name}"}, source=self) + api_version = Version(self.inventory.netbox_api_version) + for region, data in snapshot.items(): + site = self.inventory.add_update_object( + NBSite, data={"name": f"AWS {region}"}, source=self) + cluster_data = { + "name": f"AWS {account_name}/{region}", "group": group, + "type": {"name": "AWS EC2"}, } - self.add_custom_attributes(NBVM, obj, vm_data) - - vm_primary_ip4 = None - vm_primary_ip6 = None - vm_primary_ip4_is_fixed = False - vm_primary_ip6_is_fixed = False - vm_nic_dict = dict() - nic_ips = dict() - count = 0 - - for network, addresses in (grab(obj, "addresses") or {}).items(): - default_mac = next((address.get("OS-EXT-IPS-MAC:mac_addr") for address in addresses - if address.get("OS-EXT-IPS-MAC:mac_addr")), None) - interfaces = {} - for address in addresses: - mac = address.get("OS-EXT-IPS-MAC:mac_addr") or default_mac - if mac not in interfaces: - count += 1 - full_name = unquote(f"vNIC{count} ({network})") - interfaces[mac] = full_name - vm_nic_dict[full_name] = { - "name": full_name, - "description": full_name, - "enabled": True, - } - if mac: - vm_nic_dict[full_name]["mac_address"] = normalize_mac_address(mac) - nic_ips[full_name] = [] - full_name = interfaces[mac] + if api_version >= Version("4.2"): + cluster_data.update(scope_type="dcim.site", scope_id=site) + else: + cluster_data["site"] = site + cluster = self.inventory.add_update_object(NBCluster, data=cluster_data, source=self) + eks_clusters = {} + inaccessible = data.get("InaccessibleServices", []) + if self.settings.sync_eks and "eks" not in inaccessible: + for resource in data["EKSClusters"]: + try: + eks_cluster = self.service_cluster( + account, region, site, group, "eks", resource["arn"], + f"EKS {account}/{region}/{resource['name']}", + metadata={key: value for key, value in resource.items() if key != "tags"}, + tags=resource.get("tags", {})) + eks_clusters[resource["name"]] = eks_cluster + except ValueError as exc: + self.retain_source_objects() + log.warning(f"Skipping EKS cluster '{resource['name']}' " + f"(account '{account}', region '{region}'): {exc}") + if self.settings.sync_lambda and "lambda" not in inaccessible: try: - ip_addr = str(ip_interface(address["addr"])) - except (KeyError, ValueError): - log.warning(f"Invalid address on OpenStack VM '{name}'; skipping.") + lambda_cluster = self.service_cluster( + account, region, site, group, "lambda", f"lambda/{account}/{region}", + f"Lambda {account}/{region}") + except ValueError as exc: + self.retain_source_objects() + log.warning(f"Skipping Lambda functions (account '{account}', region '{region}'): {exc}") + else: + for function in data["Functions"]: + try: + self.sync_lambda(account, region, site, lambda_cluster, function) + except ValueError as exc: + self.retain_source_objects() + log.warning(f"Skipping function '{function['FunctionName']}' " + f"(account '{account}', region '{region}'): {exc}") + if self.settings.sync_s3 and "s3" not in inaccessible: + self.sync_s3(account, region, site, group, data["Buckets"]) + name_counts = {} + for instance in data["instances"]: + name = next((tag["Value"].strip() for tag in instance.get("Tags", []) + if tag["Key"] == "Name" and tag["Value"].strip()), instance["InstanceId"]) + key = name.casefold() + name_counts[key] = name_counts.get(key, 0) + 1 + for instance in data["instances"]: + tags = {tag["Key"]: tag["Value"] for tag in instance.get("Tags", [])} + vm_name = tags.get("Name", "").strip() or instance["InstanceId"] + if name_counts[vm_name.casefold()] > 1: + vm_name = f"{vm_name} ({instance['InstanceId']})" + eks_cluster_name = data.get("EKSWorkers", {}).get(instance["InstanceId"]) + if eks_cluster_name is not None and eks_cluster_name not in eks_clusters and \ + self.settings.sync_eks and "eks" not in inaccessible: + log.warning(f"Skipping instance '{instance['InstanceId']}' (account '{account}', " + f"region '{region}'): its EKS cluster '{eks_cluster_name}' was not synced.") continue - if not self.settings.permitted_subnets.permitted(ip_addr, interface_name=full_name): + self.sync_instance( + account, region, instance, data, site, + eks_clusters.get(eks_cluster_name, cluster), + api_version, vm_name=vm_name) + + def sync_instance(self, account, region, instance, data, site, cluster, api_version, vm_name=None): + instance_id = instance["InstanceId"] + if "eks" in data.get("InaccessibleServices", []): + existing = next((item for item in self.inventory.get_all_items(NBVM) + if (item.data.get("custom_fields") or {}).get("aws_account_id") == account and + (item.data.get("custom_fields") or {}).get("aws_instance_id") == instance_id), None) + if existing is not None and existing.data.get("cluster") is not None: + cluster = existing.data["cluster"] + tags = {tag["Key"]: tag["Value"] for tag in instance.get("Tags", [])} + spec = data["InstanceTypes"][instance["InstanceType"]] + vm_data = { + "name": vm_name or tags.get("Name", "").strip() or instance_id, "cluster": cluster, + "status": "active" if instance["State"]["Name"] == "running" else "offline", + "vcpus": spec["VCpuInfo"]["DefaultVCpus"], + "memory": spec["MemoryInfo"]["SizeInMiB"], + "custom_fields": { + "aws_account_id": account, "aws_region": region, + "aws_instance_id": instance_id, "aws_name": tags.get("Name", ""), + "aws_instance_type": instance["InstanceType"], + "aws_availability_zone": instance["Placement"]["AvailabilityZone"], + "aws_vpc_id": instance.get("VpcId", ""), + }, + "tags": [f"AWS:{key}={tags[key]}" for key in self.settings.tag_keys if key in tags], + } + cpu = instance.get("CpuOptions", {}) + if cpu.get("CoreCount") is not None and cpu.get("ThreadsPerCore") is not None: + vm_data["vcpus"] = cpu["CoreCount"] * cpu["ThreadsPerCore"] + if api_version >= Version("3.3"): + vm_data["site"] = site + if api_version >= Version("4.1"): + vm_data["serial"] = instance_id + if self.settings.tenant: + vm_data["tenant"] = {"name": self.settings.tenant} + disks = [ + data["Volumes"][entry["Ebs"]["VolumeId"]] + for entry in instance.get("BlockDeviceMappings", []) if "Ebs" in entry + ] + if api_version < Version("3.7"): + vm_data["disk"] = sum(volume["Size"] for volume in disks) + vm = next((item for item in self.inventory.get_all_items(NBVM) + if (item.data.get("custom_fields") or {}).get("aws_account_id") == account and + (item.data.get("custom_fields") or {}).get("aws_instance_id") == instance_id), None) + if vm is None: + vm = self.inventory.add_update_object(NBVM, data=vm_data, source=self) + else: + vm.update(data=vm_data, source=self) + self.reconcile_tags(vm, vm_data["tags"]) + if api_version >= Version("3.7"): + for volume in disks: + self.add_update_vm_child(NBVirtualDisk, vm, { + "name": volume["VolumeId"], "virtual_machine": vm, + "size": volume["Size"] * (1024 if api_version >= Version("4.1") else 1), + "description": f"AWS EBS {volume['VolumeType']}", + }) + for reference in instance.get("NetworkInterfaces", []): + eni = data["NetworkInterfaces"][reference["NetworkInterfaceId"]] + self.sync_interface(account, region, vm, eni, data, api_version) + + def add_update_vm_child(self, object_type, vm, data): + # Display-name matching loses the VM's account/cluster when names repeat. + existing = next((item for item in self.inventory.get_all_items(object_type) + if item.data.get("virtual_machine") is vm and + item.data.get("name") == data["name"]), None) + if existing is None: + return self.inventory.add_object(object_type, data=data, source=self) + existing.update(data=data, source=self) + return existing + + def sync_interface(self, account, region, vm, eni, data, api_version): + interface_data = { + "name": eni["NetworkInterfaceId"], "virtual_machine": vm, "enabled": True, + "description": eni.get("Description", ""), + "custom_fields": { + "aws_eni_id": eni["NetworkInterfaceId"], "aws_vpc_id": eni.get("VpcId", ""), + "aws_subnet_id": eni.get("SubnetId", ""), + "aws_public_ipv4": ", ".join(sorted({ + address["Association"]["PublicIp"] + for address in eni.get("PrivateIpAddresses", []) + if address.get("Association", {}).get("PublicIp") + })), + }, + } + mac = normalize_mac_address(eni.get("MacAddress")) + if api_version < Version("4.2"): + interface_data["mac_address"] = mac + interface = self.add_update_vm_child(NBVMInterface, vm, interface_data) + if api_version >= Version("4.2") and mac is not None: + mac_object = next((item for item in self.inventory.get_all_items(NBMACAddress) + if item.data.get("mac_address") == mac and + item.data.get("assigned_object_id") in (None, interface)), None) + mac_data = {"mac_address": mac, "assigned_object_id": interface, + "assigned_object_type": NBVMInterface} + if mac_object is None: + mac_object = self.inventory.add_object(NBMACAddress, data=mac_data, source=self) + else: + mac_object.update(data=mac_data, source=self) + for old in interface.get_mac_addresses(): + if old is not mac_object: + old.remove_interface_association() + interface.update(data={"primary_mac_address": mac_object}, source=self) + if not self.settings.sync_private_ips: + for address in interface.get_ip_addresses(): + if self.source_tag in address.get_tags(): + address.set_source(self) + return + subnet = data["Subnets"][eni["SubnetId"]] + addresses = self.addresses(eni, subnet) + discovered = [] + for address, primary in addresses: + owner = data.get("VpcOwners", {}).get(eni["VpcId"], account) + vrf = self.get_vrf(owner, region, eni["VpcId"]) + parsed = ip_interface(address) + candidates = [ + item for item in self.inventory.get_all_items(NBIPAddress) + if ip_interface(item.data["address"]).ip == parsed.ip and item.data.get("vrf") is vrf + ] + if len(candidates) > 1: + raise ValueError(f"Multiple NetBox IPs match '{address}' in VRF '{vrf.data['name']}'") + existing = candidates[0] if candidates else None + if existing is not None and existing.data.get("assigned_object_id") is not None and \ + existing.get_interface() is not interface: + owner = existing.get_device_vm() + owner_tags = owner.get_tags() if owner is not None else [] + same_source = owner is not None and ( + owner.source is self or self.source_tag in owner_tags) + managed_owner = same_source or NetBoxHandler.primary_tag in owner_tags + managed_orphan = NetBoxHandler.primary_tag in owner_tags and \ + NetBoxHandler.orphaned_tag in owner_tags + owner_status = owner.data.get("status") if owner is not None else None + if isinstance(owner_status, dict): + owner_status = owner_status.get("value") + if managed_owner and owner_status == "active" and vm.data["status"] == "offline": + log.error( + f"AWS source '{self.name}': IP '{address}' in VRF '{vrf.data['name']}' " + f"is assigned to active owner '{owner.get_display_name()}'. " + f"Skipping reassignment to offline ENI '{eni['NetworkInterfaceId']}'.") continue - - matched_prefix = self.return_longest_matching_prefix_for_ip(ip_interface(ip_addr), site_name) - if matched_prefix is None: - matched_prefix = self.return_longest_matching_prefix_for_ip(ip_interface(ip_addr)) - if matched_prefix is not None: - prefix = matched_prefix.data["prefix"].prefixlen - ip_addr = f"{ip_interface(ip_addr).ip}/{prefix}" - - nic_ips[full_name].append(ip_addr) - is_fixed = address.get("OS-EXT-IPS:type") == "fixed" - if ip_interface(ip_addr).version == 4 and ( - vm_primary_ip4 is None or (is_fixed and not vm_primary_ip4_is_fixed)): - vm_primary_ip4 = ip_addr - vm_primary_ip4_is_fixed = is_fixed - if ip_interface(ip_addr).version == 6 and ( - vm_primary_ip6 is None or (is_fixed and not vm_primary_ip6_is_fixed)): - vm_primary_ip6 = ip_addr - vm_primary_ip6_is_fixed = is_fixed - - # add VM to inventory - vm = self.add_device_vm_to_inventory(NBVM, object_data=vm_data, vnic_data=vm_nic_dict, - nic_ips=nic_ips, p_ipv4=vm_primary_ip4, p_ipv6=vm_primary_ip6) - if version.parse(self.inventory.netbox_api_version) >= version.parse("3.7.0"): - for disk_name, disk_size in disks.items(): - data = {"name": disk_name, "virtual_machine": vm} - if disk_size is None: - existing = self.inventory.get_by_data(NBVirtualDisk, data=data) - if existing is not None: - existing.update(data=data, source=self) + offline_to_active = managed_owner and owner_status == "offline" and vm.data["status"] == "active" + if not (same_source or managed_orphan or offline_to_active): + owner_name = owner.get_display_name() if owner is not None else "unresolved owner" + log.error( + f"AWS source '{self.name}': IP '{address}' in VRF '{vrf.data['name']}' " + f"belongs to another source (owner '{owner_name}'). " + f"Skipping reassignment to ENI '{eni['NetworkInterfaceId']}'.") continue - multiplier = 1000 if version.parse(self.inventory.netbox_api_version) >= version.parse("4.1.0") else 1 - data["size"] = disk_size * multiplier - self.inventory.add_update_object(NBVirtualDisk, data=data, source=self) - - return - - def update_basic_data(self): - """ - - Returns - ------- - - """ - - # add source identification tag - self.inventory.add_update_object(NBTag, data={ - "name": self.source_tag, - "description": f"Marks objects synced from Openstack '{self.name}' " - f"({self.settings.auth_url}) to this NetBox Instance." - }) - - # update virtual site if present - this_site_object = self.inventory.get_by_data(NBSite, data={"name": self.site_name}) - - if this_site_object is not None: - this_site_object.update(data={ - "name": self.site_name, - "comments": "A default virtual site created to house objects " - "that have been synced from this Openstack instance " - "and have no predefined site assigned." - }) - - server_role_object = self.inventory.get_by_data(NBDeviceRole, data={"name": "Server"}) - - if server_role_object is not None: - role_data = {"name": "Server", "vm_role": True} - if server_role_object.is_new is True: - role_data["color"] = "9e9e9e" - - server_role_object.update(data=role_data) - -# EOF + if managed_orphan or offline_to_active: + reason = "orphaned" if managed_orphan else "offline" + log.warning( + f"AWS source '{self.name}': Reclaiming IP '{address}' in VRF '{vrf.data['name']}' " + f"from {reason} owner '{owner.get_display_name()}' " + f"for ENI '{eni['NetworkInterfaceId']}'.") + ip_data = {"address": address, "vrf": vrf, "assigned_object_id": interface, + "assigned_object_type": NBVMInterface} + if self.settings.tenant: + ip_data["tenant"] = {"name": self.settings.tenant} + if existing is None: + existing = self.inventory.add_object(NBIPAddress, data=ip_data, source=self) + else: + existing.update(data=ip_data, source=self) + discovered.append(existing) + if eni.get("Attachment", {}).get("DeviceIndex") == 0 and primary: + vm.update(data={f"primary_ip{parsed.version}": existing}, source=self) + for old in interface.get_ip_addresses(): + if old not in discovered and (old.source is self or self.source_tag in old.get_tags()): + old.remove_interface_association() From bb1fb4fd6e4cc639c5e5fa5ac779462a062eca4b Mon Sep 17 00:00:00 2001 From: Eric Krichbaum Date: Sat, 14 Nov 2026 15:52:55 -0600 Subject: [PATCH 18/21] Refactor service_object method in connection.py Refactor service_object method to simplify matching logic and remove legacy handling. Remove patched version from other fix --- module/sources/aws/connection.py | 24 +++--------------------- 1 file changed, 3 insertions(+), 21 deletions(-) diff --git a/module/sources/aws/connection.py b/module/sources/aws/connection.py index 8df4e270..25ece72c 100644 --- a/module/sources/aws/connection.py +++ b/module/sources/aws/connection.py @@ -342,29 +342,10 @@ def resource_name(self, name, identity, limit): return f"{name[:limit - 13]}-{suffix}" def service_object(self, object_type, identity, data): - """Adopt tagged legacy objects; EC2 VMs carry aws_instance_id, not aws_resource_id.""" fields = data["custom_fields"] fields["aws_metadata"] = json_safe(fields["aws_metadata"]) - items = self.inventory.get_all_items(object_type) - matches = [item for item in items + matches = [item for item in self.inventory.get_all_items(object_type) if (item.data.get("custom_fields") or {}).get("aws_resource_id") == identity] - parent = "virtual_machine" if object_type is NBVirtualDisk else "cluster" if object_type is NBVM else "group" - if not matches: - name_matches = [item for item in items - if item.data.get("name") == data["name"] and - item.data.get(parent) is data.get(parent)] - if len(name_matches) == 1: - candidate = name_matches[0] - fields = candidate.data.get("custom_fields") or {} - candidate_identity = fields.get("aws_resource_id") - candidate_service = fields.get("aws_service") - identity_missing = candidate_identity is None or ( - isinstance(candidate_identity, str) and not candidate_identity.strip()) - service = (data.get("custom_fields") or {}).get("aws_service") - service_matches = candidate_service in (None, "", service) - source_managed = candidate.source is self or self.source_tag in candidate.get_tags() - if identity_missing and service_matches and source_managed: - matches = [candidate] if len(matches) > 1: raise ValueError(f"Multiple NetBox {object_type.name} objects match AWS resource '{identity}'") if matches: @@ -373,8 +354,9 @@ def service_object(self, object_type, identity, data): raise ValueError(f"AWS resource '{identity}' belongs to another source") item.update(data=data, source=self) else: + parent = "virtual_machine" if object_type is NBVirtualDisk else "cluster" if object_type is NBVM else "group" if any(item.data.get("name") == data["name"] and item.data.get(parent) is data.get(parent) - for item in items): + for item in self.inventory.get_all_items(object_type)): raise ValueError(f"NetBox {object_type.name} name collision for AWS resource '{identity}'") item = self.inventory.add_object(object_type, data=data, source=self) self.reconcile_tags(item, data["tags"]) From 3938e8f61a06b1e3a83ef0b0b4bd8833fd321b8d Mon Sep 17 00:00:00 2001 From: Sergey Sannikov Date: Sun, 15 Nov 2026 12:28:06 +0400 Subject: [PATCH 19/21] openstack: don't abort the run on a MAC two interfaces share, test the stale IP guards Also drops the local copy of object_synced_by_other_source, dumps service_details as the dict it is and sends the netbox-sync version. --- docs/source_openstack.md | 5 ++- module/sources/openstack/connection.py | 19 +++------ tests/test_openstack_source.py | 59 ++++++++++++++++++++++++++ 3 files changed, 69 insertions(+), 14 deletions(-) diff --git a/docs/source_openstack.md b/docs/source_openstack.md index 3617eb1d..3a5cac32 100644 --- a/docs/source_openstack.md +++ b/docs/source_openstack.md @@ -15,7 +15,7 @@ pip install -r requirements-openstack.txt uv sync --extra openstack ``` -The supported SDK range is `openstacksdk>=4.0,<5`. Selecting an OpenStack source +The tested SDK version is `openstacksdk==4.22.0`. Selecting an OpenStack source without the SDK logs an error and skips that source. ## Authentication and permissions @@ -120,7 +120,8 @@ names before positional interface matching is attempted. Detaching an earlier port therefore does not renumber or steal the later port's interface, MAC, or IPs; the detached interface is left for normal orphan handling. An interface paired without a MAC match is renamed to the discovered network -name. Ports without MAC data still rely on positional/shared matching. +name. Ports without MAC data, or with a MAC that more than one interface of the VM +carries, still rely on positional/shared matching. Virtual disks are matched by disk name within the resolved VM, so same-named VMs in different availability zones retain separate disks. For `all_projects = True`, the source can repair an IP still assigned to an diff --git a/module/sources/openstack/connection.py b/module/sources/openstack/connection.py index 2586d021..6450526b 100644 --- a/module/sources/openstack/connection.py +++ b/module/sources/openstack/connection.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright (c) 2020 - 2021 Ricardo Bartels. All rights reserved. +# Copyright (c) 2020 - 2026 netbox-sync team. All rights reserved. # # netbox-sync.py # @@ -18,6 +18,7 @@ from module.sources.common.source_base import SourceBase from module.sources.openstack.config import OpenStackConfig +from module import __version__ from module.common.logging import get_logger, DEBUG3 from module.common.misc import grab, dump, get_string_or_none from module.common.support import normalize_mac_address @@ -138,7 +139,7 @@ def create_openstack_session(self): project_domain_name=self.settings.project_domain, verify=self.settings.validate_tls_certs, app_name='netbox-sync', - app_version='0.1', + app_version=__version__, ) self.session.authorize() except Exception as e: @@ -615,12 +616,6 @@ def get_compute_host_names(hosts): return compute_host_names - def object_synced_by_other_source(self, nb_object): - if nb_object.source is not None: - return nb_object.source is not self - other_tags = [source.source_tag for source in self.inventory.source_list if source is not self] - return bool(set(nb_object.get_tags()).intersection(other_tags)) - def add_custom_attributes(self, object_type, obj, data): attributes = self.settings.host_custom_object_attributes if object_type == NBDevice else \ self.settings.vm_custom_object_attributes @@ -794,9 +789,9 @@ def add_device_vm_to_inventory(self, object_type, object_data, pnic_data=None, v mac = int_data.get("mac_address") matches = current_by_mac.get(normalize_mac_address(mac), []) if mac else [] if len(matches) > 1: - message = f"Multiple interfaces of OpenStack VM '{object_name}' match MAC '{mac}'." - log.error(message) - raise ValueError(message) + log.warning(f"Multiple interfaces of OpenStack VM '{object_name}' match MAC '{mac}', " + f"matching this port by name instead.") + continue if matches: names_by_mac[int_name] = matches[0].data["name"] mac_matched_interfaces.add(matches[0]) @@ -1078,7 +1073,7 @@ def add_host(self, obj): if log.level == DEBUG3: try: log.info("Cluster data") - dump(obj.service_details.to_dict()) + pprint.pprint(obj.service_details) except Exception as e: log.error(e) diff --git a/tests/test_openstack_source.py b/tests/test_openstack_source.py index 79e3026b..9f88ac6c 100644 --- a/tests/test_openstack_source.py +++ b/tests/test_openstack_source.py @@ -355,6 +355,65 @@ def test_stale_ip_repair_requires_complete_all_projects_discovery(cloud, all_pro assert assigned_vm.data["name"] == expected_vm_name +def test_two_interfaces_with_the_same_mac_do_not_stop_the_run(cloud): + handler = OpenStackHandler("cloud") + handler.apply() + vm = cloud.inventory.get_all_items(NBVM)[0] + interface = cloud.inventory.get_all_interfaces(vm)[0] + # a second interface with the same MAC, e.g. made by hand + cloud.inventory.add_object(NBVMInterface, data={ + "name": "eth-copy", "virtual_machine": vm, "primary_mac_address": interface.data["primary_mac_address"], + }, read_from_netbox=True) + + handler.apply() + + assert interface.data["name"] == "vNIC1 (private)" + assert "10.0.0.8/32" in {ip.data["address"] for ip in interface.get_ip_addresses()} + + +@pytest.mark.parametrize("blocker", ["old-server-still-listed", "address-reported-twice", + "ip-tagged-by-another-source", "incomplete-server-list"]) +def test_stale_ip_repair_is_blocked(cloud, blocker): + old_id = "00000000-0000-0000-0000-000000000001" + new_id = "00000000-0000-0000-0000-000000000002" + other_id = "00000000-0000-0000-0000-000000000003" + old_port = {"addr": "10.0.0.42", "version": 4, "OS-EXT-IPS-MAC:mac_addr": "fa:16:3e:00:00:01"} + cloud.session.compute.servers.return_value = [ + server(id=old_id, name="deleted.example.org", addresses={"private": [old_port]}) + ] + handler = OpenStackHandler("cloud") + handler.apply() + old_vm = cloud.inventory.get_all_items(NBVM)[0] + stale_ip = old_vm.data["primary_ip4"] + old_vm.add_tags(handler.source_tag) + stale_ip.add_tags(handler.source_tag) + old_vm.source = None + stale_ip.source = None + + servers = [server(id=new_id, name="replacement.example.org", addresses={"private": [ + {"addr": "10.0.0.43", "version": 4, "OS-EXT-IPS-MAC:mac_addr": "fa:16:3e:00:00:02"}, + {"addr": "10.0.0.42", "version": 4, "OS-EXT-IPS-MAC:mac_addr": "fa:16:3e:00:00:03"}, + ]})] + if blocker == "old-server-still-listed": + # the old server still exists, it is only filtered out of this sync + cloud.settings["vm_exclude_filter"] = "deleted.*" + servers.append(server(id=old_id, name="deleted.example.org", addresses={"private": [ + {"addr": "10.0.0.44", "version": 4, "OS-EXT-IPS-MAC:mac_addr": "fa:16:3e:00:00:04"}]})) + elif blocker == "address-reported-twice": + servers.append(server(id=other_id, name="other.example.org", addresses={"private": [ + {"addr": "10.0.0.42", "version": 4, "OS-EXT-IPS-MAC:mac_addr": "fa:16:3e:00:00:05"}]})) + elif blocker == "ip-tagged-by-another-source": + stale_ip.add_tags("Source: other") + elif blocker == "incomplete-server-list": + servers.append(server(id="not-a-uuid", name="broken.example.org", addresses={"private": [ + {"addr": "10.0.0.45", "version": 4, "OS-EXT-IPS-MAC:mac_addr": "fa:16:3e:00:00:06"}]})) + cloud.session.compute.servers.return_value = servers + handler = OpenStackHandler("cloud") + handler.apply() + + assert stale_ip.get_device_vm() is old_vm + + def test_network_ports_invalid_addresses_and_floating_primary_preference(cloud): cloud.settings["permitted_subnets"] += ", 192.0.2.0/24" vm_resource = server() From 22ddf8973d74bf973a96cb3f0e43872a22b7a25c Mon Sep 17 00:00:00 2001 From: Sergey Sannikov Date: Sun, 15 Nov 2026 12:28:15 +0400 Subject: [PATCH 20/21] openstack: pin openstacksdk 4.22.0, drop the removed settings, add CODEOWNERS --- CODEOWNERS | 2 + README.md | 2 +- module/sources/openstack/config.py | 22 +-- pyproject.toml | 2 +- requirements-openstack.txt | 2 +- tests/test_openstack_source.py | 3 +- uv.lock | 242 ++++++++++++++++++++++++++++- 7 files changed, 248 insertions(+), 27 deletions(-) diff --git a/CODEOWNERS b/CODEOWNERS index 5c64bb30..5c6e5276 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -6,3 +6,5 @@ /docs/source_hetzner.md @SerhiiZahuba /module/sources/ovirt/ @ekrichbaum /docs/source_ovirt.md @ekrichbaum +/module/sources/openstack/ @ekrichbaum +/docs/source_openstack.md @ekrichbaum diff --git a/README.md b/README.md index 3e2c882a..cba24118 100644 --- a/README.md +++ b/README.md @@ -50,7 +50,7 @@ This ensures stale objects are removed from NetBox keeping an accurate current s * oVirt / Red Hat Virtualization Engine v4 API * Optional SDK and native libraries: see [installation and configuration](docs/source_ovirt.md) #### Source: OpenStack (if used) -* Identity v3, Nova and Cinder APIs; `openstacksdk>=4.0,<5` +* Identity v3, Nova and Cinder APIs; `openstacksdk==4.22.0` * See [OpenStack installation and configuration](docs/source_openstack.md) for credentials, permissions, and inventory mapping # Installing diff --git a/module/sources/openstack/config.py b/module/sources/openstack/config.py index 63238580..f3a329b0 100644 --- a/module/sources/openstack/config.py +++ b/module/sources/openstack/config.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright (c) 2020 - 2023 Ricardo Bartels. All rights reserved. +# Copyright (c) 2020 - 2026 netbox-sync team. All rights reserved. # # netbox-sync.py # @@ -262,26 +262,6 @@ def __init__(self): bool, description="Set the name in Netbox to the VM UUID instead of name", default_value=False), - - # removed settings - ConfigOption("netbox_host_device_role", - str, - deprecation_message="You need to switch to 'host_role_relation'.", - removed=True), - ConfigOption("netbox_vm_device_role", - str, - deprecation_message="You need to switch to 'vm_role_relation'.", - removed=True), - ConfigOption("sync_tags", - bool, - deprecation_message="You need to switch to 'host_tag_relation', " + - "'vm_tag_relation' or 'cluster_tag_relation'", - removed=True), - ConfigOption("sync_parent_tags", - bool, - deprecation_message="You need to switch to 'host_tag_relation', " + - "'vm_tag_relation' or 'cluster_tag_relation'", - removed=True) ] super().__init__() diff --git a/pyproject.toml b/pyproject.toml index c8d6c1a1..90ba1ac2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,7 +34,7 @@ aws = [ "boto3>=1.40,<2", ] openstack = [ - "openstacksdk>=4.0,<5", + "openstacksdk==4.22.0", ] ovirt = [ "ovirt-engine-sdk-python==4.6.3", diff --git a/requirements-openstack.txt b/requirements-openstack.txt index 99abdacb..00e6942d 100644 --- a/requirements-openstack.txt +++ b/requirements-openstack.txt @@ -1,2 +1,2 @@ -r requirements.txt -openstacksdk>=4.0,<5 +openstacksdk==4.22.0 diff --git a/tests/test_openstack_source.py b/tests/test_openstack_source.py index 9f88ac6c..d0ef022d 100644 --- a/tests/test_openstack_source.py +++ b/tests/test_openstack_source.py @@ -6,6 +6,7 @@ import pytest +from module import __version__ from module.config.file_output import ConfigFileOutput from module.config.files import ConfigFileINI, ConfigFileYAML from module.config.parser import ConfigParser @@ -151,7 +152,7 @@ def test_registration_authentication_and_project_scope(cloud): auth_url=cloud.settings["auth_url"], project_name="admin", username="sync", password="test-only", region_name="RegionOne", user_domain_name="Default", project_domain_name="Default", verify=False, load_yaml_config=False, load_envvars=False, - app_name="netbox-sync", app_version="0.1", + app_name="netbox-sync", app_version=__version__, ) cloud.session.authorize.assert_called_once() handler.apply() diff --git a/uv.lock b/uv.lock index 990556d0..80a22e81 100644 --- a/uv.lock +++ b/uv.lock @@ -280,6 +280,91 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, ] +[[package]] +name = "cryptography" +version = "50.0.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cffi", marker = "platform_python_implementation != 'PyPy'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9d/af/182eb91b0df3fe75c4d9f26fe70684569566745f6ba7e5c9c73a862c5252/cryptography-50.0.2.tar.gz", hash = "sha256:7b46165bb56eb4704e2eaaf86f3c940d19154535d9b0ca7d6d590b04060e00d5", size = 880623, upload-time = "2026-09-30T15:30:04.884Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e5/56/d194340cc4a57535e82e1bee9e89667ac4b7c13b5d3f59686deae3094dd5/cryptography-50.0.2-cp311-abi3-macosx_11_0_arm64.whl", hash = "sha256:fa8f5efb344d6908a1ce62f4a24e2e5780f825d6f53f5f50ec5ffacac72936cb", size = 3914904, upload-time = "2026-09-30T14:43:44.339Z" }, + { url = "https://files.pythonhosted.org/packages/d9/69/c9bd862c3bf43d6399c433caf002df16e2dffd4be49bdf515cda38038711/cryptography-50.0.2-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:79def8d059362e7831389ed3be0ecdf58a89386e1271e35dd9f5af84e81bffd0", size = 4731146, upload-time = "2026-09-30T14:43:47.113Z" }, + { url = "https://files.pythonhosted.org/packages/21/69/64cef1f702bf6657e0cc186ed1a2891d50d29fb41586b254e1c07adea261/cryptography-50.0.2-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:630ebfea3bf689d075f82316324ff7433dc447fe6bc1bfc76524b74b4a9567d2", size = 4719841, upload-time = "2026-09-30T14:43:49.01Z" }, + { url = "https://files.pythonhosted.org/packages/38/6b/61a3f8d8c5e1e49a6cddccafc4015cc1c0021360ab0acb4080e7a423644a/cryptography-50.0.2-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:f9f6143a8c75945eb960d9eb98905a441394abfa24afaae239d514ffb2586480", size = 4738340, upload-time = "2026-09-30T14:43:50.932Z" }, + { url = "https://files.pythonhosted.org/packages/7b/2e/7212ca32fd43dc91f2f41db20160b268098874b4c9a0e7be94d6835f5b2e/cryptography-50.0.2-cp311-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:a582ab2ae1d34f67112cadc86702774c9ea4374df6bca6afe672817203c99134", size = 5367029, upload-time = "2026-09-30T14:43:52.911Z" }, + { url = "https://files.pythonhosted.org/packages/1a/f1/b474e930c4d910328780e3940da76f5aa5cbc48ce1fc14e44d239d9ea9db/cryptography-50.0.2-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:4061c0079120205fb760c58acab6443e217307dcf05e3702cf970e0689972856", size = 4753050, upload-time = "2026-09-30T14:43:55.272Z" }, + { url = "https://files.pythonhosted.org/packages/7c/52/9af10e80ac16b0fcc2123f9cbd5e7afbd0fd5075bb7a607c592258a39cda/cryptography-50.0.2-cp311-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:ac9ed99d81760c62fe89d5f0815cdfa1ba9a35141cf30f1c2d044f04b4803d2e", size = 4376724, upload-time = "2026-09-30T14:43:57.24Z" }, + { url = "https://files.pythonhosted.org/packages/71/37/6202e488cc1eb625ea110c292c6bda92823176e023f427d8d5660ce8d632/cryptography-50.0.2-cp311-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:87e9ce85beb6b328ba370cc6e6aea483c92617b4c95b1d33a49297eb662bfb04", size = 4737859, upload-time = "2026-09-30T14:43:59.541Z" }, + { url = "https://files.pythonhosted.org/packages/8f/30/e86d7d518489b0ae2497091a35287abcb1a2ce4037837a34afbe9b1d6964/cryptography-50.0.2-cp311-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:f265528741e048bce55c3463ed721fb0aa45a5888d8add8cfeccb3035451bbdc", size = 5324103, upload-time = "2026-09-30T14:44:01.901Z" }, + { url = "https://files.pythonhosted.org/packages/d3/69/2c833a049475e0a3444e94c7d0aca0aa51d166374a449b09e92ac98138de/cryptography-50.0.2-cp311-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:9dab55f57c74c3cad24c323bacbbd04be4705ba6eb0d92e920b1fc4837ed5079", size = 4752576, upload-time = "2026-09-30T14:44:04.545Z" }, + { url = "https://files.pythonhosted.org/packages/6c/5d/906970b83bbfc1f5bbfb677a143c181f2801f23b6a7204a3b47c42c97e65/cryptography-50.0.2-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:25784ce8b9621c90c643efb9e1e2162ab3b0224cae446ad5e70e7fcb1ce18b51", size = 4870819, upload-time = "2026-09-30T14:44:06.884Z" }, + { url = "https://files.pythonhosted.org/packages/68/e3/f2298d3bb55e0c4a91841ec4d01b3f020ba8c5fbf15ccdcc6dcf03f97025/cryptography-50.0.2-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:85d0d9a31b9098e98534226d5686b47264b95e62ce459dc2e62fdfc809f9fe93", size = 5030152, upload-time = "2026-09-30T14:44:09.443Z" }, + { url = "https://files.pythonhosted.org/packages/9a/4f/adfc442765721292fff86d314ce385d3249d22db42295c0dd057727b60f3/cryptography-50.0.2-cp311-abi3-win_amd64.whl", hash = "sha256:7afa5a6602a9f29af1f3a2965f831bae7c9d5d597b7cbb716d41ab3b7d89879c", size = 3824692, upload-time = "2026-09-30T14:44:11.671Z" }, + { url = "https://files.pythonhosted.org/packages/ce/cb/52eb3770c0d0be2702a98c6e96065ddc0a2877cf0845aa9c23397c142cd4/cryptography-50.0.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:f785f6161f202ab04d8ca194158968798e480ca058943907972da5f12e2881e8", size = 3892731, upload-time = "2026-09-30T14:44:13.485Z" }, + { url = "https://files.pythonhosted.org/packages/19/8e/aa1fc533d4546b127b45de8aa024eb5933d23eff9debfe25931e56861095/cryptography-50.0.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:0ecbc5652bdb6fc9eaf89a7d196e20941adfe812f43bc4ca05d9150496821047", size = 4710431, upload-time = "2026-09-30T14:44:15.427Z" }, + { url = "https://files.pythonhosted.org/packages/6a/64/72bc3f75176e7e406b748a3e3830432b8c51297b38368713df04dc04898a/cryptography-50.0.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ab50ee449bf968271e820086f10a33d101dd060370abc10bcd22279be2656539", size = 4694824, upload-time = "2026-09-30T14:44:17.69Z" }, + { url = "https://files.pythonhosted.org/packages/4e/c6/62c77550edfa5ca3f14bf44a1e6739b9fa09d6e998a11d97ed8213bccc98/cryptography-50.0.2-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:a9f7355e6fab51f6c369b86fb7571cffa05edee2c2121e0380a37fb9ac1cd5c1", size = 4716967, upload-time = "2026-09-30T14:44:19.661Z" }, + { url = "https://files.pythonhosted.org/packages/f4/37/cce70f150c432914460157a6ecc161752e053aa5ec0ef3b3f7dc6e31039a/cryptography-50.0.2-cp314-cp314t-manylinux_2_28_ppc64le.whl", hash = "sha256:94e5e9f108ee10471288214d3d233fbfbb492840a8457eb85178d643ddeb32c7", size = 5328676, upload-time = "2026-09-30T14:44:21.744Z" }, + { url = "https://files.pythonhosted.org/packages/aa/9a/6f2f0304d634ceafdeaf23e84537336664ac419b5d07611675c2ad3f6b7a/cryptography-50.0.2-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:241449bf940a5d27309bd317e6f9a2af6932113818bb2b8f5c59ddc7ef16da18", size = 4727698, upload-time = "2026-09-30T14:44:24.178Z" }, + { url = "https://files.pythonhosted.org/packages/1d/de/66bcf9244d118663b2e1aaded8990f4640e3d7b7411870a5765f252074d2/cryptography-50.0.2-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:d8947001be83df1394050758ce0e745dd74fb134eef0a4b5124208dfc3a68c37", size = 4354821, upload-time = "2026-09-30T14:44:26.263Z" }, + { url = "https://files.pythonhosted.org/packages/bd/e6/db28a28c7b6c676addce89136de3d8db49ea825a8c863472e36e42ead4ad/cryptography-50.0.2-cp314-cp314t-manylinux_2_34_aarch64.whl", hash = "sha256:4a20ce1e5cb4284a86692fdcba7cb8754185c6b2e5c56fcef3751cf451d3cdc2", size = 4716748, upload-time = "2026-09-30T14:44:28.447Z" }, + { url = "https://files.pythonhosted.org/packages/30/96/01546c7f69ea0e2ab790a2e4f0934a4052fb9b388147fbf83c2fd72f1e57/cryptography-50.0.2-cp314-cp314t-manylinux_2_34_ppc64le.whl", hash = "sha256:84f964e537f916e2cc85199e5a88742e964939b575ac8598b3f9d6cc416cdaf1", size = 5285085, upload-time = "2026-09-30T14:44:30.704Z" }, + { url = "https://files.pythonhosted.org/packages/6c/01/03263395f74d50b071e9e66daace3f8bef80493e5d410726f2ba8554736b/cryptography-50.0.2-cp314-cp314t-manylinux_2_34_x86_64.whl", hash = "sha256:828d49b0ff5a0e3975865571c5d91dbbdd0d38d8289b249a163e9425413a5e05", size = 4727268, upload-time = "2026-09-30T14:44:32.92Z" }, + { url = "https://files.pythonhosted.org/packages/eb/94/2bfe8f29ec0cc9c0d99359c4161adf32858e4934b72c6d100d2ac0bbe962/cryptography-50.0.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:deb9fde5c60e437ee4821bc9bc39ff31b42135c27e1dc61ef0a629389c1de62e", size = 4849503, upload-time = "2026-09-30T14:44:34.969Z" }, + { url = "https://files.pythonhosted.org/packages/54/44/e80651ecbf0e42b62e2bb5f5768916e07eea72e1297338956a61df361f88/cryptography-50.0.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:8c71ba2cd31fc93748c38e1b613200ff1c2665cbfd5341fe3a61cfde35a1430e", size = 5004057, upload-time = "2026-09-30T14:44:37.064Z" }, + { url = "https://files.pythonhosted.org/packages/f8/cc/1d33befb3cd7ea7e77d2d73f43f2066471da1b21f24a6156efcaabf6d2e8/cryptography-50.0.2-cp314-cp314t-win_amd64.whl", hash = "sha256:78198641e5be9521beea5aa782bb551a58068d10e6eb04c9c680c1b69f2e7d45", size = 3795868, upload-time = "2026-09-30T14:44:39.71Z" }, + { url = "https://files.pythonhosted.org/packages/2d/49/93f6a6e7a87c9aa68d44d3e1cdb5fe8f60c90d5d2f46acae9a56892816b8/cryptography-50.0.2-cp315-abi3.abi3t-macosx_11_0_arm64.whl", hash = "sha256:edc3342adf8f697fc5f59c887a304356f147b397809440ed64e2fa6af2f50f37", size = 4133708, upload-time = "2026-09-30T14:44:41.807Z" }, + { url = "https://files.pythonhosted.org/packages/8c/75/32ac2a56243d778805c16ca6a32b8f74fb757df7e28d7ecb560afafb59cf/cryptography-50.0.2-cp315-abi3.abi3t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d370b8d1dfcdf7130178137f6fbee6140774a1acc6cacefc4b42643ec11d0a3a", size = 4956267, upload-time = "2026-09-30T14:44:43.693Z" }, + { url = "https://files.pythonhosted.org/packages/aa/a4/2c8d734e43d97f0842ee9f1b7b4bfb3d0cf5e19edebf43c2afe6675c2320/cryptography-50.0.2-cp315-abi3.abi3t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f2f9bd7f90c64fe89253f0a2c05e3c4856072660429ce8831b4235bf29403a67", size = 4966465, upload-time = "2026-09-30T14:44:45.769Z" }, + { url = "https://files.pythonhosted.org/packages/c2/58/ee288c829a6f41f6235ae9dd33d82fd19b45442b65b4c8a3da36963d9f7a/cryptography-50.0.2-cp315-abi3.abi3t-manylinux_2_28_aarch64.whl", hash = "sha256:e275096ea1e60cc595cda2836fd4a6c725d1125108b868be17f53684d164e2cc", size = 4959356, upload-time = "2026-09-30T14:44:48.211Z" }, + { url = "https://files.pythonhosted.org/packages/92/20/9ded6d51ddd9897f6b6e81fb9ebea7951d7cc5d6c890b0ed8abf77a51a80/cryptography-50.0.2-cp315-abi3.abi3t-manylinux_2_28_ppc64le.whl", hash = "sha256:b13478603dcd0a2479ff8e87e2c19a7d525734686fe3c49542472293a204212d", size = 5548822, upload-time = "2026-09-30T14:44:50.86Z" }, + { url = "https://files.pythonhosted.org/packages/02/a8/8df951850d6b31d2a00218f19e2b3f999523437ed7a819df7fa427942fca/cryptography-50.0.2-cp315-abi3.abi3t-manylinux_2_28_x86_64.whl", hash = "sha256:58a0c478eeca76fe5e07993c5a0703def34a6dc6a0cda4f5564639b33112ffe7", size = 5001199, upload-time = "2026-09-30T14:44:53.379Z" }, + { url = "https://files.pythonhosted.org/packages/8b/f9/36b3022218ce75b7cdf068fb95f809f9bd0d820e4955ef43b90c255cc7ac/cryptography-50.0.2-cp315-abi3.abi3t-manylinux_2_31_armv7l.whl", hash = "sha256:d38cdff612d06fa6a32840d5e1b1f7a27cee4a349aa9085d94a67789d6bfd408", size = 4629333, upload-time = "2026-09-30T14:44:55.635Z" }, + { url = "https://files.pythonhosted.org/packages/8c/72/20f99a219f6af47cdd1cbd978c243b92d71496e168a746138af44ded4f29/cryptography-50.0.2-cp315-abi3.abi3t-manylinux_2_34_aarch64.whl", hash = "sha256:fdd28f912fccfec1846a94e2e1e8f9b0012f557f0c46fe4f3eb0d7a87afcf90b", size = 4958822, upload-time = "2026-09-30T14:44:59.639Z" }, + { url = "https://files.pythonhosted.org/packages/f2/20/196f112617fb08eb4d608a2a6c422373d46f9cc2857f38fc0667033c0899/cryptography-50.0.2-cp315-abi3.abi3t-manylinux_2_34_ppc64le.whl", hash = "sha256:cbc8738fd8526d80f35cb3a40d41f41a2e7030bb3b18b09a6778ef63d291c2fd", size = 5506351, upload-time = "2026-09-30T14:45:02.267Z" }, + { url = "https://files.pythonhosted.org/packages/24/95/83378121ef3eaaaf71d4b781577ff794acb39b9e1b87a3f156898c8497ed/cryptography-50.0.2-cp315-abi3.abi3t-manylinux_2_34_x86_64.whl", hash = "sha256:e105ab60406787da31fccc883fc0f733af1efd78f0136a4599692c4083a73d0c", size = 5000859, upload-time = "2026-09-30T14:45:05.009Z" }, + { url = "https://files.pythonhosted.org/packages/22/f7/70fd7ae4d1dbfa7ba29b02e1b9068771519a86027756510b700ce81086a8/cryptography-50.0.2-cp315-abi3.abi3t-musllinux_1_2_aarch64.whl", hash = "sha256:6f8700550aa1474a91e5dc07049c46f98b423b5b1ddd0483e0b51362eeeaf5be", size = 5092151, upload-time = "2026-09-30T15:29:15.932Z" }, + { url = "https://files.pythonhosted.org/packages/d4/be/688367b74de86984bd58d8efacfc7c9e68b89a6a22ced0fb4f38db50254a/cryptography-50.0.2-cp315-abi3.abi3t-musllinux_1_2_x86_64.whl", hash = "sha256:c71be1cbfa5cd9a41ee452acf1eccd82b2c05950358b106ec8ceb83411d1a020", size = 5286120, upload-time = "2026-09-30T15:29:18.309Z" }, + { url = "https://files.pythonhosted.org/packages/39/d1/55f8a3f2ef5d1529e16835ef10cf0fe3d559ce237b46dddc440c0bba3649/cryptography-50.0.2-cp315-abi3.abi3t-win_amd64.whl", hash = "sha256:c423ab384a46c4dff7217b2ea5ba2e11cffdeab6441acd04cf65a369caf0366c", size = 4111557, upload-time = "2026-09-30T15:29:20.155Z" }, + { url = "https://files.pythonhosted.org/packages/23/ad/ac987755d00e1e64273760228d2635ae38dae2be83e3c6e0d3289d91dec3/cryptography-50.0.2-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:0ec5f09541743261e66e291b4a0cbf0fb2997aeaab6d9e9c740b9dba1b58d1c2", size = 3943588, upload-time = "2026-09-30T15:29:22.265Z" }, + { url = "https://files.pythonhosted.org/packages/d5/8d/6d585339bedf85d45044c85d8412dac53f2bb6f918e8b7777efba1787844/cryptography-50.0.2-cp39-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:c5e67125c7dca78d199ec4e116aa93dbb83494808ecbb8211a2cb09b1bf41dbd", size = 4756166, upload-time = "2026-09-30T15:29:24.58Z" }, + { url = "https://files.pythonhosted.org/packages/bf/f1/1c1f6874e8550cfddd4b688ceb38cefb6ed15ceed224d56f133f3d88c214/cryptography-50.0.2-cp39-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ee247f5c245c9a2fe7c8e2214e295918838e44e00a45a6718451e4004219e767", size = 4749145, upload-time = "2026-09-30T15:29:26.807Z" }, + { url = "https://files.pythonhosted.org/packages/c1/63/61b15dc1a8de03fe0adbe3fd7608b3ad5c73bf50993bbcb1faaa930afe33/cryptography-50.0.2-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:dfe9763530994147d9af1def057a5b9658b00e8f8fe8743d144d1e0911c2e454", size = 4763638, upload-time = "2026-09-30T15:29:28.588Z" }, + { url = "https://files.pythonhosted.org/packages/fc/35/b345bdfa40c9126df1a9d33236aa98418367931b8725f84fc3ae2b98dc59/cryptography-50.0.2-cp39-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:58ddb5a8e3179d12f19e4ea34d2d32e9d63a4baa142c875c1eb59f41b7243acd", size = 5382217, upload-time = "2026-09-30T15:29:30.589Z" }, + { url = "https://files.pythonhosted.org/packages/4f/87/ef344a9e616871f2519c22d6afcda79ddd5d35e9592d95eb6e677608d055/cryptography-50.0.2-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:f21e8a22c8605750c7af886bab299a363721264061b4ac0a30efb73cfd58efc5", size = 4781387, upload-time = "2026-09-30T15:29:32.605Z" }, + { url = "https://files.pythonhosted.org/packages/90/5b/f2fdb13cd0b96f6f932c8627bb292a45f11c64d21620a8e120aee9a3b848/cryptography-50.0.2-cp39-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:9c8402a82ea0dc4ceeab793db05f0fafa8ca139ca34fcde5df0f596103c74107", size = 4403790, upload-time = "2026-09-30T15:29:34.374Z" }, + { url = "https://files.pythonhosted.org/packages/bc/ce/7e4f662b1e3c393513569e402cfc85ac7da0bd3d5435e122a3140219eb2d/cryptography-50.0.2-cp39-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:0ddc924c04591c2811ca024d62ecad4f7f6f08af8939c211438f48a16bd23602", size = 4764319, upload-time = "2026-09-30T15:29:36.149Z" }, + { url = "https://files.pythonhosted.org/packages/3c/3f/86ff33ce34cc0de6847fb96e035a1a760d81652e38643f617c02ad32ef7a/cryptography-50.0.2-cp39-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:a6557e5f38e065ca9fbdaf7cfc7435ecb1d113aa81a022d1b51921ee7432e227", size = 5338560, upload-time = "2026-09-30T15:29:39.053Z" }, + { url = "https://files.pythonhosted.org/packages/40/cf/6b5c8e2fd9202d98988ab7cb5cc5c991704c4ad55f492ff408e4969f83f1/cryptography-50.0.2-cp39-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:1981f1db4630889b9ef7803fadef12b056f428cb6b85c27ba57b774793b6093c", size = 4780973, upload-time = "2026-09-30T15:29:41.251Z" }, + { url = "https://files.pythonhosted.org/packages/10/bf/8d6ebc7dded797bd0f0160d52188021211f011a2b164ef0ae1dac4587465/cryptography-50.0.2-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:7a8701d6b584d76e909e3d305b7d126b41439876a5aaf76cddc67fc230eafa2e", size = 4897738, upload-time = "2026-09-30T15:29:43.106Z" }, + { url = "https://files.pythonhosted.org/packages/d4/aa/f3f6e0de7e6253b8baa8b2d8fb9d50924fa75cee3d4624bd4bc1208ee923/cryptography-50.0.2-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:ce47f66801c20ec6c6632453bb5960fe38939e9306970b48b3a5a26de7745d94", size = 5058280, upload-time = "2026-09-30T15:29:44.827Z" }, + { url = "https://files.pythonhosted.org/packages/f6/b6/a1faf3a27ae9405fb34b1713cc73b2d8a26b04d5c561578fa2e6ef3e5bb9/cryptography-50.0.2-cp39-abi3-win_amd64.whl", hash = "sha256:4e81d95e5bafc2d6e34e4bed780e53e4d5b9a2f928573428aa4d35fbec1eb0de", size = 3854095, upload-time = "2026-09-30T15:29:46.782Z" }, +] + +[[package]] +name = "decorator" +version = "5.3.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/60/8b/32f9823da46cde7df2087faa08cd98d01b908f8dcab982cdba9c84e85355/decorator-5.3.1.tar.gz", hash = "sha256:4cbcdd55a6efadb9dbea26b858f4fb3264567b52d69ca0d25b721b553f60ea82", size = 58084, upload-time = "2026-05-18T06:03:28.057Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/05/7f/798705f5296a58ca505d600456748d1be48078eac8a7050d8a98bc9edb89/decorator-5.3.1-py3-none-any.whl", hash = "sha256:f47fe6fdbd2edd623ecfe36875d37aba411624e2670dd395dddae1358689bb3c", size = 10365, upload-time = "2026-05-18T06:03:26.517Z" }, +] + +[[package]] +name = "dogpile-cache" +version = "1.5.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "decorator" }, + { name = "stevedore" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e7/c8/301ff89746e76745b937606df4753c032787c59ecb37dd4d4250bddc8929/dogpile_cache-1.5.0.tar.gz", hash = "sha256:849c5573c9a38f155cd4173103c702b637ede0361c12e864876877d0cd125eec", size = 947962, upload-time = "2025-10-11T17:35:36.898Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/dc/80/12235e5b75bb2c586733280854f131b86051e0bbdfb55349ff70d0f72cf9/dogpile_cache-1.5.0-py3-none-any.whl", hash = "sha256:dc7b47d37844db15e8fdc0243c1b58857a2ddc52a5118237a97127bac200e18d", size = 64447, upload-time = "2025-10-11T17:35:38.573Z" }, +] + [[package]] name = "hcloud" version = "2.25.1" @@ -311,6 +396,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, ] +[[package]] +name = "iso8601" +version = "2.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b9/f3/ef59cee614d5e0accf6fd0cbba025b93b272e626ca89fb70a3e9187c5d15/iso8601-2.1.0.tar.gz", hash = "sha256:6b1d3829ee8921c4301998c909f7829fa9ed3cbdac0d3b16af2d743aed1ba8df", size = 6522, upload-time = "2023-10-03T00:25:39.317Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6c/0c/f37b6a241f0759b7653ffa7213889d89ad49a2b76eb2ddf3b57b2738c347/iso8601-2.1.0-py3-none-any.whl", hash = "sha256:aac4145c4dcb66ad8b648a02830f5e2ff6c24af20f4f482689be402db2429242", size = 7545, upload-time = "2023-10-03T00:25:32.304Z" }, +] + [[package]] name = "jmespath" version = "1.1.0" @@ -320,9 +414,46 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/14/2f/967ba146e6d58cf6a652da73885f52fc68001525b4197effc174321d70b4/jmespath-1.1.0-py3-none-any.whl", hash = "sha256:a5663118de4908c91729bea0acadca56526eb2698e83de10cd116ae0f4e97c64", size = 20419, upload-time = "2026-01-22T16:35:24.919Z" }, ] +[[package]] +name = "jsonpatch" +version = "1.47" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "jsonpointer" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/8f/13/43ae689f9340dc7339dc7f5a3b1772edcd48fb1eb62fd5d801a2aa09d382/jsonpatch-1.47.tar.gz", hash = "sha256:5ed650b7517789e0f43428aa7b09442c0b90de0a0fd17a53fd0ebf55205c601e", size = 80423, upload-time = "2026-11-11T10:13:30.696Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5d/e9/a09b859f5fbd7a192ed40f9098fb746607ab46eaacc1eecaea713e4df0de/jsonpatch-1.47-py3-none-any.whl", hash = "sha256:c9ba6de59c481bf2be05041514ecb0d97f994aa645c4e00831805dbe2281811a", size = 25270, upload-time = "2026-11-11T10:13:29.316Z" }, +] + +[[package]] +name = "jsonpointer" +version = "3.12" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/46/cb/2c3112a061a718d93866c2555dad1e9d53865e4703d0f3be550d252b277b/jsonpointer-3.12.tar.gz", hash = "sha256:7a3dcc11b64053eaa9c549e85e413f8d40a075215f793a331bed9182a14251ae", size = 23217, upload-time = "2026-11-11T09:34:22.894Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/72/07/3c953fceee478798f6bdf7b39e0da0793f437e8b5f9e9e1360fba32fb25c/jsonpointer-3.12-py3-none-any.whl", hash = "sha256:89f391335ee2719ae72149e1bf30a4d4c7c5d92f15a3f53faa8d20cc7c6c7f19", size = 10562, upload-time = "2026-11-11T09:34:21.711Z" }, +] + +[[package]] +name = "keystoneauth1" +version = "5.18.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "iso8601" }, + { name = "os-service-types" }, + { name = "requests" }, + { name = "stevedore" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/04/af/268e14554c6bd4a2614872fb2b8431e466ce01415a9783b260ce7fb7ae5c/keystoneauth1-5.18.1.tar.gz", hash = "sha256:8970f9d103f335c39151366d70c001d1bb99b00d73845f03d47825e8dc35ff78", size = 319181, upload-time = "2026-10-12T09:04:48.166Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/78/24/38ccf4476c81435c834d4791804099f305b91087fad171ac17d18fc84ca9/keystoneauth1-5.18.1-py3-none-any.whl", hash = "sha256:2440f1f29e4aaaaf771893e8da7c098ca27548210dc13e964b53fd2aaaedd6b0", size = 367402, upload-time = "2026-10-12T09:04:46.753Z" }, +] + [[package]] name = "netbox-sync" -version = "1.9.1" +version = "1.9.3" source = { virtual = "." } dependencies = [ { name = "aiodns" }, @@ -348,6 +479,9 @@ dependencies = [ aws = [ { name = "boto3" }, ] +openstack = [ + { name = "openstacksdk" }, +] ovirt = [ { name = "ovirt-engine-sdk-python" }, ] @@ -364,6 +498,7 @@ requires-dist = [ { name = "charset-normalizer", specifier = "==3.5.2" }, { name = "hcloud", specifier = "==2.25.1" }, { name = "idna", specifier = "==3.20" }, + { name = "openstacksdk", marker = "extra == 'openstack'", specifier = "==4.22.0" }, { name = "ovirt-engine-sdk-python", marker = "extra == 'ovirt'", specifier = "==4.6.3" }, { name = "packaging", specifier = "==26.3" }, { name = "pycares", specifier = "==5.0.1" }, @@ -378,7 +513,43 @@ requires-dist = [ { name = "urllib3", specifier = "==2.8.0" }, { name = "wheel", specifier = "==0.48.0" }, ] -provides-extras = ["aws", "ovirt", "test"] +provides-extras = ["aws", "openstack", "ovirt", "test"] + +[[package]] +name = "openstacksdk" +version = "4.22.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cryptography" }, + { name = "decorator" }, + { name = "dogpile-cache" }, + { name = "iso8601" }, + { name = "jmespath" }, + { name = "jsonpatch" }, + { name = "keystoneauth1" }, + { name = "os-service-types" }, + { name = "pbr" }, + { name = "platformdirs" }, + { name = "psutil" }, + { name = "pyyaml" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/36/de/da119a0fd6ad3794ccc55c4723a50e6db415d495f77ae083622a275d94a3/openstacksdk-4.22.0.tar.gz", hash = "sha256:757f399346f804d0a70b83fead919c498521e509889cd8973ece5cf7a640e6f9", size = 1473175, upload-time = "2026-10-22T13:33:28.491Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/39/ce/62e8f3a5ffc6769dcc666826178a7bf572b8e01b41a74dc893203ed1d5ec/openstacksdk-4.22.0-py3-none-any.whl", hash = "sha256:e5aafe38e4c338bcb3844f1f446d7761c68477db8cec0e4bb91f4de9d35877c1", size = 2057402, upload-time = "2026-10-22T13:33:26.678Z" }, +] + +[[package]] +name = "os-service-types" +version = "1.9.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pbr" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/86/ae/fe7ac23155ae0b4b9779e06e9c5bb4070f2315dc4ca886a88fa3230d344b/os_service_types-1.9.0.tar.gz", hash = "sha256:1f2e5fb71d1f6f4ff31d8992674f2368465bc2f25cd94018015c3ddbfc5c617f", size = 28087, upload-time = "2026-08-04T13:07:43.171Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e3/79/f8ae99ae9bb54cd63e7ab89251b2a4ff4ff07f02847403948b6a6f50f816/os_service_types-1.9.0-py3-none-any.whl", hash = "sha256:f268545c896434177bf55c43f7d5129e65210d5d456df3f00a1c9832a71c3a4b", size = 25282, upload-time = "2026-08-04T13:07:42.258Z" }, +] [[package]] name = "ovirt-engine-sdk-python" @@ -398,6 +569,27 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/63/34/ba1c580383c9eada3711951fef0795c80b829a078d72188184bcab9dd527/packaging-26.3-py3-none-any.whl", hash = "sha256:d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c", size = 129956, upload-time = "2026-08-04T18:15:27.159Z" }, ] +[[package]] +name = "pbr" +version = "7.1.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "setuptools" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/6b/8d/ce438c28c7958e33184e8ac851ea2225b47a41e5e9e708fa3bddba631135/pbr-7.1.3.tar.gz", hash = "sha256:9a4a85b84e906337708009af0b5f5cdabeeb72d4dc213c9e97974da54fd9acc5", size = 140100, upload-time = "2026-10-07T10:38:15.526Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/bb/a2/79a926b7ab54b247c3419bfa00cfbeb78ad495d21c6d787c978c6261623d/pbr-7.1.3-py2.py3-none-any.whl", hash = "sha256:6583e878a1d97cb135fdc509811f31b9235905cde8d4dacd3dbadf9efc45d745", size = 136717, upload-time = "2026-10-07T10:38:14.069Z" }, +] + +[[package]] +name = "platformdirs" +version = "4.14.5" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d7/9a/8475be631fb33ca1466915e70fd594c42a4e84b512db650eaa14f90fe75d/platformdirs-4.14.5.tar.gz", hash = "sha256:32e9921db7385aa8169bee4fe9501923e5715ba9fb7ea3b9c69b0d8e0639f7a1", size = 71464, upload-time = "2026-11-04T16:54:50.358Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7c/ae/ecbbbfd0e4343a6a521d49b3c8656fb4643dee2a4f6eaf941cf3280391a8/platformdirs-4.14.5-py3-none-any.whl", hash = "sha256:8e778c6fa421516f4d3369449f8df3b0e1b72c0e0005fe88e3ec772d0a14e750", size = 35402, upload-time = "2026-11-04T16:54:48.875Z" }, +] + [[package]] name = "pluggy" version = "1.6.0" @@ -407,6 +599,34 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, ] +[[package]] +name = "psutil" +version = "7.2.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/aa/c6/d1ddf4abb55e93cebc4f2ed8b5d6dbad109ecb8d63748dd2b20ab5e57ebe/psutil-7.2.2.tar.gz", hash = "sha256:0746f5f8d406af344fd547f1c8daa5f5c33dbc293bb8d6a16d80b4bb88f59372", size = 493740, upload-time = "2026-01-28T18:14:54.428Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/51/08/510cbdb69c25a96f4ae523f733cdc963ae654904e8db864c07585ef99875/psutil-7.2.2-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:2edccc433cbfa046b980b0df0171cd25bcaeb3a68fe9022db0979e7aa74a826b", size = 130595, upload-time = "2026-01-28T18:14:57.293Z" }, + { url = "https://files.pythonhosted.org/packages/d6/f5/97baea3fe7a5a9af7436301f85490905379b1c6f2dd51fe3ecf24b4c5fbf/psutil-7.2.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:e78c8603dcd9a04c7364f1a3e670cea95d51ee865e4efb3556a3a63adef958ea", size = 131082, upload-time = "2026-01-28T18:14:59.732Z" }, + { url = "https://files.pythonhosted.org/packages/37/d6/246513fbf9fa174af531f28412297dd05241d97a75911ac8febefa1a53c6/psutil-7.2.2-cp313-cp313t-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1a571f2330c966c62aeda00dd24620425d4b0cc86881c89861fbc04549e5dc63", size = 181476, upload-time = "2026-01-28T18:15:01.884Z" }, + { url = "https://files.pythonhosted.org/packages/b8/b5/9182c9af3836cca61696dabe4fd1304e17bc56cb62f17439e1154f225dd3/psutil-7.2.2-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:917e891983ca3c1887b4ef36447b1e0873e70c933afc831c6b6da078ba474312", size = 184062, upload-time = "2026-01-28T18:15:04.436Z" }, + { url = "https://files.pythonhosted.org/packages/16/ba/0756dca669f5a9300d0cbcbfae9a4c30e446dfc7440ffe43ded5724bfd93/psutil-7.2.2-cp313-cp313t-win_amd64.whl", hash = "sha256:ab486563df44c17f5173621c7b198955bd6b613fb87c71c161f827d3fb149a9b", size = 139893, upload-time = "2026-01-28T18:15:06.378Z" }, + { url = "https://files.pythonhosted.org/packages/1c/61/8fa0e26f33623b49949346de05ec1ddaad02ed8ba64af45f40a147dbfa97/psutil-7.2.2-cp313-cp313t-win_arm64.whl", hash = "sha256:ae0aefdd8796a7737eccea863f80f81e468a1e4cf14d926bd9b6f5f2d5f90ca9", size = 135589, upload-time = "2026-01-28T18:15:08.03Z" }, + { url = "https://files.pythonhosted.org/packages/81/69/ef179ab5ca24f32acc1dac0c247fd6a13b501fd5534dbae0e05a1c48b66d/psutil-7.2.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:eed63d3b4d62449571547b60578c5b2c4bcccc5387148db46e0c2313dad0ee00", size = 130664, upload-time = "2026-01-28T18:15:09.469Z" }, + { url = "https://files.pythonhosted.org/packages/7b/64/665248b557a236d3fa9efc378d60d95ef56dd0a490c2cd37dafc7660d4a9/psutil-7.2.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:7b6d09433a10592ce39b13d7be5a54fbac1d1228ed29abc880fb23df7cb694c9", size = 131087, upload-time = "2026-01-28T18:15:11.724Z" }, + { url = "https://files.pythonhosted.org/packages/d5/2e/e6782744700d6759ebce3043dcfa661fb61e2fb752b91cdeae9af12c2178/psutil-7.2.2-cp314-cp314t-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1fa4ecf83bcdf6e6c8f4449aff98eefb5d0604bf88cb883d7da3d8d2d909546a", size = 182383, upload-time = "2026-01-28T18:15:13.445Z" }, + { url = "https://files.pythonhosted.org/packages/57/49/0a41cefd10cb7505cdc04dab3eacf24c0c2cb158a998b8c7b1d27ee2c1f5/psutil-7.2.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e452c464a02e7dc7822a05d25db4cde564444a67e58539a00f929c51eddda0cf", size = 185210, upload-time = "2026-01-28T18:15:16.002Z" }, + { url = "https://files.pythonhosted.org/packages/dd/2c/ff9bfb544f283ba5f83ba725a3c5fec6d6b10b8f27ac1dc641c473dc390d/psutil-7.2.2-cp314-cp314t-win_amd64.whl", hash = "sha256:c7663d4e37f13e884d13994247449e9f8f574bc4655d509c3b95e9ec9e2b9dc1", size = 141228, upload-time = "2026-01-28T18:15:18.385Z" }, + { url = "https://files.pythonhosted.org/packages/f2/fc/f8d9c31db14fcec13748d373e668bc3bed94d9077dbc17fb0eebc073233c/psutil-7.2.2-cp314-cp314t-win_arm64.whl", hash = "sha256:11fe5a4f613759764e79c65cf11ebdf26e33d6dd34336f8a337aa2996d71c841", size = 136284, upload-time = "2026-01-28T18:15:19.912Z" }, + { url = "https://files.pythonhosted.org/packages/e7/36/5ee6e05c9bd427237b11b3937ad82bb8ad2752d72c6969314590dd0c2f6e/psutil-7.2.2-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:ed0cace939114f62738d808fdcecd4c869222507e266e574799e9c0faa17d486", size = 129090, upload-time = "2026-01-28T18:15:22.168Z" }, + { url = "https://files.pythonhosted.org/packages/80/c4/f5af4c1ca8c1eeb2e92ccca14ce8effdeec651d5ab6053c589b074eda6e1/psutil-7.2.2-cp36-abi3-macosx_11_0_arm64.whl", hash = "sha256:1a7b04c10f32cc88ab39cbf606e117fd74721c831c98a27dc04578deb0c16979", size = 129859, upload-time = "2026-01-28T18:15:23.795Z" }, + { url = "https://files.pythonhosted.org/packages/b5/70/5d8df3b09e25bce090399cf48e452d25c935ab72dad19406c77f4e828045/psutil-7.2.2-cp36-abi3-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:076a2d2f923fd4821644f5ba89f059523da90dc9014e85f8e45a5774ca5bc6f9", size = 155560, upload-time = "2026-01-28T18:15:25.976Z" }, + { url = "https://files.pythonhosted.org/packages/63/65/37648c0c158dc222aba51c089eb3bdfa238e621674dc42d48706e639204f/psutil-7.2.2-cp36-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b0726cecd84f9474419d67252add4ac0cd9811b04d61123054b9fb6f57df6e9e", size = 156997, upload-time = "2026-01-28T18:15:27.794Z" }, + { url = "https://files.pythonhosted.org/packages/8e/13/125093eadae863ce03c6ffdbae9929430d116a246ef69866dad94da3bfbc/psutil-7.2.2-cp36-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:fd04ef36b4a6d599bbdb225dd1d3f51e00105f6d48a28f006da7f9822f2606d8", size = 148972, upload-time = "2026-01-28T18:15:29.342Z" }, + { url = "https://files.pythonhosted.org/packages/04/78/0acd37ca84ce3ddffaa92ef0f571e073faa6d8ff1f0559ab1272188ea2be/psutil-7.2.2-cp36-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:b58fabe35e80b264a4e3bb23e6b96f9e45a3df7fb7eed419ac0e5947c61e47cc", size = 148266, upload-time = "2026-01-28T18:15:31.597Z" }, + { url = "https://files.pythonhosted.org/packages/b4/90/e2159492b5426be0c1fef7acba807a03511f97c5f86b3caeda6ad92351a7/psutil-7.2.2-cp37-abi3-win_amd64.whl", hash = "sha256:eb7e81434c8d223ec4a219b5fc1c47d0417b12be7ea866e24fb5ad6e84b3d988", size = 137737, upload-time = "2026-01-28T18:15:33.849Z" }, + { url = "https://files.pythonhosted.org/packages/8c/c7/7bb2e321574b10df20cbde462a94e2b71d05f9bbda251ef27d104668306a/psutil-7.2.2-cp37-abi3-win_arm64.whl", hash = "sha256:8c233660f575a5a89e6d4cb65d9f938126312bca76d8fe087b947b3a1aaac9ee", size = 134617, upload-time = "2026-01-28T18:15:36.514Z" }, +] + [[package]] name = "pycares" version = "5.0.1" @@ -655,6 +875,24 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050, upload-time = "2024-12-04T17:35:26.475Z" }, ] +[[package]] +name = "stevedore" +version = "5.10.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7a/ec/b628c3b1613fe9340de1c5daba957f5b152159ac350b6ce4ad3d2469f822/stevedore-5.10.0.tar.gz", hash = "sha256:6005f32c09dc3159802a61c04862770e439e32db22f6da78ba56e8a735466882", size = 520466, upload-time = "2026-11-12T11:47:05.38Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/10/78/918aaff2864a8829d061db40acb9da04c89e5c3a01d65932b34432ebd78b/stevedore-5.10.0-py3-none-any.whl", hash = "sha256:47c48dc09e75933106c1bba0a7690eef3b5acecdb807df6d47cc40b2532fa927", size = 56789, upload-time = "2026-11-12T11:47:04.273Z" }, +] + +[[package]] +name = "typing-extensions" +version = "4.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f6/cc/6253133b5bb138fc3306cebfbda2c520f545d36b5be2c7255cc528bb45d6/typing_extensions-4.16.0.tar.gz", hash = "sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5", size = 113555, upload-time = "2026-07-02T08:40:05.92Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/49/d3/b8441a820a491ddfc024b0b0cf0393375b75ea13866d9c66727e54c2fc80/typing_extensions-4.16.0-py3-none-any.whl", hash = "sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8", size = 45571, upload-time = "2026-07-02T08:40:04.659Z" }, +] + [[package]] name = "urllib3" version = "2.8.0" From 0613a6bf8c8895bf3304791bd9034c4a87ea226a Mon Sep 17 00:00:00 2001 From: Sergey Sannikov Date: Sun, 15 Nov 2026 13:08:35 +0400 Subject: [PATCH 21/21] openstack: don't take over the VM of another server by MAC or primary IP A new server that got the address of a deleted one was matched to the deleted server's VM by its primary IP and renamed, so the stale IP repair never ran. A VM found that way is skipped now if it stores another server UUID. The kept interface also gets its name as description again. --- docs/source_openstack.md | 4 ++++ module/sources/openstack/connection.py | 29 ++++++++++++++++++++++++-- tests/test_openstack_source.py | 22 +++++++++++++++++++ 3 files changed, 53 insertions(+), 2 deletions(-) diff --git a/docs/source_openstack.md b/docs/source_openstack.md index 3a5cac32..199042c8 100644 --- a/docs/source_openstack.md +++ b/docs/source_openstack.md @@ -185,6 +185,10 @@ inventories where names repeat; the original name is then stored in the `openstack_vm_name` custom field. Changing naming settings can affect matching, so review a dry run before applying them to existing inventory. +A VM that matches a server only by MAC or primary IP is not reused if it stores a +different server UUID. A new server that got the address of a deleted one gets its +own VM, and the address moves over through the stale IP repair described above. + `skip_vm_platform` disables flavor-to-platform mapping. `skip_vm_comments` disables VM UUID comments. `strip_host_domain_name` and `strip_vm_domain_name` control domain stripping. diff --git a/module/sources/openstack/connection.py b/module/sources/openstack/connection.py index 6450526b..b3687ae6 100644 --- a/module/sources/openstack/connection.py +++ b/module/sources/openstack/connection.py @@ -579,6 +579,30 @@ def get_cluster_for_host(self, hostname): return cluster return None + def server_uuid(self, vm_data): + """The server UUID stored on a VM, None if this source stores none (skip_vm_comments).""" + key = "name" if self.settings.set_vm_name_to_uuid else "comments" + if key == "comments" and self.settings.skip_vm_comments: + return None + try: + return str(UUID(str((vm_data or {}).get(key)))) + except ValueError: + return None + + def same_server_or_none(self, object_type, candidate, object_data): + """ + A VM found by MAC or primary IP that stores another server UUID belongs to another + server, usually a deleted one whose address was handed out again. Don't take it over. + """ + if object_type != NBVM or candidate is None: + return candidate + existing, current = self.server_uuid(candidate.data), self.server_uuid(object_data) + if existing is not None and current is not None and existing != current: + log.debug(f"{candidate.name} '{candidate.get_display_name()}' matched by MAC or primary IP belongs " + f"to server {existing}, not {current}. Not using it.") + return None + return candidate + def get_server_priority(self, server): name = get_string_or_none(grab(server, "name")) if name is not None and self.settings.strip_vm_domain_name: @@ -730,6 +754,7 @@ def add_device_vm_to_inventory(self, object_type, object_data, pnic_data=None, v nic_macs = [x.get("mac_address") for x in mac_source_data.values() if x.get("mac_address") is not None] device_vm_object = self.get_object_based_on_macs(object_type, nic_macs) + device_vm_object = self.same_server_or_none(object_type, device_vm_object, object_data) if device_vm_object is not None: log.debug2("Found a matching %s object: %s" % @@ -741,6 +766,7 @@ def add_device_vm_to_inventory(self, object_type, object_data, pnic_data=None, v log.debug2(f"No match found. Trying to find {object_type.name} based on primary IP addresses") device_vm_object = self.get_object_based_on_primary_ip(object_type, p_ipv4, p_ipv6) + device_vm_object = self.same_server_or_none(object_type, device_vm_object, object_data) if device_vm_object is None: object_name = object_data.get(object_type.primary_key) @@ -807,8 +833,7 @@ def add_device_vm_to_inventory(self, object_type, object_data, pnic_data=None, v target_name = f"vNIC{index} {network_label}" index += 1 used_names.add(target_name) - if int_name not in names_by_mac: - int_data["description"] = target_name + int_data["description"] = target_name int_data["name"] = target_name remapped_data[target_name] = int_data remapped_ips[target_name] = nic_ips.get(int_name, []) diff --git a/tests/test_openstack_source.py b/tests/test_openstack_source.py index d0ef022d..63942342 100644 --- a/tests/test_openstack_source.py +++ b/tests/test_openstack_source.py @@ -297,6 +297,7 @@ def test_detaching_first_port_keeps_second_port_interface_and_ip(cloud): handler.apply() assert second_interface.data["name"] == "vNIC2 (private)" + assert second_interface.data["description"] == "vNIC2 (private)" assert {ip.data["address"] for ip in second_interface.get_ip_addresses()} == {"10.0.0.42/32"} @@ -415,6 +416,27 @@ def test_stale_ip_repair_is_blocked(cloud, blocker): assert stale_ip.get_device_vm() is old_vm +def test_new_server_with_the_address_of_a_deleted_one_gets_its_own_vm(cloud): + old_id = "00000000-0000-0000-0000-000000000001" + new_id = "00000000-0000-0000-0000-000000000002" + port = {"addr": "10.0.0.42", "version": 4, "OS-EXT-IPS-MAC:mac_addr": "fa:16:3e:00:00:01"} + cloud.session.compute.servers.return_value = [server(id=old_id, name="old.example.org", addresses={"private": [port]})] + handler = OpenStackHandler("cloud") + handler.apply() + old_vm = cloud.inventory.get_all_items(NBVM)[0] + old_vm.add_tags(handler.source_tag) + old_vm.source = None + + new_port = {"addr": "10.0.0.42", "version": 4, "OS-EXT-IPS-MAC:mac_addr": "fa:16:3e:00:00:02"} + cloud.session.compute.servers.return_value = [server(id=new_id, name="new.example.org", addresses={"private": [new_port]})] + handler = OpenStackHandler("cloud") + handler.apply() + + assert old_vm.data["name"] == "old.example.org" + assert old_vm.data["comments"] == old_id + assert sorted(vm.data["name"] for vm in cloud.inventory.get_all_items(NBVM)) == ["new.example.org", "old.example.org"] + + def test_network_ports_invalid_addresses_and_floating_primary_preference(cloud): cloud.settings["permitted_subnets"] += ", 192.0.2.0/24" vm_resource = server()