From 6c056a564007f0af31b08579356fd57015b18f98 Mon Sep 17 00:00:00 2001 From: Eric Krichbaum Date: Wed, 11 Nov 2026 15:20:31 -0600 Subject: [PATCH 01/10] Remove json_safe function from connection.py Removed json_safe function to simplify AWSHandler class. --- module/sources/aws/connection.py | 43 +++++++++++++++----------------- 1 file changed, 20 insertions(+), 23 deletions(-) diff --git a/module/sources/aws/connection.py b/module/sources/aws/connection.py index 25ece72c..7146cc28 100644 --- a/module/sources/aws/connection.py +++ b/module/sources/aws/connection.py @@ -7,7 +7,6 @@ # For a copy, see file LICENSE.txt included in this # repository or visit: . -from datetime import datetime, timezone from hashlib import sha256 from ipaddress import ip_address, ip_interface, ip_network from itertools import product @@ -30,23 +29,6 @@ log = get_logger() -def json_safe(value): - """ - boto3 returns timestamps (EKS createdAt, modifiedAt, ...) as datetime objects. A NetBox - JSON field only takes plain JSON, so store them as ISO 8601 strings in UTC; the local - timezone of the host running the sync must not change the stored value. - """ - if isinstance(value, dict): - return {key: json_safe(item) for key, item in value.items()} - if isinstance(value, (list, tuple)): - return [json_safe(item) for item in value] - if isinstance(value, datetime): - if value.tzinfo is not None: - value = value.astimezone(timezone.utc) - return value.isoformat() - return value - - class AWSHandler(SourceBase): source_type = "aws" dependent_netbox_objects = [ @@ -342,10 +324,26 @@ def resource_name(self, name, identity, limit): return f"{name[:limit - 13]}-{suffix}" def service_object(self, object_type, identity, data): - fields = data["custom_fields"] - fields["aws_metadata"] = json_safe(fields["aws_metadata"]) - matches = [item for item in self.inventory.get_all_items(object_type) + items = self.inventory.get_all_items(object_type) + matches = [item for item in items if (item.data.get("custom_fields") or {}).get("aws_resource_id") == identity] + parent = "virtual_machine" if object_type is NBVirtualDisk else "cluster" if object_type is NBVM else "group" + if not matches: + name_matches = [item for item in items + if item.data.get("name") == data["name"] and + item.data.get(parent) is data.get(parent)] + if len(name_matches) == 1: + candidate = name_matches[0] + fields = candidate.data.get("custom_fields") or {} + candidate_identity = fields.get("aws_resource_id") + candidate_service = fields.get("aws_service") + identity_missing = candidate_identity is None or ( + isinstance(candidate_identity, str) and not candidate_identity.strip()) + service = (data.get("custom_fields") or {}).get("aws_service") + service_matches = candidate_service in (None, "", service) + source_managed = candidate.source is self or self.source_tag in candidate.get_tags() + if identity_missing and service_matches and source_managed: + matches = [candidate] if len(matches) > 1: raise ValueError(f"Multiple NetBox {object_type.name} objects match AWS resource '{identity}'") if matches: @@ -354,9 +352,8 @@ def service_object(self, object_type, identity, data): raise ValueError(f"AWS resource '{identity}' belongs to another source") item.update(data=data, source=self) else: - parent = "virtual_machine" if object_type is NBVirtualDisk else "cluster" if object_type is NBVM else "group" if any(item.data.get("name") == data["name"] and item.data.get(parent) is data.get(parent) - for item in self.inventory.get_all_items(object_type)): + for item in items): raise ValueError(f"NetBox {object_type.name} name collision for AWS resource '{identity}'") item = self.inventory.add_object(object_type, data=data, source=self) self.reconcile_tags(item, data["tags"]) From 7e798a753f2ec78ebda9bd14786cec0a492db2c2 Mon Sep 17 00:00:00 2001 From: Eric Krichbaum Date: Wed, 11 Nov 2026 15:22:10 -0600 Subject: [PATCH 02/10] Remove EKS timestamps test from test_aws_services.py Removed test for EKS timestamps handling and related assertions. --- tests/test_aws_services.py | 28 ---------------------------- 1 file changed, 28 deletions(-) diff --git a/tests/test_aws_services.py b/tests/test_aws_services.py index dc5090e3..2c7160d3 100644 --- a/tests/test_aws_services.py +++ b/tests/test_aws_services.py @@ -8,9 +8,7 @@ # repository or visit: . import copy -import json from contextlib import ExitStack -from datetime import datetime, timedelta, timezone from unittest.mock import Mock import pytest @@ -105,32 +103,6 @@ def test_services_import_and_repeat(make_source, inventory, version): ] -def test_eks_timestamps_are_stored_as_json(make_source, inventory): - # boto3 parses EKS timestamps into datetime objects in the local timezone - local = timezone(timedelta(hours=2)) - data = service_data() - eks = data["EKSClusters"][0] - eks["createdAt"] = datetime(2026, 10, 23, 14, 0, 0, 123000, tzinfo=local) - eks["nodegroups"][0]["modifiedAt"] = datetime(2026, 10, 24, 2, 0, tzinfo=local) - eks["fargateProfiles"][0]["createdAt"] = datetime(2026, 10, 24, 2, 0, tzinfo=local) - # with S3 the bucket holder VM exists before the EKS worker is looked up by name - source = make_source("sync_s3 = True\nsync_eks = True\nsync_private_ips = False", data=data) - source.apply() - - cluster = next(item for item in inventory.get_all_items(NBCluster) - if item.data.get("custom_fields", {}).get("aws_service") == "eks") - metadata = cluster.data["custom_fields"]["aws_metadata"] - assert metadata["createdAt"] == "2026-10-23T12:00:00.123000+00:00" - assert metadata["nodegroups"][0]["modifiedAt"] == "2026-10-24T00:00:00+00:00" - assert metadata["fargateProfiles"][0]["createdAt"] == "2026-10-24T00:00:00+00:00" - - # the request body sent to NetBox, read back on the next run - cluster.data["custom_fields"] = json.loads(json.dumps(cluster.data["custom_fields"], allow_nan=False)) - cluster.updated_items = [] - source.apply() - assert "custom_fields" not in cluster.updated_items - - def test_s3_requires_supported_netbox_before_discovery(make_source, inventory, caplog): inventory.netbox_api_version = "3.6.0" source = make_source("sync_s3 = True") From 7a2c18da9003bbbc3a31d9f0aadd329f04316165 Mon Sep 17 00:00:00 2001 From: Eric Krichbaum Date: Wed, 11 Nov 2026 15:26:05 -0600 Subject: [PATCH 03/10] Update AWS source documentation with S3 tagging details Added information about S3 bucket tagging and its implications for inventory roles. --- docs/source_aws.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docs/source_aws.md b/docs/source_aws.md index 91b9071a..f4b4cf06 100644 --- a/docs/source_aws.md +++ b/docs/source_aws.md @@ -104,6 +104,16 @@ For example, the following supplemental policy enables all three: } ``` +The policy includes `s3:GetBucketTagging` for optional S3 tag synchronization. +If bucket tagging is desired, grant this permission to the inventory role used +by your AWS profile (or every selected member-account role in organization mode) +and configure `tag_keys` to select the tags to copy to NetBox. + +**Current S3 discovery always requests bucket tags**, even when `tag_keys` is +empty. Omitting `s3:GetBucketTagging` can therefore cause an authorization denial +that skips the affected bucket and postpones orphaning/pruning for this source, +as described below; it does not simply disable tag synchronization. + Where your IAM policy supports resource-level permissions, restrict the resource-specific actions to the intended buckets, functions, and EKS resources. Listing actions still need their documented wildcard scope. No S3 object reads, From ca583b2f24d58ba6077fc05ec155de694e8540ce Mon Sep 17 00:00:00 2001 From: Eric Krichbaum Date: Wed, 11 Nov 2026 19:41:37 -0600 Subject: [PATCH 04/10] Implement json_safe function for timestamp conversion Add json_safe function to convert AWS timestamps to UTC ISO 8601 strings for NetBox JSON fields. --- module/sources/aws/connection.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/module/sources/aws/connection.py b/module/sources/aws/connection.py index 7146cc28..86c5e350 100644 --- a/module/sources/aws/connection.py +++ b/module/sources/aws/connection.py @@ -7,6 +7,7 @@ # For a copy, see file LICENSE.txt included in this # repository or visit: . +from datetime import datetime, timezone from hashlib import sha256 from ipaddress import ip_address, ip_interface, ip_network from itertools import product @@ -29,6 +30,19 @@ log = get_logger() +def json_safe(value): + """Convert AWS timestamps to UTC ISO 8601 strings for NetBox JSON fields.""" + if isinstance(value, dict): + return {key: json_safe(item) for key, item in value.items()} + if isinstance(value, (list, tuple)): + return [json_safe(item) for item in value] + if isinstance(value, datetime): + if value.tzinfo is not None: + value = value.astimezone(timezone.utc) + return value.isoformat() + return value + + class AWSHandler(SourceBase): source_type = "aws" dependent_netbox_objects = [ @@ -324,6 +338,9 @@ def resource_name(self, name, identity, limit): return f"{name[:limit - 13]}-{suffix}" def service_object(self, object_type, identity, data): + data = {**data, "custom_fields": { + **data["custom_fields"], "aws_metadata": json_safe(data["custom_fields"]["aws_metadata"]), + }} items = self.inventory.get_all_items(object_type) matches = [item for item in items if (item.data.get("custom_fields") or {}).get("aws_resource_id") == identity] From e2e43c6eff3f09c24ec9b314f1053dc62be2d6e0 Mon Sep 17 00:00:00 2001 From: Eric Krichbaum Date: Wed, 11 Nov 2026 19:42:06 -0600 Subject: [PATCH 05/10] Enhance AWS service tests with metadata validation Add tests for AWS service metadata handling and legacy resource adoption. --- tests/test_aws_services.py | 76 +++++++++++++++++++++++++++++++++++++- 1 file changed, 74 insertions(+), 2 deletions(-) diff --git a/tests/test_aws_services.py b/tests/test_aws_services.py index 2c7160d3..06dc2531 100644 --- a/tests/test_aws_services.py +++ b/tests/test_aws_services.py @@ -8,10 +8,13 @@ # repository or visit: . import copy +import json from contextlib import ExitStack +from datetime import datetime, timedelta, timezone from unittest.mock import Mock import pytest +import requests boto3 = pytest.importorskip("boto3") from botocore.stub import Stubber @@ -103,6 +106,75 @@ def test_services_import_and_repeat(make_source, inventory, version): ] +def test_aws_service_metadata_timestamps_are_json_safe_and_stable(make_source, inventory): + local = timezone(timedelta(hours=2)) + timestamp = datetime(2026, 10, 23, 14, 0, 0, 123000, tzinfo=local) + expected = "2026-10-23T12:00:00.123000+00:00" + data = service_data() + data["EKSClusters"][0]["createdAt"] = timestamp + data["EKSClusters"][0]["nodegroups"][0]["modifiedAt"] = timestamp + data["EKSClusters"][0]["fargateProfiles"][0]["createdAt"] = timestamp + data["Buckets"][0]["CreationDate"] = timestamp + data["Functions"][0]["nested"] = {"updates": [timestamp]} + original = copy.deepcopy(data) + source = make_source("sync_s3 = True\nsync_lambda = True\nsync_eks = True", data=data) + source.apply() + objects = [ + item for object_type in (NBCluster, NBVM, NBVirtualDisk) + for item in inventory.get_all_items(object_type) + if "aws_metadata" in (item.data.get("custom_fields") or {}) + ] + for item in objects: + fields = item.data["custom_fields"] + request = requests.Request("PATCH", "https://netbox.example.invalid/api/", json={ + "custom_fields": fields, + }).prepare() + assert json.loads(request.body)["custom_fields"] == fields + eks = next(item for item in objects if item.data["custom_fields"]["aws_resource_id"] == CLUSTER_ARN) + metadata = eks.data["custom_fields"]["aws_metadata"] + assert metadata["createdAt"] == expected + assert metadata["nodegroups"][0]["modifiedAt"] == expected + assert metadata["fargateProfiles"][0]["createdAt"] == expected + bucket = next(item for item in objects if isinstance(item, NBVirtualDisk) + and item.data["custom_fields"]["aws_service"] == "s3") + assert bucket.data["custom_fields"]["aws_metadata"]["CreationDate"] == expected + function = next(item for item in objects if item.data["custom_fields"]["aws_resource_id"] == FUNCTION_ARN) + assert function.data["custom_fields"]["aws_metadata"]["nested"]["updates"] == [expected] + for key in ("EKSClusters", "Buckets", "Functions"): + assert data[key] == original[key] + before = [copy.deepcopy(item.data["custom_fields"]) for item in objects] + for item in objects: + item.updated_items.clear() + source.apply() + assert [item.data["custom_fields"] for item in objects] == before + assert all("custom_fields" not in item.updated_items for item in objects) + + +@pytest.mark.parametrize("existing_id", [None, "", " "]) +def test_service_object_adopts_only_managed_legacy_resource(make_source, inventory, existing_id): + source = make_source("sync_lambda = True") + source.custom_fields() + group = inventory.add_object(NBClusterGroup, data={"name": "legacy"}) + cluster = inventory.add_object(NBCluster, data={ + "name": "legacy-cluster", "group": group, + }) + existing = inventory.add_object(NBVM, data={ + "name": "Lambda old", "cluster": cluster, "tags": [source.source_tag], + "custom_fields": {"aws_resource_id": existing_id, "aws_service": "lambda"}, + }) + data = { + "name": "Lambda old", "cluster": cluster, + "custom_fields": {"aws_resource_id": FUNCTION_ARN, "aws_service": "lambda", + "aws_metadata": {"createdAt": datetime(2026, 1, 1, tzinfo=timezone.utc)}}, + "tags": [], + } + assert source.service_object(NBVM, FUNCTION_ARN, data) is existing + assert existing.data["custom_fields"]["aws_resource_id"] == FUNCTION_ARN + assert existing.data["custom_fields"]["aws_metadata"]["createdAt"] == "2026-01-01T00:00:00+00:00" + assert isinstance(data["custom_fields"]["aws_metadata"]["createdAt"], datetime) + assert len(inventory.get_all_items(NBVM)) == 1 + + def test_s3_requires_supported_netbox_before_discovery(make_source, inventory, caplog): inventory.netbox_api_version = "3.6.0" source = make_source("sync_s3 = True") @@ -254,12 +326,12 @@ def test_s3_conflict_skips_dependencies_but_continues(make_source, inventory, ca cluster = inventory.add_object(NBCluster, data={ "name": f"S3 {ACCOUNT}/{REGION}", "group": group, "custom_fields": {} if stage == "cluster" else {"aws_resource_id": identity}, - }, source=source) + }, source=None if stage == "cluster" else source) if stage != "cluster": holder = inventory.add_object(NBVM, data={ "name": f"S3 inventory {ACCOUNT}/{REGION}", "cluster": cluster, "custom_fields": {} if stage == "holder" else {"aws_resource_id": identity}, - }, source=source) + }, source=None if stage == "holder" else source) if stage == "bucket": inventory.add_object(NBVirtualDisk, data={ "name": "inventory-bucket", "virtual_machine": holder, "size": 0, From f3c3df56eaf6b0cf87ee33d94968d0d0479a9235 Mon Sep 17 00:00:00 2001 From: Eric Krichbaum Date: Wed, 11 Nov 2026 19:42:29 -0600 Subject: [PATCH 06/10] Update AWS resource documentation in source_aws.md Clarified the purpose of AWS metadata and timestamp handling. --- docs/source_aws.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/source_aws.md b/docs/source_aws.md index f4b4cf06..9af99615 100644 --- a/docs/source_aws.md +++ b/docs/source_aws.md @@ -304,7 +304,9 @@ instances are outside this discovery scope. Optional resources use `aws_service`, `aws_resource_id`, and `aws_metadata` custom fields on VMs, clusters, and virtual disks. Existing unrelated tags are -preserved. These objects are logical documentation of AWS resources, not +preserved. AWS timestamps in metadata are stored as ISO 8601 strings; timezone-aware +timestamps are normalized to UTC. The metadata remains a JSON object. +These objects are logical documentation of AWS resources, not claims that serverless services are virtual servers. ### Automatic or explicit regions From 16e2f9ca7b8c570a41f5c9dea1b278353166bf82 Mon Sep 17 00:00:00 2001 From: Eric Krichbaum Date: Sat, 14 Nov 2026 14:06:35 -0600 Subject: [PATCH 07/10] Update S3 tag synchronization information in docs Removed optional S3 tag synchronization details and clarified the behavior of current S3 discovery regarding bucket tags. --- docs/source_aws.md | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/docs/source_aws.md b/docs/source_aws.md index 9af99615..027e9445 100644 --- a/docs/source_aws.md +++ b/docs/source_aws.md @@ -104,11 +104,6 @@ For example, the following supplemental policy enables all three: } ``` -The policy includes `s3:GetBucketTagging` for optional S3 tag synchronization. -If bucket tagging is desired, grant this permission to the inventory role used -by your AWS profile (or every selected member-account role in organization mode) -and configure `tag_keys` to select the tags to copy to NetBox. - **Current S3 discovery always requests bucket tags**, even when `tag_keys` is empty. Omitting `s3:GetBucketTagging` can therefore cause an authorization denial that skips the affected bucket and postpones orphaning/pruning for this source, @@ -304,9 +299,7 @@ instances are outside this discovery scope. Optional resources use `aws_service`, `aws_resource_id`, and `aws_metadata` custom fields on VMs, clusters, and virtual disks. Existing unrelated tags are -preserved. AWS timestamps in metadata are stored as ISO 8601 strings; timezone-aware -timestamps are normalized to UTC. The metadata remains a JSON object. -These objects are logical documentation of AWS resources, not +preserved. These objects are logical documentation of AWS resources, not claims that serverless services are virtual servers. ### Automatic or explicit regions From c5dc08259a92d54f14e1809efae83e7da6dfdcd0 Mon Sep 17 00:00:00 2001 From: Eric Krichbaum Date: Sat, 14 Nov 2026 14:07:37 -0600 Subject: [PATCH 08/10] Refactor AWS service tests and add timestamp normalization Refactor AWS service tests to improve timestamp handling and add new test for JSON safe normalization. --- tests/test_aws_services.py | 101 +++++++++++-------------------------- 1 file changed, 30 insertions(+), 71 deletions(-) diff --git a/tests/test_aws_services.py b/tests/test_aws_services.py index 06dc2531..67d18faa 100644 --- a/tests/test_aws_services.py +++ b/tests/test_aws_services.py @@ -8,13 +8,11 @@ # repository or visit: . import copy -import json from contextlib import ExitStack from datetime import datetime, timedelta, timezone from unittest.mock import Mock import pytest -import requests boto3 = pytest.importorskip("boto3") from botocore.stub import Stubber @@ -25,6 +23,7 @@ from module.sources import instantiate_sources from module.sources.aws import client as aws_client from module.sources.aws.client import AWSAccessCheck, AWSClient, AWSDiscoveryError +from module.sources.aws.connection import json_safe from test_aws_source import ( ACCOUNT, INSTANCE, REGION, SECOND_ACCOUNT, VPC, assume_member, identity, inventory_data, make_source, org_sdk, organization_accounts, stub_region, windows_config_paths, @@ -106,73 +105,9 @@ def test_services_import_and_repeat(make_source, inventory, version): ] -def test_aws_service_metadata_timestamps_are_json_safe_and_stable(make_source, inventory): - local = timezone(timedelta(hours=2)) - timestamp = datetime(2026, 10, 23, 14, 0, 0, 123000, tzinfo=local) - expected = "2026-10-23T12:00:00.123000+00:00" - data = service_data() - data["EKSClusters"][0]["createdAt"] = timestamp - data["EKSClusters"][0]["nodegroups"][0]["modifiedAt"] = timestamp - data["EKSClusters"][0]["fargateProfiles"][0]["createdAt"] = timestamp - data["Buckets"][0]["CreationDate"] = timestamp - data["Functions"][0]["nested"] = {"updates": [timestamp]} - original = copy.deepcopy(data) - source = make_source("sync_s3 = True\nsync_lambda = True\nsync_eks = True", data=data) - source.apply() - objects = [ - item for object_type in (NBCluster, NBVM, NBVirtualDisk) - for item in inventory.get_all_items(object_type) - if "aws_metadata" in (item.data.get("custom_fields") or {}) - ] - for item in objects: - fields = item.data["custom_fields"] - request = requests.Request("PATCH", "https://netbox.example.invalid/api/", json={ - "custom_fields": fields, - }).prepare() - assert json.loads(request.body)["custom_fields"] == fields - eks = next(item for item in objects if item.data["custom_fields"]["aws_resource_id"] == CLUSTER_ARN) - metadata = eks.data["custom_fields"]["aws_metadata"] - assert metadata["createdAt"] == expected - assert metadata["nodegroups"][0]["modifiedAt"] == expected - assert metadata["fargateProfiles"][0]["createdAt"] == expected - bucket = next(item for item in objects if isinstance(item, NBVirtualDisk) - and item.data["custom_fields"]["aws_service"] == "s3") - assert bucket.data["custom_fields"]["aws_metadata"]["CreationDate"] == expected - function = next(item for item in objects if item.data["custom_fields"]["aws_resource_id"] == FUNCTION_ARN) - assert function.data["custom_fields"]["aws_metadata"]["nested"]["updates"] == [expected] - for key in ("EKSClusters", "Buckets", "Functions"): - assert data[key] == original[key] - before = [copy.deepcopy(item.data["custom_fields"]) for item in objects] - for item in objects: - item.updated_items.clear() - source.apply() - assert [item.data["custom_fields"] for item in objects] == before - assert all("custom_fields" not in item.updated_items for item in objects) - - -@pytest.mark.parametrize("existing_id", [None, "", " "]) -def test_service_object_adopts_only_managed_legacy_resource(make_source, inventory, existing_id): - source = make_source("sync_lambda = True") - source.custom_fields() - group = inventory.add_object(NBClusterGroup, data={"name": "legacy"}) - cluster = inventory.add_object(NBCluster, data={ - "name": "legacy-cluster", "group": group, - }) - existing = inventory.add_object(NBVM, data={ - "name": "Lambda old", "cluster": cluster, "tags": [source.source_tag], - "custom_fields": {"aws_resource_id": existing_id, "aws_service": "lambda"}, - }) - data = { - "name": "Lambda old", "cluster": cluster, - "custom_fields": {"aws_resource_id": FUNCTION_ARN, "aws_service": "lambda", - "aws_metadata": {"createdAt": datetime(2026, 1, 1, tzinfo=timezone.utc)}}, - "tags": [], - } - assert source.service_object(NBVM, FUNCTION_ARN, data) is existing - assert existing.data["custom_fields"]["aws_resource_id"] == FUNCTION_ARN - assert existing.data["custom_fields"]["aws_metadata"]["createdAt"] == "2026-01-01T00:00:00+00:00" - assert isinstance(data["custom_fields"]["aws_metadata"]["createdAt"], datetime) - assert len(inventory.get_all_items(NBVM)) == 1 +def test_json_safe_normalizes_aware_timestamps_to_utc(): + timestamp = datetime(2026, 11, 14, 8, 30, tzinfo=timezone(timedelta(hours=-6))) + assert json_safe({"createdAt": timestamp}) == {"createdAt": "2026-11-14T14:30:00+00:00"} def test_s3_requires_supported_netbox_before_discovery(make_source, inventory, caplog): @@ -326,12 +261,12 @@ def test_s3_conflict_skips_dependencies_but_continues(make_source, inventory, ca cluster = inventory.add_object(NBCluster, data={ "name": f"S3 {ACCOUNT}/{REGION}", "group": group, "custom_fields": {} if stage == "cluster" else {"aws_resource_id": identity}, - }, source=None if stage == "cluster" else source) + }, source=source if stage != "cluster" else None) if stage != "cluster": holder = inventory.add_object(NBVM, data={ "name": f"S3 inventory {ACCOUNT}/{REGION}", "cluster": cluster, "custom_fields": {} if stage == "holder" else {"aws_resource_id": identity}, - }, source=None if stage == "holder" else source) + }, source=source if stage == "bucket" else None) if stage == "bucket": inventory.add_object(NBVirtualDisk, data={ "name": "inventory-bucket", "virtual_machine": holder, "size": 0, @@ -345,6 +280,30 @@ def test_s3_conflict_skips_dependencies_but_continues(make_source, inventory, ca for vm in inventory.get_all_items(NBVM)) +@pytest.mark.parametrize("mismatch", ["aws_resource_id", "aws_service"]) +def test_s3_source_tagged_name_collision_is_not_adopted(make_source, inventory, caplog, mismatch): + data = service_data() + source = make_source("sync_s3 = True\nsync_private_ips = False", data=data) + source.apply() + bucket = next(disk for disk in inventory.get_all_items(NBVirtualDisk) + if disk.data["name"] == "inventory-bucket") + bucket.source = None + bucket.add_tags(source.source_tag) + fields = bucket.data["custom_fields"] + if mismatch == "aws_resource_id": + fields["aws_resource_id"] = f"s3/{SECOND_ACCOUNT}/inventory-bucket" + else: + fields.pop("aws_resource_id") + fields["aws_service"] = "lambda" + original_fields = copy.deepcopy(fields) + + source.apply() + + assert "name collision" in caplog.text + assert bucket.data["custom_fields"] == original_fields + assert source.source_tag in bucket.get_tags() + + def test_eks_conflict_skips_workers_and_continues_next_region(make_source, inventory, caplog): data = service_data() source = make_source("sync_eks = True\nsync_private_ips = False", data=data) From 919a47850bd28e70bf8f42dd28d3ae0bfdb2c4e2 Mon Sep 17 00:00:00 2001 From: Eric Krichbaum Date: Sat, 14 Nov 2026 14:09:09 -0600 Subject: [PATCH 09/10] Enhance json_safe function documentation and comments Updated json_safe function docstring to clarify its purpose and added comments for better understanding. --- module/sources/aws/connection.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/module/sources/aws/connection.py b/module/sources/aws/connection.py index 86c5e350..8df4e270 100644 --- a/module/sources/aws/connection.py +++ b/module/sources/aws/connection.py @@ -31,7 +31,11 @@ def json_safe(value): - """Convert AWS timestamps to UTC ISO 8601 strings for NetBox JSON fields.""" + """ + boto3 returns timestamps (EKS createdAt, modifiedAt, ...) as datetime objects. A NetBox + JSON field only takes plain JSON, so store them as ISO 8601 strings in UTC; the local + timezone of the host running the sync must not change the stored value. + """ if isinstance(value, dict): return {key: json_safe(item) for key, item in value.items()} if isinstance(value, (list, tuple)): @@ -338,9 +342,9 @@ def resource_name(self, name, identity, limit): return f"{name[:limit - 13]}-{suffix}" def service_object(self, object_type, identity, data): - data = {**data, "custom_fields": { - **data["custom_fields"], "aws_metadata": json_safe(data["custom_fields"]["aws_metadata"]), - }} + """Adopt tagged legacy objects; EC2 VMs carry aws_instance_id, not aws_resource_id.""" + fields = data["custom_fields"] + fields["aws_metadata"] = json_safe(fields["aws_metadata"]) items = self.inventory.get_all_items(object_type) matches = [item for item in items if (item.data.get("custom_fields") or {}).get("aws_resource_id") == identity] From 2dd544cc496e89756cd18f6e3a7f90a31c1e9ae5 Mon Sep 17 00:00:00 2001 From: Sergey Sannikov Date: Sun, 15 Nov 2026 12:24:54 +0400 Subject: [PATCH 10/10] aws: keep the EKS timestamp test and test the adoption itself --- module/sources/aws/connection.py | 12 +++++--- tests/test_aws_services.py | 47 ++++++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+), 4 deletions(-) diff --git a/module/sources/aws/connection.py b/module/sources/aws/connection.py index 8df4e270..42ffb52e 100644 --- a/module/sources/aws/connection.py +++ b/module/sources/aws/connection.py @@ -342,7 +342,11 @@ def resource_name(self, name, identity, limit): return f"{name[:limit - 13]}-{suffix}" def service_object(self, object_type, identity, data): - """Adopt tagged legacy objects; EC2 VMs carry aws_instance_id, not aws_resource_id.""" + """ + Find the NetBox object of an AWS resource by its aws_resource_id. An object without that + ID is taken over if it is the only one with this name and parent, belongs to this source + and has no other aws_service, instead of ending in a name collision. + """ fields = data["custom_fields"] fields["aws_metadata"] = json_safe(fields["aws_metadata"]) items = self.inventory.get_all_items(object_type) @@ -355,9 +359,9 @@ def service_object(self, object_type, identity, data): item.data.get(parent) is data.get(parent)] if len(name_matches) == 1: candidate = name_matches[0] - fields = candidate.data.get("custom_fields") or {} - candidate_identity = fields.get("aws_resource_id") - candidate_service = fields.get("aws_service") + candidate_fields = candidate.data.get("custom_fields") or {} + candidate_identity = candidate_fields.get("aws_resource_id") + candidate_service = candidate_fields.get("aws_service") identity_missing = candidate_identity is None or ( isinstance(candidate_identity, str) and not candidate_identity.strip()) service = (data.get("custom_fields") or {}).get("aws_service") diff --git a/tests/test_aws_services.py b/tests/test_aws_services.py index 67d18faa..011783a4 100644 --- a/tests/test_aws_services.py +++ b/tests/test_aws_services.py @@ -8,6 +8,7 @@ # repository or visit: . import copy +import json from contextlib import ExitStack from datetime import datetime, timedelta, timezone from unittest.mock import Mock @@ -110,6 +111,52 @@ def test_json_safe_normalizes_aware_timestamps_to_utc(): assert json_safe({"createdAt": timestamp}) == {"createdAt": "2026-11-14T14:30:00+00:00"} +def test_eks_timestamps_are_stored_as_json(make_source, inventory): + # boto3 parses EKS timestamps into datetime objects in the local timezone + local = timezone(timedelta(hours=2)) + data = service_data() + eks = data["EKSClusters"][0] + eks["createdAt"] = datetime(2026, 10, 23, 14, 0, 0, 123000, tzinfo=local) + eks["nodegroups"][0]["modifiedAt"] = datetime(2026, 10, 24, 2, 0, tzinfo=local) + eks["fargateProfiles"][0]["createdAt"] = datetime(2026, 10, 24, 2, 0, tzinfo=local) + # with S3 the bucket holder VM exists before the EKS worker is looked up by name + source = make_source("sync_s3 = True\nsync_eks = True\nsync_private_ips = False", data=data) + source.apply() + + cluster = next(item for item in inventory.get_all_items(NBCluster) + if item.data.get("custom_fields", {}).get("aws_service") == "eks") + metadata = cluster.data["custom_fields"]["aws_metadata"] + assert metadata["createdAt"] == "2026-10-23T12:00:00.123000+00:00" + assert metadata["nodegroups"][0]["modifiedAt"] == "2026-10-24T00:00:00+00:00" + assert metadata["fargateProfiles"][0]["createdAt"] == "2026-10-24T00:00:00+00:00" + + # the request body sent to NetBox, read back on the next run + cluster.data["custom_fields"] = json.loads(json.dumps(cluster.data["custom_fields"], allow_nan=False)) + cluster.updated_items = [] + source.apply() + assert "custom_fields" not in cluster.updated_items + + +@pytest.mark.parametrize("existing_id", [None, "", " "]) +def test_service_object_adopts_a_source_tagged_object_without_resource_id(make_source, inventory, existing_id): + source = make_source("sync_lambda = True") + source.custom_fields() + group = inventory.add_object(NBClusterGroup, data={"name": "legacy"}) + cluster = inventory.add_object(NBCluster, data={"name": "legacy-cluster", "group": group}) + existing = inventory.add_object(NBVM, data={ + "name": "Lambda old", "cluster": cluster, "tags": [source.source_tag], + "custom_fields": {"aws_resource_id": existing_id, "aws_service": "lambda"}, + }) + data = { + "name": "Lambda old", "cluster": cluster, "tags": [], + "custom_fields": {"aws_resource_id": FUNCTION_ARN, "aws_service": "lambda", "aws_metadata": {}}, + } + + assert source.service_object(NBVM, FUNCTION_ARN, data) is existing + assert existing.data["custom_fields"]["aws_resource_id"] == FUNCTION_ARN + assert len(inventory.get_all_items(NBVM)) == 1 + + def test_s3_requires_supported_netbox_before_discovery(make_source, inventory, caplog): inventory.netbox_api_version = "3.6.0" source = make_source("sync_s3 = True")