Skip to content
Merged
5 changes: 5 additions & 0 deletions docs/source_aws.md
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,11 @@ For example, the following supplemental policy enables all three:
}
```

**Current S3 discovery always requests bucket tags**, even when `tag_keys` is
empty. Omitting `s3:GetBucketTagging` can therefore cause an authorization denial
that skips the affected bucket and postpones orphaning/pruning for this source,
as described below; it does not simply disable tag synchronization.

Where your IAM policy supports resource-level permissions, restrict the
resource-specific actions to the intended buckets, functions, and EKS resources.
Listing actions still need their documented wildcard scope. No S3 object reads,
Expand Down
28 changes: 25 additions & 3 deletions module/sources/aws/connection.py
Original file line number Diff line number Diff line change
Expand Up @@ -342,10 +342,33 @@ def resource_name(self, name, identity, limit):
return f"{name[:limit - 13]}-{suffix}"

def service_object(self, object_type, identity, data):
"""
Find the NetBox object of an AWS resource by its aws_resource_id. An object without that
ID is taken over if it is the only one with this name and parent, belongs to this source
and has no other aws_service, instead of ending in a name collision.
"""
fields = data["custom_fields"]
fields["aws_metadata"] = json_safe(fields["aws_metadata"])
matches = [item for item in self.inventory.get_all_items(object_type)
items = self.inventory.get_all_items(object_type)
matches = [item for item in items
if (item.data.get("custom_fields") or {}).get("aws_resource_id") == identity]
parent = "virtual_machine" if object_type is NBVirtualDisk else "cluster" if object_type is NBVM else "group"
if not matches:
name_matches = [item for item in items
if item.data.get("name") == data["name"] and
item.data.get(parent) is data.get(parent)]
if len(name_matches) == 1:
candidate = name_matches[0]
candidate_fields = candidate.data.get("custom_fields") or {}
candidate_identity = candidate_fields.get("aws_resource_id")
candidate_service = candidate_fields.get("aws_service")
identity_missing = candidate_identity is None or (
isinstance(candidate_identity, str) and not candidate_identity.strip())
service = (data.get("custom_fields") or {}).get("aws_service")
service_matches = candidate_service in (None, "", service)
source_managed = candidate.source is self or self.source_tag in candidate.get_tags()
if identity_missing and service_matches and source_managed:
matches = [candidate]
if len(matches) > 1:
raise ValueError(f"Multiple NetBox {object_type.name} objects match AWS resource '{identity}'")
if matches:
Expand All @@ -354,9 +377,8 @@ def service_object(self, object_type, identity, data):
raise ValueError(f"AWS resource '{identity}' belongs to another source")
item.update(data=data, source=self)
else:
parent = "virtual_machine" if object_type is NBVirtualDisk else "cluster" if object_type is NBVM else "group"
if any(item.data.get("name") == data["name"] and item.data.get(parent) is data.get(parent)
for item in self.inventory.get_all_items(object_type)):
for item in items):
raise ValueError(f"NetBox {object_type.name} name collision for AWS resource '{identity}'")
item = self.inventory.add_object(object_type, data=data, source=self)
self.reconcile_tags(item, data["tags"])
Expand Down
54 changes: 52 additions & 2 deletions tests/test_aws_services.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
from module.sources import instantiate_sources
from module.sources.aws import client as aws_client
from module.sources.aws.client import AWSAccessCheck, AWSClient, AWSDiscoveryError
from module.sources.aws.connection import json_safe
from test_aws_source import (
ACCOUNT, INSTANCE, REGION, SECOND_ACCOUNT, VPC, assume_member, identity, inventory_data,
make_source, org_sdk, organization_accounts, stub_region, windows_config_paths,
Expand Down Expand Up @@ -105,6 +106,11 @@ def test_services_import_and_repeat(make_source, inventory, version):
]


def test_json_safe_normalizes_aware_timestamps_to_utc():
timestamp = datetime(2026, 11, 14, 8, 30, tzinfo=timezone(timedelta(hours=-6)))
assert json_safe({"createdAt": timestamp}) == {"createdAt": "2026-11-14T14:30:00+00:00"}


def test_eks_timestamps_are_stored_as_json(make_source, inventory):
# boto3 parses EKS timestamps into datetime objects in the local timezone
local = timezone(timedelta(hours=2))
Expand All @@ -131,6 +137,26 @@ def test_eks_timestamps_are_stored_as_json(make_source, inventory):
assert "custom_fields" not in cluster.updated_items


@pytest.mark.parametrize("existing_id", [None, "", " "])
def test_service_object_adopts_a_source_tagged_object_without_resource_id(make_source, inventory, existing_id):
source = make_source("sync_lambda = True")
source.custom_fields()
group = inventory.add_object(NBClusterGroup, data={"name": "legacy"})
cluster = inventory.add_object(NBCluster, data={"name": "legacy-cluster", "group": group})
existing = inventory.add_object(NBVM, data={
"name": "Lambda old", "cluster": cluster, "tags": [source.source_tag],
"custom_fields": {"aws_resource_id": existing_id, "aws_service": "lambda"},
})
data = {
"name": "Lambda old", "cluster": cluster, "tags": [],
"custom_fields": {"aws_resource_id": FUNCTION_ARN, "aws_service": "lambda", "aws_metadata": {}},
}

assert source.service_object(NBVM, FUNCTION_ARN, data) is existing
assert existing.data["custom_fields"]["aws_resource_id"] == FUNCTION_ARN
assert len(inventory.get_all_items(NBVM)) == 1


def test_s3_requires_supported_netbox_before_discovery(make_source, inventory, caplog):
inventory.netbox_api_version = "3.6.0"
source = make_source("sync_s3 = True")
Expand Down Expand Up @@ -282,12 +308,12 @@ def test_s3_conflict_skips_dependencies_but_continues(make_source, inventory, ca
cluster = inventory.add_object(NBCluster, data={
"name": f"S3 {ACCOUNT}/{REGION}", "group": group,
"custom_fields": {} if stage == "cluster" else {"aws_resource_id": identity},
}, source=source)
}, source=source if stage != "cluster" else None)
if stage != "cluster":
holder = inventory.add_object(NBVM, data={
"name": f"S3 inventory {ACCOUNT}/{REGION}", "cluster": cluster,
"custom_fields": {} if stage == "holder" else {"aws_resource_id": identity},
}, source=source)
}, source=source if stage == "bucket" else None)
if stage == "bucket":
inventory.add_object(NBVirtualDisk, data={
"name": "inventory-bucket", "virtual_machine": holder, "size": 0,
Expand All @@ -301,6 +327,30 @@ def test_s3_conflict_skips_dependencies_but_continues(make_source, inventory, ca
for vm in inventory.get_all_items(NBVM))


@pytest.mark.parametrize("mismatch", ["aws_resource_id", "aws_service"])
def test_s3_source_tagged_name_collision_is_not_adopted(make_source, inventory, caplog, mismatch):
data = service_data()
source = make_source("sync_s3 = True\nsync_private_ips = False", data=data)
source.apply()
bucket = next(disk for disk in inventory.get_all_items(NBVirtualDisk)
if disk.data["name"] == "inventory-bucket")
bucket.source = None
bucket.add_tags(source.source_tag)
fields = bucket.data["custom_fields"]
if mismatch == "aws_resource_id":
fields["aws_resource_id"] = f"s3/{SECOND_ACCOUNT}/inventory-bucket"
else:
fields.pop("aws_resource_id")
fields["aws_service"] = "lambda"
original_fields = copy.deepcopy(fields)

source.apply()

assert "name collision" in caplog.text
assert bucket.data["custom_fields"] == original_fields
assert source.source_tag in bucket.get_tags()


def test_eks_conflict_skips_workers_and_continues_next_region(make_source, inventory, caplog):
data = service_data()
source = make_source("sync_eks = True\nsync_private_ips = False", data=data)
Expand Down
Loading