diff --git a/docs/source_aws.md b/docs/source_aws.md index 91b9071a..027e9445 100644 --- a/docs/source_aws.md +++ b/docs/source_aws.md @@ -104,6 +104,11 @@ For example, the following supplemental policy enables all three: } ``` +**Current S3 discovery always requests bucket tags**, even when `tag_keys` is +empty. Omitting `s3:GetBucketTagging` can therefore cause an authorization denial +that skips the affected bucket and postpones orphaning/pruning for this source, +as described below; it does not simply disable tag synchronization. + Where your IAM policy supports resource-level permissions, restrict the resource-specific actions to the intended buckets, functions, and EKS resources. Listing actions still need their documented wildcard scope. No S3 object reads, diff --git a/module/sources/aws/connection.py b/module/sources/aws/connection.py index 25ece72c..42ffb52e 100644 --- a/module/sources/aws/connection.py +++ b/module/sources/aws/connection.py @@ -342,10 +342,33 @@ def resource_name(self, name, identity, limit): return f"{name[:limit - 13]}-{suffix}" def service_object(self, object_type, identity, data): + """ + Find the NetBox object of an AWS resource by its aws_resource_id. An object without that + ID is taken over if it is the only one with this name and parent, belongs to this source + and has no other aws_service, instead of ending in a name collision. + """ fields = data["custom_fields"] fields["aws_metadata"] = json_safe(fields["aws_metadata"]) - matches = [item for item in self.inventory.get_all_items(object_type) + items = self.inventory.get_all_items(object_type) + matches = [item for item in items if (item.data.get("custom_fields") or {}).get("aws_resource_id") == identity] + parent = "virtual_machine" if object_type is NBVirtualDisk else "cluster" if object_type is NBVM else "group" + if not matches: + name_matches = [item for item in items + if item.data.get("name") == data["name"] and + item.data.get(parent) is data.get(parent)] + if len(name_matches) == 1: + candidate = name_matches[0] + candidate_fields = candidate.data.get("custom_fields") or {} + candidate_identity = candidate_fields.get("aws_resource_id") + candidate_service = candidate_fields.get("aws_service") + identity_missing = candidate_identity is None or ( + isinstance(candidate_identity, str) and not candidate_identity.strip()) + service = (data.get("custom_fields") or {}).get("aws_service") + service_matches = candidate_service in (None, "", service) + source_managed = candidate.source is self or self.source_tag in candidate.get_tags() + if identity_missing and service_matches and source_managed: + matches = [candidate] if len(matches) > 1: raise ValueError(f"Multiple NetBox {object_type.name} objects match AWS resource '{identity}'") if matches: @@ -354,9 +377,8 @@ def service_object(self, object_type, identity, data): raise ValueError(f"AWS resource '{identity}' belongs to another source") item.update(data=data, source=self) else: - parent = "virtual_machine" if object_type is NBVirtualDisk else "cluster" if object_type is NBVM else "group" if any(item.data.get("name") == data["name"] and item.data.get(parent) is data.get(parent) - for item in self.inventory.get_all_items(object_type)): + for item in items): raise ValueError(f"NetBox {object_type.name} name collision for AWS resource '{identity}'") item = self.inventory.add_object(object_type, data=data, source=self) self.reconcile_tags(item, data["tags"]) diff --git a/tests/test_aws_services.py b/tests/test_aws_services.py index dc5090e3..011783a4 100644 --- a/tests/test_aws_services.py +++ b/tests/test_aws_services.py @@ -24,6 +24,7 @@ from module.sources import instantiate_sources from module.sources.aws import client as aws_client from module.sources.aws.client import AWSAccessCheck, AWSClient, AWSDiscoveryError +from module.sources.aws.connection import json_safe from test_aws_source import ( ACCOUNT, INSTANCE, REGION, SECOND_ACCOUNT, VPC, assume_member, identity, inventory_data, make_source, org_sdk, organization_accounts, stub_region, windows_config_paths, @@ -105,6 +106,11 @@ def test_services_import_and_repeat(make_source, inventory, version): ] +def test_json_safe_normalizes_aware_timestamps_to_utc(): + timestamp = datetime(2026, 11, 14, 8, 30, tzinfo=timezone(timedelta(hours=-6))) + assert json_safe({"createdAt": timestamp}) == {"createdAt": "2026-11-14T14:30:00+00:00"} + + def test_eks_timestamps_are_stored_as_json(make_source, inventory): # boto3 parses EKS timestamps into datetime objects in the local timezone local = timezone(timedelta(hours=2)) @@ -131,6 +137,26 @@ def test_eks_timestamps_are_stored_as_json(make_source, inventory): assert "custom_fields" not in cluster.updated_items +@pytest.mark.parametrize("existing_id", [None, "", " "]) +def test_service_object_adopts_a_source_tagged_object_without_resource_id(make_source, inventory, existing_id): + source = make_source("sync_lambda = True") + source.custom_fields() + group = inventory.add_object(NBClusterGroup, data={"name": "legacy"}) + cluster = inventory.add_object(NBCluster, data={"name": "legacy-cluster", "group": group}) + existing = inventory.add_object(NBVM, data={ + "name": "Lambda old", "cluster": cluster, "tags": [source.source_tag], + "custom_fields": {"aws_resource_id": existing_id, "aws_service": "lambda"}, + }) + data = { + "name": "Lambda old", "cluster": cluster, "tags": [], + "custom_fields": {"aws_resource_id": FUNCTION_ARN, "aws_service": "lambda", "aws_metadata": {}}, + } + + assert source.service_object(NBVM, FUNCTION_ARN, data) is existing + assert existing.data["custom_fields"]["aws_resource_id"] == FUNCTION_ARN + assert len(inventory.get_all_items(NBVM)) == 1 + + def test_s3_requires_supported_netbox_before_discovery(make_source, inventory, caplog): inventory.netbox_api_version = "3.6.0" source = make_source("sync_s3 = True") @@ -282,12 +308,12 @@ def test_s3_conflict_skips_dependencies_but_continues(make_source, inventory, ca cluster = inventory.add_object(NBCluster, data={ "name": f"S3 {ACCOUNT}/{REGION}", "group": group, "custom_fields": {} if stage == "cluster" else {"aws_resource_id": identity}, - }, source=source) + }, source=source if stage != "cluster" else None) if stage != "cluster": holder = inventory.add_object(NBVM, data={ "name": f"S3 inventory {ACCOUNT}/{REGION}", "cluster": cluster, "custom_fields": {} if stage == "holder" else {"aws_resource_id": identity}, - }, source=source) + }, source=source if stage == "bucket" else None) if stage == "bucket": inventory.add_object(NBVirtualDisk, data={ "name": "inventory-bucket", "virtual_machine": holder, "size": 0, @@ -301,6 +327,30 @@ def test_s3_conflict_skips_dependencies_but_continues(make_source, inventory, ca for vm in inventory.get_all_items(NBVM)) +@pytest.mark.parametrize("mismatch", ["aws_resource_id", "aws_service"]) +def test_s3_source_tagged_name_collision_is_not_adopted(make_source, inventory, caplog, mismatch): + data = service_data() + source = make_source("sync_s3 = True\nsync_private_ips = False", data=data) + source.apply() + bucket = next(disk for disk in inventory.get_all_items(NBVirtualDisk) + if disk.data["name"] == "inventory-bucket") + bucket.source = None + bucket.add_tags(source.source_tag) + fields = bucket.data["custom_fields"] + if mismatch == "aws_resource_id": + fields["aws_resource_id"] = f"s3/{SECOND_ACCOUNT}/inventory-bucket" + else: + fields.pop("aws_resource_id") + fields["aws_service"] = "lambda" + original_fields = copy.deepcopy(fields) + + source.apply() + + assert "name collision" in caplog.text + assert bucket.data["custom_fields"] == original_fields + assert source.source_tag in bucket.get_tags() + + def test_eks_conflict_skips_workers_and_continues_next_region(make_source, inventory, caplog): data = service_data() source = make_source("sync_eks = True\nsync_private_ips = False", data=data)