From fc3b4a5af1d7eb6d3ce4186fda3723aa2bad9d4f Mon Sep 17 00:00:00 2001 From: k_s_neeraj Date: Sun, 1 Nov 2026 22:50:40 +0000 Subject: [PATCH] fix(idempotency): add deterministic support for set, frozenset, and bytes types (#8725) --- .../utilities/idempotency/persistence/base.py | 26 +++++-- .../test_idempotency_hash_serialization.py | 75 ++++++++++++++++++- 2 files changed, 93 insertions(+), 8 deletions(-) diff --git a/aws_lambda_powertools/utilities/idempotency/persistence/base.py b/aws_lambda_powertools/utilities/idempotency/persistence/base.py index dcc5c932a9d..e0d1fde4340 100644 --- a/aws_lambda_powertools/utilities/idempotency/persistence/base.py +++ b/aws_lambda_powertools/utilities/idempotency/persistence/base.py @@ -4,6 +4,7 @@ from __future__ import annotations +import base64 import datetime as dt import hashlib import json @@ -46,13 +47,24 @@ def default(self, obj: Any) -> Any: # Preserve existing encodings, including Decimal subclasses mixed with Enum. return super().default(obj) except TypeError: - if isinstance(obj, Enum): - return obj.value - if isinstance(obj, (dt.date, dt.time)): - return obj.isoformat() - if isinstance(obj, (UUID, PurePath)): - return str(obj) - raise + return self._encode_custom_type(obj) + + @staticmethod + def _encode_custom_type(obj: Any) -> Any: + if isinstance(obj, Enum): + return obj.value + if isinstance(obj, (dt.date, dt.time)): + return obj.isoformat() + if isinstance(obj, (UUID, PurePath)): + return str(obj) + if isinstance(obj, (set, frozenset)): + try: + return sorted(obj) + except TypeError: + return sorted(obj, key=repr) + if isinstance(obj, bytes): + return base64.b64encode(obj).decode("utf-8") + raise TypeError(f"Object of type {type(obj).__name__} is not JSON serializable") class BasePersistenceLayer(ABC): diff --git a/tests/functional/idempotency/_boto3/test_idempotency_hash_serialization.py b/tests/functional/idempotency/_boto3/test_idempotency_hash_serialization.py index 95c3c165e3b..af333335839 100644 --- a/tests/functional/idempotency/_boto3/test_idempotency_hash_serialization.py +++ b/tests/functional/idempotency/_boto3/test_idempotency_hash_serialization.py @@ -238,7 +238,7 @@ def process(payload): stubber.assert_no_pending_responses() -@pytest.mark.parametrize("value", [b"bytes", {1, 2}, complex(1, 2)], ids=["bytes", "set", "complex"]) +@pytest.mark.parametrize("value", [complex(1, 2)], ids=["complex"]) def test_unsupported_key_types_still_raise(persistence_store, lambda_context, value): @idempotent_function( data_keyword_argument="payload", @@ -254,3 +254,76 @@ def process(payload): assert isinstance(error.value.__cause__, TypeError) assert "is not JSON serializable" in str(error.value.__cause__) stubber.assert_no_pending_responses() + + +@pytest.mark.parametrize("as_dataclass", [True, False], ids=["dataclass", "dict"]) +def test_set_frozenset_and_bytes_support( + persistence_store, + lambda_context, + timestamp_future, + as_dataclass, +): + """Asserts that sets, frozensets with different insertion orders produce the same key, and bytes work.""" + # Payload 1 with set and frozenset in one insertion order + value_1 = { + "tags": {"b", "a", "c"}, + "frozen_tags": frozenset(["z", "y"]), + "token": b"secret-token", + } + payload_1 = Payload("order-1", value_1) + + # Payload 2 with the exact same set and frozenset in a DIFFERENT insertion order + value_2 = { + "tags": {"c", "a", "b"}, + "frozen_tags": frozenset(["y", "z"]), + "token": b"secret-token", + } + payload_2 = Payload("order-1", value_2) + + prepared_value = { + "tags": ["a", "b", "c"], + "frozen_tags": ["y", "z"], + "token": "c2VjcmV0LXRva2Vu", + } + + config = IdempotencyConfig( + event_key_jmespath="order_id", + payload_validation_jmespath="value", + lambda_context=lambda_context, + use_local_cache=False, + ) + + @idempotent_function( + data_keyword_argument="payload", + persistence_store=persistence_store, + config=config, + key_prefix="orders", + ) + def process(payload): + return {"status": "already processed"} + + expected_update = build_idempotency_update_item_stub("order-1", {"status": "already processed"}) + expected_update["Key"]["id"] = {"S": f"orders#{hash_idempotency_key('order-1')}"} + expected_update["ExpressionAttributeNames"]["#validation_key"] = "validation" + expected_update["ExpressionAttributeValues"][":validation_key"] = {"S": hash_idempotency_key(prepared_value)} + expected_update["UpdateExpression"] += ", #validation_key = :validation_key" + + with stub.Stubber(persistence_store.client) as stubber: + # Test first insertion order (puts item into the store) + stubber.add_response("put_item", {}, _expected_put("order-1", prepared_value)) + stubber.add_response("update_item", {}, expected_update) + + request_1 = payload_1 if as_dataclass else dataclasses.asdict(payload_1) + assert process(payload=request_1) == {"status": "already processed"} + + # Test second insertion order (produces the exact same hash/key, successfully reusing the record) + stubber.add_client_error( + "put_item", + "ConditionalCheckFailedException", + modeled_fields=_completed_record("order-1", prepared_value, timestamp_future), + expected_params=_expected_put("order-1", prepared_value), + ) + request_2 = payload_2 if as_dataclass else dataclasses.asdict(payload_2) + assert process(payload=request_2) == {"status": "already processed"} + + stubber.assert_no_pending_responses()