Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 19 additions & 7 deletions aws_lambda_powertools/utilities/idempotency/persistence/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@

from __future__ import annotations

import base64
import datetime as dt
import hashlib
import json
Expand Down Expand Up @@ -46,13 +47,24 @@ def default(self, obj: Any) -> Any:
# Preserve existing encodings, including Decimal subclasses mixed with Enum.
return super().default(obj)
except TypeError:
if isinstance(obj, Enum):
return obj.value
if isinstance(obj, (dt.date, dt.time)):
return obj.isoformat()
if isinstance(obj, (UUID, PurePath)):
return str(obj)
raise
return self._encode_custom_type(obj)

@staticmethod
def _encode_custom_type(obj: Any) -> Any:
if isinstance(obj, Enum):
return obj.value
if isinstance(obj, (dt.date, dt.time)):
return obj.isoformat()
if isinstance(obj, (UUID, PurePath)):
return str(obj)
if isinstance(obj, (set, frozenset)):
try:
return sorted(obj)
except TypeError:
return sorted(obj, key=repr)
Comment on lines +60 to +64

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

sorted() doesn't raise for a set of frozensets, because frozensets compare as subsets, so it just returns them in whatever order the set iterates. String hashes change between processes, so the same payload can get a different idempotency key after a cold start. I ran {"groups": {frozenset({"admins"}), frozenset({"billing"}), frozenset({"ops", "dev"})}} through the encoder with PYTHONHASHSEED 1 to 6 and got three different orders. The repr fallback has the same problem, since a frozenset's repr follows hash order too.

Sorting by each item's JSON gives the same order every time. With this, all six seeds give [["admins"], ["billing"], ["dev", "ops"]], and test_idempotency_hash_serialization.py still passes:

Suggested change
if isinstance(obj, (set, frozenset)):
try:
return sorted(obj)
except TypeError:
return sorted(obj, key=repr)
if isinstance(obj, (set, frozenset)):
# Sort by each item's JSON so the order doesn't depend on hashing. sorted() alone isn't enough:
# frozensets compare as subsets, and their repr() follows hash order, which changes between processes.
return sorted(obj, key=lambda item: json.dumps(item, cls=_IdempotencyKeyEncoder, sort_keys=True))

if isinstance(obj, bytes):
return base64.b64encode(obj).decode("utf-8")
raise TypeError(f"Object of type {type(obj).__name__} is not JSON serializable")


class BasePersistenceLayer(ABC):
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -238,7 +238,7 @@ def process(payload):
stubber.assert_no_pending_responses()


@pytest.mark.parametrize("value", [b"bytes", {1, 2}, complex(1, 2)], ids=["bytes", "set", "complex"])
@pytest.mark.parametrize("value", [complex(1, 2)], ids=["complex"])
def test_unsupported_key_types_still_raise(persistence_store, lambda_context, value):
@idempotent_function(
data_keyword_argument="payload",
Expand All @@ -254,3 +254,76 @@ def process(payload):
assert isinstance(error.value.__cause__, TypeError)
assert "is not JSON serializable" in str(error.value.__cause__)
stubber.assert_no_pending_responses()


@pytest.mark.parametrize("as_dataclass", [True, False], ids=["dataclass", "dict"])
def test_set_frozenset_and_bytes_support(
persistence_store,
lambda_context,
timestamp_future,
as_dataclass,
):
"""Asserts that sets, frozensets with different insertion orders produce the same key, and bytes work."""
# Payload 1 with set and frozenset in one insertion order
value_1 = {
"tags": {"b", "a", "c"},
"frozen_tags": frozenset(["z", "y"]),
"token": b"secret-token",
}
payload_1 = Payload("order-1", value_1)

# Payload 2 with the exact same set and frozenset in a DIFFERENT insertion order
value_2 = {
"tags": {"c", "a", "b"},
"frozen_tags": frozenset(["y", "z"]),
"token": b"secret-token",
}
payload_2 = Payload("order-1", value_2)

prepared_value = {
"tags": ["a", "b", "c"],
"frozen_tags": ["y", "z"],
"token": "c2VjcmV0LXRva2Vu",
}

config = IdempotencyConfig(
event_key_jmespath="order_id",
payload_validation_jmespath="value",
lambda_context=lambda_context,
use_local_cache=False,
)

@idempotent_function(
data_keyword_argument="payload",
persistence_store=persistence_store,
config=config,
key_prefix="orders",
)
def process(payload):
return {"status": "already processed"}

expected_update = build_idempotency_update_item_stub("order-1", {"status": "already processed"})
expected_update["Key"]["id"] = {"S": f"orders#{hash_idempotency_key('order-1')}"}
expected_update["ExpressionAttributeNames"]["#validation_key"] = "validation"
expected_update["ExpressionAttributeValues"][":validation_key"] = {"S": hash_idempotency_key(prepared_value)}
expected_update["UpdateExpression"] += ", #validation_key = :validation_key"

with stub.Stubber(persistence_store.client) as stubber:
# Test first insertion order (puts item into the store)
stubber.add_response("put_item", {}, _expected_put("order-1", prepared_value))
stubber.add_response("update_item", {}, expected_update)

request_1 = payload_1 if as_dataclass else dataclasses.asdict(payload_1)
assert process(payload=request_1) == {"status": "already processed"}

# Test second insertion order (produces the exact same hash/key, successfully reusing the record)
stubber.add_client_error(
"put_item",
"ConditionalCheckFailedException",
modeled_fields=_completed_record("order-1", prepared_value, timestamp_future),
expected_params=_expected_put("order-1", prepared_value),
)
request_2 = payload_2 if as_dataclass else dataclasses.asdict(payload_2)
assert process(payload=request_2) == {"status": "already processed"}

stubber.assert_no_pending_responses()
Loading