Skip to content

Commit ebfc9f0

Browse files
fix(idempotency): use the record key for DynamoDB composite items (#8739)
Read the idempotency key from the sort key for composite tables. Replayed records then use the actual operation key for response hooks, local cache lookups and in-progress diagnostics. Cover both conditional-write response and get-item paths through the public decorator, retaining simple-key controls. Co-authored-by: Leandro Damascena <lcdama@amazon.pt>
1 parent 1e42825 commit ebfc9f0

4 files changed

Lines changed: 138 additions & 2 deletions

File tree

‎aws_lambda_powertools/utilities/idempotency/persistence/dynamodb.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -162,7 +162,7 @@ def _item_to_data_record(self, item: dict[str, Any]) -> DataRecord:
162162
"""
163163
data = self._deserializer.deserialize({"M": item})
164164
return DataRecord(
165-
idempotency_key=data[self.key_attr],
165+
idempotency_key=data[self.sort_key_attr] if self.sort_key_attr else data[self.key_attr],
166166
status=data[self.status_attr],
167167
expiry_timestamp=data[self.expiry_attr],
168168
in_progress_expiry_timestamp=data.get(self.in_progress_expiry_attr),

‎docs/utilities/idempotency.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -983,6 +983,8 @@ You can create your own persistent store from scratch by inheriting the `BasePer
983983

984984
### Manipulating the Idempotent Response
985985

986+
With a DynamoDB composite key, `DataRecord.idempotency_key` contains the sort key value identifying the operation, rather than the shared static partition key. Replayed records use this same key for local caching.
987+
986988
You can set up a `response_hook` in the `IdempotentConfig` class to manipulate the returned data when an operation is idempotent. The hook function will be called with the current deserialized response object and the Idempotency record.
987989

988990
=== "Using an Idempotent Response Hook"

‎examples/idempotency/src/working_with_response_hook.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@
1616

1717

1818
def my_response_hook(response: dict, idempotent_data: DataRecord) -> dict:
19-
# Return inserted Header data into the Idempotent Response
19+
# With a composite key, this is the operation's sort key, not the shared partition key
2020
response["x-idempotent-key"] = idempotent_data.idempotency_key
2121

2222
# expiry_timestamp can be None so include if set
Lines changed: 134 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,134 @@
1+
import pytest
2+
from botocore.stub import Stubber
3+
4+
from aws_lambda_powertools.utilities.idempotency import DynamoDBPersistenceLayer, IdempotencyConfig, idempotent
5+
from aws_lambda_powertools.utilities.idempotency.exceptions import IdempotencyAlreadyInProgressError
6+
7+
8+
@pytest.fixture(params=[None, "sk", "record_key"])
9+
def record_store(request, config):
10+
return DynamoDBPersistenceLayer(
11+
table_name="TEST_TABLE",
12+
boto_config=config,
13+
sort_key_attr=request.param,
14+
static_pk_value="shared-partition",
15+
)
16+
17+
18+
def stored_item(store, key, expiry, status="COMPLETED"):
19+
item = {
20+
"id": {"S": "shared-partition" if store.sort_key_attr else key},
21+
"expiration": {"N": expiry},
22+
"status": {"S": status},
23+
"data": {"S": '{"message": "stored"}'},
24+
}
25+
if store.sort_key_attr:
26+
item[store.sort_key_attr] = {"S": key}
27+
return item
28+
29+
30+
def stub_existing_record(stubber, store, item, source):
31+
if source == "condition":
32+
stubber.add_client_error("put_item", "ConditionalCheckFailedException", modeled_fields={"Item": item})
33+
else:
34+
stubber.add_client_error("put_item", "ConditionalCheckFailedException")
35+
key = {"id": item["id"]}
36+
if store.sort_key_attr:
37+
key[store.sort_key_attr] = item[store.sort_key_attr]
38+
stubber.add_response(
39+
"get_item",
40+
{"Item": item},
41+
{
42+
"TableName": "TEST_TABLE",
43+
"Key": key,
44+
"ConsistentRead": True,
45+
},
46+
)
47+
48+
49+
@pytest.mark.parametrize("source", ["condition", "get_item"])
50+
@pytest.mark.parametrize("use_local_cache", [False, True])
51+
def test_response_hook_receives_record_key(
52+
record_store,
53+
source,
54+
use_local_cache,
55+
lambda_apigw_event,
56+
lambda_context,
57+
timestamp_future,
58+
hashed_idempotency_key,
59+
default_jmespath,
60+
):
61+
keys = []
62+
63+
def response_hook(response, record):
64+
keys.append((record.idempotency_key, record.sort_key))
65+
return response
66+
67+
config = IdempotencyConfig(
68+
event_key_jmespath=default_jmespath,
69+
use_local_cache=use_local_cache,
70+
response_hook=response_hook,
71+
)
72+
73+
@idempotent(config=config, persistence_store=record_store)
74+
def lambda_handler(event, context):
75+
pytest.fail("A stored response must bypass the handler")
76+
77+
item = stored_item(record_store, hashed_idempotency_key, timestamp_future)
78+
with Stubber(record_store.client) as stubber:
79+
stub_existing_record(stubber, record_store, item, source)
80+
assert lambda_handler(lambda_apigw_event, lambda_context) == {"message": "stored"}
81+
stubber.assert_no_pending_responses()
82+
83+
expected_sort_key = hashed_idempotency_key if record_store.sort_key_attr else None
84+
assert keys == [(hashed_idempotency_key, expected_sort_key)]
85+
86+
87+
@pytest.mark.parametrize("source", ["condition", "get_item"])
88+
def test_stored_response_is_replayed_from_local_cache(
89+
record_store,
90+
source,
91+
lambda_apigw_event,
92+
lambda_context,
93+
timestamp_future,
94+
hashed_idempotency_key,
95+
default_jmespath,
96+
):
97+
config = IdempotencyConfig(event_key_jmespath=default_jmespath, use_local_cache=True)
98+
99+
@idempotent(config=config, persistence_store=record_store)
100+
def lambda_handler(event, context):
101+
pytest.fail("A stored response must bypass the handler")
102+
103+
item = stored_item(record_store, hashed_idempotency_key, timestamp_future)
104+
with Stubber(record_store.client) as stubber:
105+
stub_existing_record(stubber, record_store, item, source)
106+
assert lambda_handler(lambda_apigw_event, lambda_context) == {"message": "stored"}
107+
# No more DynamoDB requests are stubbed: the second invocation must use the cache.
108+
assert lambda_handler(lambda_apigw_event, lambda_context) == {"message": "stored"}
109+
stubber.assert_no_pending_responses()
110+
111+
112+
@pytest.mark.parametrize("source", ["condition", "get_item"])
113+
def test_in_progress_error_identifies_record_key(
114+
record_store,
115+
source,
116+
lambda_apigw_event,
117+
lambda_context,
118+
timestamp_future,
119+
hashed_idempotency_key,
120+
default_jmespath,
121+
):
122+
config = IdempotencyConfig(event_key_jmespath=default_jmespath)
123+
124+
@idempotent(config=config, persistence_store=record_store)
125+
def lambda_handler(event, context):
126+
pytest.fail("An in-progress invocation must bypass the handler")
127+
128+
item = stored_item(record_store, hashed_idempotency_key, timestamp_future, status="INPROGRESS")
129+
with Stubber(record_store.client) as stubber:
130+
stub_existing_record(stubber, record_store, item, source)
131+
with pytest.raises(IdempotencyAlreadyInProgressError) as exc:
132+
lambda_handler(lambda_apigw_event, lambda_context)
133+
assert f"{default_jmespath}={hashed_idempotency_key}" in str(exc.value)
134+
stubber.assert_no_pending_responses()

0 commit comments

Comments
 (0)