From ceb919cd6b78cfad00daf7945a449ff0b675d852 Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Thu, 12 Nov 2026 10:05:43 +0000 Subject: [PATCH 1/5] feat(aws-lambda): Link SQS and SNS invocations to their producers The boto3 integration propagates sentry-trace in SQS and SNS message attributes (#8145). Read it back in the Lambda integration and link the invocation span to each distinct producer span, like the boto3 receive span does. Messages never parent the invocation. Co-Authored-By: Alexander Webb --- .../integrations/aws_lambda/_handler.py | 67 +++++++++- .../aws_lambda/test_aws_lambda.py | 126 ++++++++++++++++++ 2 files changed, 191 insertions(+), 2 deletions(-) diff --git a/sentry_sdk/integrations/aws_lambda/_handler.py b/sentry_sdk/integrations/aws_lambda/_handler.py index 89ba0089b7..c143195bde 100644 --- a/sentry_sdk/integrations/aws_lambda/_handler.py +++ b/sentry_sdk/integrations/aws_lambda/_handler.py @@ -1,4 +1,5 @@ import functools +import json import sys from os import environ from typing import TYPE_CHECKING @@ -20,7 +21,8 @@ ORIGIN, ) from sentry_sdk.scope import Scope -from sentry_sdk.traces import SegmentNameSource +from sentry_sdk.traces import SENTRY_TRACE_HEADER_NAME, SegmentNameSource +from sentry_sdk.tracing_utils import extract_sentrytrace_data from sentry_sdk.utils import ( TimeoutThread, capture_internal_exceptions, @@ -31,12 +33,67 @@ if TYPE_CHECKING: from typing import Any, Callable, Mapping, TypeVar - from sentry_sdk._types import Attributes + from sentry_sdk._types import Attributes, SpanLink F = TypeVar("F", bound=Callable[..., Any]) TIMEOUT_WARNING_BUFFER = 1500 MILLIS_TO_SECONDS = 1000.0 +# OTel's default link limit; Lambda delivers up to 10,000 SQS messages at once. +# https://opentelemetry.io/docs/specs/otel/trace/sdk/#span-limits +MAX_PRODUCER_LINKS = 128 + + +def _get_producer_links(aws_event: "Any") -> "list[SpanLink]": + """Links to the spans that sent the SQS or SNS messages this invocation got.""" + records = aws_event.get("Records") if isinstance(aws_event, dict) else None + links: "list[SpanLink]" = [] + for record in records or (): + if "Sns" in record: + # https://docs.aws.amazon.com/lambda/latest/dg/with-sns.html + sentry_trace = ( + record["Sns"] + .get("MessageAttributes", {}) + .get(SENTRY_TRACE_HEADER_NAME, {}) + .get("Value") + ) + else: + # https://docs.aws.amazon.com/lambda/latest/dg/with-sqs.html + sentry_trace = ( + record.get("messageAttributes", {}) + .get(SENTRY_TRACE_HEADER_NAME, {}) + .get("stringValue") + ) + # SNS wraps the message and its attributes in a JSON envelope when raw message delivery is off. + # https://docs.aws.amazon.com/sns/latest/dg/sns-large-payload-raw-message-delivery.html + body = record.get("body", "") + if sentry_trace is None and f'"{SENTRY_TRACE_HEADER_NAME}"' in body: + envelope = json.loads(body) + if envelope.get("Type") == "Notification": + sentry_trace = ( + envelope.get("MessageAttributes", {}) + .get(SENTRY_TRACE_HEADER_NAME, {}) + .get("Value") + ) + + data = extract_sentrytrace_data(sentry_trace) + if data is None or not data["trace_id"] or not data["parent_span_id"]: + continue + + link: "SpanLink" = { + "trace_id": str(data["trace_id"]), + "span_id": str(data["parent_span_id"]), + "attributes": {SPANDATA.SENTRY_LINK_TYPE: "message_origin"}, + } + if data["parent_sampled"] is not None: + link["sampled"] = bool(data["parent_sampled"]) + # a batch send gives many messages the same producer span. + if link not in links: + links.append(link) + if len(links) == MAX_PRODUCER_LINKS: + break + + return links def _wrap_handler(handler: "F") -> "F": @@ -187,12 +244,18 @@ def sentry_handler( } ) + # SQS and SNS messages link to their producers; they never parent the invocation. + producer_links: "list[SpanLink]" = [] + with capture_internal_exceptions(): + producer_links = _get_producer_links(aws_event) + sentry_sdk.continue_trace(headers) Scope.set_custom_sampling_context(sampling_context) with sentry_sdk.start_span( name=function_name, parent_span=None, attributes=attributes, + links=producer_links, ): try: return handler(aws_event, aws_context, *args, **kwargs) diff --git a/tests/integrations/aws_lambda/test_aws_lambda.py b/tests/integrations/aws_lambda/test_aws_lambda.py index 51cf7c6f79..7c3ea7566f 100644 --- a/tests/integrations/aws_lambda/test_aws_lambda.py +++ b/tests/integrations/aws_lambda/test_aws_lambda.py @@ -870,6 +870,132 @@ def test_trace_continuation(lambda_client, test_environment): assert SPANDATA.FAAS_INVOCATION_ID in attrs +MESSAGE_ORIGIN_LINK = { + SPANDATA.SENTRY_LINK_TYPE: {"value": "message_origin", "type": "string"} +} + + +def _sqs_record(sentry_trace=None, body="{}"): + record = { + "messageId": "059f36b4-87a3-44ab-83d2-661975830a7d", + "body": body, + "messageAttributes": {}, + "eventSource": "aws:sqs", + "eventSourceARN": "arn:aws:sqs:us-east-1:012345678912:orders", + } + if sentry_trace is not None: + record["messageAttributes"]["sentry-trace"] = { + "stringValue": sentry_trace, + "stringListValues": [], + "binaryListValues": [], + "dataType": "String", + } + return record + + +def _sns_notification(sentry_trace): + return { + "Type": "Notification", + "MessageId": "95df01b4-ee98-5cb9-9903-4c221d41eb5e", + "TopicArn": "arn:aws:sns:us-east-1:012345678912:alerts", + "Message": "{}", + "MessageAttributes": { + "sentry-trace": {"Type": "String", "Value": sentry_trace} + }, + } + + +@pytest.mark.parametrize( + "aws_event,links", + [ + pytest.param( + { + "Records": [ + _sqs_record("471a43a4192642f0b136d5159a501701-6e8f22c393e68f19-1"), + # the same producer span: linked once. + _sqs_record("471a43a4192642f0b136d5159a501701-6e8f22c393e68f19-1"), + _sqs_record("0f4c2a5b8d9e4f1a2b3c4d5e6f7a8b9c-1a2b3c4d5e6f7a8b-0"), + _sqs_record(), + # incomplete context: not linked. + _sqs_record("-1"), + ] + }, + [ + { + "trace_id": "471a43a4192642f0b136d5159a501701", + "span_id": "6e8f22c393e68f19", + "attributes": MESSAGE_ORIGIN_LINK, + "sampled": True, + }, + { + "trace_id": "0f4c2a5b8d9e4f1a2b3c4d5e6f7a8b9c", + "span_id": "1a2b3c4d5e6f7a8b", + "attributes": MESSAGE_ORIGIN_LINK, + "sampled": False, + }, + ], + id="sqs", + ), + pytest.param( + { + "Records": [ + { + "EventSource": "aws:sns", + "Sns": _sns_notification( + "471a43a4192642f0b136d5159a501701-6e8f22c393e68f19-1" + ), + } + ] + }, + [ + { + "trace_id": "471a43a4192642f0b136d5159a501701", + "span_id": "6e8f22c393e68f19", + "attributes": MESSAGE_ORIGIN_LINK, + "sampled": True, + } + ], + id="sns", + ), + pytest.param( + { + "Records": [ + # SNS to SQS without raw message delivery. + _sqs_record( + body=json.dumps( + _sns_notification( + "471a43a4192642f0b136d5159a501701-6e8f22c393e68f19-1" + ) + ) + ) + ] + }, + [ + { + "trace_id": "471a43a4192642f0b136d5159a501701", + "span_id": "6e8f22c393e68f19", + "attributes": MESSAGE_ORIGIN_LINK, + "sampled": True, + } + ], + id="sns-to-sqs", + ), + ], +) +def test_producer_links(lambda_client, test_environment, aws_event, links): + lambda_client.invoke( + FunctionName="BasicOkSpanStreaming", + Payload=json.dumps(aws_event), + ) + + (segment_span,) = [ + s for s in test_environment["server"].span_items if s["is_segment"] + ] + # messages are linked, never the parent of the invocation. + assert "parent_span_id" not in segment_span + assert segment_span["links"] == links + + def test_request_attributes(lambda_client, test_environment): payload = { "headers": { From b43bfc65afc1a8824fc4d4a86b81e425d7aba488 Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Thu, 12 Nov 2026 10:58:58 +0000 Subject: [PATCH 2/5] fix(aws-lambda): Skip SQS bodies that aren't an SNS envelope A body that mentions sentry-trace but isn't a JSON object raised in json.loads, and the whole link list was dropped. Skip that record instead, like boto3's _message_link does. --- sentry_sdk/integrations/aws_lambda/_handler.py | 10 ++++++++-- tests/integrations/aws_lambda/test_aws_lambda.py | 2 ++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/sentry_sdk/integrations/aws_lambda/_handler.py b/sentry_sdk/integrations/aws_lambda/_handler.py index c143195bde..50fe983634 100644 --- a/sentry_sdk/integrations/aws_lambda/_handler.py +++ b/sentry_sdk/integrations/aws_lambda/_handler.py @@ -68,8 +68,14 @@ def _get_producer_links(aws_event: "Any") -> "list[SpanLink]": # https://docs.aws.amazon.com/sns/latest/dg/sns-large-payload-raw-message-delivery.html body = record.get("body", "") if sentry_trace is None and f'"{SENTRY_TRACE_HEADER_NAME}"' in body: - envelope = json.loads(body) - if envelope.get("Type") == "Notification": + try: + envelope = json.loads(body) + except ValueError: + envelope = None + if ( + isinstance(envelope, dict) + and envelope.get("Type") == "Notification" + ): sentry_trace = ( envelope.get("MessageAttributes", {}) .get(SENTRY_TRACE_HEADER_NAME, {}) diff --git a/tests/integrations/aws_lambda/test_aws_lambda.py b/tests/integrations/aws_lambda/test_aws_lambda.py index 7c3ea7566f..8f32b7d59d 100644 --- a/tests/integrations/aws_lambda/test_aws_lambda.py +++ b/tests/integrations/aws_lambda/test_aws_lambda.py @@ -918,6 +918,8 @@ def _sns_notification(sentry_trace): _sqs_record(), # incomplete context: not linked. _sqs_record("-1"), + # a body that isn't JSON: skipped, other links kept. + _sqs_record(body='not json: "sentry-trace"'), ] }, [ From 6bf1e294bd91115e33fbf490a89624237937d062 Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Thu, 12 Nov 2026 12:11:55 +0000 Subject: [PATCH 3/5] ref(aws-lambda): Simplify reading Records for producer links --- sentry_sdk/integrations/aws_lambda/_handler.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/sentry_sdk/integrations/aws_lambda/_handler.py b/sentry_sdk/integrations/aws_lambda/_handler.py index b81fb7f025..c600c5d030 100644 --- a/sentry_sdk/integrations/aws_lambda/_handler.py +++ b/sentry_sdk/integrations/aws_lambda/_handler.py @@ -46,9 +46,12 @@ def _get_producer_links(aws_event: "Any") -> "list[SpanLink]": """Links to the spans that sent the SQS or SNS messages this invocation got.""" - records = aws_event.get("Records") if isinstance(aws_event, dict) else None + records: "list[Any]" = [] + if isinstance(aws_event, dict): + records = aws_event.get("Records") or [] + links: "list[SpanLink]" = [] - for record in records or (): + for record in records: if "Sns" in record: # https://docs.aws.amazon.com/lambda/latest/dg/with-sns.html sentry_trace = ( From 364aca5477daabc209fb97af9f24f4bd62170cac Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Thu, 12 Nov 2026 12:18:47 +0000 Subject: [PATCH 4/5] fix(aws-lambda): Don't link producers in the continued trace The span links spec says not to add a message_origin link when the consumer continues the producer's trace. Build the links after continue_trace and skip any whose trace_id matches the invocation's. Also test a sentry-trace without a sampling flag, which links without sampled. Co-Authored-By: Ivana Kellyer <131587164+sentrivana@users.noreply.github.com> --- .../integrations/aws_lambda/_handler.py | 21 +++++++++++++------ .../aws_lambda/test_aws_lambda.py | 20 ++++++++++++++++++ 2 files changed, 35 insertions(+), 6 deletions(-) diff --git a/sentry_sdk/integrations/aws_lambda/_handler.py b/sentry_sdk/integrations/aws_lambda/_handler.py index c600c5d030..080d9a6765 100644 --- a/sentry_sdk/integrations/aws_lambda/_handler.py +++ b/sentry_sdk/integrations/aws_lambda/_handler.py @@ -44,7 +44,7 @@ MAX_PRODUCER_LINKS = 128 -def _get_producer_links(aws_event: "Any") -> "list[SpanLink]": +def _get_producer_links(aws_event: "Any", trace_id: str) -> "list[SpanLink]": """Links to the spans that sent the SQS or SNS messages this invocation got.""" records: "list[Any]" = [] if isinstance(aws_event, dict): @@ -88,6 +88,9 @@ def _get_producer_links(aws_event: "Any") -> "list[SpanLink]": data = extract_sentrytrace_data(sentry_trace) if data is None or not data["trace_id"] or not data["parent_span_id"]: continue + # the invocation continues this trace, so a link would point into its own trace. + if data["trace_id"] == trace_id: + continue link: "SpanLink" = { "trace_id": str(data["trace_id"]), @@ -253,11 +256,6 @@ def sentry_handler( } ) - # SQS and SNS messages link to their producers; they never parent the invocation. - producer_links: "list[SpanLink]" = [] - with capture_internal_exceptions(): - producer_links = _get_producer_links(aws_event) - # a synchronous `Invoke` passes the invoking span in `ClientContext`; it wins over # event headers, which a forwarded HTTP event keeps from further upstream. # https://docs.aws.amazon.com/lambda/latest/dg/python-context.html @@ -267,6 +265,17 @@ def sentry_handler( sentry_sdk.continue_trace(custom) else: sentry_sdk.continue_trace(headers) + + # SQS and SNS messages link to their producers; they never parent the invocation. + producer_links: "list[SpanLink]" = [] + with capture_internal_exceptions(): + producer_links = _get_producer_links( + aws_event, + sentry_sdk.get_current_scope() + .get_active_propagation_context() + .trace_id, + ) + Scope.set_custom_sampling_context(sampling_context) with sentry_sdk.start_span( name=function_name, diff --git a/tests/integrations/aws_lambda/test_aws_lambda.py b/tests/integrations/aws_lambda/test_aws_lambda.py index 023bd18a47..aa548d3241 100644 --- a/tests/integrations/aws_lambda/test_aws_lambda.py +++ b/tests/integrations/aws_lambda/test_aws_lambda.py @@ -918,6 +918,8 @@ def _sns_notification(sentry_trace): _sqs_record(), # incomplete context: not linked. _sqs_record("-1"), + # no sampling decision: linked without `sampled`. + _sqs_record("5c6d7e8f9a0b4c1d8e2f3a4b5c6d7e8f-2b3c4d5e6f7a8b9c"), # a body that isn't JSON: skipped, other links kept. _sqs_record(body='not json: "sentry-trace"'), ] @@ -935,6 +937,11 @@ def _sns_notification(sentry_trace): "attributes": MESSAGE_ORIGIN_LINK, "sampled": False, }, + { + "trace_id": "5c6d7e8f9a0b4c1d8e2f3a4b5c6d7e8f", + "span_id": "2b3c4d5e6f7a8b9c", + "attributes": MESSAGE_ORIGIN_LINK, + }, ], id="sqs", ), @@ -1020,6 +1027,11 @@ def test_trace_continuation_from_client_context( "sentry-trace": "471a43a4192642f0b136d5159a501701-6e8f22c393e68f19-1" } }, + "Records": [ + # a producer in the continued trace: not linked, it's already in the trace. + _sqs_record("471a43a4192642f0b136d5159a501701-1a2b3c4d5e6f7a8b-1"), + _sqs_record("5c6d7e8f9a0b4c1d8e2f3a4b5c6d7e8f-2b3c4d5e6f7a8b9c-1"), + ], } lambda_client.invoke(FunctionName="ClientContextTrace", Payload=json.dumps(payload)) @@ -1029,6 +1041,14 @@ def test_trace_continuation_from_client_context( ] assert segment_span["trace_id"] == "471a43a4192642f0b136d5159a501701" assert segment_span["parent_span_id"] == "6e8f22c393e68f19" + assert segment_span["links"] == [ + { + "trace_id": "5c6d7e8f9a0b4c1d8e2f3a4b5c6d7e8f", + "span_id": "2b3c4d5e6f7a8b9c", + "attributes": MESSAGE_ORIGIN_LINK, + "sampled": True, + } + ] def test_request_attributes(lambda_client, test_environment): From ee3ca80fa58da4590dacb6e7f0aa99774d62b57d Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Thu, 12 Nov 2026 12:43:41 +0000 Subject: [PATCH 5/5] ref(aws-lambda): Stop at MAX_PRODUCER_LINKS with >= --- sentry_sdk/integrations/aws_lambda/_handler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sentry_sdk/integrations/aws_lambda/_handler.py b/sentry_sdk/integrations/aws_lambda/_handler.py index 080d9a6765..bde707c339 100644 --- a/sentry_sdk/integrations/aws_lambda/_handler.py +++ b/sentry_sdk/integrations/aws_lambda/_handler.py @@ -102,7 +102,7 @@ def _get_producer_links(aws_event: "Any", trace_id: str) -> "list[SpanLink]": # a batch send gives many messages the same producer span. if link not in links: links.append(link) - if len(links) == MAX_PRODUCER_LINKS: + if len(links) >= MAX_PRODUCER_LINKS: break return links