From 30d436afe93cbdfe9651ceca1925621e7d0b87f4 Mon Sep 17 00:00:00 2001 From: Pablo Deputter Date: Mon, 19 Oct 2026 14:47:23 +0200 Subject: [PATCH 01/21] wip --- sentry_sdk/consts.py | 30 +++ sentry_sdk/integrations/boto3/_client.py | 6 +- .../integrations/boto3/_services/base.py | 17 +- .../integrations/boto3/_services/lambda_.py | 103 +++++++- sentry_sdk/integrations/boto3/_utils.py | 21 ++ tests/integrations/boto3/test_client.py | 10 +- tests/integrations/boto3/test_lambda.py | 232 +++++++++++++++++- 7 files changed, 393 insertions(+), 26 deletions(-) diff --git a/sentry_sdk/consts.py b/sentry_sdk/consts.py index 45c1a68fae..53b0a5c412 100644 --- a/sentry_sdk/consts.py +++ b/sentry_sdk/consts.py @@ -375,6 +375,18 @@ class SPANDATA: Example: "arn:aws:lambda:us-east-1:123456789012:function:my-function" """ + AWS_LAMBDA_RESOURCE_MAPPING_ID = "aws.lambda.resource_mapping.id" + """ + The UUID of the AWS Lambda event source mapping the request refers to. + Example: "587b4f6c-1d0e-4d48-8a3f-2c6f9a1e0b7d" + """ + + AWS_LAMBDA_TARGET_FUNCTION_NAME = "aws.lambda.target_function.name" + """ + The name of the AWS Lambda function an AWS SDK call acts on, without a qualifier. + Example: "billing" + """ + AWS_LOG_GROUP_NAMES = "aws.log.group.names" """ The name(s) of the AWS log group(s) an application is writing to. @@ -670,6 +682,24 @@ class SPANDATA: Example: "af9d5aa4-a685-4c5f-a22b-444f80b3cc28" """ + FAAS_INVOKED_NAME = "faas.invoked_name" + """ + The name of the invoked function, without a qualifier. + Example: "billing" + """ + + FAAS_INVOKED_PROVIDER = "faas.invoked_provider" + """ + The cloud provider of the invoked function. + Example: "aws" + """ + + FAAS_INVOKED_REGION = "faas.invoked_region" + """ + The cloud region of the invoked function. + Example: "eu-north-1" + """ + FAAS_NAME = "faas.name" """ The name of the serverless function. diff --git a/sentry_sdk/integrations/boto3/_client.py b/sentry_sdk/integrations/boto3/_client.py index 35ea846e22..9c17de9bc8 100644 --- a/sentry_sdk/integrations/boto3/_client.py +++ b/sentry_sdk/integrations/boto3/_client.py @@ -96,6 +96,9 @@ def sentry_patched_make_api_call( ctx.add_metadata(self) service_ext = _resolve_service(ctx.service_name) span = _start_client_span(ctx, service_ext) + if service_ext is not None: + with capture_internal_exceptions(): + api_params = service_ext.get_request_params(ctx, span) or api_params if span is None: return orig_make_api_call(self, operation_name, api_params) @@ -136,7 +139,8 @@ def sentry_patched_make_api_call( streaming_body_instrumented = False with capture_internal_exceptions(): - streaming_body_instrumented = _instrument_streaming_body(span, parsed) + if service_ext is None or service_ext.has_streaming_response(ctx): + streaming_body_instrumented = _instrument_streaming_body(span, parsed) # `StreamingBody`s finish their span when consumed or closed. if not streaming_body_instrumented: diff --git a/sentry_sdk/integrations/boto3/_services/base.py b/sentry_sdk/integrations/boto3/_services/base.py index ec7ac1df1c..2afe3be484 100644 --- a/sentry_sdk/integrations/boto3/_services/base.py +++ b/sentry_sdk/integrations/boto3/_services/base.py @@ -1,10 +1,11 @@ from typing import TYPE_CHECKING if TYPE_CHECKING: - from typing import Any, Optional + from typing import Any, Dict, Optional from sentry_sdk._types import Attributes from sentry_sdk.integrations.boto3._context import AwsCallContext + from sentry_sdk.traces import Span class _ServiceExtension: @@ -25,8 +26,22 @@ def get_request_attributes(self, ctx: "AwsCallContext") -> "Attributes": """Return request attributes to add before the AWS request is made.""" return {} + def get_request_params( + self, ctx: "AwsCallContext", span: "Optional[Span]" + ) -> "Optional[Dict[str, Any]]": + """ + Return params to send instead of `ctx.params`, e.g. with trace propagation added, + or `None` to send them unchanged. Runs for every call, also when no span was + started (`span` is `None`), and must not mutate `ctx.params`. + """ + return None + def get_response_attributes( self, ctx: "AwsCallContext", response: "Any" ) -> "Attributes": """Return response attributes to add after the AWS request is made.""" return {} + + def has_streaming_response(self, ctx: "AwsCallContext") -> bool: + """Return whether a `StreamingBody` response is read over time, so the span ends when it's read or closed, or `False` to end it with the call.""" + return True diff --git a/sentry_sdk/integrations/boto3/_services/lambda_.py b/sentry_sdk/integrations/boto3/_services/lambda_.py index c40b969762..85f36b40d9 100644 --- a/sentry_sdk/integrations/boto3/_services/lambda_.py +++ b/sentry_sdk/integrations/boto3/_services/lambda_.py @@ -1,22 +1,109 @@ +import base64 +import json from typing import TYPE_CHECKING +from sentry_sdk.consts import SPANDATA from sentry_sdk.integrations.boto3._services.base import _ServiceExtension -from sentry_sdk.integrations.boto3._utils import _get_aws_arn_attributes +from sentry_sdk.integrations.boto3._utils import ( + _extract_attributes, + _get_aws_arn_attributes, + _get_propagation_headers, +) +from sentry_sdk.integrations.boto3.consts import CLOUD_PROVIDER from sentry_sdk.utils import capture_internal_exceptions if TYPE_CHECKING: + from typing import Any, Dict, Optional, Sequence + from sentry_sdk._types import Attributes from sentry_sdk.integrations.boto3._context import AwsCallContext + from sentry_sdk.integrations.boto3._utils import _AttributeSpec + from sentry_sdk.traces import Span + +# set on both request and response of the event source mapping apis. +_RESOURCE_MAPPING_ATTRIBUTES: "Sequence[_AttributeSpec]" = ( + ("UUID", SPANDATA.AWS_LAMBDA_RESOURCE_MAPPING_ID), +) class _LambdaExtension(_ServiceExtension): def get_request_attributes(self, ctx: "AwsCallContext") -> "Attributes": - attributes: "Attributes" = {} + attributes: "Attributes" = _extract_attributes( + ctx.params, _RESOURCE_MAPPING_ATTRIBUTES + ) + if "FunctionName" not in ctx.params: + return attributes + + # a name, a partial arn (`:function:`) or an arn, each optionally + # followed by `:`. + function_name = ctx.params["FunctionName"] + name = function_name.split(":function:")[-1].split(":", 1)[0] + invoke = ctx.operation_name in ( + "Invoke", + "InvokeAsync", + "InvokeWithResponseStream", + ) + if invoke: + attributes[SPANDATA.FAAS_INVOKED_NAME] = name + attributes[SPANDATA.FAAS_INVOKED_PROVIDER] = CLOUD_PROVIDER + if ctx.region_name is not None: + attributes[SPANDATA.FAAS_INVOKED_REGION] = ctx.region_name + else: + attributes[SPANDATA.AWS_LAMBDA_TARGET_FUNCTION_NAME] = name + + # params aren't validated yet by botocore. with capture_internal_exceptions(): - attributes.update( - _get_aws_arn_attributes( - (ctx.params.get("FunctionName"),), - ("lambda",), - ) - ) + arn_attributes = _get_aws_arn_attributes((function_name,), ("lambda",)) + attributes.update(arn_attributes) + if invoke and arn_attributes: + attributes[SPANDATA.FAAS_INVOKED_REGION] = function_name.split(":")[3] + + return attributes + + def get_request_params( + self, ctx: "AwsCallContext", span: "Optional[Span]" + ) -> "Optional[Dict[str, Any]]": + # lambda only passes `ClientContext` to the function on synchronous invocations. + # https://docs.aws.amazon.com/lambda/latest/api/API_Invoke.html#lambda-Invoke-request-ClientContext + if ( + ctx.operation_name not in ("Invoke", "InvokeWithResponseStream") + or ctx.params.get("InvocationType", "RequestResponse") != "RequestResponse" + ): + return None + + headers = _get_propagation_headers(ctx, span) + client_context = ( + json.loads(base64.b64decode(ctx.params["ClientContext"])) + if "ClientContext" in ctx.params + else {} + ) + custom = client_context.get("custom", {}) + # never mix with or replace trace data the caller propagates itself. + if not headers or any(key in custom for key in headers): + return None + + client_context["custom"] = {**custom, **headers} + encoded = base64.b64encode( + json.dumps(client_context, separators=(",", ":")).encode() + ).decode() + # it takes at most 3,583 bytes of base64. + if len(encoded) > 3583: + return None + return {**ctx.params, "ClientContext": encoded} + + def has_streaming_response(self, ctx: "AwsCallContext") -> bool: + # `Invoke` returns once the function finished; its `Payload` is the complete + # result, not a stream. ending with the call keeps the span of a caller that + # never reads it. + return ctx.operation_name != "Invoke" + + def get_response_attributes( + self, ctx: "AwsCallContext", response: "Any" + ) -> "Attributes": + attributes = _extract_attributes(response, _RESOURCE_MAPPING_ATTRIBUTES) + # `Invoke` returns `200 OK` when the function fails, with `FunctionError` set. the + # payload holds the error and is never read. + # https://docs.aws.amazon.com/lambda/latest/api/API_Invoke.html#lambda-Invoke-response-FunctionError + if "FunctionError" in response: + attributes[SPANDATA.ERROR_TYPE] = "lambda_function_error" return attributes diff --git a/sentry_sdk/integrations/boto3/_utils.py b/sentry_sdk/integrations/boto3/_utils.py index 9f124639a6..f3597fb94e 100644 --- a/sentry_sdk/integrations/boto3/_utils.py +++ b/sentry_sdk/integrations/boto3/_utils.py @@ -1,11 +1,15 @@ from typing import TYPE_CHECKING +import sentry_sdk from sentry_sdk.consts import SPANDATA +from sentry_sdk.tracing_utils import should_propagate_trace if TYPE_CHECKING: from typing import Any, Dict, Iterable, Optional, Sequence, Tuple from sentry_sdk._types import Attributes + from sentry_sdk.integrations.boto3._context import AwsCallContext + from sentry_sdk.traces import Span # tuple of (request param name, span attribute name). _AttributeSpec = Tuple[str, str] @@ -38,3 +42,20 @@ def _get_aws_arn_attributes( return attributes return {} + + +def _get_propagation_headers( + ctx: "AwsCallContext", span: "Optional[Span]" +) -> "Dict[str, str]": + """ + Return the `sentry-trace` and `baggage` (and `traceparent`, if enabled) a call + propagates to its consumer, from its span or else the scope, like the HTTP headers. + Empty when `trace_propagation_targets` excludes the endpoint. + """ + if ctx.endpoint_url is None or not should_propagate_trace( + sentry_sdk.get_client(), ctx.endpoint_url + ): + return {} + return dict( + sentry_sdk.get_current_scope().iter_trace_propagation_headers(span=span) + ) diff --git a/tests/integrations/boto3/test_client.py b/tests/integrations/boto3/test_client.py index 999d3876f4..7d336c3cae 100644 --- a/tests/integrations/boto3/test_client.py +++ b/tests/integrations/boto3/test_client.py @@ -181,11 +181,11 @@ def record_client_span(request, **kwargs): "service_name,method_name,api_params,payload_field", [ pytest.param( - "lambda", - "invoke", - {"FunctionName": "function"}, - "Payload", - id="payload", + "polly", + "synthesize_speech", + {"Text": "hi", "OutputFormat": "mp3", "VoiceId": "Joanna"}, + "AudioStream", + id="audio-stream", ), pytest.param( "s3", diff --git a/tests/integrations/boto3/test_lambda.py b/tests/integrations/boto3/test_lambda.py index 00c5d58df1..0c722e5ee9 100644 --- a/tests/integrations/boto3/test_lambda.py +++ b/tests/integrations/boto3/test_lambda.py @@ -1,22 +1,232 @@ +import base64 +import io +import json +from copy import deepcopy + +import pytest +from botocore.response import StreamingBody from botocore.stub import Stubber +import sentry_sdk from sentry_sdk.consts import OP, SPANDATA -from tests.integrations.boto3.helpers import capture_spans_by_op +from tests.integrations.boto3.helpers import ( + capture_spans_by_op, + require_botocore_model_fields, +) from tests.integrations.boto3.helpers import client_factory as client_factory +FUNCTION_ARN = "arn:aws:lambda:eu-north-1:123456789012:function:billing" +MAPPING_ID = "587b4f6c-1d0e-4d48-8a3f-2c6f9a1e0b7d" -def test_request_attributes(client_factory, capture_items): - client = client_factory("lambda") - function_arn = "arn:aws:lambda:eu-north-1:123456789012:function:orders" +def _stubbed_span(client_factory, capture_items, method, params, response): + """Return the span and the params botocore sent.""" + client = client_factory("lambda") + require_botocore_model_fields( + client, method, input_fields=tuple(params), output_fields=tuple(response) + ) + sent = [] + client.meta.events.register( + "before-parameter-build", lambda params, **kwargs: sent.append(params) + ) with Stubber(client) as stubber: - stubber.add_response("delete_function", {}, {"FunctionName": function_arn}) + stubber.add_response(method, response) spans = capture_spans_by_op( - lambda: client.delete_function(**{"FunctionName": function_arn}), - capture_items, + lambda: getattr(client, method)(**params), capture_items ) - (span,) = spans[OP.RPC] - attributes = span["attributes"] - assert attributes[SPANDATA.CLOUD_ACCOUNT_ID] == "123456789012" - assert attributes[SPANDATA.CLOUD_RESOURCE_ID] == function_arn + return span, sent[0] + + +def _client_context(value): + return json.loads(base64.b64decode(value)) + + +@pytest.mark.parametrize( + "method,params,response,expected", + [ + pytest.param( + "invoke", + {"FunctionName": "billing:prod"}, + {"StatusCode": 200}, + { + SPANDATA.FAAS_INVOKED_NAME: "billing", + SPANDATA.FAAS_INVOKED_PROVIDER: "aws", + SPANDATA.FAAS_INVOKED_REGION: "eu-north-1", + }, + id="invoke-name", + ), + pytest.param( + "invoke", + { + "FunctionName": "arn:aws:lambda:us-east-1:123456789012:function:billing:7" + }, + {"StatusCode": 200}, + { + SPANDATA.CLOUD_RESOURCE_ID: "arn:aws:lambda:us-east-1:123456789012:function:billing:7", + SPANDATA.FAAS_INVOKED_NAME: "billing", + SPANDATA.FAAS_INVOKED_PROVIDER: "aws", + SPANDATA.FAAS_INVOKED_REGION: "us-east-1", + }, + id="invoke-arn", + ), + pytest.param( + "delete_function", + {"FunctionName": FUNCTION_ARN}, + {}, + { + SPANDATA.CLOUD_RESOURCE_ID: FUNCTION_ARN, + SPANDATA.AWS_LAMBDA_TARGET_FUNCTION_NAME: "billing", + }, + id="function-arn", + ), + pytest.param( + "create_event_source_mapping", + { + "FunctionName": "billing", + "EventSourceArn": "arn:aws:sqs:eu-north-1:123456789012:orders", + }, + {"UUID": MAPPING_ID}, + { + SPANDATA.AWS_LAMBDA_TARGET_FUNCTION_NAME: "billing", + SPANDATA.AWS_LAMBDA_RESOURCE_MAPPING_ID: MAPPING_ID, + }, + id="mapping-from-response", + ), + pytest.param( + "get_event_source_mapping", + {"UUID": MAPPING_ID}, + {}, + {SPANDATA.AWS_LAMBDA_RESOURCE_MAPPING_ID: MAPPING_ID}, + id="mapping-from-request", + ), + ], +) +def test_function_attributes( + client_factory, capture_items, method, params, response, expected +): + span, _ = _stubbed_span(client_factory, capture_items, method, params, response) + + assert { + key: value + for key, value in span["attributes"].items() + if key.startswith(("faas.invoked_", "aws.lambda.")) + or key == SPANDATA.CLOUD_RESOURCE_ID + } == expected + + +@pytest.mark.parametrize( + "response,expected_error_type", + [ + pytest.param({"StatusCode": 200}, None, id="ok"), + pytest.param( + {"StatusCode": 200, "FunctionError": "Unhandled"}, + "lambda_function_error", + id="function-error", + ), + # the payload is never read; the span still ends with the call. + pytest.param( + {"StatusCode": 200, "Payload": StreamingBody(io.BytesIO(b"null"), 4)}, + None, + id="unread-payload", + ), + ], +) +def test_invoke_outcome(client_factory, capture_items, response, expected_error_type): + span, _ = _stubbed_span( + client_factory, capture_items, "invoke", {"FunctionName": "billing"}, response + ) + + assert span["status"] == ("ok" if expected_error_type is None else "error") + assert span["attributes"].get(SPANDATA.ERROR_TYPE) == expected_error_type + + +def test_invoke_propagates_trace(client_factory, capture_items): + span, sent = _stubbed_span( + client_factory, + capture_items, + "invoke", + {"FunctionName": "billing"}, + {"StatusCode": 200}, + ) + + custom = _client_context(sent["ClientContext"])["custom"] + assert custom["sentry-trace"] == f"{span['trace_id']}-{span['span_id']}-1" + assert f"sentry-trace_id={span['trace_id']}" in custom["baggage"] + + +def test_invoke_keeps_client_context(client_factory, capture_items): + client_context = { + "client": {"app_title": "shop"}, + "custom": {"tenant": "a"}, + } + params = { + "FunctionName": "billing", + "ClientContext": base64.b64encode(json.dumps(client_context).encode()).decode(), + } + original = deepcopy(params) + _, sent = _stubbed_span( + client_factory, capture_items, "invoke", params, {"StatusCode": 200} + ) + + # only the trace data is added; the caller's values and params are unchanged. + assert params == original + sent_context = _client_context(sent["ClientContext"]) + del sent_context["custom"]["sentry-trace"], sent_context["custom"]["baggage"] + assert sent_context == client_context + + +def test_invoke_propagates_without_span(client_factory): + client = client_factory("lambda") + sent = [] + client.meta.events.register( + "before-parameter-build", lambda params, **kwargs: sent.append(params) + ) + with Stubber(client) as stubber: + stubber.add_response("invoke", {"StatusCode": 200}) + client.invoke(FunctionName="billing") + + # without a parent, no span is started; the scope's trace is propagated instead, + # like the HTTP headers. + (params,) = sent + trace_id = sentry_sdk.get_current_scope().get_active_propagation_context().trace_id + assert _client_context(params["ClientContext"])["custom"][ + "sentry-trace" + ].startswith(trace_id) + + +@pytest.mark.parametrize( + "params", + [ + # lambda doesn't pass `ClientContext` to an asynchronously invoked function. + pytest.param( + {"FunctionName": "billing", "InvocationType": "Event"}, id="async" + ), + # with trace data, it would exceed the 3,583 bytes lambda accepts. + pytest.param( + { + "FunctionName": "billing", + "ClientContext": base64.b64encode( + json.dumps({"custom": {"padding": "x" * 2600}}).encode() + ).decode(), + }, + id="too-large", + ), + # the caller propagates its own trace. + pytest.param( + { + "FunctionName": "billing", + "ClientContext": base64.b64encode( + json.dumps({"custom": {"sentry-trace": "set-by-caller"}}).encode() + ).decode(), + }, + id="caller-trace", + ), + ], +) +def test_invoke_without_propagation(client_factory, capture_items, params): + _, sent = _stubbed_span( + client_factory, capture_items, "invoke", params, {"StatusCode": 202} + ) + + assert sent == params From d5767c4ac38b72a8eb4fafd16fc32201599bb8bd Mon Sep 17 00:00:00 2001 From: Pablo Deputter Date: Fri, 23 Oct 2026 15:23:43 +0200 Subject: [PATCH 02/21] fix wrong parent ID --- sentry_sdk/integrations/boto3/_utils.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/sentry_sdk/integrations/boto3/_utils.py b/sentry_sdk/integrations/boto3/_utils.py index f3597fb94e..1ca3fa5f7e 100644 --- a/sentry_sdk/integrations/boto3/_utils.py +++ b/sentry_sdk/integrations/boto3/_utils.py @@ -2,6 +2,7 @@ import sentry_sdk from sentry_sdk.consts import SPANDATA +from sentry_sdk.traces import NoOpSpan from sentry_sdk.tracing_utils import should_propagate_trace if TYPE_CHECKING: @@ -56,6 +57,10 @@ def _get_propagation_headers( sentry_sdk.get_client(), ctx.endpoint_url ): return {} + # `NoOpSpan` is never sent, use scope's span instead. + # which should be the parent normally. + if isinstance(span, NoOpSpan): + span = None return dict( sentry_sdk.get_current_scope().iter_trace_propagation_headers(span=span) ) From ae57d8b500256cb161d42e017b328fef661bcb60 Mon Sep 17 00:00:00 2001 From: Pablo Deputter Date: Fri, 23 Oct 2026 16:36:46 +0200 Subject: [PATCH 03/21] wip --- sentry_sdk/integrations/boto3/_client.py | 17 +- .../integrations/boto3/_instrumentation.py | 15 +- .../integrations/boto3/_services/base.py | 24 +- .../integrations/boto3/_services/lambda_.py | 68 +++--- .../integrations/boto3/_services/registry.py | 13 +- sentry_sdk/integrations/boto3/_utils.py | 9 +- tests/integrations/boto3/test_client.py | 44 +--- tests/integrations/boto3/test_lambda.py | 206 +++++++++--------- 8 files changed, 180 insertions(+), 216 deletions(-) diff --git a/sentry_sdk/integrations/boto3/_client.py b/sentry_sdk/integrations/boto3/_client.py index 9c17de9bc8..25af44c02d 100644 --- a/sentry_sdk/integrations/boto3/_client.py +++ b/sentry_sdk/integrations/boto3/_client.py @@ -96,9 +96,9 @@ def sentry_patched_make_api_call( ctx.add_metadata(self) service_ext = _resolve_service(ctx.service_name) span = _start_client_span(ctx, service_ext) - if service_ext is not None: - with capture_internal_exceptions(): - api_params = service_ext.get_request_params(ctx, span) or api_params + with capture_internal_exceptions(): + # override the request params for e.g. trace propagation. + api_params = service_ext.get_request_params(ctx, span) or api_params if span is None: return orig_make_api_call(self, operation_name, api_params) @@ -110,7 +110,7 @@ def sentry_patched_make_api_call( try: parsed = orig_make_api_call(self, operation_name, api_params) except BaseException as error: - if service_ext is not None and isinstance(error, ClientError): + if isinstance(error, ClientError): with capture_internal_exceptions(): attributes.update( service_ext.get_response_attributes(ctx, error.response) @@ -120,11 +120,8 @@ def sentry_patched_make_api_call( with capture_internal_exceptions(): span.set_attributes(attributes) raise - if service_ext is not None: - with capture_internal_exceptions(): - attributes.update( - service_ext.get_response_attributes(ctx, parsed) - ) + with capture_internal_exceptions(): + attributes.update(service_ext.get_response_attributes(ctx, parsed)) with capture_internal_exceptions(): attributes.update(_get_response_attributes(parsed)) with capture_internal_exceptions(): @@ -139,7 +136,7 @@ def sentry_patched_make_api_call( streaming_body_instrumented = False with capture_internal_exceptions(): - if service_ext is None or service_ext.has_streaming_response(ctx): + if service_ext.has_streaming_response(ctx): streaming_body_instrumented = _instrument_streaming_body(span, parsed) # `StreamingBody`s finish their span when consumed or closed. diff --git a/sentry_sdk/integrations/boto3/_instrumentation.py b/sentry_sdk/integrations/boto3/_instrumentation.py index 731dac6fa3..777780fa50 100644 --- a/sentry_sdk/integrations/boto3/_instrumentation.py +++ b/sentry_sdk/integrations/boto3/_instrumentation.py @@ -148,7 +148,7 @@ def _get_error_attributes(exception: "BaseException") -> "Attributes": def _start_client_span( ctx: "AwsCallContext", - service_ext: "Optional[_ServiceExtension]" = None, + service_ext: "_ServiceExtension", ) -> "Optional[Span]": if sentry_sdk.get_current_span() is None: return None @@ -171,14 +171,13 @@ def _start_client_span( # with `get_span_op()`. span_op = OP.RPC - if service_ext is not None: - with capture_internal_exceptions(): - service_op = service_ext.get_span_op(ctx) - if service_op is not None: - span_op = service_op + with capture_internal_exceptions(): + service_op = service_ext.get_span_op(ctx) + if service_op is not None: + span_op = service_op - with capture_internal_exceptions(): - attributes.update(service_ext.get_request_attributes(ctx)) + with capture_internal_exceptions(): + attributes.update(service_ext.get_request_attributes(ctx)) # `start_span()` evaluates `ignore_spans` against the initial attributes. # https://opentelemetry.io/docs/specs/semconv/rpc/rpc-spans/#rpc-client-span diff --git a/sentry_sdk/integrations/boto3/_services/base.py b/sentry_sdk/integrations/boto3/_services/base.py index 2afe3be484..910e7a2e0e 100644 --- a/sentry_sdk/integrations/boto3/_services/base.py +++ b/sentry_sdk/integrations/boto3/_services/base.py @@ -12,8 +12,8 @@ class _ServiceExtension: """ Optional hooks for adding service-specific behavior to AWS client span; non-overridden methods keep the generic instrumentation. - Services without a registered extension in `_SERVICE_EXTENSIONS` continue - to use the generic instrumentation. + Services without a registered extension in `_SERVICE_EXTENSIONS` use this + class as is, i.e. the generic instrumentation. """ __slots__ = () @@ -26,22 +26,24 @@ def get_request_attributes(self, ctx: "AwsCallContext") -> "Attributes": """Return request attributes to add before the AWS request is made.""" return {} + def get_response_attributes( + self, ctx: "AwsCallContext", response: "Any" + ) -> "Attributes": + """Return response attributes to add after the AWS request is made.""" + return {} + def get_request_params( self, ctx: "AwsCallContext", span: "Optional[Span]" ) -> "Optional[Dict[str, Any]]": """ Return params to send instead of `ctx.params`, e.g. with trace propagation added, - or `None` to send them unchanged. Runs for every call, also when no span was - started (`span` is `None`), and must not mutate `ctx.params`. + or `None` to send them unchanged; must not mutate `ctx.params`. """ return None - def get_response_attributes( - self, ctx: "AwsCallContext", response: "Any" - ) -> "Attributes": - """Return response attributes to add after the AWS request is made.""" - return {} - def has_streaming_response(self, ctx: "AwsCallContext") -> bool: - """Return whether a `StreamingBody` response is read over time, so the span ends when it's read or closed, or `False` to end it with the call.""" + """ + Return whether a `StreamingBody` response is read over time, so the span ends + when it's read or closed, or `False` to end it with the call. + """ return True diff --git a/sentry_sdk/integrations/boto3/_services/lambda_.py b/sentry_sdk/integrations/boto3/_services/lambda_.py index 85f36b40d9..9d1e65bcad 100644 --- a/sentry_sdk/integrations/boto3/_services/lambda_.py +++ b/sentry_sdk/integrations/boto3/_services/lambda_.py @@ -5,7 +5,6 @@ from sentry_sdk.consts import SPANDATA from sentry_sdk.integrations.boto3._services.base import _ServiceExtension from sentry_sdk.integrations.boto3._utils import ( - _extract_attributes, _get_aws_arn_attributes, _get_propagation_headers, ) @@ -13,37 +12,35 @@ from sentry_sdk.utils import capture_internal_exceptions if TYPE_CHECKING: - from typing import Any, Dict, Optional, Sequence + from typing import Any, Dict, Optional from sentry_sdk._types import Attributes from sentry_sdk.integrations.boto3._context import AwsCallContext - from sentry_sdk.integrations.boto3._utils import _AttributeSpec from sentry_sdk.traces import Span -# set on both request and response of the event source mapping apis. -_RESOURCE_MAPPING_ATTRIBUTES: "Sequence[_AttributeSpec]" = ( - ("UUID", SPANDATA.AWS_LAMBDA_RESOURCE_MAPPING_ID), -) - class _LambdaExtension(_ServiceExtension): def get_request_attributes(self, ctx: "AwsCallContext") -> "Attributes": - attributes: "Attributes" = _extract_attributes( - ctx.params, _RESOURCE_MAPPING_ATTRIBUTES - ) + attributes: "Attributes" = {} + + # event source mapping apis. + if "UUID" in ctx.params: + attributes[SPANDATA.AWS_LAMBDA_RESOURCE_MAPPING_ID] = ctx.params["UUID"] if "FunctionName" not in ctx.params: return attributes - # a name, a partial arn (`:function:`) or an arn, each optionally - # followed by `:`. + # a name, a partial arn (`:function:`) or an + # arn (`arn::lambda:::function:`) + # each optionally followed by `:`. function_name = ctx.params["FunctionName"] name = function_name.split(":function:")[-1].split(":", 1)[0] - invoke = ctx.operation_name in ( + + is_invoke = ctx.operation_name in ( "Invoke", "InvokeAsync", "InvokeWithResponseStream", ) - if invoke: + if is_invoke: attributes[SPANDATA.FAAS_INVOKED_NAME] = name attributes[SPANDATA.FAAS_INVOKED_PROVIDER] = CLOUD_PROVIDER if ctx.region_name is not None: @@ -51,15 +48,29 @@ def get_request_attributes(self, ctx: "AwsCallContext") -> "Attributes": else: attributes[SPANDATA.AWS_LAMBDA_TARGET_FUNCTION_NAME] = name - # params aren't validated yet by botocore. with capture_internal_exceptions(): arn_attributes = _get_aws_arn_attributes((function_name,), ("lambda",)) attributes.update(arn_attributes) - if invoke and arn_attributes: + if is_invoke and arn_attributes: attributes[SPANDATA.FAAS_INVOKED_REGION] = function_name.split(":")[3] return attributes + def get_response_attributes( + self, ctx: "AwsCallContext", response: "Any" + ) -> "Attributes": + attributes: "Attributes" = {} + + if "UUID" in response: + attributes[SPANDATA.AWS_LAMBDA_RESOURCE_MAPPING_ID] = response["UUID"] + + # `Invoke` returns `200 OK` when the function fails; reading it would leave + # the caller with an empty body. + if "FunctionError" in response: + attributes[SPANDATA.ERROR_TYPE] = "lambda_function_error" + + return attributes + def get_request_params( self, ctx: "AwsCallContext", span: "Optional[Span]" ) -> "Optional[Dict[str, Any]]": @@ -72,6 +83,8 @@ def get_request_params( return None headers = _get_propagation_headers(ctx, span) + + # https://docs.aws.amazon.com/lambda/latest/dg/python-context.html client_context = ( json.loads(base64.b64decode(ctx.params["ClientContext"])) if "ClientContext" in ctx.params @@ -86,24 +99,15 @@ def get_request_params( encoded = base64.b64encode( json.dumps(client_context, separators=(",", ":")).encode() ).decode() - # it takes at most 3,583 bytes of base64. + + # lambda rejects more than 3,583 bytes of base64; skip propagation rather + # than fail the call. if len(encoded) > 3583: return None + return {**ctx.params, "ClientContext": encoded} def has_streaming_response(self, ctx: "AwsCallContext") -> bool: - # `Invoke` returns once the function finished; its `Payload` is the complete - # result, not a stream. ending with the call keeps the span of a caller that - # never reads it. + # `Invoke`'s `Payload` is a `StreamingBody`, but span + # shouldn't be kept open until the caller reads it. return ctx.operation_name != "Invoke" - - def get_response_attributes( - self, ctx: "AwsCallContext", response: "Any" - ) -> "Attributes": - attributes = _extract_attributes(response, _RESOURCE_MAPPING_ATTRIBUTES) - # `Invoke` returns `200 OK` when the function fails, with `FunctionError` set. the - # payload holds the error and is never read. - # https://docs.aws.amazon.com/lambda/latest/api/API_Invoke.html#lambda-Invoke-response-FunctionError - if "FunctionError" in response: - attributes[SPANDATA.ERROR_TYPE] = "lambda_function_error" - return attributes diff --git a/sentry_sdk/integrations/boto3/_services/registry.py b/sentry_sdk/integrations/boto3/_services/registry.py index 84dba8b8f8..ace34aa023 100644 --- a/sentry_sdk/integrations/boto3/_services/registry.py +++ b/sentry_sdk/integrations/boto3/_services/registry.py @@ -1,5 +1,6 @@ from typing import TYPE_CHECKING +from sentry_sdk.integrations.boto3._services.base import _ServiceExtension from sentry_sdk.integrations.boto3._services.dynamodb import _DynamoDBExtension from sentry_sdk.integrations.boto3._services.kinesis import _KinesisExtension from sentry_sdk.integrations.boto3._services.lambda_ import _LambdaExtension @@ -16,14 +17,12 @@ if TYPE_CHECKING: from typing import Dict, Optional - from sentry_sdk.integrations.boto3._services.base import _ServiceExtension - # add a ServiceExtension here when one is implemented. for example: # _SERVICE_EXTENSIONS = {"s3": _S3Extension()} # when py 3.15 drops, we might want to take a look at using # a lazy-loading approach using the new `lazy` keyword. -_SERVICE_EXTENSIONS: "Dict[str, _ServiceExtension]" = { +_SERVICE_EXTENSIONS: "Dict[Optional[str], _ServiceExtension]" = { "dynamodb": _DynamoDBExtension(), "kinesis": _KinesisExtension(), "lambda": _LambdaExtension(), @@ -37,8 +36,6 @@ def _resolve_service( service_name: "Optional[str]", -) -> "Optional[_ServiceExtension]": - """Return the extension for a service, or `None` for generic instrumentation.""" - if service_name is None: - return None - return _SERVICE_EXTENSIONS.get(service_name) +) -> "_ServiceExtension": + """Return the extension for a service, or `_ServiceExtension()` for generic instrumentation.""" + return _SERVICE_EXTENSIONS.get(service_name, _ServiceExtension()) diff --git a/sentry_sdk/integrations/boto3/_utils.py b/sentry_sdk/integrations/boto3/_utils.py index 1ca3fa5f7e..7567547587 100644 --- a/sentry_sdk/integrations/boto3/_utils.py +++ b/sentry_sdk/integrations/boto3/_utils.py @@ -48,19 +48,16 @@ def _get_aws_arn_attributes( def _get_propagation_headers( ctx: "AwsCallContext", span: "Optional[Span]" ) -> "Dict[str, str]": - """ - Return the `sentry-trace` and `baggage` (and `traceparent`, if enabled) a call - propagates to its consumer, from its span or else the scope, like the HTTP headers. - Empty when `trace_propagation_targets` excludes the endpoint. - """ + """Return the `sentry-trace` and `baggage` headers a call propagates to its consumer.""" if ctx.endpoint_url is None or not should_propagate_trace( sentry_sdk.get_client(), ctx.endpoint_url ): return {} + # `NoOpSpan` is never sent, use scope's span instead. - # which should be the parent normally. if isinstance(span, NoOpSpan): span = None + return dict( sentry_sdk.get_current_scope().iter_trace_propagation_headers(span=span) ) diff --git a/tests/integrations/boto3/test_client.py b/tests/integrations/boto3/test_client.py index 7d336c3cae..f8ccb8261d 100644 --- a/tests/integrations/boto3/test_client.py +++ b/tests/integrations/boto3/test_client.py @@ -177,39 +177,11 @@ def record_client_span(request, **kwargs): assert span.get("end_timestamp") is not None -@pytest.mark.parametrize( - "service_name,method_name,api_params,payload_field", - [ - pytest.param( - "polly", - "synthesize_speech", - {"Text": "hi", "OutputFormat": "mp3", "VoiceId": "Joanna"}, - "AudioStream", - id="audio-stream", - ), - pytest.param( - "s3", - "get_object_annotation", - { - "Bucket": "bucket", - "Key": "file.txt", - "AnnotationName": "annotation", - }, - "AnnotationPayload", - id="annotation-payload", - ), - ], -) -def test_non_body_stream_delays_client_span( - capture_items, - client_factory, - service_name, - method_name, - api_params, - payload_field, -): - client = client_factory(service_name) - require_botocore_model_fields(client, method_name, output_fields=(payload_field,)) +def test_non_body_stream_delays_client_span(capture_items, client_factory): + client = client_factory("s3") + require_botocore_model_fields( + client, "get_object_annotation", output_fields=("AnnotationPayload",) + ) request_client_spans = [] def record_client_span(request, **kwargs): @@ -220,8 +192,10 @@ def record_client_span(request, **kwargs): def invoke(): parent = sentry_sdk.get_current_span() with MockResponse(client, 200, {"content-length": "1"}, b"x"): - response = getattr(client, method_name)(**api_params) - body = response[payload_field] + response = client.get_object_annotation( + Bucket="bucket", Key="file.txt", AnnotationName="annotation" + ) + body = response["AnnotationPayload"] assert isinstance(body, StreamingBody) (request_client_span,) = request_client_spans assert request_client_span.end_timestamp is None diff --git a/tests/integrations/boto3/test_lambda.py b/tests/integrations/boto3/test_lambda.py index 0c722e5ee9..94d8697723 100644 --- a/tests/integrations/boto3/test_lambda.py +++ b/tests/integrations/boto3/test_lambda.py @@ -1,6 +1,7 @@ import base64 import io import json +from contextlib import nullcontext from copy import deepcopy import pytest @@ -9,15 +10,13 @@ import sentry_sdk from sentry_sdk.consts import OP, SPANDATA +from sentry_sdk.integrations.boto3 import Boto3Integration from tests.integrations.boto3.helpers import ( capture_spans_by_op, require_botocore_model_fields, ) from tests.integrations.boto3.helpers import client_factory as client_factory -FUNCTION_ARN = "arn:aws:lambda:eu-north-1:123456789012:function:billing" -MAPPING_ID = "587b4f6c-1d0e-4d48-8a3f-2c6f9a1e0b7d" - def _stubbed_span(client_factory, capture_items, method, params, response): """Return the span and the params botocore sent.""" @@ -38,167 +37,162 @@ def _stubbed_span(client_factory, capture_items, method, params, response): return span, sent[0] -def _client_context(value): - return json.loads(base64.b64decode(value)) - - @pytest.mark.parametrize( - "method,params,response,expected", + "function_name,region,resource_id", [ + # fall back to client's region. + pytest.param("billing:prod", "eu-north-1", None, id="name"), pytest.param( - "invoke", - {"FunctionName": "billing:prod"}, - {"StatusCode": 200}, - { - SPANDATA.FAAS_INVOKED_NAME: "billing", - SPANDATA.FAAS_INVOKED_PROVIDER: "aws", - SPANDATA.FAAS_INVOKED_REGION: "eu-north-1", - }, - id="invoke-name", - ), - pytest.param( - "invoke", - { - "FunctionName": "arn:aws:lambda:us-east-1:123456789012:function:billing:7" - }, - {"StatusCode": 200}, - { - SPANDATA.CLOUD_RESOURCE_ID: "arn:aws:lambda:us-east-1:123456789012:function:billing:7", - SPANDATA.FAAS_INVOKED_NAME: "billing", - SPANDATA.FAAS_INVOKED_PROVIDER: "aws", - SPANDATA.FAAS_INVOKED_REGION: "us-east-1", - }, - id="invoke-arn", - ), - pytest.param( - "delete_function", - {"FunctionName": FUNCTION_ARN}, - {}, - { - SPANDATA.CLOUD_RESOURCE_ID: FUNCTION_ARN, - SPANDATA.AWS_LAMBDA_TARGET_FUNCTION_NAME: "billing", - }, - id="function-arn", + "arn:aws:lambda:us-east-1:123456789012:function:billing:7", + "us-east-1", + "arn:aws:lambda:us-east-1:123456789012:function:billing:7", + id="arn", ), + ], +) +def test_invoke_attributes( + client_factory, capture_items, function_name, region, resource_id +): + span, _ = _stubbed_span( + client_factory, + capture_items, + "invoke", + {"FunctionName": function_name}, + {"StatusCode": 200}, + ) + + attributes = span["attributes"] + assert attributes[SPANDATA.FAAS_INVOKED_NAME] == "billing" + assert attributes[SPANDATA.FAAS_INVOKED_REGION] == region + assert attributes.get(SPANDATA.CLOUD_RESOURCE_ID) == resource_id + assert SPANDATA.AWS_LAMBDA_TARGET_FUNCTION_NAME not in attributes + + +def test_function_attributes(client_factory, capture_items): + arn = "arn:aws:lambda:eu-north-1:123456789012:function:billing" + span, _ = _stubbed_span( + client_factory, capture_items, "delete_function", {"FunctionName": arn}, {} + ) + + attributes = span["attributes"] + assert attributes[SPANDATA.AWS_LAMBDA_TARGET_FUNCTION_NAME] == "billing" + assert attributes[SPANDATA.CLOUD_RESOURCE_ID] == arn + assert SPANDATA.FAAS_INVOKED_NAME not in attributes + + +@pytest.mark.parametrize( + "method,params,response", + [ pytest.param( "create_event_source_mapping", { "FunctionName": "billing", "EventSourceArn": "arn:aws:sqs:eu-north-1:123456789012:orders", }, - {"UUID": MAPPING_ID}, - { - SPANDATA.AWS_LAMBDA_TARGET_FUNCTION_NAME: "billing", - SPANDATA.AWS_LAMBDA_RESOURCE_MAPPING_ID: MAPPING_ID, - }, - id="mapping-from-response", + {"UUID": "587b4f6c-1d0e-4d48-8a3f-2c6f9a1e0b7d"}, + id="from-response", ), pytest.param( "get_event_source_mapping", - {"UUID": MAPPING_ID}, + {"UUID": "587b4f6c-1d0e-4d48-8a3f-2c6f9a1e0b7d"}, {}, - {SPANDATA.AWS_LAMBDA_RESOURCE_MAPPING_ID: MAPPING_ID}, - id="mapping-from-request", + id="from-request", ), ], ) -def test_function_attributes( - client_factory, capture_items, method, params, response, expected +def test_event_source_mapping_attributes( + client_factory, capture_items, method, params, response ): span, _ = _stubbed_span(client_factory, capture_items, method, params, response) - assert { - key: value - for key, value in span["attributes"].items() - if key.startswith(("faas.invoked_", "aws.lambda.")) - or key == SPANDATA.CLOUD_RESOURCE_ID - } == expected - - -@pytest.mark.parametrize( - "response,expected_error_type", - [ - pytest.param({"StatusCode": 200}, None, id="ok"), - pytest.param( - {"StatusCode": 200, "FunctionError": "Unhandled"}, - "lambda_function_error", - id="function-error", - ), - # the payload is never read; the span still ends with the call. - pytest.param( - {"StatusCode": 200, "Payload": StreamingBody(io.BytesIO(b"null"), 4)}, - None, - id="unread-payload", - ), - ], -) -def test_invoke_outcome(client_factory, capture_items, response, expected_error_type): - span, _ = _stubbed_span( - client_factory, capture_items, "invoke", {"FunctionName": "billing"}, response + assert ( + span["attributes"][SPANDATA.AWS_LAMBDA_RESOURCE_MAPPING_ID] + == "587b4f6c-1d0e-4d48-8a3f-2c6f9a1e0b7d" ) - assert span["status"] == ("ok" if expected_error_type is None else "error") - assert span["attributes"].get(SPANDATA.ERROR_TYPE) == expected_error_type - -def test_invoke_propagates_trace(client_factory, capture_items): - span, sent = _stubbed_span( +def test_invoke_function_error(client_factory, capture_items): + span, _ = _stubbed_span( client_factory, capture_items, "invoke", {"FunctionName": "billing"}, - {"StatusCode": 200}, + { + "StatusCode": 200, + "FunctionError": "Unhandled", + "Payload": StreamingBody(io.BytesIO(b"{}"), 2), + }, ) - custom = _client_context(sent["ClientContext"])["custom"] - assert custom["sentry-trace"] == f"{span['trace_id']}-{span['span_id']}-1" - assert f"sentry-trace_id={span['trace_id']}" in custom["baggage"] + assert span["status"] == "error" + assert span["attributes"][SPANDATA.ERROR_TYPE] == "lambda_function_error" -def test_invoke_keeps_client_context(client_factory, capture_items): +def test_invoke_propagates_trace(client_factory, capture_items): client_context = { "client": {"app_title": "shop"}, - "custom": {"tenant": "a"}, + "custom": {"customer_id": "1234567890"}, } params = { "FunctionName": "billing", "ClientContext": base64.b64encode(json.dumps(client_context).encode()).decode(), } - original = deepcopy(params) - _, sent = _stubbed_span( + original_params = deepcopy(params) + span, sent = _stubbed_span( client_factory, capture_items, "invoke", params, {"StatusCode": 200} ) - # only the trace data is added; the caller's values and params are unchanged. - assert params == original - sent_context = _client_context(sent["ClientContext"]) - del sent_context["custom"]["sentry-trace"], sent_context["custom"]["baggage"] + sent_context = json.loads(base64.b64decode(sent["ClientContext"])) + custom = sent_context["custom"] + assert custom.pop("sentry-trace") == f"{span['trace_id']}-{span['span_id']}-1" + assert f"sentry-trace_id={span['trace_id']}" in custom.pop("baggage") + # caller's values and params should be left unchanged. assert sent_context == client_context + assert params == original_params -def test_invoke_propagates_without_span(client_factory): +@pytest.mark.parametrize( + "ignore_spans,parent", + [ + pytest.param([], nullcontext, id="no-parent"), + pytest.param( + ["Lambda.Invoke"], + lambda: sentry_sdk.start_span(name="parent"), + id="ignored-span", + ), + ], +) +def test_invoke_propagates_scope_trace( + sentry_init, client_factory, ignore_spans, parent +): client = client_factory("lambda") + sentry_init( + traces_sample_rate=1.0, + integrations=[Boto3Integration()], + ignore_spans=ignore_spans, + ) sent = [] client.meta.events.register( "before-parameter-build", lambda params, **kwargs: sent.append(params) ) with Stubber(client) as stubber: stubber.add_response("invoke", {"StatusCode": 200}) - client.invoke(FunctionName="billing") + with parent(): + client.invoke(FunctionName="billing") + expected = dict( + sentry_sdk.get_current_scope().iter_trace_propagation_headers() + ) - # without a parent, no span is started; the scope's trace is propagated instead, - # like the HTTP headers. + # the scope's trace is propagated instead. (params,) = sent - trace_id = sentry_sdk.get_current_scope().get_active_propagation_context().trace_id - assert _client_context(params["ClientContext"])["custom"][ - "sentry-trace" - ].startswith(trace_id) + custom = json.loads(base64.b64decode(params["ClientContext"]))["custom"] + assert custom["sentry-trace"] == expected["sentry-trace"] @pytest.mark.parametrize( "params", [ - # lambda doesn't pass `ClientContext` to an asynchronously invoked function. + # lambda doesn't pass `ClientContext` to an async. invoked function. pytest.param( {"FunctionName": "billing", "InvocationType": "Event"}, id="async" ), From 7ea52042793f3e5004b1be4b675c0d32cbc530db Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Mon, 9 Nov 2026 07:11:47 +0000 Subject: [PATCH 04/21] feat(aws-lambda): Continue the trace from Invoke ClientContext MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A synchronous boto3 Invoke now carries sentry-trace and baggage in ClientContext.custom (added with the Lambda service extension). The Lambda integration continues the trace from there. ClientContext wins over event headers, because a forwarded HTTP event can keep headers from further upstream. The ClientContextTrace test function works around sam local, which drops ClientContext, by handing the event's client_context to the runtime. Co-Authored-By: Johannes Daxböck <114897+dingsdax@users.noreply.github.com> --- .../integrations/aws_lambda/_handler.py | 15 ++++++- .../ClientContextTrace/.gitignore | 11 +++++ .../ClientContextTrace/index.py | 45 +++++++++++++++++++ .../aws_lambda/test_aws_lambda.py | 33 ++++++++++++++ 4 files changed, 102 insertions(+), 2 deletions(-) create mode 100644 tests/integrations/aws_lambda/lambda_functions_with_embedded_sdk/ClientContextTrace/.gitignore create mode 100644 tests/integrations/aws_lambda/lambda_functions_with_embedded_sdk/ClientContextTrace/index.py diff --git a/sentry_sdk/integrations/aws_lambda/_handler.py b/sentry_sdk/integrations/aws_lambda/_handler.py index 89ba0089b7..98209b9104 100644 --- a/sentry_sdk/integrations/aws_lambda/_handler.py +++ b/sentry_sdk/integrations/aws_lambda/_handler.py @@ -20,7 +20,7 @@ ORIGIN, ) from sentry_sdk.scope import Scope -from sentry_sdk.traces import SegmentNameSource +from sentry_sdk.traces import SENTRY_TRACE_HEADER_NAME, SegmentNameSource from sentry_sdk.utils import ( TimeoutThread, capture_internal_exceptions, @@ -187,7 +187,18 @@ def sentry_handler( } ) - sentry_sdk.continue_trace(headers) + # a synchronous `Invoke` passes the invoking span in `ClientContext`; it wins over + # event headers, which a forwarded HTTP event keeps from further upstream. + # https://docs.aws.amazon.com/lambda/latest/dg/python-context.html + client_context = aws_context.client_context + if ( + client_context is not None + and client_context.custom + and SENTRY_TRACE_HEADER_NAME in client_context.custom + ): + sentry_sdk.continue_trace(client_context.custom) + else: + sentry_sdk.continue_trace(headers) Scope.set_custom_sampling_context(sampling_context) with sentry_sdk.start_span( name=function_name, diff --git a/tests/integrations/aws_lambda/lambda_functions_with_embedded_sdk/ClientContextTrace/.gitignore b/tests/integrations/aws_lambda/lambda_functions_with_embedded_sdk/ClientContextTrace/.gitignore new file mode 100644 index 0000000000..1c56884372 --- /dev/null +++ b/tests/integrations/aws_lambda/lambda_functions_with_embedded_sdk/ClientContextTrace/.gitignore @@ -0,0 +1,11 @@ +# Need to add some ignore rules in this directory, because the unit tests will add the Sentry SDK and its dependencies +# into this directory to create a Lambda function package that contains everything needed to instrument a Lambda function using Sentry. + +# Ignore everything +* + +# But not index.py +!index.py + +# And not .gitignore itself +!.gitignore diff --git a/tests/integrations/aws_lambda/lambda_functions_with_embedded_sdk/ClientContextTrace/index.py b/tests/integrations/aws_lambda/lambda_functions_with_embedded_sdk/ClientContextTrace/index.py new file mode 100644 index 0000000000..043961d343 --- /dev/null +++ b/tests/integrations/aws_lambda/lambda_functions_with_embedded_sdk/ClientContextTrace/index.py @@ -0,0 +1,45 @@ +import json +import os + +import sentry_sdk +from sentry_sdk.integrations.aws_lambda import AwsLambdaIntegration +from sentry_sdk.integrations.aws_lambda._runtime import get_lambda_bootstrap + +sentry_sdk.init( + dsn=os.environ.get("SENTRY_DSN"), + traces_sample_rate=1.0, + integrations=[AwsLambdaIntegration()], +) + +# `sam local` drops `ClientContext`; pass the event's `client_context` to the runtime +# instead, like lambda does for a synchronous `Invoke`. +bootstrap = get_lambda_bootstrap() +handle_event_request = bootstrap.handle_event_request + + +def handle_event_request_with_client_context( + lambda_runtime_client, + request_handler, + invoke_id, + event_body, + content_type, + client_context_json, + *args, +): + client_context_json = json.dumps(json.loads(event_body)["client_context"]) + return handle_event_request( + lambda_runtime_client, + request_handler, + invoke_id, + event_body, + content_type, + client_context_json, + *args, + ) + + +bootstrap.handle_event_request = handle_event_request_with_client_context + + +def handler(event, context): + return {"event": event} diff --git a/tests/integrations/aws_lambda/test_aws_lambda.py b/tests/integrations/aws_lambda/test_aws_lambda.py index 2d7712b2d2..4b253f67cf 100644 --- a/tests/integrations/aws_lambda/test_aws_lambda.py +++ b/tests/integrations/aws_lambda/test_aws_lambda.py @@ -855,6 +855,39 @@ def test_trace_continuation(lambda_client, test_environment): assert SPANDATA.FAAS_INVOCATION_ID in attrs +@pytest.mark.parametrize( + "headers", + [ + pytest.param({}, id="client-context"), + # `ClientContext` wins over event headers. + pytest.param( + {"sentry-trace": "0f4c2a5b8d9e4f1a2b3c4d5e6f7a8b9c-1a2b3c4d5e6f7a8b-1"}, + id="client-context-and-event-headers", + ), + ], +) +def test_trace_continuation_from_client_context( + lambda_client, test_environment, headers +): + payload = { + "headers": headers, + # `ClientContextTrace` passes it to the runtime as the invocation's `ClientContext`. + "client_context": { + "custom": { + "sentry-trace": "471a43a4192642f0b136d5159a501701-6e8f22c393e68f19-1" + } + }, + } + + lambda_client.invoke(FunctionName="ClientContextTrace", Payload=json.dumps(payload)) + + (segment_span,) = [ + s for s in test_environment["server"].span_items if s["is_segment"] + ] + assert segment_span["trace_id"] == "471a43a4192642f0b136d5159a501701" + assert segment_span["parent_span_id"] == "6e8f22c393e68f19" + + def test_request_attributes(lambda_client, test_environment): payload = { "headers": { From 61bf99840f4e1a683d75b9e73ca94bb2ffa00159 Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Mon, 9 Nov 2026 07:11:53 +0000 Subject: [PATCH 05/21] feat(aws-lambda): Detect Lambda triggers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Classify the delivered event into faas.trigger: http (API Gateway, function URLs, ALB, CloudFront, AppSync), pubsub (SQS, SNS, Kinesis, Kafka, MQ, SES, EventBridge), datasource (S3, DynamoDB Streams, DocumentDB), timer (EventBridge schedules) or other. The trigger is read from the event envelope, never from its payload. messaging.batch.message_count is now only set for messaging deliveries, with the number of delivered messages. Co-Authored-By: Johannes Daxböck <114897+dingsdax@users.noreply.github.com> --- sentry_sdk/consts.py | 6 + .../integrations/aws_lambda/_handler.py | 16 +- .../integrations/aws_lambda/_trigger.py | 123 +++++++++++++ .../aws_lambda/test_aws_lambda.py | 169 +++++++++++++++++- 4 files changed, 309 insertions(+), 5 deletions(-) create mode 100644 sentry_sdk/integrations/aws_lambda/_trigger.py diff --git a/sentry_sdk/consts.py b/sentry_sdk/consts.py index f6cb52d45c..a87a4e4243 100644 --- a/sentry_sdk/consts.py +++ b/sentry_sdk/consts.py @@ -706,6 +706,12 @@ class SPANDATA: Example: "my_function" """ + FAAS_TRIGGER = "faas.trigger" + """ + The type of the trigger on which the function is executed. + Example: "pubsub" + """ + FAAS_VERSION = "faas.version" """ The version of the function that was invoked. diff --git a/sentry_sdk/integrations/aws_lambda/_handler.py b/sentry_sdk/integrations/aws_lambda/_handler.py index 98209b9104..f22c174cf3 100644 --- a/sentry_sdk/integrations/aws_lambda/_handler.py +++ b/sentry_sdk/integrations/aws_lambda/_handler.py @@ -12,6 +12,10 @@ _get_user_from_event, _make_request_event_processor, ) +from sentry_sdk.integrations.aws_lambda._trigger import ( + _get_messaging_batch_size, + _get_trigger, +) from sentry_sdk.integrations.aws_lambda.consts import ( CLOUD_PLATFORM, CLOUD_PROVIDER, @@ -64,10 +68,8 @@ def sentry_handler( if isinstance(aws_event, list) and len(aws_event) >= 1: request_data = aws_event[0] - batch_size = len(aws_event) else: request_data = aws_event - batch_size = 1 if not isinstance(request_data, dict): # If we're not dealing with a dictionary, we won't be able to get @@ -160,11 +162,19 @@ def sentry_handler( SPANDATA.SENTRY_SEGMENT_NAME_SOURCE: SegmentNameSource.COMPONENT, SPANDATA.CLOUD_PROVIDER: CLOUD_PROVIDER, SPANDATA.CLOUD_PLATFORM: CLOUD_PLATFORM, - SPANDATA.MESSAGING_BATCH_MESSAGE_COUNT: batch_size, **header_attributes, **additional_attributes, } + with capture_internal_exceptions(): + attributes[SPANDATA.FAAS_TRIGGER] = _get_trigger(aws_event) + # HTTP and direct invocations aren't batches, even when the payload is a list. + messaging_batch_size = _get_messaging_batch_size(aws_event) + if messaging_batch_size is not None: + attributes[SPANDATA.MESSAGING_BATCH_MESSAGE_COUNT] = ( + messaging_batch_size + ) + with capture_internal_exceptions(): invoked_function_arn = aws_context.invoked_function_arn function_version = aws_context.function_version diff --git a/sentry_sdk/integrations/aws_lambda/_trigger.py b/sentry_sdk/integrations/aws_lambda/_trigger.py new file mode 100644 index 0000000000..a43dbe5ccf --- /dev/null +++ b/sentry_sdk/integrations/aws_lambda/_trigger.py @@ -0,0 +1,123 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from typing import Any, Optional + +# `faas.trigger` of each `eventSource` of `Records` or of a top-level `eventSource`. +# https://docs.aws.amazon.com/lambda/latest/dg/lambda-services.html +_TRIGGERS: "dict[Optional[str], str]" = { + "aws:sqs": "pubsub", + "aws:sns": "pubsub", + "aws:kinesis": "pubsub", + "aws:ses": "pubsub", + "aws:kafka": "pubsub", + "SelfManagedKafka": "pubsub", + "aws:mq": "pubsub", + "aws:rmq": "pubsub", + "aws:s3": "datasource", + "aws:dynamodb": "datasource", + "aws:docdb": "datasource", +} + +# event sources that group their messages by topic-partition or queue. +# https://docs.aws.amazon.com/lambda/latest/dg/with-msk.html +# https://docs.aws.amazon.com/lambda/latest/dg/with-mq.html +_GROUPED_MESSAGES_KEYS: "dict[Optional[str], str]" = { + "aws:kafka": "records", + "SelfManagedKafka": "records", + "aws:rmq": "rmqMessagesByQueue", +} + +# https://docs.aws.amazon.com/eventbridge/latest/ref/overiew-event-structure.html +_EVENTBRIDGE_KEYS = frozenset( + ( + "version", + "id", + "detail-type", + "source", + "account", + "time", + "region", + "resources", + "detail", + ) +) + + +def _get_records_source(records: "list[Any]") -> "Optional[str]": + """The records' `eventSource`, one per event; SNS spells it `EventSource`.""" + return records[0].get("eventSource") or records[0].get("EventSource") + + +def _is_appsync_event(aws_event: "Any") -> bool: + # https://docs.aws.amazon.com/appsync/latest/devguide/resolver-context-reference-js.html + return ( + isinstance(aws_event, dict) + and "arguments" in aws_event + and "fieldName" in aws_event.get("info", {}) + ) + + +def _get_trigger(aws_event: "Any") -> str: + """`faas.trigger` from the envelope of the delivered event, never from its payload.""" + if isinstance(aws_event, list): + # an AppSync batch resolver delivers a list of resolver events. + if aws_event and all(_is_appsync_event(event) for event in aws_event): + return "http" + return "other" + + if not isinstance(aws_event, dict): + return "other" + + # API Gateway REST APIs and ALB send `httpMethod`, HTTP APIs and function URLs + # `requestContext.http`. + request_context = aws_event.get("requestContext") + if request_context is not None and ( + "httpMethod" in aws_event or "http" in request_context + ): + return "http" + + if _is_appsync_event(aws_event): + return "http" + + records = aws_event.get("Records") + if records: + # CloudFront and Lambda@Edge records have no `eventSource`. + if "cf" in records[0]: + return "http" + return _TRIGGERS.get(_get_records_source(records), "other") + + if _EVENTBRIDGE_KEYS <= aws_event.keys(): + if ( + aws_event["source"] == "aws.events" + and aws_event["detail-type"] == "Scheduled Event" + ): + return "timer" + return "pubsub" + + return _TRIGGERS.get(aws_event.get("eventSource"), "other") + + +def _get_messaging_batch_size(aws_event: "Any") -> "Optional[int]": + """ + The number of messages delivered by a messaging event source, or `None` for any + other event (HTTP, direct invocation, datasource, ...). + """ + if not isinstance(aws_event, dict): + return None + + records = aws_event.get("Records") + if records: + if _get_records_source(records) in ("aws:sqs", "aws:sns"): + return len(records) + return None + + event_source = aws_event.get("eventSource") + if event_source == "aws:mq": + return len(aws_event["messages"]) + + grouped_messages_key = _GROUPED_MESSAGES_KEYS.get(event_source) + if grouped_messages_key is None: + return None + + return sum(len(messages) for messages in aws_event[grouped_messages_key].values()) diff --git a/tests/integrations/aws_lambda/test_aws_lambda.py b/tests/integrations/aws_lambda/test_aws_lambda.py index 4b253f67cf..0487c7a304 100644 --- a/tests/integrations/aws_lambda/test_aws_lambda.py +++ b/tests/integrations/aws_lambda/test_aws_lambda.py @@ -756,7 +756,8 @@ def test_no_error(lambda_client, test_environment): assert _get_span_attr(attrs, SPANDATA.AWS_LOG_STREAM_NAMES) == [ LATEST_FUNCTION_VERSION ] - assert _get_span_attr(attrs, SPANDATA.MESSAGING_BATCH_MESSAGE_COUNT) == 1 + assert _get_span_attr(attrs, SPANDATA.FAAS_TRIGGER) == "other" + assert SPANDATA.MESSAGING_BATCH_MESSAGE_COUNT not in attrs def test_error(lambda_client, test_environment): @@ -810,7 +811,171 @@ def test_error(lambda_client, test_environment): assert _get_span_attr(attrs, SPANDATA.AWS_LOG_STREAM_NAMES) == [ LATEST_FUNCTION_VERSION ] - assert _get_span_attr(attrs, SPANDATA.MESSAGING_BATCH_MESSAGE_COUNT) == 1 + assert _get_span_attr(attrs, SPANDATA.FAAS_TRIGGER) == "other" + assert SPANDATA.MESSAGING_BATCH_MESSAGE_COUNT not in attrs + + +def _sqs_record(): + return { + "messageId": "059f36b4-87a3-44ab-83d2-661975830a7d", + "body": "{}", + "eventSource": "aws:sqs", + "eventSourceARN": "arn:aws:sqs:us-east-1:012345678912:orders", + } + + +def _sns_record(): + return { + "EventSource": "aws:sns", + "Sns": {"TopicArn": "arn:aws:sns:us-east-1:012345678912:alerts"}, + } + + +def _appsync_event(): + return { + "arguments": {"id": "1"}, + "info": {"fieldName": "getOrder", "parentTypeName": "Query"}, + } + + +def _eventbridge_event(source, detail_type): + return { + "version": "0", + "id": "53dc4d37-cffa-4f76-80c9-8b7d4a4d2eaa", + "detail-type": detail_type, + "source": source, + "account": "012345678912", + "time": "2026-10-26T12:00:00Z", + "region": "us-east-1", + "resources": [], + "detail": {}, + } + + +@pytest.mark.parametrize( + "aws_event,trigger,batch_count", + [ + pytest.param( + {"httpMethod": "GET", "path": "/", "requestContext": {"stage": "prod"}}, + "http", + None, + id="api-gateway-rest", + ), + pytest.param( + { + "version": "2.0", + "routeKey": "GET /", + "requestContext": {"http": {"method": "GET"}}, + }, + "http", + None, + id="api-gateway-http", + ), + pytest.param( + {"Records": [{"cf": {"request": {"uri": "/"}}}]}, + "http", + None, + id="cloudfront", + ), + pytest.param(_appsync_event(), "http", None, id="appsync"), + pytest.param( + [_appsync_event(), _appsync_event()], "http", None, id="appsync-batch" + ), + pytest.param( + {"Records": [_sqs_record(), _sqs_record()]}, "pubsub", 2, id="sqs" + ), + pytest.param({"Records": [_sns_record()]}, "pubsub", 1, id="sns"), + pytest.param( + {"Records": [{"eventSource": "aws:kinesis"}]}, + "pubsub", + None, + id="kinesis", + ), + pytest.param( + { + "eventSource": "aws:kafka", + "records": { + "orders-0": [{"offset": 1}, {"offset": 2}], + "orders-1": [{"offset": 1}], + }, + }, + "pubsub", + 3, + id="msk", + ), + pytest.param( + {"eventSource": "SelfManagedKafka", "records": {"orders-0": [{}]}}, + "pubsub", + 1, + id="self-managed-kafka", + ), + pytest.param( + {"eventSource": "aws:mq", "messages": [{}, {}]}, + "pubsub", + 2, + id="activemq", + ), + pytest.param( + { + "eventSource": "aws:rmq", + "rmqMessagesByQueue": {"orders::/": [{}, {}], "alerts::/": [{}]}, + }, + "pubsub", + 3, + id="rabbitmq", + ), + pytest.param( + _eventbridge_event("aws.s3", "Object Created"), + "pubsub", + None, + id="eventbridge", + ), + pytest.param( + _eventbridge_event("aws.events", "Scheduled Event"), + "timer", + None, + id="eventbridge-schedule", + ), + pytest.param( + {"Records": [{"eventSource": "aws:s3"}]}, "datasource", None, id="s3" + ), + pytest.param( + {"Records": [{"eventSource": "aws:dynamodb"}]}, + "datasource", + None, + id="dynamodb-streams", + ), + pytest.param( + {"eventSource": "aws:docdb", "events": [{}]}, + "datasource", + None, + id="documentdb", + ), + pytest.param( + [{"good dog": "Maisey"}, {"good dog": "Charlie"}], + "other", + None, + id="list", + ), + ], +) +def test_trigger(lambda_client, test_environment, aws_event, trigger, batch_count): + lambda_client.invoke( + FunctionName="BasicOkSpanStreaming", + Payload=json.dumps(aws_event), + ) + + (segment_span,) = [ + s for s in test_environment["server"].span_items if s["is_segment"] + ] + attrs = segment_span["attributes"] + assert _get_span_attr(attrs, SPANDATA.FAAS_TRIGGER) == trigger + if batch_count is None: + assert SPANDATA.MESSAGING_BATCH_MESSAGE_COUNT not in attrs + else: + assert ( + _get_span_attr(attrs, SPANDATA.MESSAGING_BATCH_MESSAGE_COUNT) == batch_count + ) def test_trace_continuation(lambda_client, test_environment): From c3003ce7ff72204ced3da3fa3303248eaa96d081 Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Mon, 9 Nov 2026 07:14:02 +0000 Subject: [PATCH 06/21] feat(aws-lambda): Add queue.process spans for messaging triggers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit For an SQS, SNS, Kafka, ActiveMQ or RabbitMQ delivery, the handler runs inside a queue.process child span (kind consumer) with messaging.system, messaging.operation.type and messaging.operation.name set to process, the batch message count, and the destination name when all records share it. The span links to each distinct producer context found in the records' Sentry message attributes: SQS messageAttributes, SNS MessageAttributes, and SQS bodies that wrap an SNS notification. Co-Authored-By: Johannes Daxböck <114897+dingsdax@users.noreply.github.com> --- sentry_sdk/consts.py | 12 ++ .../integrations/aws_lambda/_handler.py | 10 +- .../integrations/aws_lambda/_trigger.py | 108 ++++++++++++- .../aws_lambda/test_aws_lambda.py | 146 +++++++++++++++++- 4 files changed, 268 insertions(+), 8 deletions(-) diff --git a/sentry_sdk/consts.py b/sentry_sdk/consts.py index a87a4e4243..bc2e786907 100644 --- a/sentry_sdk/consts.py +++ b/sentry_sdk/consts.py @@ -1126,6 +1126,18 @@ class SPANDATA: Number of retries/attempts to process a message. """ + MESSAGING_OPERATION_NAME = "messaging.operation.name" + """ + The system-specific name of the messaging operation. + Example: "publish" + """ + + MESSAGING_OPERATION_TYPE = "messaging.operation.type" + """ + The type of the messaging operation: `create`, `send`, `receive`, `process` or `settle`. + Example: "send" + """ + MESSAGING_SYSTEM = "messaging.system" """ The messaging system's name, e.g. `kafka`, `aws_sqs` diff --git a/sentry_sdk/integrations/aws_lambda/_handler.py b/sentry_sdk/integrations/aws_lambda/_handler.py index f22c174cf3..bae67cfcbb 100644 --- a/sentry_sdk/integrations/aws_lambda/_handler.py +++ b/sentry_sdk/integrations/aws_lambda/_handler.py @@ -1,5 +1,6 @@ import functools import sys +from contextlib import nullcontext from os import environ from typing import TYPE_CHECKING from urllib.parse import urlencode @@ -15,6 +16,7 @@ from sentry_sdk.integrations.aws_lambda._trigger import ( _get_messaging_batch_size, _get_trigger, + _start_process_span, ) from sentry_sdk.integrations.aws_lambda.consts import ( CLOUD_PLATFORM, @@ -33,7 +35,7 @@ ) if TYPE_CHECKING: - from typing import Any, Callable, Mapping, TypeVar + from typing import Any, Callable, ContextManager, Mapping, TypeVar from sentry_sdk._types import Attributes @@ -216,7 +218,11 @@ def sentry_handler( attributes=attributes, ): try: - return handler(aws_event, aws_context, *args, **kwargs) + process_span: "ContextManager[Any]" = nullcontext() + with capture_internal_exceptions(): + process_span = _start_process_span(aws_event) + with process_span: + return handler(aws_event, aws_context, *args, **kwargs) except Exception: exc_info = sys.exc_info() sentry_event, hint = event_from_exception( diff --git a/sentry_sdk/integrations/aws_lambda/_trigger.py b/sentry_sdk/integrations/aws_lambda/_trigger.py index a43dbe5ccf..bf7a9eabf5 100644 --- a/sentry_sdk/integrations/aws_lambda/_trigger.py +++ b/sentry_sdk/integrations/aws_lambda/_trigger.py @@ -1,7 +1,18 @@ +import json +from contextlib import nullcontext from typing import TYPE_CHECKING +import sentry_sdk +from sentry_sdk.consts import OP, SPANDATA +from sentry_sdk.integrations.aws_lambda.consts import ORIGIN +from sentry_sdk.traces import SENTRY_TRACE_HEADER_NAME +from sentry_sdk.tracing_utils import extract_sentrytrace_data +from sentry_sdk.utils import capture_internal_exceptions + if TYPE_CHECKING: - from typing import Any, Optional + from typing import Any, ContextManager, Optional + + from sentry_sdk._types import SpanLink # `faas.trigger` of each `eventSource` of `Records` or of a top-level `eventSource`. # https://docs.aws.amazon.com/lambda/latest/dg/lambda-services.html @@ -28,6 +39,13 @@ "aws:rmq": "rmqMessagesByQueue", } +# `messaging.system` of the event sources whose producers can propagate a trace. +_MESSAGING_SYSTEMS = {"aws:sqs": "aws_sqs", "aws:sns": "aws.sns"} + +# OTel's default limit; Lambda delivers up to 10,000 SQS messages at once. +# https://opentelemetry.io/docs/specs/otel/trace/sdk/#span-limits +MAX_PRODUCER_LINKS = 128 + # https://docs.aws.amazon.com/eventbridge/latest/ref/overiew-event-structure.html _EVENTBRIDGE_KEYS = frozenset( ( @@ -121,3 +139,91 @@ def _get_messaging_batch_size(aws_event: "Any") -> "Optional[int]": return None return sum(len(messages) for messages in aws_event[grouped_messages_key].values()) + + +def _get_producer_trace(record: "dict[str, Any]") -> "Optional[str]": + """The `sentry-trace` the producer put in a record's message attributes.""" + # https://docs.aws.amazon.com/lambda/latest/dg/with-sns.html + if record.get("EventSource") == "aws:sns": + attribute = ( + record["Sns"].get("MessageAttributes", {}).get(SENTRY_TRACE_HEADER_NAME, {}) + ) + return attribute.get("Value") + + # https://docs.aws.amazon.com/lambda/latest/dg/with-sqs.html + attribute = record.get("messageAttributes", {}).get(SENTRY_TRACE_HEADER_NAME) + if attribute is not None: + return attribute.get("stringValue") + + # without raw message delivery, an SNS subscription delivers the notification, + # with its message attributes, as the body. + # https://docs.aws.amazon.com/sns/latest/dg/sns-sqs-as-subscriber.html + if '"TopicArn"' in record["body"]: + notification = json.loads(record["body"]) + attribute = notification.get("MessageAttributes", {}).get( + SENTRY_TRACE_HEADER_NAME, {} + ) + return attribute.get("Value") + + return None + + +def _get_producer_links(records: "list[Any]") -> "list[SpanLink]": + """One link per distinct producer context; messages never parent the invocation.""" + links: "list[SpanLink]" = [] + linked = set() + for record in records: + with capture_internal_exceptions(): + producer = extract_sentrytrace_data(_get_producer_trace(record)) + if producer is None: + continue + + trace_id = str(producer["trace_id"]) + span_id = str(producer["parent_span_id"]) + if (trace_id, span_id) in linked: + continue + linked.add((trace_id, span_id)) + + link: "SpanLink" = {"trace_id": trace_id, "span_id": span_id} + if producer["parent_sampled"] is not None: + link["sampled"] = bool(producer["parent_sampled"]) + links.append(link) + if len(links) == MAX_PRODUCER_LINKS: + break + + return links + + +def _start_process_span(aws_event: "Any") -> "ContextManager[Any]": + """A `queue.process` span around the handler for an SQS or SNS delivery.""" + records = aws_event.get("Records") if isinstance(aws_event, dict) else None + if not records: + return nullcontext() + + event_source = _get_records_source(records) + if event_source not in _MESSAGING_SYSTEMS: + return nullcontext() + + # one queue or topic per event: `arn:aws:sqs:::`. + arn = ( + records[0]["eventSourceARN"] + if event_source == "aws:sqs" + else records[0]["Sns"]["TopicArn"] + ) + destination = arn.split(":")[5] + + return sentry_sdk.start_span( + name=f"process {destination}", + attributes={ + SPANDATA.SENTRY_OP: OP.QUEUE_PROCESS, + SPANDATA.SENTRY_ORIGIN: ORIGIN, + SPANDATA.SENTRY_KIND: "consumer", + SPANDATA.MESSAGING_SYSTEM: _MESSAGING_SYSTEMS[event_source], + SPANDATA.MESSAGING_OPERATION_TYPE: "process", + SPANDATA.MESSAGING_OPERATION_NAME: "process", + SPANDATA.MESSAGING_DESTINATION_NAME: destination, + SPANDATA.MESSAGING_BATCH_MESSAGE_COUNT: len(records), + SPANDATA.FAAS_TRIGGER: "pubsub", + }, + links=_get_producer_links(records), + ) diff --git a/tests/integrations/aws_lambda/test_aws_lambda.py b/tests/integrations/aws_lambda/test_aws_lambda.py index 0487c7a304..6404e41df9 100644 --- a/tests/integrations/aws_lambda/test_aws_lambda.py +++ b/tests/integrations/aws_lambda/test_aws_lambda.py @@ -815,20 +815,41 @@ def test_error(lambda_client, test_environment): assert SPANDATA.MESSAGING_BATCH_MESSAGE_COUNT not in attrs -def _sqs_record(): - return { +def _sqs_record(sentry_trace=None): + record = { "messageId": "059f36b4-87a3-44ab-83d2-661975830a7d", "body": "{}", + "messageAttributes": {}, "eventSource": "aws:sqs", "eventSourceARN": "arn:aws:sqs:us-east-1:012345678912:orders", } + if sentry_trace is not None: + record["messageAttributes"]["sentry-trace"] = { + "stringValue": sentry_trace, + "stringListValues": [], + "binaryListValues": [], + "dataType": "String", + } + return record -def _sns_record(): - return { +def _sns_record(sentry_trace=None): + record = { "EventSource": "aws:sns", - "Sns": {"TopicArn": "arn:aws:sns:us-east-1:012345678912:alerts"}, + "Sns": { + "Type": "Notification", + "MessageId": "95df01b4-ee98-5cb9-9903-4c221d41eb5e", + "TopicArn": "arn:aws:sns:us-east-1:012345678912:alerts", + "Message": "{}", + "MessageAttributes": {}, + }, } + if sentry_trace is not None: + record["Sns"]["MessageAttributes"]["sentry-trace"] = { + "Type": "String", + "Value": sentry_trace, + } + return record def _appsync_event(): @@ -978,6 +999,121 @@ def test_trigger(lambda_client, test_environment, aws_event, trigger, batch_coun ) +@pytest.mark.parametrize( + "aws_event,system,destination,links", + [ + pytest.param( + { + "Records": [ + _sqs_record("471a43a4192642f0b136d5159a501701-6e8f22c393e68f19-1"), + # the same send context: linked once. + _sqs_record("471a43a4192642f0b136d5159a501701-6e8f22c393e68f19-1"), + _sqs_record("0f4c2a5b8d9e4f1a2b3c4d5e6f7a8b9c-1a2b3c4d5e6f7a8b-0"), + _sqs_record(), + ] + }, + "aws_sqs", + "orders", + [ + { + "trace_id": "471a43a4192642f0b136d5159a501701", + "span_id": "6e8f22c393e68f19", + "sampled": True, + }, + { + "trace_id": "0f4c2a5b8d9e4f1a2b3c4d5e6f7a8b9c", + "span_id": "1a2b3c4d5e6f7a8b", + "sampled": False, + }, + ], + id="sqs", + ), + pytest.param( + { + "Records": [ + _sns_record("471a43a4192642f0b136d5159a501701-6e8f22c393e68f19-1") + ] + }, + "aws.sns", + "alerts", + [ + { + "trace_id": "471a43a4192642f0b136d5159a501701", + "span_id": "6e8f22c393e68f19", + "sampled": True, + } + ], + id="sns", + ), + pytest.param( + { + "Records": [ + { + **_sqs_record(), + "body": json.dumps( + _sns_record( + "471a43a4192642f0b136d5159a501701-6e8f22c393e68f19-1" + )["Sns"] + ), + } + ] + }, + "aws_sqs", + "orders", + [ + { + "trace_id": "471a43a4192642f0b136d5159a501701", + "span_id": "6e8f22c393e68f19", + "sampled": True, + } + ], + id="sns-to-sqs", + ), + ], +) +def test_messaging_process_span( + lambda_client, test_environment, aws_event, system, destination, links +): + lambda_client.invoke( + FunctionName="BasicOkSpanStreaming", + Payload=json.dumps(aws_event), + ) + + span_items = test_environment["server"].span_items + (segment_span,) = [s for s in span_items if s["is_segment"]] + (process_span,) = [ + s + for s in span_items + if _get_span_attr(s["attributes"], SPANDATA.SENTRY_OP) == OP.QUEUE_PROCESS + ] + # messages are linked, never the parent of the invocation. + assert "parent_span_id" not in segment_span + assert process_span["parent_span_id"] == segment_span["span_id"] + assert process_span["trace_id"] == segment_span["trace_id"] + assert process_span["name"] == f"process {destination}" + assert process_span["links"] == links + + attrs = process_span["attributes"] + assert _get_span_attr(attrs, SPANDATA.SENTRY_KIND) == "consumer" + assert _get_span_attr(attrs, SPANDATA.MESSAGING_SYSTEM) == system + assert _get_span_attr(attrs, SPANDATA.MESSAGING_OPERATION_TYPE) == "process" + assert _get_span_attr(attrs, SPANDATA.MESSAGING_OPERATION_NAME) == "process" + assert _get_span_attr(attrs, SPANDATA.MESSAGING_DESTINATION_NAME) == destination + assert _get_span_attr(attrs, SPANDATA.MESSAGING_BATCH_MESSAGE_COUNT) == len( + aws_event["Records"] + ) + + +def test_no_process_span_without_messages(lambda_client, test_environment): + lambda_client.invoke( + FunctionName="BasicOkSpanStreaming", + Payload=json.dumps({"Records": [{"eventSource": "aws:kinesis"}]}), + ) + + (span,) = test_environment["server"].span_items + assert span["is_segment"] + + def test_trace_continuation(lambda_client, test_environment): trace_id = "471a43a4192642f0b136d5159a501701" parent_span_id = "6e8f22c393e68f19" From f864c510e50b0f5e43fb6e4a849a6eb6071b548c Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Mon, 9 Nov 2026 07:57:42 +0000 Subject: [PATCH 07/21] ref(aws-lambda): Don't set messaging.batch.message_count on the Lambda span Keep the count on messaging spans only (boto3 receive and batch-send spans, and queue.process for SQS/SNS triggers), as in the OTel messaging conventions and the JS SDK. This drops the batch counting for Kafka, MQ and RabbitMQ deliveries, and matches #8219 so both PRs change these lines the same way. Co-Authored-By: johdax <114897+dingsdax@users.noreply.github.com> --- .../integrations/aws_lambda/_handler.py | 11 +----- .../integrations/aws_lambda/_trigger.py | 34 ---------------- .../aws_lambda/test_aws_lambda.py | 39 ++++--------------- 3 files changed, 8 insertions(+), 76 deletions(-) diff --git a/sentry_sdk/integrations/aws_lambda/_handler.py b/sentry_sdk/integrations/aws_lambda/_handler.py index f22c174cf3..affc83977d 100644 --- a/sentry_sdk/integrations/aws_lambda/_handler.py +++ b/sentry_sdk/integrations/aws_lambda/_handler.py @@ -12,10 +12,7 @@ _get_user_from_event, _make_request_event_processor, ) -from sentry_sdk.integrations.aws_lambda._trigger import ( - _get_messaging_batch_size, - _get_trigger, -) +from sentry_sdk.integrations.aws_lambda._trigger import _get_trigger from sentry_sdk.integrations.aws_lambda.consts import ( CLOUD_PLATFORM, CLOUD_PROVIDER, @@ -168,12 +165,6 @@ def sentry_handler( with capture_internal_exceptions(): attributes[SPANDATA.FAAS_TRIGGER] = _get_trigger(aws_event) - # HTTP and direct invocations aren't batches, even when the payload is a list. - messaging_batch_size = _get_messaging_batch_size(aws_event) - if messaging_batch_size is not None: - attributes[SPANDATA.MESSAGING_BATCH_MESSAGE_COUNT] = ( - messaging_batch_size - ) with capture_internal_exceptions(): invoked_function_arn = aws_context.invoked_function_arn diff --git a/sentry_sdk/integrations/aws_lambda/_trigger.py b/sentry_sdk/integrations/aws_lambda/_trigger.py index a43dbe5ccf..1d3d527a87 100644 --- a/sentry_sdk/integrations/aws_lambda/_trigger.py +++ b/sentry_sdk/integrations/aws_lambda/_trigger.py @@ -19,15 +19,6 @@ "aws:docdb": "datasource", } -# event sources that group their messages by topic-partition or queue. -# https://docs.aws.amazon.com/lambda/latest/dg/with-msk.html -# https://docs.aws.amazon.com/lambda/latest/dg/with-mq.html -_GROUPED_MESSAGES_KEYS: "dict[Optional[str], str]" = { - "aws:kafka": "records", - "SelfManagedKafka": "records", - "aws:rmq": "rmqMessagesByQueue", -} - # https://docs.aws.amazon.com/eventbridge/latest/ref/overiew-event-structure.html _EVENTBRIDGE_KEYS = frozenset( ( @@ -96,28 +87,3 @@ def _get_trigger(aws_event: "Any") -> str: return "pubsub" return _TRIGGERS.get(aws_event.get("eventSource"), "other") - - -def _get_messaging_batch_size(aws_event: "Any") -> "Optional[int]": - """ - The number of messages delivered by a messaging event source, or `None` for any - other event (HTTP, direct invocation, datasource, ...). - """ - if not isinstance(aws_event, dict): - return None - - records = aws_event.get("Records") - if records: - if _get_records_source(records) in ("aws:sqs", "aws:sns"): - return len(records) - return None - - event_source = aws_event.get("eventSource") - if event_source == "aws:mq": - return len(aws_event["messages"]) - - grouped_messages_key = _GROUPED_MESSAGES_KEYS.get(event_source) - if grouped_messages_key is None: - return None - - return sum(len(messages) for messages in aws_event[grouped_messages_key].values()) diff --git a/tests/integrations/aws_lambda/test_aws_lambda.py b/tests/integrations/aws_lambda/test_aws_lambda.py index 0487c7a304..d30cd7d403 100644 --- a/tests/integrations/aws_lambda/test_aws_lambda.py +++ b/tests/integrations/aws_lambda/test_aws_lambda.py @@ -853,12 +853,11 @@ def _eventbridge_event(source, detail_type): @pytest.mark.parametrize( - "aws_event,trigger,batch_count", + "aws_event,trigger", [ pytest.param( {"httpMethod": "GET", "path": "/", "requestContext": {"stage": "prod"}}, "http", - None, id="api-gateway-rest", ), pytest.param( @@ -868,27 +867,20 @@ def _eventbridge_event(source, detail_type): "requestContext": {"http": {"method": "GET"}}, }, "http", - None, id="api-gateway-http", ), pytest.param( {"Records": [{"cf": {"request": {"uri": "/"}}}]}, "http", - None, id="cloudfront", ), - pytest.param(_appsync_event(), "http", None, id="appsync"), - pytest.param( - [_appsync_event(), _appsync_event()], "http", None, id="appsync-batch" - ), - pytest.param( - {"Records": [_sqs_record(), _sqs_record()]}, "pubsub", 2, id="sqs" - ), - pytest.param({"Records": [_sns_record()]}, "pubsub", 1, id="sns"), + pytest.param(_appsync_event(), "http", id="appsync"), + pytest.param([_appsync_event(), _appsync_event()], "http", id="appsync-batch"), + pytest.param({"Records": [_sqs_record(), _sqs_record()]}, "pubsub", id="sqs"), + pytest.param({"Records": [_sns_record()]}, "pubsub", id="sns"), pytest.param( {"Records": [{"eventSource": "aws:kinesis"}]}, "pubsub", - None, id="kinesis", ), pytest.param( @@ -900,19 +892,16 @@ def _eventbridge_event(source, detail_type): }, }, "pubsub", - 3, id="msk", ), pytest.param( {"eventSource": "SelfManagedKafka", "records": {"orders-0": [{}]}}, "pubsub", - 1, id="self-managed-kafka", ), pytest.param( {"eventSource": "aws:mq", "messages": [{}, {}]}, "pubsub", - 2, id="activemq", ), pytest.param( @@ -921,45 +910,37 @@ def _eventbridge_event(source, detail_type): "rmqMessagesByQueue": {"orders::/": [{}, {}], "alerts::/": [{}]}, }, "pubsub", - 3, id="rabbitmq", ), pytest.param( _eventbridge_event("aws.s3", "Object Created"), "pubsub", - None, id="eventbridge", ), pytest.param( _eventbridge_event("aws.events", "Scheduled Event"), "timer", - None, id="eventbridge-schedule", ), - pytest.param( - {"Records": [{"eventSource": "aws:s3"}]}, "datasource", None, id="s3" - ), + pytest.param({"Records": [{"eventSource": "aws:s3"}]}, "datasource", id="s3"), pytest.param( {"Records": [{"eventSource": "aws:dynamodb"}]}, "datasource", - None, id="dynamodb-streams", ), pytest.param( {"eventSource": "aws:docdb", "events": [{}]}, "datasource", - None, id="documentdb", ), pytest.param( [{"good dog": "Maisey"}, {"good dog": "Charlie"}], "other", - None, id="list", ), ], ) -def test_trigger(lambda_client, test_environment, aws_event, trigger, batch_count): +def test_trigger(lambda_client, test_environment, aws_event, trigger): lambda_client.invoke( FunctionName="BasicOkSpanStreaming", Payload=json.dumps(aws_event), @@ -970,12 +951,6 @@ def test_trigger(lambda_client, test_environment, aws_event, trigger, batch_coun ] attrs = segment_span["attributes"] assert _get_span_attr(attrs, SPANDATA.FAAS_TRIGGER) == trigger - if batch_count is None: - assert SPANDATA.MESSAGING_BATCH_MESSAGE_COUNT not in attrs - else: - assert ( - _get_span_attr(attrs, SPANDATA.MESSAGING_BATCH_MESSAGE_COUNT) == batch_count - ) def test_trace_continuation(lambda_client, test_environment): From 5a479b9624163f12c7438393be0e830a936065a6 Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Mon, 9 Nov 2026 07:59:42 +0000 Subject: [PATCH 08/21] test(aws-lambda): Cover queue.process when the handler raises The error is captured, both the process span and the invocation span are marked as errors, and the exception still reaches the Lambda runtime. Co-Authored-By: johdax <114897+dingsdax@users.noreply.github.com> --- .../aws_lambda/test_aws_lambda.py | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/tests/integrations/aws_lambda/test_aws_lambda.py b/tests/integrations/aws_lambda/test_aws_lambda.py index 49df8435cd..97ec409a09 100644 --- a/tests/integrations/aws_lambda/test_aws_lambda.py +++ b/tests/integrations/aws_lambda/test_aws_lambda.py @@ -1089,6 +1089,32 @@ def test_no_process_span_without_messages(lambda_client, test_environment): assert span["is_segment"] +def test_process_span_error(lambda_client, test_environment): + response = lambda_client.invoke( + FunctionName="RaiseErrorSpanStreaming", + Payload=json.dumps({"Records": [_sqs_record()]}), + ) + + # the handler's exception still reaches the Lambda runtime. + assert json.loads(response["Payload"].read())["errorMessage"] == "Oh!" + + (error_event,) = test_environment["server"].envelopes + (exception,) = error_event["exception"]["values"] + assert exception["value"] == "Oh!" + assert not exception["mechanism"]["handled"] + + span_items = test_environment["server"].span_items + (segment_span,) = [s for s in span_items if s["is_segment"]] + (process_span,) = [ + s + for s in span_items + if _get_span_attr(s["attributes"], SPANDATA.SENTRY_OP) == OP.QUEUE_PROCESS + ] + assert process_span["parent_span_id"] == segment_span["span_id"] + assert process_span["status"] == "error" + assert segment_span["status"] == "error" + + def test_trace_continuation(lambda_client, test_environment): trace_id = "471a43a4192642f0b136d5159a501701" parent_span_id = "6e8f22c393e68f19" From 120a4417d61fe00544f150a01418209c220cdcb9 Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Mon, 9 Nov 2026 12:17:47 +0000 Subject: [PATCH 09/21] fix(aws-lambda): Don't assume the context has client_context MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Read client_context and its custom dict with getattr, so a context object without them falls back to the event headers. Co-Authored-By: Johannes Daxböck <114897+dingsdax@users.noreply.github.com> --- sentry_sdk/integrations/aws_lambda/_handler.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/sentry_sdk/integrations/aws_lambda/_handler.py b/sentry_sdk/integrations/aws_lambda/_handler.py index 98209b9104..3d006ad19e 100644 --- a/sentry_sdk/integrations/aws_lambda/_handler.py +++ b/sentry_sdk/integrations/aws_lambda/_handler.py @@ -190,13 +190,10 @@ def sentry_handler( # a synchronous `Invoke` passes the invoking span in `ClientContext`; it wins over # event headers, which a forwarded HTTP event keeps from further upstream. # https://docs.aws.amazon.com/lambda/latest/dg/python-context.html - client_context = aws_context.client_context - if ( - client_context is not None - and client_context.custom - and SENTRY_TRACE_HEADER_NAME in client_context.custom - ): - sentry_sdk.continue_trace(client_context.custom) + client_context = getattr(aws_context, "client_context", None) + custom = getattr(client_context, "custom", None) + if custom and SENTRY_TRACE_HEADER_NAME in custom: + sentry_sdk.continue_trace(custom) else: sentry_sdk.continue_trace(headers) Scope.set_custom_sampling_context(sampling_context) From d877f00a65923085321672ad9aeab6dc0086e3fa Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Mon, 9 Nov 2026 12:18:07 +0000 Subject: [PATCH 10/21] test(aws-lambda): Keep one test_trigger case per code path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Each case is a SAM invoke. The removed cases only checked other keys of the same lookup table. Co-Authored-By: Johannes Daxböck <114897+dingsdax@users.noreply.github.com> --- .../aws_lambda/test_aws_lambda.py | 41 +------------------ 1 file changed, 1 insertion(+), 40 deletions(-) diff --git a/tests/integrations/aws_lambda/test_aws_lambda.py b/tests/integrations/aws_lambda/test_aws_lambda.py index d30cd7d403..ccee9d0cb4 100644 --- a/tests/integrations/aws_lambda/test_aws_lambda.py +++ b/tests/integrations/aws_lambda/test_aws_lambda.py @@ -879,39 +879,10 @@ def _eventbridge_event(source, detail_type): pytest.param({"Records": [_sqs_record(), _sqs_record()]}, "pubsub", id="sqs"), pytest.param({"Records": [_sns_record()]}, "pubsub", id="sns"), pytest.param( - {"Records": [{"eventSource": "aws:kinesis"}]}, - "pubsub", - id="kinesis", - ), - pytest.param( - { - "eventSource": "aws:kafka", - "records": { - "orders-0": [{"offset": 1}, {"offset": 2}], - "orders-1": [{"offset": 1}], - }, - }, + {"eventSource": "aws:kafka", "records": {"orders-0": [{}]}}, "pubsub", id="msk", ), - pytest.param( - {"eventSource": "SelfManagedKafka", "records": {"orders-0": [{}]}}, - "pubsub", - id="self-managed-kafka", - ), - pytest.param( - {"eventSource": "aws:mq", "messages": [{}, {}]}, - "pubsub", - id="activemq", - ), - pytest.param( - { - "eventSource": "aws:rmq", - "rmqMessagesByQueue": {"orders::/": [{}, {}], "alerts::/": [{}]}, - }, - "pubsub", - id="rabbitmq", - ), pytest.param( _eventbridge_event("aws.s3", "Object Created"), "pubsub", @@ -923,16 +894,6 @@ def _eventbridge_event(source, detail_type): id="eventbridge-schedule", ), pytest.param({"Records": [{"eventSource": "aws:s3"}]}, "datasource", id="s3"), - pytest.param( - {"Records": [{"eventSource": "aws:dynamodb"}]}, - "datasource", - id="dynamodb-streams", - ), - pytest.param( - {"eventSource": "aws:docdb", "events": [{}]}, - "datasource", - id="documentdb", - ), pytest.param( [{"good dog": "Maisey"}, {"good dog": "Charlie"}], "other", From feaef1c6429c83309228b22a618c02eb5d3cff65 Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Mon, 9 Nov 2026 12:19:13 +0000 Subject: [PATCH 11/21] ref(aws-lambda): Keep faas.trigger on the invocation span only MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit OTel sets faas.trigger on the FaaS span, which already has it. Also make the producer link cap private, like the other module constants. Co-Authored-By: Johannes Daxböck <114897+dingsdax@users.noreply.github.com> --- sentry_sdk/integrations/aws_lambda/_trigger.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/sentry_sdk/integrations/aws_lambda/_trigger.py b/sentry_sdk/integrations/aws_lambda/_trigger.py index eea0f525fd..bbb81585f4 100644 --- a/sentry_sdk/integrations/aws_lambda/_trigger.py +++ b/sentry_sdk/integrations/aws_lambda/_trigger.py @@ -35,7 +35,7 @@ # OTel's default limit; Lambda delivers up to 10,000 SQS messages at once. # https://opentelemetry.io/docs/specs/otel/trace/sdk/#span-limits -MAX_PRODUCER_LINKS = 128 +_MAX_PRODUCER_LINKS = 128 # https://docs.aws.amazon.com/eventbridge/latest/ref/overiew-event-structure.html _EVENTBRIDGE_KEYS = frozenset( @@ -154,7 +154,7 @@ def _get_producer_links(records: "list[Any]") -> "list[SpanLink]": if producer["parent_sampled"] is not None: link["sampled"] = bool(producer["parent_sampled"]) links.append(link) - if len(links) == MAX_PRODUCER_LINKS: + if len(links) == _MAX_PRODUCER_LINKS: break return links @@ -189,7 +189,6 @@ def _start_process_span(aws_event: "Any") -> "ContextManager[Any]": SPANDATA.MESSAGING_OPERATION_NAME: "process", SPANDATA.MESSAGING_DESTINATION_NAME: destination, SPANDATA.MESSAGING_BATCH_MESSAGE_COUNT: len(records), - SPANDATA.FAAS_TRIGGER: "pubsub", }, links=_get_producer_links(records), ) From 02918244e520acfbe8f74207d719648f3caf8c76 Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Wed, 11 Nov 2026 14:45:11 +0000 Subject: [PATCH 12/21] feat(aws-lambda): Mark producer links as message_origin Matches the boto3 receive links from #8145. --- sentry_sdk/integrations/aws_lambda/_trigger.py | 6 +++++- tests/integrations/aws_lambda/test_aws_lambda.py | 7 +++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/sentry_sdk/integrations/aws_lambda/_trigger.py b/sentry_sdk/integrations/aws_lambda/_trigger.py index bbb81585f4..8d50140d9e 100644 --- a/sentry_sdk/integrations/aws_lambda/_trigger.py +++ b/sentry_sdk/integrations/aws_lambda/_trigger.py @@ -150,7 +150,11 @@ def _get_producer_links(records: "list[Any]") -> "list[SpanLink]": continue linked.add((trace_id, span_id)) - link: "SpanLink" = {"trace_id": trace_id, "span_id": span_id} + link: "SpanLink" = { + "trace_id": trace_id, + "span_id": span_id, + "attributes": {SPANDATA.SENTRY_LINK_TYPE: "message_origin"}, + } if producer["parent_sampled"] is not None: link["sampled"] = bool(producer["parent_sampled"]) links.append(link) diff --git a/tests/integrations/aws_lambda/test_aws_lambda.py b/tests/integrations/aws_lambda/test_aws_lambda.py index baf0c2fd57..faa17c580b 100644 --- a/tests/integrations/aws_lambda/test_aws_lambda.py +++ b/tests/integrations/aws_lambda/test_aws_lambda.py @@ -24,6 +24,9 @@ DOCKER_NETWORK_NAME = "lambda-test-network" SAM_TEMPLATE_FILE = "sam.template.yaml" SAM_SHUTDOWN_TIMEOUT = 10 +MESSAGE_ORIGIN_LINK = { + SPANDATA.SENTRY_LINK_TYPE: {"value": "message_origin", "type": "string"} +} @pytest.fixture(scope="session", autouse=True) @@ -969,11 +972,13 @@ def test_trigger(lambda_client, test_environment, aws_event, trigger): { "trace_id": "471a43a4192642f0b136d5159a501701", "span_id": "6e8f22c393e68f19", + "attributes": MESSAGE_ORIGIN_LINK, "sampled": True, }, { "trace_id": "0f4c2a5b8d9e4f1a2b3c4d5e6f7a8b9c", "span_id": "1a2b3c4d5e6f7a8b", + "attributes": MESSAGE_ORIGIN_LINK, "sampled": False, }, ], @@ -991,6 +996,7 @@ def test_trigger(lambda_client, test_environment, aws_event, trigger): { "trace_id": "471a43a4192642f0b136d5159a501701", "span_id": "6e8f22c393e68f19", + "attributes": MESSAGE_ORIGIN_LINK, "sampled": True, } ], @@ -1015,6 +1021,7 @@ def test_trigger(lambda_client, test_environment, aws_event, trigger): { "trace_id": "471a43a4192642f0b136d5159a501701", "span_id": "6e8f22c393e68f19", + "attributes": MESSAGE_ORIGIN_LINK, "sampled": True, } ], From 6bd4fd0f33018a739b7e4894e91a946e9aff7df0 Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Wed, 11 Nov 2026 17:59:05 +0000 Subject: [PATCH 13/21] fix(aws-lambda): Skip producer links without trace or span id A sentry-trace like "-1" parses with no trace_id or parent_span_id, which became a link to "None". Skip it, as boto3 receive links do. --- sentry_sdk/integrations/aws_lambda/_trigger.py | 6 +++++- tests/integrations/aws_lambda/test_aws_lambda.py | 2 ++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/sentry_sdk/integrations/aws_lambda/_trigger.py b/sentry_sdk/integrations/aws_lambda/_trigger.py index 8d50140d9e..9a007fca73 100644 --- a/sentry_sdk/integrations/aws_lambda/_trigger.py +++ b/sentry_sdk/integrations/aws_lambda/_trigger.py @@ -141,7 +141,11 @@ def _get_producer_links(records: "list[Any]") -> "list[SpanLink]": for record in records: with capture_internal_exceptions(): producer = extract_sentrytrace_data(_get_producer_trace(record)) - if producer is None: + if ( + producer is None + or not producer["trace_id"] + or not producer["parent_span_id"] + ): continue trace_id = str(producer["trace_id"]) diff --git a/tests/integrations/aws_lambda/test_aws_lambda.py b/tests/integrations/aws_lambda/test_aws_lambda.py index faa17c580b..c6642639f1 100644 --- a/tests/integrations/aws_lambda/test_aws_lambda.py +++ b/tests/integrations/aws_lambda/test_aws_lambda.py @@ -964,6 +964,8 @@ def test_trigger(lambda_client, test_environment, aws_event, trigger): _sqs_record("471a43a4192642f0b136d5159a501701-6e8f22c393e68f19-1"), _sqs_record("0f4c2a5b8d9e4f1a2b3c4d5e6f7a8b9c-1a2b3c4d5e6f7a8b-0"), _sqs_record(), + # incomplete context: not linked. + _sqs_record("-1"), ] }, "aws_sqs", From 8fad3d878a87ba5784cc9e2a672a958cb55b02ad Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Thu, 12 Nov 2026 11:33:22 +0000 Subject: [PATCH 14/21] fix(aws-lambda): Detect ActiveMQ triggers sent as aws:amq --- sentry_sdk/integrations/aws_lambda/_trigger.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sentry_sdk/integrations/aws_lambda/_trigger.py b/sentry_sdk/integrations/aws_lambda/_trigger.py index 1d3d527a87..10b6063e51 100644 --- a/sentry_sdk/integrations/aws_lambda/_trigger.py +++ b/sentry_sdk/integrations/aws_lambda/_trigger.py @@ -12,7 +12,9 @@ "aws:ses": "pubsub", "aws:kafka": "pubsub", "SelfManagedKafka": "pubsub", + # ActiveMQ: the Lambda docs show `aws:mq`, real deliveries use `aws:amq`. "aws:mq": "pubsub", + "aws:amq": "pubsub", "aws:rmq": "pubsub", "aws:s3": "datasource", "aws:dynamodb": "datasource", From 2a29e8ef7191049ddf3042f518feae189f7576a8 Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Thu, 12 Nov 2026 13:16:57 +0000 Subject: [PATCH 15/21] ref(aws-lambda): Leave messaging.batch.message_count to #8229 Restores the batch count on the invocation span, as on major/3.0, so this PR only adds faas.trigger. --- sentry_sdk/integrations/aws_lambda/_handler.py | 3 +++ tests/integrations/aws_lambda/test_aws_lambda.py | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/sentry_sdk/integrations/aws_lambda/_handler.py b/sentry_sdk/integrations/aws_lambda/_handler.py index 9485f10169..77d405cfc1 100644 --- a/sentry_sdk/integrations/aws_lambda/_handler.py +++ b/sentry_sdk/integrations/aws_lambda/_handler.py @@ -65,8 +65,10 @@ def sentry_handler( if isinstance(aws_event, list) and len(aws_event) >= 1: request_data = aws_event[0] + batch_size = len(aws_event) else: request_data = aws_event + batch_size = 1 if not isinstance(request_data, dict): # If we're not dealing with a dictionary, we won't be able to get @@ -159,6 +161,7 @@ def sentry_handler( SPANDATA.SENTRY_SEGMENT_NAME_SOURCE: SegmentNameSource.COMPONENT, SPANDATA.CLOUD_PROVIDER: CLOUD_PROVIDER, SPANDATA.CLOUD_PLATFORM: CLOUD_PLATFORM, + SPANDATA.MESSAGING_BATCH_MESSAGE_COUNT: batch_size, **header_attributes, **additional_attributes, } diff --git a/tests/integrations/aws_lambda/test_aws_lambda.py b/tests/integrations/aws_lambda/test_aws_lambda.py index 7c09d034e2..2df33e1828 100644 --- a/tests/integrations/aws_lambda/test_aws_lambda.py +++ b/tests/integrations/aws_lambda/test_aws_lambda.py @@ -772,7 +772,7 @@ def test_no_error(lambda_client, test_environment): LATEST_FUNCTION_VERSION ] assert _get_span_attr(attrs, SPANDATA.FAAS_TRIGGER) == "other" - assert SPANDATA.MESSAGING_BATCH_MESSAGE_COUNT not in attrs + assert _get_span_attr(attrs, SPANDATA.MESSAGING_BATCH_MESSAGE_COUNT) == 1 def test_error(lambda_client, test_environment): @@ -827,7 +827,7 @@ def test_error(lambda_client, test_environment): LATEST_FUNCTION_VERSION ] assert _get_span_attr(attrs, SPANDATA.FAAS_TRIGGER) == "other" - assert SPANDATA.MESSAGING_BATCH_MESSAGE_COUNT not in attrs + assert _get_span_attr(attrs, SPANDATA.MESSAGING_BATCH_MESSAGE_COUNT) == 1 def _sqs_record(): From 6592fcd3d34e65d55a9697630ec5794939e647ee Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Thu, 12 Nov 2026 13:36:32 +0000 Subject: [PATCH 16/21] ref(aws-lambda): Inline records source lookup, drop _TRIGGERS annotation --- sentry_sdk/integrations/aws_lambda/_trigger.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/sentry_sdk/integrations/aws_lambda/_trigger.py b/sentry_sdk/integrations/aws_lambda/_trigger.py index 10b6063e51..bce42e624c 100644 --- a/sentry_sdk/integrations/aws_lambda/_trigger.py +++ b/sentry_sdk/integrations/aws_lambda/_trigger.py @@ -1,11 +1,11 @@ from typing import TYPE_CHECKING if TYPE_CHECKING: - from typing import Any, Optional + from typing import Any # `faas.trigger` of each `eventSource` of `Records` or of a top-level `eventSource`. # https://docs.aws.amazon.com/lambda/latest/dg/lambda-services.html -_TRIGGERS: "dict[Optional[str], str]" = { +_TRIGGERS = { "aws:sqs": "pubsub", "aws:sns": "pubsub", "aws:kinesis": "pubsub", @@ -37,11 +37,6 @@ ) -def _get_records_source(records: "list[Any]") -> "Optional[str]": - """The records' `eventSource`, one per event; SNS spells it `EventSource`.""" - return records[0].get("eventSource") or records[0].get("EventSource") - - def _is_appsync_event(aws_event: "Any") -> bool: # https://docs.aws.amazon.com/appsync/latest/devguide/resolver-context-reference-js.html return ( @@ -78,7 +73,9 @@ def _get_trigger(aws_event: "Any") -> str: # CloudFront and Lambda@Edge records have no `eventSource`. if "cf" in records[0]: return "http" - return _TRIGGERS.get(_get_records_source(records), "other") + # SNS spells it `EventSource`. + source = records[0].get("eventSource") or records[0].get("EventSource") + return _TRIGGERS.get(source, "other") if _EVENTBRIDGE_KEYS <= aws_event.keys(): if ( @@ -88,4 +85,4 @@ def _get_trigger(aws_event: "Any") -> str: return "timer" return "pubsub" - return _TRIGGERS.get(aws_event.get("eventSource"), "other") + return _TRIGGERS.get(aws_event.get("eventSource", ""), "other") From 4aad8372ecc49e73d846e63a5eb1ce941c400dc5 Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Thu, 12 Nov 2026 13:45:48 +0000 Subject: [PATCH 17/21] fix(aws-lambda): Only parse SNS envelopes in SQS bodies that are JSON notifications --- .../integrations/aws_lambda/_trigger.py | 21 ++++++++++++------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/sentry_sdk/integrations/aws_lambda/_trigger.py b/sentry_sdk/integrations/aws_lambda/_trigger.py index 4ac812d31a..58e28ed1bf 100644 --- a/sentry_sdk/integrations/aws_lambda/_trigger.py +++ b/sentry_sdk/integrations/aws_lambda/_trigger.py @@ -123,14 +123,19 @@ def _get_producer_trace(record: "dict[str, Any]") -> "Optional[str]": # without raw message delivery, an SNS subscription delivers the notification, # with its message attributes, as the body. # https://docs.aws.amazon.com/sns/latest/dg/sns-sqs-as-subscriber.html - if '"TopicArn"' in record["body"]: - notification = json.loads(record["body"]) - attribute = notification.get("MessageAttributes", {}).get( - SENTRY_TRACE_HEADER_NAME, {} - ) - return attribute.get("Value") - - return None + body = record.get("body", "") + if '"TopicArn"' not in body: + return None + try: + notification = json.loads(body) + except ValueError: + return None + if not isinstance(notification, dict) or notification.get("Type") != "Notification": + return None + attribute = notification.get("MessageAttributes", {}).get( + SENTRY_TRACE_HEADER_NAME, {} + ) + return attribute.get("Value") def _get_producer_links(records: "list[Any]") -> "list[SpanLink]": From 61c3b58b4bf448a7d59441a8a71a80304cc72a01 Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Thu, 12 Nov 2026 13:46:51 +0000 Subject: [PATCH 18/21] test(aws-lambda): Use constants for trigger test events --- .../aws_lambda/test_aws_lambda.py | 79 +++++++++---------- 1 file changed, 36 insertions(+), 43 deletions(-) diff --git a/tests/integrations/aws_lambda/test_aws_lambda.py b/tests/integrations/aws_lambda/test_aws_lambda.py index 30ffacf2e1..5b63696bc4 100644 --- a/tests/integrations/aws_lambda/test_aws_lambda.py +++ b/tests/integrations/aws_lambda/test_aws_lambda.py @@ -25,6 +25,32 @@ SAM_TEMPLATE_FILE = "sam.template.yaml" SAM_SHUTDOWN_TIMEOUT = 10 +SQS_RECORD = { + "messageId": "059f36b4-87a3-44ab-83d2-661975830a7d", + "body": "{}", + "eventSource": "aws:sqs", + "eventSourceARN": "arn:aws:sqs:us-east-1:012345678912:orders", +} +SNS_RECORD = { + "EventSource": "aws:sns", + "Sns": {"TopicArn": "arn:aws:sns:us-east-1:012345678912:alerts"}, +} +APPSYNC_EVENT = { + "arguments": {"id": "1"}, + "info": {"fieldName": "getOrder", "parentTypeName": "Query"}, +} +EVENTBRIDGE_EVENT = { + "version": "0", + "id": "53dc4d37-cffa-4f76-80c9-8b7d4a4d2eaa", + "detail-type": "Object Created", + "source": "aws.s3", + "account": "012345678912", + "time": "2026-10-26T12:00:00Z", + "region": "us-east-1", + "resources": [], + "detail": {}, +} + @pytest.fixture(scope="session", autouse=True) def test_environment(): @@ -824,43 +850,6 @@ def test_error(lambda_client, test_environment): assert _get_span_attr(attrs, SPANDATA.FAAS_TRIGGER) == "other" -def _sqs_record(): - return { - "messageId": "059f36b4-87a3-44ab-83d2-661975830a7d", - "body": "{}", - "eventSource": "aws:sqs", - "eventSourceARN": "arn:aws:sqs:us-east-1:012345678912:orders", - } - - -def _sns_record(): - return { - "EventSource": "aws:sns", - "Sns": {"TopicArn": "arn:aws:sns:us-east-1:012345678912:alerts"}, - } - - -def _appsync_event(): - return { - "arguments": {"id": "1"}, - "info": {"fieldName": "getOrder", "parentTypeName": "Query"}, - } - - -def _eventbridge_event(source, detail_type): - return { - "version": "0", - "id": "53dc4d37-cffa-4f76-80c9-8b7d4a4d2eaa", - "detail-type": detail_type, - "source": source, - "account": "012345678912", - "time": "2026-10-26T12:00:00Z", - "region": "us-east-1", - "resources": [], - "detail": {}, - } - - @pytest.mark.parametrize( "aws_event,trigger", [ @@ -883,22 +872,26 @@ def _eventbridge_event(source, detail_type): "http", id="cloudfront", ), - pytest.param(_appsync_event(), "http", id="appsync"), - pytest.param([_appsync_event(), _appsync_event()], "http", id="appsync-batch"), - pytest.param({"Records": [_sqs_record(), _sqs_record()]}, "pubsub", id="sqs"), - pytest.param({"Records": [_sns_record()]}, "pubsub", id="sns"), + pytest.param(APPSYNC_EVENT, "http", id="appsync"), + pytest.param([APPSYNC_EVENT, APPSYNC_EVENT], "http", id="appsync-batch"), + pytest.param({"Records": [SQS_RECORD, SQS_RECORD]}, "pubsub", id="sqs"), + pytest.param({"Records": [SNS_RECORD]}, "pubsub", id="sns"), pytest.param( {"eventSource": "aws:kafka", "records": {"orders-0": [{}]}}, "pubsub", id="msk", ), pytest.param( - _eventbridge_event("aws.s3", "Object Created"), + EVENTBRIDGE_EVENT, "pubsub", id="eventbridge", ), pytest.param( - _eventbridge_event("aws.events", "Scheduled Event"), + { + **EVENTBRIDGE_EVENT, + "source": "aws.events", + "detail-type": "Scheduled Event", + }, "timer", id="eventbridge-schedule", ), From ab2381a500faee8dd0b9dfbfb736d6298de7bb89 Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Thu, 12 Nov 2026 13:56:17 +0000 Subject: [PATCH 19/21] ref(aws-lambda): Drop the producer links that #8257 already sets #8257 links SQS/SNS invocations to their producers on the invocation span. Remove this PR's second copy of that code from _trigger.py, keep the queue.process span without links, and reuse the test helpers and records already on major/3.0. Co-Authored-By: Ivana Kellyer <131587164+sentrivana@users.noreply.github.com> --- .../integrations/aws_lambda/_trigger.py | 81 +----------- .../aws_lambda/test_aws_lambda.py | 124 +----------------- 2 files changed, 9 insertions(+), 196 deletions(-) diff --git a/sentry_sdk/integrations/aws_lambda/_trigger.py b/sentry_sdk/integrations/aws_lambda/_trigger.py index 58e28ed1bf..a66acebf43 100644 --- a/sentry_sdk/integrations/aws_lambda/_trigger.py +++ b/sentry_sdk/integrations/aws_lambda/_trigger.py @@ -1,18 +1,12 @@ -import json from contextlib import nullcontext from typing import TYPE_CHECKING import sentry_sdk from sentry_sdk.consts import OP, SPANDATA from sentry_sdk.integrations.aws_lambda.consts import ORIGIN -from sentry_sdk.traces import SENTRY_TRACE_HEADER_NAME -from sentry_sdk.tracing_utils import extract_sentrytrace_data -from sentry_sdk.utils import capture_internal_exceptions if TYPE_CHECKING: - from typing import Any, ContextManager, Optional - - from sentry_sdk._types import SpanLink + from typing import Any, ContextManager # `faas.trigger` of each `eventSource` of `Records` or of a top-level `eventSource`. # https://docs.aws.amazon.com/lambda/latest/dg/lambda-services.html @@ -32,13 +26,9 @@ "aws:docdb": "datasource", } -# `messaging.system` of the event sources whose producers can propagate a trace. +# `messaging.system` of the messaging event sources that get a `queue.process` span. _MESSAGING_SYSTEMS = {"aws:sqs": "aws_sqs", "aws:sns": "aws.sns"} -# OTel's default limit; Lambda delivers up to 10,000 SQS messages at once. -# https://opentelemetry.io/docs/specs/otel/trace/sdk/#span-limits -_MAX_PRODUCER_LINKS = 128 - # https://docs.aws.amazon.com/eventbridge/latest/ref/overiew-event-structure.html _EVENTBRIDGE_KEYS = frozenset( ( @@ -106,72 +96,6 @@ def _get_trigger(aws_event: "Any") -> str: return _TRIGGERS.get(aws_event.get("eventSource", ""), "other") -def _get_producer_trace(record: "dict[str, Any]") -> "Optional[str]": - """The `sentry-trace` the producer put in a record's message attributes.""" - # https://docs.aws.amazon.com/lambda/latest/dg/with-sns.html - if record.get("EventSource") == "aws:sns": - attribute = ( - record["Sns"].get("MessageAttributes", {}).get(SENTRY_TRACE_HEADER_NAME, {}) - ) - return attribute.get("Value") - - # https://docs.aws.amazon.com/lambda/latest/dg/with-sqs.html - attribute = record.get("messageAttributes", {}).get(SENTRY_TRACE_HEADER_NAME) - if attribute is not None: - return attribute.get("stringValue") - - # without raw message delivery, an SNS subscription delivers the notification, - # with its message attributes, as the body. - # https://docs.aws.amazon.com/sns/latest/dg/sns-sqs-as-subscriber.html - body = record.get("body", "") - if '"TopicArn"' not in body: - return None - try: - notification = json.loads(body) - except ValueError: - return None - if not isinstance(notification, dict) or notification.get("Type") != "Notification": - return None - attribute = notification.get("MessageAttributes", {}).get( - SENTRY_TRACE_HEADER_NAME, {} - ) - return attribute.get("Value") - - -def _get_producer_links(records: "list[Any]") -> "list[SpanLink]": - """One link per distinct producer context; messages never parent the invocation.""" - links: "list[SpanLink]" = [] - linked = set() - for record in records: - with capture_internal_exceptions(): - producer = extract_sentrytrace_data(_get_producer_trace(record)) - if ( - producer is None - or not producer["trace_id"] - or not producer["parent_span_id"] - ): - continue - - trace_id = str(producer["trace_id"]) - span_id = str(producer["parent_span_id"]) - if (trace_id, span_id) in linked: - continue - linked.add((trace_id, span_id)) - - link: "SpanLink" = { - "trace_id": trace_id, - "span_id": span_id, - "attributes": {SPANDATA.SENTRY_LINK_TYPE: "message_origin"}, - } - if producer["parent_sampled"] is not None: - link["sampled"] = bool(producer["parent_sampled"]) - links.append(link) - if len(links) == _MAX_PRODUCER_LINKS: - break - - return links - - def _start_process_span(aws_event: "Any") -> "ContextManager[Any]": """A `queue.process` span around the handler for an SQS or SNS delivery.""" records = aws_event.get("Records") if isinstance(aws_event, dict) else None @@ -203,5 +127,4 @@ def _start_process_span(aws_event: "Any") -> "ContextManager[Any]": SPANDATA.MESSAGING_DESTINATION_NAME: destination, SPANDATA.MESSAGING_BATCH_MESSAGE_COUNT: len(records), }, - links=_get_producer_links(records), ) diff --git a/tests/integrations/aws_lambda/test_aws_lambda.py b/tests/integrations/aws_lambda/test_aws_lambda.py index 40f92f5c58..d2da44e25d 100644 --- a/tests/integrations/aws_lambda/test_aws_lambda.py +++ b/tests/integrations/aws_lambda/test_aws_lambda.py @@ -24,9 +24,6 @@ DOCKER_NETWORK_NAME = "lambda-test-network" SAM_TEMPLATE_FILE = "sam.template.yaml" SAM_SHUTDOWN_TIMEOUT = 10 -MESSAGE_ORIGIN_LINK = { - SPANDATA.SENTRY_LINK_TYPE: {"value": "message_origin", "type": "string"} -} SQS_RECORD = { "messageId": "059f36b4-87a3-44ab-83d2-661975830a7d", @@ -853,43 +850,6 @@ def test_error(lambda_client, test_environment): assert _get_span_attr(attrs, SPANDATA.FAAS_TRIGGER) == "other" -def _sqs_record(sentry_trace=None): - record = { - "messageId": "059f36b4-87a3-44ab-83d2-661975830a7d", - "body": "{}", - "messageAttributes": {}, - "eventSource": "aws:sqs", - "eventSourceARN": "arn:aws:sqs:us-east-1:012345678912:orders", - } - if sentry_trace is not None: - record["messageAttributes"]["sentry-trace"] = { - "stringValue": sentry_trace, - "stringListValues": [], - "binaryListValues": [], - "dataType": "String", - } - return record - - -def _sns_record(sentry_trace=None): - record = { - "EventSource": "aws:sns", - "Sns": { - "Type": "Notification", - "MessageId": "95df01b4-ee98-5cb9-9903-4c221d41eb5e", - "TopicArn": "arn:aws:sns:us-east-1:012345678912:alerts", - "Message": "{}", - "MessageAttributes": {}, - }, - } - if sentry_trace is not None: - record["Sns"]["MessageAttributes"]["sentry-trace"] = { - "Type": "String", - "Value": sentry_trace, - } - return record - - @pytest.mark.parametrize( "aws_event,trigger", [ @@ -957,85 +917,16 @@ def test_trigger(lambda_client, test_environment, aws_event, trigger): @pytest.mark.parametrize( - "aws_event,system,destination,links", + "aws_event,system,destination", [ pytest.param( - { - "Records": [ - _sqs_record("471a43a4192642f0b136d5159a501701-6e8f22c393e68f19-1"), - # the same send context: linked once. - _sqs_record("471a43a4192642f0b136d5159a501701-6e8f22c393e68f19-1"), - _sqs_record("0f4c2a5b8d9e4f1a2b3c4d5e6f7a8b9c-1a2b3c4d5e6f7a8b-0"), - _sqs_record(), - # incomplete context: not linked. - _sqs_record("-1"), - ] - }, - "aws_sqs", - "orders", - [ - { - "trace_id": "471a43a4192642f0b136d5159a501701", - "span_id": "6e8f22c393e68f19", - "attributes": MESSAGE_ORIGIN_LINK, - "sampled": True, - }, - { - "trace_id": "0f4c2a5b8d9e4f1a2b3c4d5e6f7a8b9c", - "span_id": "1a2b3c4d5e6f7a8b", - "attributes": MESSAGE_ORIGIN_LINK, - "sampled": False, - }, - ], - id="sqs", - ), - pytest.param( - { - "Records": [ - _sns_record("471a43a4192642f0b136d5159a501701-6e8f22c393e68f19-1") - ] - }, - "aws.sns", - "alerts", - [ - { - "trace_id": "471a43a4192642f0b136d5159a501701", - "span_id": "6e8f22c393e68f19", - "attributes": MESSAGE_ORIGIN_LINK, - "sampled": True, - } - ], - id="sns", - ), - pytest.param( - { - "Records": [ - { - **_sqs_record(), - "body": json.dumps( - _sns_record( - "471a43a4192642f0b136d5159a501701-6e8f22c393e68f19-1" - )["Sns"] - ), - } - ] - }, - "aws_sqs", - "orders", - [ - { - "trace_id": "471a43a4192642f0b136d5159a501701", - "span_id": "6e8f22c393e68f19", - "attributes": MESSAGE_ORIGIN_LINK, - "sampled": True, - } - ], - id="sns-to-sqs", + {"Records": [SQS_RECORD, SQS_RECORD]}, "aws_sqs", "orders", id="sqs" ), + pytest.param({"Records": [SNS_RECORD]}, "aws.sns", "alerts", id="sns"), ], ) def test_messaging_process_span( - lambda_client, test_environment, aws_event, system, destination, links + lambda_client, test_environment, aws_event, system, destination ): lambda_client.invoke( FunctionName="BasicOkSpanStreaming", @@ -1049,12 +940,11 @@ def test_messaging_process_span( for s in span_items if _get_span_attr(s["attributes"], SPANDATA.SENTRY_OP) == OP.QUEUE_PROCESS ] - # messages are linked, never the parent of the invocation. - assert "parent_span_id" not in segment_span assert process_span["parent_span_id"] == segment_span["span_id"] assert process_span["trace_id"] == segment_span["trace_id"] assert process_span["name"] == f"process {destination}" - assert process_span["links"] == links + # producer links stay on the invocation span, see `test_producer_links`. + assert "links" not in process_span attrs = process_span["attributes"] assert _get_span_attr(attrs, SPANDATA.SENTRY_KIND) == "consumer" @@ -1080,7 +970,7 @@ def test_no_process_span_without_messages(lambda_client, test_environment): def test_process_span_error(lambda_client, test_environment): response = lambda_client.invoke( FunctionName="RaiseErrorSpanStreaming", - Payload=json.dumps({"Records": [_sqs_record()]}), + Payload=json.dumps({"Records": [SQS_RECORD]}), ) # the handler's exception still reaches the Lambda runtime. From 544b0083bf25a5b74017919fe48aa41c75693080 Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Thu, 12 Nov 2026 14:06:09 +0000 Subject: [PATCH 20/21] feat(aws-lambda): Link producers from the queue.process span Following the OTel messaging conventions, the process span carries the links to each message's creation context. Move the links that #8257 put on the invocation span to the queue.process span, reusing the same _get_producer_links. Co-Authored-By: Ivana Kellyer <131587164+sentrivana@users.noreply.github.com> --- .../integrations/aws_lambda/_handler.py | 6 ++--- .../integrations/aws_lambda/_trigger.py | 11 ++++++-- .../aws_lambda/test_aws_lambda.py | 25 ++++++++++++------- 3 files changed, 28 insertions(+), 14 deletions(-) diff --git a/sentry_sdk/integrations/aws_lambda/_handler.py b/sentry_sdk/integrations/aws_lambda/_handler.py index 02cdc811d4..25a01e236c 100644 --- a/sentry_sdk/integrations/aws_lambda/_handler.py +++ b/sentry_sdk/integrations/aws_lambda/_handler.py @@ -271,7 +271,8 @@ def sentry_handler( else: sentry_sdk.continue_trace(headers) - # SQS and SNS messages link to their producers; they never parent the invocation. + # SQS and SNS messages link to their producers from the `queue.process` span; + # they never parent the invocation. producer_links: "list[SpanLink]" = [] with capture_internal_exceptions(): producer_links = _get_producer_links( @@ -286,12 +287,11 @@ def sentry_handler( name=function_name, parent_span=None, attributes=attributes, - links=producer_links, ): try: process_span: "ContextManager[Any]" = nullcontext() with capture_internal_exceptions(): - process_span = _start_process_span(aws_event) + process_span = _start_process_span(aws_event, producer_links) with process_span: return handler(aws_event, aws_context, *args, **kwargs) except Exception: diff --git a/sentry_sdk/integrations/aws_lambda/_trigger.py b/sentry_sdk/integrations/aws_lambda/_trigger.py index a66acebf43..fab22631bd 100644 --- a/sentry_sdk/integrations/aws_lambda/_trigger.py +++ b/sentry_sdk/integrations/aws_lambda/_trigger.py @@ -8,6 +8,8 @@ if TYPE_CHECKING: from typing import Any, ContextManager + from sentry_sdk._types import SpanLink + # `faas.trigger` of each `eventSource` of `Records` or of a top-level `eventSource`. # https://docs.aws.amazon.com/lambda/latest/dg/lambda-services.html _TRIGGERS = { @@ -96,8 +98,10 @@ def _get_trigger(aws_event: "Any") -> str: return _TRIGGERS.get(aws_event.get("eventSource", ""), "other") -def _start_process_span(aws_event: "Any") -> "ContextManager[Any]": - """A `queue.process` span around the handler for an SQS or SNS delivery.""" +def _start_process_span( + aws_event: "Any", links: "list[SpanLink]" +) -> "ContextManager[Any]": + """A `queue.process` span around the handler for an SQS or SNS delivery, linked to its producers.""" records = aws_event.get("Records") if isinstance(aws_event, dict) else None if not records: return nullcontext() @@ -127,4 +131,7 @@ def _start_process_span(aws_event: "Any") -> "ContextManager[Any]": SPANDATA.MESSAGING_DESTINATION_NAME: destination, SPANDATA.MESSAGING_BATCH_MESSAGE_COUNT: len(records), }, + # OTel links a consumer's process span to each message's creation context. + # https://opentelemetry.io/docs/specs/semconv/messaging/messaging-spans/ + links=links, ) diff --git a/tests/integrations/aws_lambda/test_aws_lambda.py b/tests/integrations/aws_lambda/test_aws_lambda.py index d2da44e25d..102e58fb9e 100644 --- a/tests/integrations/aws_lambda/test_aws_lambda.py +++ b/tests/integrations/aws_lambda/test_aws_lambda.py @@ -943,8 +943,6 @@ def test_messaging_process_span( assert process_span["parent_span_id"] == segment_span["span_id"] assert process_span["trace_id"] == segment_span["trace_id"] assert process_span["name"] == f"process {destination}" - # producer links stay on the invocation span, see `test_producer_links`. - assert "links" not in process_span attrs = process_span["attributes"] assert _get_span_attr(attrs, SPANDATA.SENTRY_KIND) == "consumer" @@ -1162,12 +1160,17 @@ def test_producer_links(lambda_client, test_environment, aws_event, links): Payload=json.dumps(aws_event), ) - (segment_span,) = [ - s for s in test_environment["server"].span_items if s["is_segment"] + span_items = test_environment["server"].span_items + (segment_span,) = [s for s in span_items if s["is_segment"]] + (process_span,) = [ + s + for s in span_items + if _get_span_attr(s["attributes"], SPANDATA.SENTRY_OP) == OP.QUEUE_PROCESS ] - # messages are linked, never the parent of the invocation. + # messages are linked from the process span, never the parent of the invocation. assert "parent_span_id" not in segment_span - assert segment_span["links"] == links + assert "links" not in segment_span + assert process_span["links"] == links @pytest.mark.parametrize( @@ -1201,12 +1204,16 @@ def test_trace_continuation_from_client_context( lambda_client.invoke(FunctionName="ClientContextTrace", Payload=json.dumps(payload)) - (segment_span,) = [ - s for s in test_environment["server"].span_items if s["is_segment"] + span_items = test_environment["server"].span_items + (segment_span,) = [s for s in span_items if s["is_segment"]] + (process_span,) = [ + s + for s in span_items + if _get_span_attr(s["attributes"], SPANDATA.SENTRY_OP) == OP.QUEUE_PROCESS ] assert segment_span["trace_id"] == "471a43a4192642f0b136d5159a501701" assert segment_span["parent_span_id"] == "6e8f22c393e68f19" - assert segment_span["links"] == [ + assert process_span["links"] == [ { "trace_id": "5c6d7e8f9a0b4c1d8e2f3a4b5c6d7e8f", "span_id": "2b3c4d5e6f7a8b9c", From 1a11d5a766e1a296322ae084cb69fe483cfd0fea Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Thu, 12 Nov 2026 14:12:28 +0000 Subject: [PATCH 21/21] ref(aws-lambda): Inline _start_process_span into the handler --- .../integrations/aws_lambda/_handler.py | 48 +++++++++++++++-- .../integrations/aws_lambda/_trigger.py | 51 +------------------ 2 files changed, 44 insertions(+), 55 deletions(-) diff --git a/sentry_sdk/integrations/aws_lambda/_handler.py b/sentry_sdk/integrations/aws_lambda/_handler.py index 25a01e236c..0a8607a6eb 100644 --- a/sentry_sdk/integrations/aws_lambda/_handler.py +++ b/sentry_sdk/integrations/aws_lambda/_handler.py @@ -14,10 +14,7 @@ _get_user_from_event, _make_request_event_processor, ) -from sentry_sdk.integrations.aws_lambda._trigger import ( - _get_trigger, - _start_process_span, -) +from sentry_sdk.integrations.aws_lambda._trigger import _get_trigger from sentry_sdk.integrations.aws_lambda.consts import ( CLOUD_PLATFORM, CLOUD_PROVIDER, @@ -47,6 +44,8 @@ # OTel's default link limit; Lambda delivers up to 10,000 SQS messages at once. # https://opentelemetry.io/docs/specs/otel/trace/sdk/#span-limits MAX_PRODUCER_LINKS = 128 +# `messaging.system` of the messaging event sources that get a `queue.process` span. +MESSAGING_SYSTEMS = {"aws:sqs": "aws_sqs", "aws:sns": "aws.sns"} def _get_producer_links(aws_event: "Any", trace_id: str) -> "list[SpanLink]": @@ -289,9 +288,48 @@ def sentry_handler( attributes=attributes, ): try: + # a `queue.process` span around the handler for an SQS or SNS delivery. process_span: "ContextManager[Any]" = nullcontext() with capture_internal_exceptions(): - process_span = _start_process_span(aws_event, producer_links) + records: "list[Any]" = [] + if isinstance(aws_event, dict): + records = aws_event.get("Records") or [] + + event_source = None + if records: + # SNS spells it `EventSource`. + event_source = records[0].get("eventSource") or records[ + 0 + ].get("EventSource") + + if event_source in MESSAGING_SYSTEMS: + # one queue or topic per event: `arn:aws:sqs:::`. + arn = ( + records[0]["eventSourceARN"] + if event_source == "aws:sqs" + else records[0]["Sns"]["TopicArn"] + ) + destination = arn.split(":")[5] + process_span = sentry_sdk.start_span( + name=f"process {destination}", + attributes={ + SPANDATA.SENTRY_OP: OP.QUEUE_PROCESS, + SPANDATA.SENTRY_ORIGIN: ORIGIN, + SPANDATA.SENTRY_KIND: "consumer", + SPANDATA.MESSAGING_SYSTEM: MESSAGING_SYSTEMS[ + event_source + ], + SPANDATA.MESSAGING_OPERATION_TYPE: "process", + SPANDATA.MESSAGING_OPERATION_NAME: "process", + SPANDATA.MESSAGING_DESTINATION_NAME: destination, + SPANDATA.MESSAGING_BATCH_MESSAGE_COUNT: len( + records + ), + }, + # OTel links a consumer's process span to each message's creation context. + # https://opentelemetry.io/docs/specs/semconv/messaging/messaging-spans/ + links=producer_links, + ) with process_span: return handler(aws_event, aws_context, *args, **kwargs) except Exception: diff --git a/sentry_sdk/integrations/aws_lambda/_trigger.py b/sentry_sdk/integrations/aws_lambda/_trigger.py index fab22631bd..bce42e624c 100644 --- a/sentry_sdk/integrations/aws_lambda/_trigger.py +++ b/sentry_sdk/integrations/aws_lambda/_trigger.py @@ -1,14 +1,7 @@ -from contextlib import nullcontext from typing import TYPE_CHECKING -import sentry_sdk -from sentry_sdk.consts import OP, SPANDATA -from sentry_sdk.integrations.aws_lambda.consts import ORIGIN - if TYPE_CHECKING: - from typing import Any, ContextManager - - from sentry_sdk._types import SpanLink + from typing import Any # `faas.trigger` of each `eventSource` of `Records` or of a top-level `eventSource`. # https://docs.aws.amazon.com/lambda/latest/dg/lambda-services.html @@ -28,9 +21,6 @@ "aws:docdb": "datasource", } -# `messaging.system` of the messaging event sources that get a `queue.process` span. -_MESSAGING_SYSTEMS = {"aws:sqs": "aws_sqs", "aws:sns": "aws.sns"} - # https://docs.aws.amazon.com/eventbridge/latest/ref/overiew-event-structure.html _EVENTBRIDGE_KEYS = frozenset( ( @@ -96,42 +86,3 @@ def _get_trigger(aws_event: "Any") -> str: return "pubsub" return _TRIGGERS.get(aws_event.get("eventSource", ""), "other") - - -def _start_process_span( - aws_event: "Any", links: "list[SpanLink]" -) -> "ContextManager[Any]": - """A `queue.process` span around the handler for an SQS or SNS delivery, linked to its producers.""" - records = aws_event.get("Records") if isinstance(aws_event, dict) else None - if not records: - return nullcontext() - - # SNS spells it `EventSource`. - event_source = records[0].get("eventSource") or records[0].get("EventSource") - if event_source not in _MESSAGING_SYSTEMS: - return nullcontext() - - # one queue or topic per event: `arn:aws:sqs:::`. - arn = ( - records[0]["eventSourceARN"] - if event_source == "aws:sqs" - else records[0]["Sns"]["TopicArn"] - ) - destination = arn.split(":")[5] - - return sentry_sdk.start_span( - name=f"process {destination}", - attributes={ - SPANDATA.SENTRY_OP: OP.QUEUE_PROCESS, - SPANDATA.SENTRY_ORIGIN: ORIGIN, - SPANDATA.SENTRY_KIND: "consumer", - SPANDATA.MESSAGING_SYSTEM: _MESSAGING_SYSTEMS[event_source], - SPANDATA.MESSAGING_OPERATION_TYPE: "process", - SPANDATA.MESSAGING_OPERATION_NAME: "process", - SPANDATA.MESSAGING_DESTINATION_NAME: destination, - SPANDATA.MESSAGING_BATCH_MESSAGE_COUNT: len(records), - }, - # OTel links a consumer's process span to each message's creation context. - # https://opentelemetry.io/docs/specs/semconv/messaging/messaging-spans/ - links=links, - )