From bb98b11c9ef581d87262ed6e9391d87a03c01190 Mon Sep 17 00:00:00 2001 From: ryanhiizy <150064366+ryanhiizy@users.noreply.github.com> Date: Sat, 7 Nov 2026 15:06:17 +1100 Subject: [PATCH 1/6] fix(tracing): Find query source across the greenlet that async SQLAlchemy runs the driver in --- sentry_sdk/tracing_utils.py | 33 ++++++++++-- .../sqlalchemy/test_sqlalchemy.py | 50 +++++++++++++++++++ 2 files changed, 79 insertions(+), 4 deletions(-) diff --git a/sentry_sdk/tracing_utils.py b/sentry_sdk/tracing_utils.py index 4d4c35ce39..cd4769038f 100644 --- a/sentry_sdk/tracing_utils.py +++ b/sentry_sdk/tracing_utils.py @@ -292,6 +292,33 @@ def _should_be_included( ) +def _caller_frames(frame: "Optional[FrameType]") -> "Iterator[FrameType]": + """ + Yields ``frame`` and the frames that called it. When the current + greenlet's frames run out, continues into those of its parent greenlet, + where they were suspended to switch to it. Async SQLAlchemy runs the + driver, and so the cursor events that create query spans, inside such a + greenlet, started from the coroutine that awaited the query. + """ + while frame is not None: + yield frame + frame = frame.f_back + + greenlet = sys.modules.get("greenlet") + if greenlet is None: + return + + try: + parent = greenlet.getcurrent().parent + frame = parent.gr_frame if parent is not None else None + except Exception: + return + + while frame is not None: + yield frame + frame = frame.f_back + + def add_source( span: "Union[sentry_sdk.tracing.Span, sentry_sdk.traces.StreamedSpan]", project_root: "Optional[str]", @@ -302,8 +329,8 @@ def add_source( Adds OTel compatible source code information to the span """ # Find the correct frame - frame: "Union[FrameType, None]" = sys._getframe() - while frame is not None: + frame: "Union[FrameType, None]" = None + for frame in _caller_frames(sys._getframe()): abs_path = _get_frame_module_abs_path(frame) try: @@ -325,8 +352,6 @@ def add_source( ) if should_be_included: break - - frame = frame.f_back else: frame = None diff --git a/tests/integrations/sqlalchemy/test_sqlalchemy.py b/tests/integrations/sqlalchemy/test_sqlalchemy.py index e08bb0a423..835d82c972 100644 --- a/tests/integrations/sqlalchemy/test_sqlalchemy.py +++ b/tests/integrations/sqlalchemy/test_sqlalchemy.py @@ -760,6 +760,56 @@ class Person(Base): raise AssertionError("No db span found") +@pytest.mark.parametrize("span_streaming", [True, False]) +def test_query_source_in_greenlet( + sentry_init, + capture_events, + capture_items, + span_streaming, +): + greenlet = pytest.importorskip("greenlet") + sentry_init( + integrations=[SqlalchemyIntegration()], + traces_sample_rate=1.0, + enable_db_query_source=True, + db_query_source_threshold_ms=0, + trace_lifecycle="stream" if span_streaming else "static", + ) + engine = create_engine("sqlite:///:memory:") + + def run_query(): + with engine.connect() as connection: + # As SQLAlchemy's asyncio extension does, the driver runs in a + # greenlet, in the caller's context, whose own frames hold no + # application code. + driver = greenlet.greenlet(connection.execute) + driver.gr_context = greenlet.getcurrent().gr_context + driver.switch(text("SELECT 1")) + + if span_streaming: + items = capture_items("span") + with sentry_sdk.traces.start_span(name="custom parent"): + run_query() + sentry_sdk.flush() + sources = [ + item.payload["attributes"].get(SPANDATA.CODE_FUNCTION) + for item in items + if item.payload["name"] == "SELECT 1" + ] + else: + events = capture_events() + with start_transaction(name="test_transaction", sampled=True): + run_query() + (event,) = events + sources = [ + span["data"].get(SPANDATA.CODE_FUNCTION) + for span in event["spans"] + if span.get("description") == "SELECT 1" + ] + + assert sources == ["run_query"] + + @pytest.mark.parametrize("span_streaming", [True, False]) def test_query_source_with_module_in_search_path( sentry_init, From dc44691c2f27231ac43afde222f2021409e8f6d2 Mon Sep 17 00:00:00 2001 From: ryanhiizy Date: Wed, 11 Nov 2026 09:15:04 +1100 Subject: [PATCH 2/6] fix(tracing): Only follow the parent greenlet from SQLAlchemy's async bridge A greenlet's parent is not necessarily its caller, so following any parent could attribute a span to unrelated code. Only SQLAlchemy's bridge greenlet, which it marks with __sqlalchemy_greenlet_provider__, is known to be spawned by the coroutine awaiting the query. Test through SQLAlchemy's real greenlet_spawn, add a case where an unrelated parent gives no source, and install greenlet for the SQLAlchemy 2.x test environments, where it is only an optional extra. --- scripts/populate_tox/config.py | 5 + sentry_sdk/tracing_utils.py | 20 +++- .../sqlalchemy/test_sqlalchemy.py | 104 +++++++++++++----- tox.ini | 12 ++ 4 files changed, 107 insertions(+), 34 deletions(-) diff --git a/scripts/populate_tox/config.py b/scripts/populate_tox/config.py index 71b6cfa14a..d5705f2684 100644 --- a/scripts/populate_tox/config.py +++ b/scripts/populate_tox/config.py @@ -509,6 +509,11 @@ }, "sqlalchemy": { "package": "sqlalchemy", + "deps": { + # Needed to test the asyncio extension, which only 1.4 installs + # greenlet for + ">=2.0": ["greenlet"], + }, "python": { "<1.4": "<3.10", }, diff --git a/sentry_sdk/tracing_utils.py b/sentry_sdk/tracing_utils.py index cd4769038f..c7c56629bb 100644 --- a/sentry_sdk/tracing_utils.py +++ b/sentry_sdk/tracing_utils.py @@ -294,11 +294,14 @@ def _should_be_included( def _caller_frames(frame: "Optional[FrameType]") -> "Iterator[FrameType]": """ - Yields ``frame`` and the frames that called it. When the current - greenlet's frames run out, continues into those of its parent greenlet, - where they were suspended to switch to it. Async SQLAlchemy runs the - driver, and so the cursor events that create query spans, inside such a - greenlet, started from the coroutine that awaited the query. + Yields ``frame`` and the frames that called it. + + Async SQLAlchemy runs the sync driver, and so the cursor events that + create query spans, in a greenlet it spawns from the coroutine awaiting + the query. That greenlet's own frames end at the bridge, so when the + current greenlet is SQLAlchemy's bridge, the walk continues into the + frames of its parent, which is suspended in the awaiting caller. Any other + greenlet's parent need not be its caller, so its frames are not followed. """ while frame is not None: yield frame @@ -309,7 +312,12 @@ def _caller_frames(frame: "Optional[FrameType]") -> "Iterator[FrameType]": return try: - parent = greenlet.getcurrent().parent + current = greenlet.getcurrent() + # SQLAlchemy marks its bridge greenlet with this attribute and checks + # for it itself. + if not getattr(current, "__sqlalchemy_greenlet_provider__", False): + return + parent = current.parent frame = parent.gr_frame if parent is not None else None except Exception: return diff --git a/tests/integrations/sqlalchemy/test_sqlalchemy.py b/tests/integrations/sqlalchemy/test_sqlalchemy.py index 835d82c972..5196e5c2b2 100644 --- a/tests/integrations/sqlalchemy/test_sqlalchemy.py +++ b/tests/integrations/sqlalchemy/test_sqlalchemy.py @@ -1,8 +1,10 @@ +import asyncio import os from datetime import datetime from unittest import mock import pytest +import sqlalchemy.util from sqlalchemy import Column, ForeignKey, Integer, String, create_engine, text from sqlalchemy.exc import IntegrityError from sqlalchemy.ext.declarative import declarative_base @@ -760,14 +762,41 @@ class Person(Base): raise AssertionError("No db span found") +def _select_one_sources(run_query, span_streaming, capture_events, capture_items): + if span_streaming: + items = capture_items("span") + with sentry_sdk.traces.start_span(name="custom parent"): + run_query() + sentry_sdk.flush() + return [ + item.payload["attributes"].get(SPANDATA.CODE_FUNCTION) + for item in items + if item.payload["name"] == "SELECT 1" + ] + + events = capture_events() + with start_transaction(name="test_transaction", sampled=True): + run_query() + (event,) = events + return [ + span["data"].get(SPANDATA.CODE_FUNCTION) + for span in event["spans"] + if span.get("description") == "SELECT 1" + ] + + @pytest.mark.parametrize("span_streaming", [True, False]) -def test_query_source_in_greenlet( +def test_query_source_through_async_bridge( sentry_init, capture_events, capture_items, span_streaming, ): - greenlet = pytest.importorskip("greenlet") + pytest.importorskip("greenlet") + greenlet_spawn = getattr(sqlalchemy.util, "greenlet_spawn", None) + if greenlet_spawn is None: + pytest.skip("SQLAlchemy has no asyncio support before 1.4") + sentry_init( integrations=[SqlalchemyIntegration()], traces_sample_rate=1.0, @@ -777,37 +806,56 @@ def test_query_source_in_greenlet( ) engine = create_engine("sqlite:///:memory:") - def run_query(): + async def query_from_coroutine(): with engine.connect() as connection: - # As SQLAlchemy's asyncio extension does, the driver runs in a - # greenlet, in the caller's context, whose own frames hold no + # As SQLAlchemy's asyncio extension does, run the sync call in + # SQLAlchemy's bridge greenlet, whose own frames hold no # application code. - driver = greenlet.greenlet(connection.execute) - driver.gr_context = greenlet.getcurrent().gr_context + await greenlet_spawn(connection.execute, text("SELECT 1")) + + def run_query(): + loop = asyncio.new_event_loop() + try: + loop.run_until_complete(query_from_coroutine()) + finally: + loop.close() + + sources = _select_one_sources( + run_query, span_streaming, capture_events, capture_items + ) + + assert sources == ["query_from_coroutine"] + + +def test_query_source_not_taken_from_unrelated_greenlet_parent( + sentry_init, capture_events +): + greenlet = pytest.importorskip("greenlet") + sentry_init( + integrations=[SqlalchemyIntegration()], + traces_sample_rate=1.0, + enable_db_query_source=True, + db_query_source_threshold_ms=0, + ) + engine = create_engine("sqlite:///:memory:") + + def unrelated_wait(): + greenlet.getcurrent().parent.switch() + + def run_query(): + with engine.connect() as connection: + # A greenlet's parent is not necessarily its caller: this one's + # parent is suspended in code unrelated to the query. + waiter = greenlet.greenlet(unrelated_wait) + waiter.switch() + driver = greenlet.greenlet(connection.execute, parent=waiter) + if hasattr(driver, "gr_context"): + driver.gr_context = greenlet.getcurrent().gr_context driver.switch(text("SELECT 1")) - if span_streaming: - items = capture_items("span") - with sentry_sdk.traces.start_span(name="custom parent"): - run_query() - sentry_sdk.flush() - sources = [ - item.payload["attributes"].get(SPANDATA.CODE_FUNCTION) - for item in items - if item.payload["name"] == "SELECT 1" - ] - else: - events = capture_events() - with start_transaction(name="test_transaction", sampled=True): - run_query() - (event,) = events - sources = [ - span["data"].get(SPANDATA.CODE_FUNCTION) - for span in event["spans"] - if span.get("description") == "SELECT 1" - ] + sources = _select_one_sources(run_query, False, capture_events, None) - assert sources == ["run_query"] + assert sources == [None] @pytest.mark.parametrize("span_streaming", [True, False]) diff --git a/tox.ini b/tox.ini index c8af226e19..38fb5f2aae 100644 --- a/tox.ini +++ b/tox.ini @@ -12874,24 +12874,36 @@ deps = sqlalchemy-v2.1.6: sqlalchemy==2.1.6 + py3.11-sqlalchemy-v2.1.6: greenlet==3.5.6 py3.11-sqlalchemy-v2.1.6: typing_extensions==4.16.0 + py3.14-sqlalchemy-v2.1.6: greenlet==3.5.6 py3.14-sqlalchemy-v2.1.6: typing_extensions==4.16.0 + py3.15-sqlalchemy-v2.1.6: greenlet==3.5.6 py3.15-sqlalchemy-v2.1.6: typing_extensions==4.16.0 + py3.15t-sqlalchemy-v2.1.6: greenlet==3.5.6 py3.15t-sqlalchemy-v2.1.6: typing_extensions==4.16.0 sqlalchemy-latest: sqlalchemy==2.1.6 + sqlalchemy-latest: greenlet + + py3.11-sqlalchemy-latest: greenlet==3.5.6 py3.11-sqlalchemy-latest: typing_extensions==4.16.0 + py3.14-sqlalchemy-latest: greenlet==3.5.6 py3.14-sqlalchemy-latest: typing_extensions==4.16.0 + py3.15-sqlalchemy-latest: greenlet==3.5.6 py3.15-sqlalchemy-latest: typing_extensions==4.16.0 + py3.15t-sqlalchemy-latest: greenlet==3.5.6 py3.15t-sqlalchemy-latest: typing_extensions==4.16.0 + sqlalchemy-v2.1.6: greenlet + sqlalchemy-latest: greenlet # ~~~ Flags ~~~ From 95dd7a34abfd1e7e300bb1ea9a39746eea74ae31 Mon Sep 17 00:00:00 2001 From: ryanhiizy Date: Wed, 11 Nov 2026 09:42:42 +1100 Subject: [PATCH 3/6] fix(tracing): Recognise SQLAlchemy's async bridge before 1.4.53 and 2.0.31 Earlier releases don't set __sqlalchemy_greenlet_provider__ on the bridge greenlet, so identify it by its class there. --- sentry_sdk/tracing_utils.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/sentry_sdk/tracing_utils.py b/sentry_sdk/tracing_utils.py index c7c56629bb..3a1b665be4 100644 --- a/sentry_sdk/tracing_utils.py +++ b/sentry_sdk/tracing_utils.py @@ -292,6 +292,17 @@ def _should_be_included( ) +def _is_sqlalchemy_async_bridge(current: "Any") -> bool: + # SQLAlchemy marks its bridge greenlet with this attribute and checks for + # it itself, from 1.4.53 and 2.0.31. Earlier releases check the class. + if getattr(current, "__sqlalchemy_greenlet_provider__", False): + return True + cls = type(current) + return cls.__name__ == "_AsyncIoGreenlet" and cls.__module__.startswith( + "sqlalchemy." + ) + + def _caller_frames(frame: "Optional[FrameType]") -> "Iterator[FrameType]": """ Yields ``frame`` and the frames that called it. @@ -313,9 +324,7 @@ def _caller_frames(frame: "Optional[FrameType]") -> "Iterator[FrameType]": try: current = greenlet.getcurrent() - # SQLAlchemy marks its bridge greenlet with this attribute and checks - # for it itself. - if not getattr(current, "__sqlalchemy_greenlet_provider__", False): + if not _is_sqlalchemy_async_bridge(current): return parent = current.parent frame = parent.gr_frame if parent is not None else None From 603bd1862028d01a82894f5c7eb5696176ecbc6b Mon Sep 17 00:00:00 2001 From: ryanhiizy Date: Wed, 11 Nov 2026 09:42:42 +1100 Subject: [PATCH 4/6] test(sqlalchemy): Fail instead of skipping the async bridge tests without greenlet Skip on SQLAlchemy before 1.4 only, so the tests can't silently stop running if greenlet drops out of the 1.4+ environments. --- .../sqlalchemy/test_sqlalchemy.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/tests/integrations/sqlalchemy/test_sqlalchemy.py b/tests/integrations/sqlalchemy/test_sqlalchemy.py index 5196e5c2b2..4a43334c11 100644 --- a/tests/integrations/sqlalchemy/test_sqlalchemy.py +++ b/tests/integrations/sqlalchemy/test_sqlalchemy.py @@ -762,6 +762,15 @@ class Person(Base): raise AssertionError("No db span found") +def _async_bridge_or_skip(): + # Not skipped when greenlet is missing: from 1.4 on, the test environments + # must install it, and greenlet_spawn fails loudly without it. + greenlet_spawn = getattr(sqlalchemy.util, "greenlet_spawn", None) + if greenlet_spawn is None: + pytest.skip("SQLAlchemy has no asyncio support before 1.4") + return greenlet_spawn + + def _select_one_sources(run_query, span_streaming, capture_events, capture_items): if span_streaming: items = capture_items("span") @@ -792,11 +801,7 @@ def test_query_source_through_async_bridge( capture_items, span_streaming, ): - pytest.importorskip("greenlet") - greenlet_spawn = getattr(sqlalchemy.util, "greenlet_spawn", None) - if greenlet_spawn is None: - pytest.skip("SQLAlchemy has no asyncio support before 1.4") - + greenlet_spawn = _async_bridge_or_skip() sentry_init( integrations=[SqlalchemyIntegration()], traces_sample_rate=1.0, @@ -830,7 +835,9 @@ def run_query(): def test_query_source_not_taken_from_unrelated_greenlet_parent( sentry_init, capture_events ): - greenlet = pytest.importorskip("greenlet") + _async_bridge_or_skip() + import greenlet + sentry_init( integrations=[SqlalchemyIntegration()], traces_sample_rate=1.0, From ceb11ac41ec531e763c08543ffd7d05463324a92 Mon Sep 17 00:00:00 2001 From: ryanhiizy Date: Wed, 11 Nov 2026 14:50:01 +1100 Subject: [PATCH 5/6] test(sqlalchemy): Install greenlet in the 1.4 test environments too SQLAlchemy 1.4 only depends on greenlet on some platforms, not macOS arm64, where the async bridge tests would otherwise fail. --- scripts/populate_tox/config.py | 6 +++--- tox.ini | 5 +++++ 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/scripts/populate_tox/config.py b/scripts/populate_tox/config.py index d5705f2684..b2db1058bb 100644 --- a/scripts/populate_tox/config.py +++ b/scripts/populate_tox/config.py @@ -510,9 +510,9 @@ "sqlalchemy": { "package": "sqlalchemy", "deps": { - # Needed to test the asyncio extension, which only 1.4 installs - # greenlet for - ">=2.0": ["greenlet"], + # Needed to test the asyncio extension. SQLAlchemy 1.4 installs + # greenlet only on some platforms, 2.x only with an extra. + ">=1.4": ["greenlet"], }, "python": { "<1.4": "<3.10", diff --git a/tox.ini b/tox.ini index 38fb5f2aae..ec72269d10 100644 --- a/tox.ini +++ b/tox.ini @@ -12872,6 +12872,10 @@ deps = sqlalchemy-v1.4.54: sqlalchemy==1.4.54 + py3.11-sqlalchemy-v1.4.54: greenlet==3.5.6 + + py3.12-sqlalchemy-v1.4.54: greenlet==3.5.6 + sqlalchemy-v2.1.6: sqlalchemy==2.1.6 py3.11-sqlalchemy-v2.1.6: greenlet==3.5.6 @@ -12902,6 +12906,7 @@ deps = py3.15t-sqlalchemy-latest: greenlet==3.5.6 py3.15t-sqlalchemy-latest: typing_extensions==4.16.0 + sqlalchemy-v1.4.54: greenlet sqlalchemy-v2.1.6: greenlet sqlalchemy-latest: greenlet From cc399009cf7d792f23a207693f8da9d2a4e165c6 Mon Sep 17 00:00:00 2001 From: ryanhiizy Date: Wed, 11 Nov 2026 15:00:18 +1100 Subject: [PATCH 6/6] test(sqlalchemy): Keep only the async bridge regression test --- .../sqlalchemy/test_sqlalchemy.py | 95 +++++-------------- 1 file changed, 26 insertions(+), 69 deletions(-) diff --git a/tests/integrations/sqlalchemy/test_sqlalchemy.py b/tests/integrations/sqlalchemy/test_sqlalchemy.py index 4a43334c11..66edb10ec4 100644 --- a/tests/integrations/sqlalchemy/test_sqlalchemy.py +++ b/tests/integrations/sqlalchemy/test_sqlalchemy.py @@ -762,38 +762,6 @@ class Person(Base): raise AssertionError("No db span found") -def _async_bridge_or_skip(): - # Not skipped when greenlet is missing: from 1.4 on, the test environments - # must install it, and greenlet_spawn fails loudly without it. - greenlet_spawn = getattr(sqlalchemy.util, "greenlet_spawn", None) - if greenlet_spawn is None: - pytest.skip("SQLAlchemy has no asyncio support before 1.4") - return greenlet_spawn - - -def _select_one_sources(run_query, span_streaming, capture_events, capture_items): - if span_streaming: - items = capture_items("span") - with sentry_sdk.traces.start_span(name="custom parent"): - run_query() - sentry_sdk.flush() - return [ - item.payload["attributes"].get(SPANDATA.CODE_FUNCTION) - for item in items - if item.payload["name"] == "SELECT 1" - ] - - events = capture_events() - with start_transaction(name="test_transaction", sampled=True): - run_query() - (event,) = events - return [ - span["data"].get(SPANDATA.CODE_FUNCTION) - for span in event["spans"] - if span.get("description") == "SELECT 1" - ] - - @pytest.mark.parametrize("span_streaming", [True, False]) def test_query_source_through_async_bridge( sentry_init, @@ -801,7 +769,12 @@ def test_query_source_through_async_bridge( capture_items, span_streaming, ): - greenlet_spawn = _async_bridge_or_skip() + # Not skipped when greenlet is missing: from 1.4 on, the test environments + # install it, and greenlet_spawn fails loudly without it. + greenlet_spawn = getattr(sqlalchemy.util, "greenlet_spawn", None) + if greenlet_spawn is None: + pytest.skip("SQLAlchemy has no asyncio support before 1.4") + sentry_init( integrations=[SqlalchemyIntegration()], traces_sample_rate=1.0, @@ -825,46 +798,30 @@ def run_query(): finally: loop.close() - sources = _select_one_sources( - run_query, span_streaming, capture_events, capture_items - ) + if span_streaming: + items = capture_items("span") + with sentry_sdk.traces.start_span(name="custom parent"): + run_query() + sentry_sdk.flush() + sources = [ + item.payload["attributes"].get(SPANDATA.CODE_FUNCTION) + for item in items + if item.payload["name"] == "SELECT 1" + ] + else: + events = capture_events() + with start_transaction(name="test_transaction", sampled=True): + run_query() + (event,) = events + sources = [ + span["data"].get(SPANDATA.CODE_FUNCTION) + for span in event["spans"] + if span.get("description") == "SELECT 1" + ] assert sources == ["query_from_coroutine"] -def test_query_source_not_taken_from_unrelated_greenlet_parent( - sentry_init, capture_events -): - _async_bridge_or_skip() - import greenlet - - sentry_init( - integrations=[SqlalchemyIntegration()], - traces_sample_rate=1.0, - enable_db_query_source=True, - db_query_source_threshold_ms=0, - ) - engine = create_engine("sqlite:///:memory:") - - def unrelated_wait(): - greenlet.getcurrent().parent.switch() - - def run_query(): - with engine.connect() as connection: - # A greenlet's parent is not necessarily its caller: this one's - # parent is suspended in code unrelated to the query. - waiter = greenlet.greenlet(unrelated_wait) - waiter.switch() - driver = greenlet.greenlet(connection.execute, parent=waiter) - if hasattr(driver, "gr_context"): - driver.gr_context = greenlet.getcurrent().gr_context - driver.switch(text("SELECT 1")) - - sources = _select_one_sources(run_query, False, capture_events, None) - - assert sources == [None] - - @pytest.mark.parametrize("span_streaming", [True, False]) def test_query_source_with_module_in_search_path( sentry_init,