feat: trace requests across the frontend and backend

Dockerfile

@@ -38,6 +38,8 @@ RUN npm install
 
 COPY /frontend/ /frontend/
 
+# Unset keeps the default in `frontend/src/instrument.ts`.
+ARG VITE_SENTRY_TRACES_SAMPLE_RATE
 RUN npm run build
 
 # Runtime container

backend/pyproject.toml

@@ -4,7 +4,7 @@ version = "0.8.2"
 description = "Backend for the Climate Rapid Evaluation Framework"
 requires-python = ">=3.12"
 dependencies = [
-    "fastapi[standard]<1.0.0,>=0.114.2",
+    "fastapi[standard]<1.0.0,>=0.162.0",
     "httpx>=0.27",
     "pydantic>2.0",
     "psycopg[binary]<4.0.0,>=3.1.13",
@@ -14,7 +14,8 @@ dependencies = [
     "loguru",
     "pyyaml>=6.0",
     "fastapi-sqlalchemy-monitor>=1.1.3",
-    "prometheus-fastapi-instrumentator>=7.0.0",
+    "cr-service[tracing]>=0.6.0",
+    "opentelemetry-instrumentation-sqlalchemy>=0.51b0",
 ]
 
 [dependency-groups]

backend/src/ref_backend/builder.py

@@ -1,6 +1,10 @@
 from dataclasses import asdict
+from importlib.metadata import version
 
 import sentry_sdk
+from cr_service import metrics, tracing
+from cr_service.health import router as health_router
+from cr_service.info import ServiceInfo
 from fastapi import FastAPI
 from fastapi.routing import APIRoute
 from fastapi_sqlalchemy_monitor import AlchemyStatistics, SQLAlchemyMonitor
@@ -19,7 +23,8 @@
 from ref_backend.api.main import api_router
 from ref_backend.caching import CacheControlMiddleware
 from ref_backend.core.config import Settings
-from ref_backend.metrics import instrument_app
+
+SERVICE = ServiceInfo(name="ref-backend", version=version("ref-backend"))
 
 description = """
 API for querying the results from the Climate Rapid Evaluation Framework (Climate REF).
@@ -36,7 +41,8 @@
 
 
 def custom_generate_unique_id(route: APIRoute) -> str:
-    return f"{route.tags[0]}-{route.name}"
+    # The cr-service health and metrics routes carry no tag.
+    return f"{route.tags[0]}-{route.name}" if route.tags else route.name
 
 
 class SlowQueryMonitor(ConditionalAction):
@@ -95,9 +101,14 @@ def build_app(settings: Settings, ref_config: Config, database: Database) -> Fas
     if settings.SENTRY_DSN and settings.ENVIRONMENT != "local":
         sentry_sdk.init(
             dsn=str(settings.SENTRY_DSN),
+            environment=settings.ENVIRONMENT,
             release=settings.GIT_COMMIT,
-            enable_tracing=True,
+            traces_sample_rate=settings.SENTRY_TRACES_SAMPLE_RATE,
+            send_default_pii=False,
         )
+    # FastAPI traces each request itself once a tracer provider is installed.
+    tracing.install_provider(SERVICE)
+    tracing.instrument_sqlalchemy(database._engine)
 
     app = FastAPI(
         title=settings.PROJECT_NAME,
@@ -141,8 +152,9 @@ def build_app(settings: Settings, ref_config: Config, database: Database) -> Fas
         results_max_age=settings.RESULTS_CACHE_MAX_AGE,
     )
 
-    # Registered before the static mount, so the SPA fallback does not swallow /metrics.
-    instrument_app(app)
+    # Registered before the static mount, so the SPA fallback does not swallow the probes or /metrics.
+    app.include_router(health_router)
+    metrics.instrument_app(app)
 
     app.include_router(api_router, prefix=settings.API_V1_STR)
 

backend/src/ref_backend/caching.py

@@ -4,6 +4,7 @@
 dashboard.climate-ref.org is fronted by Cloudflare, which handles the caching according to the headers we set.
 """
 
+from cr_service.health import PROBE_PATHS
 from starlette import status
 from starlette.datastructures import MutableHeaders
 from starlette.types import ASGIApp, Message, Receive, Scope, Send
@@ -33,7 +34,7 @@ def __init__(self, app: ASGIApp, api_prefix: str, api_max_age: int, results_max_
         self.app = app
         self.api_prefix = api_prefix.rstrip("/") + "/"
         self.results_prefix = self.api_prefix + "results/"
-        self.uncached_paths = frozenset({"/metrics", *(self.api_prefix + p for p in UNCACHED_API_PATHS)})
+        self.uncached_paths = frozenset({*PROBE_PATHS, *(self.api_prefix + p for p in UNCACHED_API_PATHS)})
         self.api_policy = f"public, max-age={api_max_age}"
         self.results_policy = f"public, max-age={results_max_age}"
 

backend/src/ref_backend/core/config.py

@@ -5,6 +5,7 @@
 from pydantic import (
     AnyUrl,
     BeforeValidator,
+    Field,
     HttpUrl,
     computed_field,
 )
@@ -66,6 +67,7 @@ def all_cors_origins(self) -> list[str]:
 
     PROJECT_NAME: str = "Climate Rapid Evaluation Framework"
     SENTRY_DSN: HttpUrl | None = None
+    SENTRY_TRACES_SAMPLE_RATE: float = Field(default=1.0, ge=0.0, le=1.0)
     REF_CONFIGURATION: str = "data"
     REF_READ_ONLY_DATABASE: bool = False
     """

backend/tests/test_api/test_metrics.py

@@ -7,8 +7,10 @@ def test_metrics_records_instrumented_requests(client):
     assert any(line.startswith("http_request_duration_seconds_count{") for line in lines)
 
 
-def test_metrics_excludes_health_check(client):
-    client.get("/api/v1/utils/health-check/")
+def test_metrics_excludes_probes(client):
+    assert client.get("/livez").status_code == 200
+    assert client.get("/readyz").status_code == 200
     response = client.get("/metrics")
-    assert "health-check" not in response.text
+    assert "/livez" not in response.text
+    assert "/readyz" not in response.text
     assert 'handler="/metrics"' not in response.text

backend/tests/test_builder.py

@@ -18,3 +18,5 @@ def test_sentry_release_is_the_git_commit(monkeypatch):
     builder.build_app(settings, config, deps._get_database_dependency(settings, config))
 
     assert calls[0]["release"] == "e145da768b1725ce708150597a565df09c923dc3"
+    assert calls[0]["environment"] == "staging"
+    assert calls[0]["traces_sample_rate"] == 1.0

backend/tests/test_caching.py

@@ -99,7 +99,9 @@ def test_writes_are_not_stored(self, spa_client: TestClient):
 
 
 class TestLiveEndpointsAreNotStored:
-    @pytest.mark.parametrize("path", ["/api/v1/utils/health-check/", "/api/v1/utils/about", "/metrics"])
+    @pytest.mark.parametrize(
+        "path", ["/api/v1/utils/health-check/", "/api/v1/utils/about", "/metrics", "/livez", "/readyz"]
+    )
     def test_no_store(self, client: TestClient, path: str):
         r = client.get(path)
         assert r.status_code == 200

backend/tests/test_tracing.py

@@ -0,0 +1,30 @@
+import pytest
+from opentelemetry import trace
+from opentelemetry.sdk.trace import TracerProvider
+from opentelemetry.sdk.trace.export import SimpleSpanProcessor
+from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter
+
+TRACE_ID = "4bf92f3577b34da6a3ce929d0e0e4736"
+
+
+@pytest.fixture(scope="module")
+def exporter():
+    exporter = InMemorySpanExporter()
+    provider = TracerProvider()
+    provider.add_span_processor(SimpleSpanProcessor(exporter))
+    trace.set_tracer_provider(provider)
+    return exporter
+
+
+def test_request_span_continues_the_browser_trace(exporter, client):
+    exporter.clear()
+
+    response = client.get(
+        "/api/v1/utils/about", headers={"traceparent": f"00-{TRACE_ID}-00f067aa0ba902b7-01"}
+    )
+
+    assert response.status_code == 200
+    spans = [span for span in exporter.get_finished_spans() if span.kind == trace.SpanKind.SERVER]
+    assert spans
+    assert trace.format_trace_id(spans[0].context.trace_id) == TRACE_ID
+    assert spans[0].attributes["http.route"] == "/api/v1/utils/about"

changelog/150.improvement.md

@@ -0,0 +1 @@
+Added request tracing from the dashboard through the API, and `/livez` and `/readyz` health probes.

frontend/src/instrument.ts

@@ -1,5 +1,7 @@
 import * as Sentry from "@sentry/react";
 
+const DEFAULT_TRACES_SAMPLE_RATE = 0.1;
+
 Sentry.init({
   // This isn't real sensitive since it's a public DSN, but don't commit your real one!
   dsn: import.meta.env.PROD
@@ -11,5 +13,17 @@ Sentry.init({
     httpHeaders: false,
     httpBodies: [],
   },
+  // The backend follows the browser's sampling decision, so this rate also sets how many API spans it records.
+  tracesSampleRate: Number(
+    import.meta.env.VITE_SENTRY_TRACES_SAMPLE_RATE ||
+      DEFAULT_TRACES_SAMPLE_RATE,
+  ),
+  // Same-origin calls plus the hosted APIs listed in `lib/apiEndpoint.ts`.
+  tracePropagationTargets: [
+    /^\/api\//,
+    /^https:\/\/[\w.-]*climate-ref\.org\/api\//,
+  ],
+  // The backend's OpenTelemetry spans join the trace through `traceparent`.
+  propagateTraceparent: true,
   integrations: [],
 });

frontend/src/main.tsx

@@ -66,6 +66,8 @@ const router = createRouter({
   defaultPreloadStaleTime: 0,
 });
 
+Sentry.addIntegration(Sentry.tanstackRouterBrowserTracingIntegration(router));
+
 declare module "@tanstack/react-router" {
   interface Register {
     router: typeof router;

frontend/src/vite-env.d.ts

@@ -3,3 +3,8 @@
 
 /** Version of the frontend bundle, injected at build time from package.json. */
 declare const __APP_VERSION__: string;
+
+interface ImportMetaEnv {
+  /** Share of page loads traced by Sentry, from 0 to 1, read at build time. */
+  readonly VITE_SENTRY_TRACES_SAMPLE_RATE?: string;
+}