From b23f5524196c59e456a35bf6b330fc934648bb68 Mon Sep 17 00:00:00 2001 From: Alex Fournier Date: Tue, 17 Nov 2026 17:12:07 -0700 Subject: [PATCH] feat(observability): trace remote HTTP middleware calls Signed-off-by: Alex Fournier --- Cargo.lock | 3 + .../Cargo.toml | 3 + .../src/remote.rs | 317 ++++++++++++++++-- docs/observability/traces.mdx | 2 + 4 files changed, 291 insertions(+), 34 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 25a7c84793..49a12d6b0a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5242,6 +5242,8 @@ dependencies = [ "miette", "openshell-core", "openshell-extension-core", + "openshell-otel", + "openshell-otel-test-support", "openshell-supervisor-middleware-builtins", "prost", "prost-types", @@ -5249,6 +5251,7 @@ dependencies = [ "tokio-stream", "tonic", "tracing", + "tracing-subscriber", ] [[package]] diff --git a/crates/openshell-supervisor-middleware/Cargo.toml b/crates/openshell-supervisor-middleware/Cargo.toml index 1685212f70..3a1bfed58e 100644 --- a/crates/openshell-supervisor-middleware/Cargo.toml +++ b/crates/openshell-supervisor-middleware/Cargo.toml @@ -13,6 +13,7 @@ rust-version.workspace = true [dependencies] openshell-core = { path = "../openshell-core", default-features = false } openshell-extension-core = { path = "../openshell-extension-core" } +openshell-otel = { path = "../openshell-otel" } miette = { workspace = true } futures = { workspace = true } @@ -24,9 +25,11 @@ tonic = { workspace = true, features = ["channel", "server", "tls-native-roots"] tracing = { workspace = true } [dev-dependencies] +openshell-otel-test-support = { path = "../openshell-otel-test-support" } openshell-supervisor-middleware-builtins = { path = "../openshell-supervisor-middleware-builtins" } tokio = { workspace = true, features = ["test-util"] } tokio-stream = { workspace = true, features = ["net"] } +tracing-subscriber = { workspace = true } [lints] workspace = true diff --git a/crates/openshell-supervisor-middleware/src/remote.rs b/crates/openshell-supervisor-middleware/src/remote.rs index 8ecc01cb5b..96e9208ca5 100644 --- a/crates/openshell-supervisor-middleware/src/remote.rs +++ b/crates/openshell-supervisor-middleware/src/remote.rs @@ -22,11 +22,53 @@ use std::sync::Arc; use tonic::service::interceptor::InterceptedService; use tonic::transport::Channel; use tonic::{Request, Response, Status}; +use tracing::{Instrument as _, Span}; use crate::MIDDLEWARE_GRPC_MESSAGE_BYTES; type ExtensionChannel = InterceptedService; +fn http_rpc_span(operation: &'static str) -> Span { + tracing::info_span!( + "middleware_rpc", + otel.name = operation, + otel.kind = "client", + otel.status_code = tracing::field::Empty, + rpc.system.name = "grpc", + rpc.method = operation, + rpc.response.status_code = tracing::field::Empty, + error.type = tracing::field::Empty, + ) +} + +fn traced_request(request: Request) -> std::result::Result, Status> { + use tonic::service::Interceptor as _; + + let (metadata, extensions, message) = request.into_parts(); + let mut context = Request::from_parts(metadata, extensions, ()); + context.metadata_mut().remove("traceparent"); + context.metadata_mut().remove("tracestate"); + let context = openshell_otel::TraceContextInterceptor.call(context)?; + let (metadata, extensions, ()) = context.into_parts(); + Ok(Request::from_parts(metadata, extensions, message)) +} + +fn traced_response_stream( + response: std::result::Result>, Status>, + span: Span, +) -> std::result::Result, Status> { + match response { + Ok(response) => Ok(Box::pin(openshell_otel::TracedGrpcStream::new( + response.into_inner(), + span, + ))), + Err(status) => { + openshell_otel::record_grpc_status(&span, status.code()); + Err(status) + } + } +} + /// Adapts the borrowed runtime request contract to the owned protobuf service /// contract only when dispatch crosses a gRPC-shaped boundary. #[derive(Clone)] @@ -207,9 +249,24 @@ impl SupervisorMiddlewareEndpoint for RemoteMiddlewareService { request: Request, ) -> std::result::Result, Status> { let mut client = self.client.clone(); - // Legacy HTTP hooks stay callable until their removal in 0.2.0. - #[allow(deprecated)] - client.evaluate_http_request(request).await + let span = + http_rpc_span("openshell.middleware.v1.SupervisorMiddleware/EvaluateHttpRequest"); + let result = async { + let guard = openshell_otel::ErrorStatusGuard::current(); + let request = traced_request(request)?; + // Legacy HTTP hooks stay callable until their removal in 0.2.0. + #[allow(deprecated)] + guard.finish(client.evaluate_http_request(request).await) + } + .instrument(span.clone()) + .await; + openshell_otel::record_grpc_status( + &span, + result + .as_ref() + .map_or_else(Status::code, |_| tonic::Code::Ok), + ); + result } async fn open_websocket_session( @@ -231,15 +288,19 @@ impl SupervisorMiddlewareEndpoint for RemoteMiddlewareService { receiver: tokio::sync::mpsc::Receiver, ) -> std::result::Result { let mut client = self.response_client.clone(); - // Legacy HTTP hooks stay callable until their removal in 0.2.0. - #[allow(deprecated)] - let responses = client - .evaluate(Request::new(tokio_stream::wrappers::ReceiverStream::new( - receiver, - ))) - .await? - .into_inner(); - Ok(Box::pin(responses)) + let span = http_rpc_span("openshell.middleware.v1.HttpResponsePreReturn/Evaluate"); + let response = async { + let guard = openshell_otel::ErrorStatusGuard::current(); + let request = traced_request(Request::new( + tokio_stream::wrappers::ReceiverStream::new(receiver), + ))?; + // Legacy HTTP hooks stay callable until their removal in 0.2.0. + #[allow(deprecated)] + guard.finish(client.evaluate(request).await) + } + .instrument(span.clone()) + .await; + traced_response_stream(response, span) } async fn open_http_request_hook( @@ -247,13 +308,17 @@ impl SupervisorMiddlewareEndpoint for RemoteMiddlewareService { receiver: tokio::sync::mpsc::Receiver, ) -> std::result::Result { let mut client = self.http_hooks_client.clone(); - let responses = client - .evaluate_http_request_hook(Request::new(tokio_stream::wrappers::ReceiverStream::new( - receiver, - ))) - .await? - .into_inner(); - Ok(Box::pin(responses)) + let span = http_rpc_span("openshell.middleware.v1.HttpHooks/EvaluateHttpRequestHook"); + let response = async { + let guard = openshell_otel::ErrorStatusGuard::current(); + let request = traced_request(Request::new( + tokio_stream::wrappers::ReceiverStream::new(receiver), + ))?; + guard.finish(client.evaluate_http_request_hook(request).await) + } + .instrument(span.clone()) + .await; + traced_response_stream(response, span) } async fn open_http_response_hook( @@ -261,13 +326,17 @@ impl SupervisorMiddlewareEndpoint for RemoteMiddlewareService { receiver: tokio::sync::mpsc::Receiver, ) -> std::result::Result { let mut client = self.http_hooks_client.clone(); - let responses = client - .evaluate_http_response_hook(Request::new(tokio_stream::wrappers::ReceiverStream::new( - receiver, - ))) - .await? - .into_inner(); - Ok(Box::pin(responses)) + let span = http_rpc_span("openshell.middleware.v1.HttpHooks/EvaluateHttpResponseHook"); + let response = async { + let guard = openshell_otel::ErrorStatusGuard::current(); + let request = traced_request(Request::new( + tokio_stream::wrappers::ReceiverStream::new(receiver), + ))?; + guard.finish(client.evaluate_http_response_hook(request).await) + } + .instrument(span.clone()) + .await; + traced_response_stream(response, span) } } @@ -276,15 +345,17 @@ mod tests { use futures::StreamExt; use openshell_core::proto::middleware::v1::http_hooks_server::{HttpHooks, HttpHooksServer}; use openshell_core::proto::{ - HttpContinue, HttpPreflightResult, HttpRequestHookPreflight, HttpRequestHookResult, - HttpResponseHookPreflight, HttpResponseHookResult, SupervisorMiddlewarePhase, - http_preflight_result, http_request_hook_event, http_request_hook_result, - http_response_hook_event, http_response_hook_result, + HttpContinue, HttpDenial, HttpPreflightResult, HttpRequestHookPreflight, + HttpRequestHookResult, HttpResponseHookPreflight, HttpResponseHookResult, + SupervisorMiddlewarePhase, http_preflight_result, http_request_hook_event, + http_request_hook_result, http_response_hook_event, http_response_hook_result, }; use tokio::net::TcpListener; use tokio::sync::{mpsc, oneshot}; use tokio_stream::wrappers::TcpListenerStream; use tonic::transport::Server; + use tracing::instrument::WithSubscriber as _; + use tracing_subscriber::layer::SubscriberExt as _; use super::*; @@ -320,11 +391,17 @@ mod tests { preflight.phase, SupervisorMiddlewarePhase::PreCredentials as i32 ); + if preflight.middleware_name == "rpc-error" { + return Err(Status::unavailable("private-error-sentinel")); + } + let result = if preflight.middleware_name == "deny" { + http_request_hook_result::Result::Deny(HttpDenial::default()) + } else { + http_request_hook_result::Result::PreflightResult(continue_without_body()) + }; Ok(Response::new(Box::pin(futures::stream::iter([Ok( HttpRequestHookResult { - result: Some(http_request_hook_result::Result::PreflightResult( - continue_without_body(), - )), + result: Some(result), }, )])))) } @@ -352,17 +429,82 @@ mod tests { } } - #[tokio::test] + #[test] + fn requests_without_active_context_discard_stale_trace_headers_only() { + tracing::subscriber::with_default(tracing::subscriber::NoSubscriber::default(), || { + let mut request = Request::new("unchanged body"); + request + .metadata_mut() + .insert("traceparent", "forged".parse().unwrap()); + request + .metadata_mut() + .insert("tracestate", "forged=value".parse().unwrap()); + request + .metadata_mut() + .insert("authorization", "Bearer unchanged".parse().unwrap()); + let request = traced_request(request).unwrap(); + assert!(request.metadata().get("traceparent").is_none()); + assert!(request.metadata().get("tracestate").is_none()); + assert_eq!( + request.metadata().get("authorization").unwrap(), + "Bearer unchanged" + ); + assert_eq!(request.into_inner(), "unchanged body"); + }); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn http_hooks_rpcs_share_channel_and_rotating_bearer() { + let _tracing_lock = openshell_otel_test_support::tracing_test_lock().await; + let collector = openshell_otel_test_support::OtlpTestServer::start().await; + let provider = openshell_otel::build_provider(&openshell_otel::OtlpExportConfig { + endpoint: collector.endpoint(), + service_name: openshell_otel::ServiceName::Fixed("middleware-rpc-test"), + service_version: None, + resource_attributes: Vec::new(), + }) + .unwrap(); + let subscriber = tracing::Dispatch::new(tracing_subscriber::registry().with( + openshell_otel::layer_for_target_prefix( + &provider, + "middleware-test", + "openshell_supervisor_middleware", + ), + )); + exercise_http_hooks(&provider, collector) + .with_subscriber(subscriber) + .await; + } + + async fn exercise_http_hooks( + provider: &openshell_otel::SdkTracerProvider, + collector: openshell_otel_test_support::OtlpTestServer, + ) { + let phase = tracing::info_span!("test.middleware.phase"); let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind"); let address = listener.local_addr().expect("address"); let (authorization_tx, mut authorization_rx) = mpsc::unbounded_channel(); + let (context_tx, mut context_rx) = mpsc::unbounded_channel(); let (shutdown_tx, shutdown_rx) = oneshot::channel(); let server = tokio::spawn(async move { Server::builder() .add_service(HttpHooksServer::with_interceptor( HttpHooksService, move |request: Request<()>| { + context_tx + .send(( + request + .metadata() + .get("traceparent") + .and_then(|value| value.to_str().ok()) + .map(str::to_owned), + request + .metadata() + .get("tracestate") + .and_then(|value| value.to_str().ok()) + .map(str::to_owned), + )) + .unwrap(); authorization_tx .send( request @@ -408,6 +550,7 @@ mod tests { drop(request_tx); let mut requests = client .open_http_request_hook(request_rx) + .instrument(phase.clone()) .await .expect("request RPC"); assert_eq!( @@ -425,6 +568,13 @@ mod tests { authorization_rx.recv().await.as_deref(), Some("Bearer request-token") ); + provider.force_flush().unwrap(); + assert!( + collector.received().spans.is_empty(), + "the RPC is not complete before stream EOS" + ); + assert!(requests.next().await.is_none()); + let mut contexts = vec![(context_rx.recv().await.unwrap(), Some("OK"))]; bearer .update("response-token", i64::MAX) @@ -444,6 +594,7 @@ mod tests { drop(response_tx); let mut responses = client .open_http_response_hook(response_rx) + .instrument(phase.clone()) .await .expect("response RPC"); assert_eq!( @@ -461,9 +612,107 @@ mod tests { authorization_rx.recv().await.as_deref(), Some("Bearer response-token") ); + assert!(responses.next().await.is_none()); + contexts.push((context_rx.recv().await.unwrap(), Some("OK"))); + + for (mode, expected_status) in [ + ("rpc-error", Some("UNAVAILABLE")), + ("deny", Some("OK")), + ("drop", None), + ] { + let (tx, rx) = mpsc::channel(1); + tx.send(HttpRequestHookEvent { + event: Some(http_request_hook_event::Event::Preflight( + HttpRequestHookPreflight { + phase: SupervisorMiddlewarePhase::PreCredentials as i32, + middleware_name: mode.into(), + ..Default::default() + }, + )), + }) + .await + .unwrap(); + drop(tx); + let result = client + .open_http_request_hook(rx) + .instrument(phase.clone()) + .await; + if mode == "rpc-error" { + let error = result.err().expect("transport failure"); + assert_eq!(error.code(), tonic::Code::Unavailable); + assert_eq!(error.message(), "private-error-sentinel"); + } else { + let mut stream = result.unwrap(); + let result = stream.next().await.unwrap().unwrap(); + if mode == "deny" { + assert!(matches!( + result.result, + Some(http_request_hook_result::Result::Deny(_)) + )); + assert!(stream.next().await.is_none()); + } + // Dropping before EOS must not invent a successful terminal status. + } + assert_eq!( + authorization_rx.recv().await.as_deref(), + Some("Bearer response-token") + ); + contexts.push((context_rx.recv().await.unwrap(), expected_status)); + } drop((requests, responses, client)); + drop(phase); shutdown_tx.send(()).expect("shutdown"); server.await.expect("server task"); + provider.force_flush().unwrap(); + provider.shutdown().unwrap(); + let received = collector.shutdown().await; + let phase = received + .spans + .iter() + .find(|span| span.name == "test.middleware.phase") + .unwrap(); + assert_eq!(received.spans.len(), 6); + for ((context, state), expected_status) in contexts { + assert!(state.as_deref().unwrap_or_default().is_empty()); + let context = context.expect("trusted RPC trace context"); + let rpc = received + .spans + .iter() + .find(|span| { + context == format!("00-{}-{}-01", hex(&span.trace_id), hex(&span.span_id)) + }) + .expect("carrier identifies the actual RPC span"); + assert_eq!(rpc.parent_span_id, phase.span_id); + assert!(rpc.name.starts_with("openshell.middleware.v1.HttpHooks/")); + if let Some(status) = expected_status { + assert!(rpc.attributes.contains( + &openshell_otel_test_support::relay::string_attribute( + "rpc.response.status_code", + status + ) + )); + } else { + assert!( + !rpc.attributes + .iter() + .any(|attribute| attribute.key == "rpc.response.status_code") + ); + } + } + let spans = format!("{:?}", received.spans); + for secret in ["request-token", "response-token", "private-error-sentinel"] { + assert!(!spans.contains(secret)); + } + } + + fn hex(bytes: &[u8]) -> String { + use std::fmt::Write as _; + + let mut value = String::with_capacity(bytes.len() * 2); + for byte in bytes { + write!(value, "{byte:02x}").unwrap(); + } + value } } diff --git a/docs/observability/traces.mdx b/docs/observability/traces.mdx index abe075e27b..e473f7aec7 100644 --- a/docs/observability/traces.mdx +++ b/docs/observability/traces.mdx @@ -75,6 +75,8 @@ The CLI forwards `TRACEPARENT` and `TRACESTATE` from its environment on gateway The gateway also passes its trace context to compute drivers, so driver spans appear in the same trace as the gateway request that caused them. Built-in drivers export their spans through the gateway's process. Managed external drivers receive the trace endpoint, gateway name, and compute driver from the gateway. Drivers that you run yourself own their telemetry configuration. +Remote HTTP middleware calls carry the supervisor's active trace context, not headers supplied by the workload. Their gRPC client spans sit under the request or response middleware stage and stay open while the response stream is consumed. A middleware policy refusal is a successful RPC; transport failures record a bounded gRPC status without the error message or payload. Dropping a stream before its terminal status does not report success. WebSocket sessions and middleware configuration calls are outside this HTTP tracing path. + ## Next Steps - Configure [OTLP export](/how-it-works/gateways/configuration#otlp-export), including sampling and the `OTEL_*` variables the gateway honors.